diff --git a/RELEASES.json b/RELEASES.json index d992658ce5f2dca95b6c044ee2f9ec92c16ac47d..a855cc075b39979fe34ac53090074f06f2c6142c 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,18 @@ { - "current": "6a642e1", + "current": "v25 (bc24829)", "releases": [ + { + "version": "v25", + "sha": "bc24829", + "date": "2026-08-14", + "subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)" + }, + { + "version": "v26", + "sha": "6a642e1", + "date": "2026-08-14", + "subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)" + }, { "version": "v24", "sha": "514d829", diff --git a/VERSION b/VERSION index 0bf110a675db86b8830d7320c052d74ddd1ea098..a91a1a22b1f8a24f211e83676adbae666a565ea0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6a642e1 +v25 (bc24829) diff --git a/api/ai_review.py b/api/ai_review.py index 488eb3f83d4c3a76d4abc4105ddfda1e3a2a6dac..5f5d12ec06e026428249349d16ac14d82c458144 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -205,258 +205,3 @@ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): "problem": reason or "the model did not choose one of the stages"} return choice, {"provider": p["name"], "model": model, "reason": reason} return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} - - -# ══════════════════════════════════════════════════════════════════ the FLOW WRITER -# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. -# -# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the -# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: -# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the -# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object -# in that vocabulary. -# -# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This -# is the one module in the API package that already holds an LLM ladder and its credentials, so a -# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs -# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 -# shipped three finished routers 404-dead behind green gates. The DOOR is a route on -# `routes_automation.py`, which is already mounted. -# -# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds -# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are -# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema -# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters -# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise -# and answers 200, so a draft accepted without the diff would show a person a flow that is not the -# flow the model described, with nothing anywhere saying so. -# -# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door -# can be proven end to end with no API key and no spend. A gate that can only run where a -# credential exists is a gate that never runs. - -#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: -#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you -#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, -#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. -FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") -MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 -MAX_PROMPT_CHARS = 2000 - - -def flow_providers(): - """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.""" - pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() - by_name = {p["name"]: p for p in PROVIDERS} - order = [pin] if pin else list(FLOW_PROVIDER_ORDER) - return [by_name[n] for n in order - if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] - - -def flow_schema(kinds, trigger_keys, table_keys): - """The tool schema the model fills — the catalog's OWN key lists as enums. - - ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the - transport rather than by a check afterwards. A model asked for "any action name" invents - `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a - 400 from `clean_actions` carrying a sentence about a word the person never typed. - ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a - schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a - transport failure (`routes_query._spec_schema` carries the same note, for the same measured - reason). - """ - return { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": ["flow", "refused"], - "description": "refused = this cannot be built from the steps available"}, - "refusal": {"type": "string", - "description": "when kind=refused: ONE plain sentence naming what is " - "missing, in the words a non-technical person would use"}, - "name": {"type": "string", "description": "a short title for the automation"}, - "trigger": {"type": "string", "enum": sorted(trigger_keys), - "description": "what starts this automation"}, - "table": {"type": "string", "enum": sorted(table_keys), - "description": "the database whose records this flow walks, if any"}, - "actions": { - "type": "array", - "maxItems": MAX_DRAFT_ACTIONS, - "items": { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": sorted(kinds)}, - "why": {"type": "string", - "description": "one short sentence: why this step is here"}, - "config": {"type": "object", "additionalProperties": True, - "description": "the step's settings, using ONLY the keys named " - "for that kind in the system message"}, - }, - "required": ["kind"], - }, - }, - }, - "required": ["kind"], - } - - -def flow_system_prompt(catalog, required, triggers, tables): - """What the model is told it may build with — DERIVED, never written down twice. - - Every list here is the server's own: the catalog rows the menu paints, the required-key table - the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases - with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to - the catalog is offered here on the same deploy and a kind removed stops being offered. - """ - lines = ["You build small automations for a business tool. You are given the EXACT set of " - "steps this tool can perform. You may use nothing else.", - "", "THE STEPS YOU MAY USE:"] - # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the - # PHRASE is the only human wording of that requirement anywhere — the one the runner's own - # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a - # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing - # what to call the setting. Handing it one and hoping is how a draft comes back configured - # against a key the validator drops. - req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} - for row in catalog or []: - if not row.get("ready"): - continue - kind = str(row.get("kind") or "") - need = req.get(kind) or [] - lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" - + (f" REQUIRED settings: {', '.join(need)}" if need else "")) - lines += ["", "WHAT CAN START AN AUTOMATION:"] - for t in triggers or []: - if t.get("planned") or not t.get("ready", True): - continue - lines.append(f"- {t.get('key')}: {t.get('label')}") - lines += ["", "THE DATABASES THIS PERSON HAS:"] - for t in (tables or [])[:40]: - cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) - lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") - if not tables: - lines.append("- (none - do not name a database)") - lines += [ - "", - "RULES:", - "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " - "answer kind=refused and say plainly which capability is missing.", - "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " - "rather than inventing a web address, a CSS selector or a column name.", - "3. A web address must start with http:// or https://.", - "4. To use a value from the record the flow is walking, write it as {{Column name}}.", - "5. Name a database only from the list above, by its key.", - "6. Keep it short. Fewer steps that work beat more steps that guess.", - ] - return "\n".join(lines) - - -def _flow_from_tool_call(obj): - """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" - if not isinstance(obj, dict): - return None, "the assistant's answer could not be read" - if str(obj.get("kind") or "") == "refused": - return None, (str(obj.get("refusal") or "").strip() - or "the assistant could not build this from the steps available") - acts = obj.get("actions") - if not isinstance(acts, list) or not acts: - return None, ("the assistant did not produce any steps - try describing what should " - "happen, one action at a time") - out = [] - for a in acts[:MAX_DRAFT_ACTIONS]: - if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): - continue - cfg = a.get("config") - out.append({"kind": str(a["kind"]).strip(), - "config": cfg if isinstance(cfg, dict) else {}, - "why": str(a.get("why") or "").strip()[:200]}) - if not out: - return None, "the assistant's steps could not be read" - return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", - "trigger": str(obj.get("trigger") or "").strip(), - "table": str(obj.get("table") or "").strip(), - # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, - # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need - # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the - # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past - # the twelfth and said nothing, so a model that answered with a twenty-step journey had - # eight steps deleted between the answer and the screen, with no key anywhere in the - # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this - # ceiling is the draft door's own, which makes reporting it the whole obligation. - # ⚠ Found by the verifier that checked this ticket, not by a gate. - "asked": len([a for a in acts if isinstance(a, dict)]), - "actions": out}, "" - - -def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None): - """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. - - Exactly one of `draft` and `refusal_sentence` is truthy — the same contract - `web_agent.run_step` keeps, so a caller has no third case to get wrong. - """ - text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] - if not text: - return None, "type what you want the automation to do", None - kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) - if not kinds: - return None, "this deployment offers no automation steps to build with", None - trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) - if t.get("key") and not t.get("planned")}) or ["manual"] - table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] - tools = [{"type": "function", "function": { - "name": "build_automation", - "description": "Emit the automation, or refuse.", - "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] - messages = [{"role": "system", - "content": flow_system_prompt(catalog, required, triggers, tables)}, - {"role": "user", "content": text}] - - if chat is not None: - # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal - # branches as a live call; only the transport is replaced. - draft, refusal = _flow_from_tool_call(chat(messages, tools)) - return draft, refusal, "injected" - - provs = flow_providers() - if not provs: - # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was - # never built [[flag-shipped-without-its-writer]]. - return None, ("the assistant is not configured on this deployment, so an automation cannot " - "be drafted from a description yet"), None - tmo = float(timeout or TIMEOUT_SECONDS) - problems = [] - for p in provs: - if p["shape"] != "openai": - # Anthropic's tool transport differs; it stays on the ladder for `decide()` and is - # SKIPPED here rather than half-supported. Named in `problems` so it is not invisible. - problems.append(f"{p['name']}: tool calls are not wired for this shape") - continue - try: - r = requests.post(p["url"], timeout=tmo, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1500}) - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: {type(e).__name__}") - continue - if r.status_code != 200: - problems.append(f"{p['name']}: HTTP {r.status_code}") - continue - try: - calls = (((r.json().get("choices") or [{}])[0].get("message") or {}) - .get("tool_calls") or []) - args = json.loads(calls[0]["function"]["arguments"]) if calls else None - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") - continue - draft, refusal = _flow_from_tool_call(args) - if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": - # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more - # expensive rung — the model has just said this cannot be built. `decide()` takes the - # same posture for the same reason. - return None, refusal, p["name"] - if draft is not None: - return draft, "", p["name"] - problems.append(f"{p['name']}: {refusal}") - return None, ("; ".join(problems)[:300] or "no provider answered"), None diff --git a/api/automation_engine.py b/api/automation_engine.py index 8e10a3845d227a38a734c89154bc3c1cf4b15a5b..7b272a33112e098767ae00cc21e1a74d188df0a5 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -1,13949 +1,13468 @@ -"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. - -THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant -store and a RUN that lives in this process. The definition is durable, small and rarely written; -the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file -is built to avoid, and it has two halves: - - * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the - store outlives the process that wrote it, so a container restart mid-run leaves an automation - that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives - in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written - exactly ONCE per run, at the end. - * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s - floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape - by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes - its whole result first and commits it in a single `update` callback. - -WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not -deployed, so its extraction core is copied into this file (the wave doc's instruction), ported -httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with -it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: -the skill version takes a URL from a developer on a CLI; this takes one from a request body. -""" -from __future__ import annotations - -import copy -import datetime as _dt -import hashlib -import hmac -import ipaddress -import json -import os -import re -import socket -import threading -import time -from urllib.parse import urljoin, urlparse - -import requests - -# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability -# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — -# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the -# connector that routes, which is the shape the split was for: this file no longer knows that -# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore -# a signal, not a convenience — it means something in the automation runtime started making a -# vendor decision, and that belongs one layer down. - -try: # the parser the extraction half needs - from bs4 import BeautifulSoup -except Exception: # pragma: no cover — deploy lag; see mailbox ② - BeautifulSoup = None - -# --------------------------------------------------------------------------------------------- -# THE BUCKET (C4-AUTO) -# --------------------------------------------------------------------------------------------- - -#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. -STORE_KEY = "automations" -#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through -#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for -#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the -#: session-D mailbox as a finding against A's file rather than fixed from here. -UT_STORE_KEY = "user_tables" -UT_PREFIX = "ut_" - -MAX_AUTOMATIONS = 40 -MAX_RUNS = 20 # trimmed history per automation -MAX_NAME = 80 -#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x -#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it -#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently -#: walking 5,000 of 32,826 connected rows. It was. -#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from -#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no -#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is -#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. -#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing -#: should not also be the widest one. And do not delete it — D-143's own row says the editable -#: substrate still needs a bound. -MAX_UT_ROWS = 5000 -MAX_UT_TABLES = 40 - -#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. -#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; -#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real -#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — -#: indistinguishable on every surface from a run that worked, and a strong candidate for why the -#: owner's enrichment kept appearing to do nothing. -#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better -#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. -ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" - -#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. -#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the -#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull -#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's -#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour -#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still -#: report success. A time series that stops after five days without saying so is worse than one -#: that was never built. -#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE -#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: -#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day -#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts -#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had -#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the -#: silent version would have flatlined a chart inside a fortnight with a green dot over it. -#: -#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises -#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, -#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 -#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a -#: smaller number. -#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) -MAX_UT_IG_ROWS = 200_000 -IG_TABLE_PREFIX = "ut_ig_" - -#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was -#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — -#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY -#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would -#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket -#: serialisation cost, purely by an accident of naming. Naming the append tables is the version -#: that stays true when the next `ut_ig_*` table is not one. -IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" -IG_POSTS_TABLE = "ut_ig_posts" -IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" -IG_COMMENTS_TABLE = "ut_ig_comments" - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY -# --------------------------------------------------------------------------------------------- -# R1: FULL PARITY — profile AND posts AND comments, not a profile tier. -# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live -# Instagram rows, and the two schemas may DIVERGE where the vendors do. -# -# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is -# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity -# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`. -# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on -# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video -# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables -# holding hundreds of thousands of rows is a migration on production data to buy a shared grid -# nobody asked for. R2 chose the version with no migration. -# -# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn -# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST -# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says -# `tt_id` rather than inheriting a column labelled "Instagram id". -TT_TABLE_PREFIX = "ut_tt_" -TT_PROFILE_TABLE = "ut_tt_profile" -TT_SNAPSHOTS_TABLE = "ut_tt_snapshots" -TT_POSTS_TABLE = "ut_tt_posts" -TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots" -TT_COMMENTS_TABLE = "ut_tt_comments" - -AUTOMATION_RECORD_MODE = "automation" -#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and -#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per -#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET, -#: which is the mechanism the note above says survives the next table that is not an append table. -APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE, - TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE}) - -#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. -#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger -#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram -#: column, an Instagram search — and they survive on the automations that already use them -#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). -#: -#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: -#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind -#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was -#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole -#: description: three machine nodes it does not have, and a one-line summary about searching -#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. -#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the -#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It -#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple -#: with no runner is a control that must refuse. -KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok") -#: ⭐⭐ WAVE 30 · T04 — THE DISCOVERY KINDS UNDER ONE NAME, and this constant is a bug fix rather -#: than tidying. `discover_tiktok` shipped in wave 29 by being added to `KINDS`, `RUNNERS`, -#: `clean_config` and `TRIGGER_*` — four sites that were found — while FIVE more tested the string -#: `"discover_instagram"` directly and were not. The visible symptom was the owner's: picking the -#: TikTok trigger 400'd with *"say how many profiles to fetch"* on the FIRST save, because the seed -#: below was one of the five. The other four are silent — a canvas with no `find` panel, a flow -#: table resolving to the wrong default, and a summary sentence announcing the automation would -#: "do nothing yet". -#: -#: ⛔ SO THE RULE IS: A DISCOVERY BRANCH TESTS MEMBERSHIP OF THIS TUPLE, NEVER A KIND STRING. -#: Three parallel string comparisons is precisely how the third platform gets missed twice more, -#: and the misses are individually invisible — each one degrades a different surface, none of them -#: raises, and every gate stays green (this whole family shipped green in wave 29). -#: ⚠ The kind ↔ platform facts that genuinely DIFFER — the dataset id, the default target table, -#: the field map — stay resolved per kind where they are used. This tuple answers *"is this a -#: corpus search?"* and nothing else; widening it into a platform registry would just move the -#: problem somewhere with a longer name. -DISCOVERY_KINDS = ("discover_instagram", "discover_tiktok") -#: ⭐ WAVE 30 · T06 — THE TRIGGER HALF, and it is a SEPARATE map because the two are NOT -#: interchangeable, which cost a red to learn. Law 1 makes a discovery TRIGGER choose its kind, so -#: `trigger ⇒ kind` always holds — but the converse does NOT: a definition can carry -#: `kind: "discover_instagram"` with no trigger at all (a direct API create does exactly that, and -#: `verify_automation`'s `_discover_defn` fixture is one). Testing the KIND where the rule is about -#: the trigger therefore fires on definitions the picker could never produce — measured: it spawned -#: a preset database under a dry-run fixture that asserts none exists. -#: ⛔ SO: "did somebody PICK a corpus search in the picker?" reads THIS. "Is this stored definition a -#: corpus search?" reads `DISCOVERY_KINDS`. Two questions, two maps, and the gate asserts this one -#: agrees with law 1 rather than trusting that it does. -DISCOVERY_TRIGGER_KIND = {"ig_profile_match": "discover_instagram", - "tiktok_profile_match": "discover_tiktok"} -#: ⭐ WAVE 30 · T08 — the ENRICH action kinds, one per network. Same argument as `DISCOVERY_KINDS` -#: one paragraph up: these two share a validator, a selection, a cooldown and a run summary, and the -#: only things that differ are which connector answers and which tables the rows land in. -#: ⛔ ONE VALIDATOR, NOT TWO. `clean_actions`' enrich branch is ~40 lines of clamps whose comments -#: record why each one is shaped the way it is (`submitted=False` because the client re-posts the -#: whole action list; `limit` clamped twice because the run sees STORED configs). A second copy for -#: TikTok would start identical and drift, and the way it fails is that one network silently accepts -#: a limit the other refuses. -ENRICH_KINDS = ("enrich_instagram", "enrich_tiktok") -#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores -#: when the body names none, which after R6 is every create the client makes. -DEFAULT_KIND = "plain" -#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — -#: the ruling retires the door, not the two automations behind it (see `create`). -#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. -RETIRED_KINDS = ("scrape_db", "field_instagram") -#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were -#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name -#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence -#: per kind, in one place, because `create` and `clean_definition` both say it. -RETIRED_KIND_REPLACEMENT = { - "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", - "scrape_db": "Add a scrape step to any automation instead", -} -#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the -#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. -#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any -#: flow can take, so refusing the kind costs a person nothing but a different click. -#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared -#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a -#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch -#: door open on purpose, and there is a gate check that says so in those words. A kind may only be -#: walled off at a door once something else answers the same need. -#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) -KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database", - "field_instagram": "Instagram profile column", - "discover_instagram": "Find Instagram profiles", - "discover_tiktok": "Find TikTok profiles"} -EXTRACTS = ("table", "jsonld") -STATES = ("idle", "running", "ok", "error", "partial") -#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). -#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a -#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). -TIERS = ("anonymous", "brightdata") -#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap -#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown -#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation -#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps -#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. -TIER_ALIASES = {"hiker": "brightdata"} - - -def clean_tier(raw): - """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" - t = str(raw or "").strip().lower() - t = TIER_ALIASES.get(t, t) - return t if t in TIERS else "" - - -def row_cap(table_key): - """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. - - An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything - else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. - """ - return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS - -#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI -#: shows and the vocabulary the parser accepts cannot drift. -CRON_PRESETS = [ - {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, - {"cron": "0 * * * *", "label": "Hourly"}, - {"cron": "0 6 * * *", "label": "Daily at 06:00"}, - {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, - {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, -] - -UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" - - -def _now(): - return _dt.datetime.now() - - -def _stamp(dt=None): - return (dt or _now()).strftime("%Y-%m-%d %H:%M") - - -def _iso(dt=None): - """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). - - ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables - and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A - naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC - minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is - not expressible at all. With the offset the same string is an instant, and relative rendering - becomes possible without migrating a single stored row. - - ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in - this module is against a naive `_now()`, and mixing aware and naive datetimes raises - `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. - """ - dt = dt or _now() - if dt.tzinfo is None: - dt = dt.astimezone() # a naive stamp from this process IS local time - return dt.isoformat(timespec="seconds") - - -#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. -#: -#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one -#: function down: `first_found` cells exist as post-D-18 offset stamps -#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s -#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A -#: converter that understood only the newest shape would blank the oldest rows — and a migration -#: that empties cells is indistinguishable from one that moved them. -#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" -#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and -#: shows up as itself instead of disappearing. -def _day(s): - """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" - raw = str(s or "").strip() - if not raw: - return "" - head = raw.replace("T", " ").split(" ")[0] - try: - _dt.date.fromisoformat(head) - except ValueError: - dt = _parse_iso(raw) - return dt.strftime("%Y-%m-%d") if dt else "" - return head - - -#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. -#: -#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` -#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every -#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse -#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 -#: dialect and a 0–100 dialect, and they meet here). -#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null -#: — and `0.0` would claim we measured zero engagement. -def _pct100(v): - """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" - if v is None or (isinstance(v, str) and not v.strip()): - return "" - try: - return _s(round(float(v) * 100.0, 4)) - except (TypeError, ValueError): - return "" - - -def _parse_iso(s): - """A stamp → a NAIVE LOCAL datetime, with or without an offset. - - Both shapes exist in the store simultaneously and always will: every run committed before - D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new - shape would silently return None for them — and `is_due` reads None as "never ran", which - would re-fire every schedule once. Reading both is what makes the change additive. - """ - raw = str(s or "").strip().replace("Z", "+00:00") - dt = None - try: - dt = _dt.datetime.fromisoformat(raw) - except Exception: # noqa: BLE001 - try: - dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") - except Exception: # noqa: BLE001 - return None - return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt - - -# --------------------------------------------------------------------------------------------- -# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher -# --------------------------------------------------------------------------------------------- -# The skill version guards the URL a developer typed. This one guards a URL that arrived in a -# request body, which changes the threat model in two ways the original does not cover: -# -# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` -# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` -# sails straight past a guard that only ever saw the first URL. The negative control ("a -# localhost URL is refused") would still pass while the rail was wide open. `fetch` below -# therefore takes the hops MANUALLY and re-guards every one. -# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the -# host and checks EVERY answer, not just the literal. -# -# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the -# record between the guard and the socket is not stopped by it. Closing that needs a pinned -# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather -# than implied away — the rail refuses the realistic cases and says what it does not cover. - -class Refused(ValueError): - """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" - - -def _ip_public(ip): - return not (ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_reserved or ip.is_unspecified or ip.is_multicast) - - -def guard(url, allowed=()): - """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers - checked too, plus the optional `--allowed` domain rail scrape.py carries.""" - p = urlparse(url or "") - if p.scheme not in ("http", "https"): - raise Refused(f"only http(s) allowed, not {p.scheme!r}") - host = (p.hostname or "").strip() - if not host: - raise Refused("no host in the URL") - low = host.lower() - if (low in ("localhost", "localhost.localdomain") - or low.endswith(".local") or low.endswith(".internal") - or low.endswith(".localhost")): - raise Refused(f"refused non-public host {host!r} (public web only)") - try: # a bare-IP host is decided on the literal - ip = ipaddress.ip_address(low) - except ValueError: - ip = None - if ip is not None and not _ip_public(ip): - raise Refused(f"refused non-public IP {host} (public web only)") - if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): - raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") - if ip is None: - try: - infos = socket.getaddrinfo(host, None) - except Exception as e: - raise Refused(f"could not resolve {host!r}: {type(e).__name__}") - for info in infos: - addr = info[4][0] - try: - resolved = ipaddress.ip_address(addr.split("%")[0]) - except ValueError: - continue - if not _ip_public(resolved): - raise Refused( - f"refused {host!r}: it resolves to the non-public address {resolved}") - return True - - -def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): - """GET `url`, taking redirects BY HAND so every hop passes `guard`. - - Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, - not a silent truncation — an endless redirect is indistinguishable from an attempt to walk - the fetcher somewhere it was told not to go. - """ - current = url - hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} - hdrs.update(headers or {}) - for _hop in range(max_hops + 1): - guard(current, allowed) - r = requests.get(current, timeout=timeout, headers=hdrs, - allow_redirects=False, stream=True) - if r.status_code in (301, 302, 303, 307, 308): - loc = r.headers.get("Location") - r.close() - if not loc: - raise Refused(f"redirect {r.status_code} with no Location header") - current = urljoin(current, loc) - continue - body = r.raw.read(max_kb * 1024, decode_content=True) or b"" - status, final = r.status_code, str(r.url) - r.close() - return status, final, body - raise Refused(f"more than {max_hops} redirects — refusing to follow further") - - -def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): - """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. - - ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by - hand because a redirected GET is ordinary. A redirected POST is not: re-sending a - credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail - exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever - re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. - (The vendor calls below are the only POSTs this module makes, and they carry the API key.) - """ - guard(url) - hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} - hdrs.update(headers or {}) - r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, - data=json.dumps(body if body is not None else {}), stream=True) - try: - if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, " - f"never re-sent to a location the server picked") - return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") - finally: - r.close() - - -# --------------------------------------------------------------------------------------------- -# EXTRACTION — vendored from scrape.py, unchanged in behaviour -# --------------------------------------------------------------------------------------------- - -def _soup(body): - if BeautifulSoup is None: - raise RuntimeError( - "beautifulsoup4 is not installed in this environment — the automation engine's " - "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " - "aios-web/requirements.txt).") - try: - return BeautifulSoup(body, "lxml") - except Exception: - return BeautifulSoup(body, "html.parser") - - -def tables(soup): - """Every HTML table as rows. Dict rows when the first row looks like a header.""" - out = [] - for t in soup.find_all("table"): - rows = [] - for tr in t.find_all("tr"): - cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] - if cells: - rows.append(cells) - if not rows: - continue - head, body = rows[0], rows[1:] - if body and len(head) == len(body[0]) and all(head): - out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) - else: - out.append(rows) - return out - - -def jsonld(soup): - out = [] - for s in soup.find_all("script", attrs={"type": "application/ld+json"}): - try: - out.append(json.loads(s.string or s.get_text())) - except Exception: - pass - return out - - -def meta(soup): - m = {} - if soup.title and soup.title.string: - m["title"] = soup.title.string.strip() - for tag in soup.find_all("meta"): - k = tag.get("name") or tag.get("property") - v = tag.get("content") - if k and v and (k in ("description", "keywords", "author") - or k.startswith(("og:", "twitter:"))): - m[k] = v.strip() - return m - - -def preview(url, extract="table", table_index=0, allowed=()): - """The field-map preview the editor calls BEFORE anything is created: what columns does this - page actually offer, and what do the first rows look like? Never writes.""" - status, final, body = fetch(url, allowed=allowed) - if not (200 <= status < 300): - return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status} — it may be bot-gated or gone", - "columns": [], "sample": [], "rowCount": 0} - soup = _soup(body) - rows = [] - if extract == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 - picked = found[idx] if found else [] - rows = [r for r in picked if isinstance(r, dict)] - cols, seen = [], set() - for r in rows[:50]: - for k in r: - if k not in seen: - seen.add(k) - cols.append(k) - return {"ok": True, "status": status, "url": final, "columns": cols, - "sample": rows[:8], "rowCount": len(rows), - "tableCount": len(tables(soup)) if extract != "jsonld" else 0, - "title": (meta(soup) or {}).get("title", "")} - - -def _scalar(v): - if isinstance(v, (str, int, float)) and not isinstance(v, bool): - return str(v) - if isinstance(v, bool): - return "1" if v else "" - if isinstance(v, dict): - return str(v.get("name") or v.get("@id") or "") - if isinstance(v, list): - return ", ".join(_scalar(x) for x in v[:8]) - return "" - - -# --------------------------------------------------------------------------------------------- -# CRON — a 5-field parser and a "what was the last fire time" walk -# --------------------------------------------------------------------------------------------- -# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward -# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute -# between the last run and now?"* — and answering it that way is what makes a missed tick -# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and -# not never. - -_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] - - -def _parse_field(spec, lo, hi): - out = set() - for part in str(spec).split(","): - part = part.strip() - if not part: - raise ValueError("empty cron field part") - step = 1 - if "/" in part: - part, _, s = part.partition("/") - step = int(s) - if step < 1: - raise ValueError("cron step must be >= 1") - if part in ("*", "?"): - a, b = lo, hi - elif "-" in part.lstrip("-"): - a_s, _, b_s = part.partition("-") - a, b = int(a_s), int(b_s) - else: - a = b = int(part) - if a < lo or b > hi or a > b: - raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") - out.update(range(a, b + 1, step)) - return out - - -def parse_cron(expr): - """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. - - Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at - WRITE time, because a cron string nobody can evaluate is an automation that silently never - runs and reports no error anywhere. - """ - parts = str(expr or "").split() - if len(parts) != 5: - raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") - sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] - dom_r = parts[2].strip() not in ("*", "?") - dow_r = parts[4].strip() not in ("*", "?") - return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), - dom_r, dow_r) - - -def _day_matches(day, doms, months, dows, dom_r, dow_r): - if day.month not in months: - return False - # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the - # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes - # a schedule that looks right fire almost never. - dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 - if dom_r and dow_r: - return day.day in doms or dow in dows - if dom_r: - return day.day in doms - if dow_r: - return dow in dows - return True - - -def prev_fire(expr, now=None, lookback_days=400): - """The most recent scheduled minute at or before `now`, or None inside the lookback. - - Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, - and only a matching day needs its hours/minutes searched. - """ - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - now = (now or _now()).replace(second=0, microsecond=0) - for back in range(lookback_days + 1): - day = (now - _dt.timedelta(days=back)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = back == 0 - for h in sorted(hours, reverse=True): - if same_day and h > now.hour: - continue - for m in sorted(minutes, reverse=True): - if same_day and h == now.hour and m > now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -def is_due(defn, now=None): - """Should the scheduler run this automation right now? - - Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the - moment the schedule was enabled, else creation) and at or before now. Anchoring on - `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at - 14:00" from running immediately and looking like a bug. - """ - if not isinstance(defn, dict): - return False - sched = defn.get("schedule") or {} - if not sched.get("enabled"): - return False - try: - fire = prev_fire(sched.get("cron"), now=now) - except ValueError: - return False - if fire is None: - return False - since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) - or _parse_iso(sched.get("enabledAt")) - or _parse_iso(defn.get("created"))) - return since is None or fire > since - - -def next_fire(expr, now=None, lookahead_days=400): - """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" - try: - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - except ValueError: - return None - now = (now or _now()).replace(second=0, microsecond=0) - for ahead in range(lookahead_days + 1): - day = (now + _dt.timedelta(days=ahead)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = ahead == 0 - for h in sorted(hours): - if same_day and h < now.hour: - continue - for m in sorted(minutes): - if same_day and h == now.hour and m <= now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -# --------------------------------------------------------------------------------------------- -# THE UPSERT — pure, so the arithmetic is testable without a store or a network -# --------------------------------------------------------------------------------------------- - -#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. -#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled -#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by -#: pre-collected corpus values of unknown vintage — no error, no visible change other than the -#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this -#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. -#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus -#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read -#: belongs. Suppressing the observation would be the same defect arriving from the other side. -CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") - - -def corpus_protect(before, _src): - """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. - - ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell - non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making - `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A - re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read - owns. - """ - return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () - - -def upsert_rows(existing, incoming, key_field, cap=None, protect=None): - """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. - - `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row - may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are - untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to - the merge, and a rule baked in here would apply to nine paths that never asked for one. - - THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped - appearing on the source page has not necessarily stopped existing — the page changed its - filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream - hiccup into silent data loss, and the row may be carrying user-typed overlay values in - columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. - - Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. - - ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on - entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to - call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` - that same loop is hundreds of millions of dict copies and the automation simply never - finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) - - ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different - facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the - DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — - a property of the SYSTEM, and never benign. One number for both meant a table hitting its - ceiling was indistinguishable from a page with a few blank cells, which is how a time series - stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. - """ - cap = MAX_UT_ROWS if cap is None else int(cap) - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, - "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} - by_key, dupe_ids = {}, set() - for rid, row in rows.items(): - kv = str(row.get(key_field, "") or "").strip() - if not kv: - continue - if kv in by_key: - dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left - continue - by_key[kv] = rid - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 - seen_keys, incoming_dupes = set(), 0 - for src in incoming or []: - kv = str((src or {}).get(key_field, "") or "").strip() - if not kv: - counts["skipped"] += 1 # no key -> cannot be upserted; never guessed - continue - if kv in seen_keys: - incoming_dupes += 1 # the SOURCE listed it twice; first wins - continue - seen_keys.add(kv) - rid = by_key.get(kv) - if rid is None: - if len(rows) >= cap: - counts["capped"] += 1 # LOUD: the runner turns this into a partial run - continue - rid = str(next_id) - next_id += 1 - rows[rid] = dict(src) - by_key[kv] = rid - counts["inserted"] += 1 - continue - before = rows[rid] - # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. - # `held` counts the cells a lower-provenance source was refused, so the run can SAY it - # rather than quietly doing the right thing — a protection nobody is told about is - # indistinguishable from a source that happened to agree. - keep = set(protect(before, src) or ()) if protect else set() - use = {k: v for k, v in src.items() if k not in keep} if keep else src - if keep: - counts["held"] = counts.get("held", 0) + sum( - 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) - changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} - if changed: - before.update(use) - counts["updated"] += 1 - else: - counts["unchanged"] += 1 - counts["duplicates"] = incoming_dupes + len(dupe_ids) - counts["orphans"] = sum( - 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) - return rows, counts - - -def dedupe_canonical_rows(existing, key_field, newest_by=""): - """Collapse duplicate logical rows while preserving the lowest stable row id. - - Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: - repeated shortcodes there are new timestamped observations, not duplicate posts. Values are - taken newest-first and then filled from older rows, so a sparse fresh projection does not - erase a field an earlier row knew. - """ - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - groups = {} - for rid, row in rows.items(): - identity = str(row.get(key_field) or "").strip() - if identity: - groups.setdefault(identity, []).append((rid, row)) - removed = 0 - for members in groups.values(): - if len(members) < 2: - continue - keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) - ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ - if newest_by else members - merged = {} - for _rid, row in ordered: - for key, value in row.items(): - if key not in merged or str(merged.get(key) or "").strip() == "": - merged[key] = value - rows[keep] = merged - for rid, _row in members: - if rid != keep: - rows.pop(rid, None) - removed += 1 - return rows, removed - - -# --------------------------------------------------------------------------------------------- -# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) -# --------------------------------------------------------------------------------------------- - -def _ut_slug(label): - s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return (s or "table")[:40] - - -def ut_all(rt): - try: - return dict(rt.get(UT_STORE_KEY) or {}) - except Exception: - return {} - - -def disable_for_table(rt, table_key, note="target database deleted"): - """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. - - The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still - shows what existed and why it stopped — silently deleting a user's automation because its - target died would read as data loss. Returns the ids it touched.""" - key = str(table_key or "") - touched = [] - - def _up(cur): - for aid, d in (cur or {}).items(): - if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: - sch = d.get("schedule") - if not isinstance(sch, dict): - sch = d["schedule"] = {} - sch["enabled"] = False - d["statusNote"] = note - touched.append(str(aid)) - return cur - - if key: - _store_update(rt, _up, flush="sync") - return touched - - -def ut_get(rt, key): - return ut_all(rt).get(str(key)) - - -def retire_automation_stage_fields(rt, tables=None): - """Delete obsolete Board-only fields and their hidden cells, never user fields. - - The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` - metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old - generated timestamp/cycle cells are cleared too, including rows whose field definition was - removed by an interrupted earlier migration. Re-running this migration is a no-op. - - ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is - O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to - deep-copy (see this module's header), and `GET /automations` was paying for THREE independent - copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the - WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent - snapshot can never be the thing that gets written back. - """ - tables = ut_all(rt) if tables is None else tables - stage_keys = set() - for table in tables.values(): - if not isinstance(table, dict): - continue - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): - stage_keys.add(str(field.get("key") or "")) - for row in (table.get("rows") or {}).values(): - for key in (row or {}): - text = str(key) - if text.startswith("stage_auto_"): - stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) - stage_keys.discard("") - if not stage_keys: - return {"tables": 0, "fields": 0, "cells": 0} - - changed = {"tables": set(), "fields": 0, "cells": 0} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for table_key, table in cur.items(): - if not isinstance(table, dict): - continue - kept, removed = [], set() - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - key = str(field.get("key") or "") if isinstance(field, dict) else "" - if key in stage_keys and isinstance(auto, dict) and \ - (auto.get("stageField") or auto.get("cyclesField")): - removed.add(key) - changed["fields"] += 1 - continue - kept.append(field) - if removed: - table["fields"] = kept - changed["tables"].add(str(table_key)) - for row in (table.get("rows") or {}).values(): - if not isinstance(row, dict): - continue - for stage_key in stage_keys: - for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): - if key in row: - row.pop(key, None) - changed["cells"] += 1 - changed["tables"].add(str(table_key)) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return {"tables": len(changed["tables"]), "fields": changed["fields"], - "cells": changed["cells"]} - - -def bind_unbound_fields(rt): - """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` - are bound to the definition that already writes them — matched on the (targetTable, - fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no - definition references is DISABLED with the reason on it rather than deleted or guessed: - the column was already dead (nothing runs a column no flow names), and now it says so. - Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" - by_binding = {} - for aid, d in all_definitions(rt).items(): - cfg = d.get("config") or {} - if cfg.get("fieldKey") and cfg.get("targetTable"): - by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) - plan = {} - for tk, t in ut_all(rt).items(): - for f in (t.get("fields") or []): - a = f.get("automation") - # An already-DISABLED bag is a decision this migration made on a previous pass — - # re-planning it every call would turn the once-per-process sweep into a write - # per read. - if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ - and not a.get("disabled"): - plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) - if not plan: - return 0, 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for (tk, fk), aid in plan.items(): - for f in ((cur.get(tk) or {}).get("fields") or []): - if f.get("key") == fk and isinstance(f.get("automation"), dict): - if aid: - f["automation"]["flowId"] = aid - else: - f["automation"]["disabled"] = True - f["automation"]["statusNote"] = ( - "not bound to any flow — create an automation for this column " - "or delete it") - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - bound = sum(1 for v in plan.values() if v) - return bound, len(plan) - bound - - -def ut_key_for(label, key=None): - """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which - must not create anything) resolves the same key by construction rather than by copying the - derivation and drifting from it.""" - k = str(key or (UT_PREFIX + _ut_slug(label))) - return k if k.startswith(UT_PREFIX) else UT_PREFIX + k - - -#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same -#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging -#: `core` into the API's boot path — and the two are held in step by a gate check. -MACHINE_OWNERS = ("automation", "scheduler") - - -def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", - record_mode="", lock_fields=False, tables=None): - """Create the table if it is missing; return its key. Idempotent — a re-run of an automation - that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) - and an existing table with that key is adopted, not duplicated. - - ⭐⭐ WAVE 31 · T30 — `tables` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING, and it is - the SKIP TEST below that it pays for. `rt.get` deep-copies the whole tenant document on every - call (documented ceiling 35.8 MB / ~1.4 s), so a caller that ensures FOUR tables in one pass - paid four copies to answer four questions about one document. Owner item 7, verbatim: *"It still - takes forever to change the Config for automation as well. It just say Saving..."* — measured - **7,912 ms** for one save on `4258a93`. - - ⚠ THE LEND IS READ-ONLY AND SAFE BY INSPECTION, not by hope — the same argument - `retire_automation_stage_fields` (this module, same parameter name, same contract) already - makes: only the `have` lookup below reads it, and the WRITE still goes through `rt.update`, - whose `_up` re-reads the live document under the store lock. **A lent snapshot can therefore - never be the thing that gets written back**, and the worst a stale one can do is spend a commit - that would have been skipped — never write a wrong value. Callers that ensure two tables - under the SAME key in one pass pass `tables=None` for the second (see `_spawn_presets`). - - Fields are MERGED, never replaced: a user who added a column to an automation's table keeps - it, and a new source column joins on the next run. - - `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped - `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their - provenance. Fields already on the table keep whatever tag they have (first flow wins; - shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not - ownership). - - ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever - or WHATEVER ran the automation, so the same table belonged to a person if its first run was - manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` - admits only the creator or an admin, so **whether you could open your own Instagram - snapshots depended on a race you never saw**. The owner is now the automation's creator, and - a table already stamped with a machine name is ADOPTED the next time a run knows a human - one. Adoption is a repair, not a widening: the automation's creator is the person who asked - for the table in the first place. - """ - key = ut_key_for(label, key) - # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. - # - # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it - # would leave every table already in production on the old `text` schema forever while new - # ones got the honest types — the split schema R3's note warned about, arriving through the - # very function the note was written on. Migrating here means a table is brought forward - # immediately BEFORE anything appends to it, so no caller has to remember anything and no - # tenant is left behind by a script nobody ran. - # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is - # already current, which after the first run is every call. - if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): - try: - migrate_ig_tables(rt, log=lambda *_a: None, only=key) - except Exception: # noqa: BLE001 - # A migration that cannot run must not stop the automation from writing its rows. - # The old schema still reads; a refused write loses the pull we just paid for. - pass - wanted = [] - for raw_field in (fields or []): - field = dict(raw_field) - if flow_tag or lock_fields: - automation = dict(field.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - if lock_fields: - automation["preset"] = True - field["automation"] = automation - wanted.append(field) - created = _iso() - human = username if username and username not in MACHINE_OWNERS else "" - - # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store - # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo - # budget, an idempotent helper that always writes is the same defect as a per-row insert. - have = ut_get(rt, key) if tables is None else tables.get(str(key)) - have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} - missing = [f for f in wanted if f.get("key") not in have_fields] - missing_locks = [f for f in wanted - if lock_fields and f.get("key") in have_fields - and (not isinstance(have_fields[f.get("key")].get("automation"), dict) - or have_fields[f.get("key")]["automation"].get("preset") is not True)] - if (have is not None and not missing and not missing_locks - and not (record_mode and have.get("recordMode") != record_mode) - and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): - return key - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - if len(cur) >= MAX_UT_TABLES: - return cur - cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", - "createdBy": username, "created": created, - "fields": wanted, "rows": {}} - if record_mode: - cur[key]["recordMode"] = record_mode - return cur - have = {f.get("key") for f in (t.get("fields") or [])} - for f in wanted: - if f.get("key") not in have: - t.setdefault("fields", []).append(f) - have.add(f.get("key")) - elif lock_fields: - stored = next((g for g in (t.get("fields") or []) - if g.get("key") == f.get("key")), None) - if stored is not None: - automation = dict(stored.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - automation["preset"] = True - stored["automation"] = automation - # ADOPTION: a machine name is not an owner. It never overwrites a human one. - if human and (t.get("createdBy") or "") in MACHINE_OWNERS: - t["createdBy"] = human - # An automation's table SAYS an automation owns it — the nav badge reads from this. - t["source"] = t.get("source") or "Automation" - if record_mode: - t["recordMode"] = record_mode - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return key - - -def ut_write_rows(rt, key, rows): - """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t["rows"] = rows - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): - """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value - for k, rws in (extra_rows or {}).items(): - tt = cur.get(k) - if tt is not None: - tt["rows"] = rws - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -IG_FIELD_DESCRIPTIONS = { - "alt_text": "Accessibility text attached to the Instagram post.", - # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would - # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED - # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off - # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same - # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a - # description that named only the first would have quietly excluded the second. - "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " - "rather than measured at the time shown.", - "avg_comments_12": "Average comments across the 12 most recent captured posts.", - "avg_engagement": "Average engagement rate reported for the profile.", - "avg_likes_12": "Average likes across the 12 most recent captured posts.", - "avg_plays_12": "Average plays across the 12 most recent captured posts.", - "avg_views_12": "Average views across the 12 most recent captured posts.", - "views": "View count for the post, as Instagram displays it.", - "video_duration": "Length of the video in seconds.", - "comments_disabled": "Checked when the creator has turned comments off for this post.", - "plays": "Times the video started playing, including replays.", - "bio": "Biography shown on the Instagram profile.", - "bio_hashtags": "Hashtags listed in the profile biography.", - "business_category": "Instagram's business category for the account.", - "caption": "Caption published with the Instagram post.", - "category": "Instagram's category for the profile.", - "comment_key": "Unique ID for the captured comment record.", - "commented_at": "Date the comment was posted.", - "comments": "Comment count reported for the post.", - "comments_captured": "Number of distinct comment records linked to this profile.", - "comments_link": "Comment records linked to this record.", - "country_code": "Country code reported for the profile.", - "created_by": "User whose automation first added this lead.", - "enriched_at": "Date the profile was last enriched.", - # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number - # beside it is the only thing that says how much of one. - "location_guess": "Most common city tagged across this profile's captured posts - a guess, " - "not a stated location.", - "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " - "city.", - "external_url": "Website linked from the Instagram biography.", - "external_url_title": "Title Instagram shows for the biography link.", - "fbid": "Facebook ID associated with the Instagram account.", - "first_found": "Date an automation first found this profile.", - "followers": "Latest reported follower count.", - "following": "Latest reported number of accounts followed.", - "found_count": "Number of times discovery automations found this profile.", - "full_name": "Display name shown on the Instagram profile.", - "handle": "Instagram username without the @ symbol.", - "has_channel": "Checked when the profile has an Instagram channel.", - "hashtags": "Hashtags extracted from the post caption.", - "highlights_count": "Total story highlight collections reported for the profile.", - "ig_id": "Instagram's internal ID for the account.", - "influencer_key": "Normalized handle linking this row to its profile.", - "is_business": "Checked when Instagram marks the account as a business.", - "is_joined_recently": "Checked when Instagram marks the account as recently joined.", - "is_private": "Checked when the Instagram profile is private.", - "is_professional": "Checked when Instagram marks the account as professional.", - "last_found": "Date an automation most recently found this profile.", - "likes": "Like count reported for this record.", - "measured_at": "Date engagement metrics on this post were last read.", - "measurements_captured": "Number of measurement rows stored for this post.", - "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", - "partner": "Brand named in the post's paid partnership metadata.", - "partner_id": "Partner ID reported for the Instagram profile.", - "platform": "Social network for this profile.", - "plays": "Video plays, including repeat plays, reported for the post.", - "post_link": "Post record linked to this measurement or comment.", - "post_measurements_captured": "Number of distinct post measurements linked to this profile.", - "post_snapshot_key": "Unique ID for this post measurement.", - "post_snapshots_link": "Engagement measurement rows linked to this record.", - "posted_at": "Date the Instagram post was published.", - "posts_captured": "Number of distinct post records captured for this profile.", - "posts_count": "Total posts reported for the Instagram profile.", - "posts_link": "Post records linked to this profile.", - "profile_name": "Profile name returned by the data provider.", - "profile_reads": "Number of profile snapshots stored for this profile.", - "profile_snapshots_link": "Profile snapshot rows linked to this profile.", - "profile_url": "Direct URL to the Instagram profile.", - "pronouns": "Pronouns shown on the Instagram profile.", - "pulled_at": "Date this snapshot was collected.", - "related_accounts": "Accounts Instagram suggests alongside this profile.", - "replies": "Reply count reported for the comment.", - "shortcode": "Instagram shortcode that uniquely identifies the post.", - "source_payload": "Full source response for this record.", - "snapshot_key": "Unique ID for this profile snapshot.", - "source": "Method or provider used to read this profile.", - "tagged_location": "Location tagged on the post; it is not the creator's residence.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "type": "Instagram media type: image, video, or carousel.", - "url": "Direct URL to the Instagram post.", - "verified": "Checked when Instagram marks the profile as verified.", -} - - -#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the -#: map above is one line of `field_def`: it defaults `description` to -#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column -#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`, -#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram -#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check. -#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting -#: the Instagram sentence is the failure this map exists to prevent. -TT_FIELD_DESCRIPTIONS = { - "platform": "Network this handle is on.", - "handle": "TikTok @name; the unique account handle.", - "full_name": "Display name shown on the TikTok profile.", - "tt_id": "TikTok's own numeric id for the account.", - "profile_url": "Direct URL to the TikTok profile.", - "bio": "Profile biography text.", - "external_url": "Link in the TikTok bio.", - "verified": "Checked when TikTok marks the account as verified.", - "is_private": "Checked when the account is private.", - "followers": "Follower count at the time of the pull.", - "following": "Accounts this profile follows.", - "posts_count": "Videos published by this account.", - "likes_received": "Total likes this account's videos have received.", - "avg_engagement": "Average engagement rate, stored as a percentage.", - "comment_engagement": "Comment engagement rate, stored as a percentage.", - "like_engagement": "Like engagement rate, stored as a percentage.", - "is_business": "Approximate: set when TikTok flags the account as a commerce user.", - "country_code": "Two-letter country code reported for the account.", - "predicted_lang": "Language TikTok predicts for this account.", - "account_created_at": "When the TikTok account itself was created.", - "region": "Region reported for the account.", - "shortcode": "TikTok's numeric id for the video; unique per post.", - "type": "TikTok post type: video or image.", - "url": "Direct URL to the TikTok post.", - "caption": "Post description text.", - "posted_at": "When the creator published the post.", - "likes": "Likes reported for the post.", - "comments": "Comment count reported for the post.", - "views": "Play count TikTok reports for the video.", - "shares": "Times the post was shared.", - "saves": "Times the post was saved to a collection.", - "video_duration": "Video length in seconds.", - "hashtags": "Hashtags used in the post.", - "tagged_location": "Commerce location reported for the post; it is not the creator's home.", - "influencer_key": "Handle of the account this record belongs to.", - "comment_key": "Unique ID for this comment.", - "commented_at": "When the comment was posted.", - "text": "What the comment says. The commenter's name is not stored as a column.", - "replies": "Reply count reported for the comment.", - "snapshot_key": "Unique ID for this profile snapshot.", - "post_snapshot_key": "Unique ID for this post measurement.", - "pulled_at": "When this measurement was read.", - "enriched_at": "When this row was last enriched.", - "source": "Method or provider used to read this record.", - "source_payload": "Full source response for this record.", -} - - -def field_def(text_key, label, ftype="text", **extra): - """One machine-spawned column definition. - - ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, - `profile`), and it stays ONE constructor rather than growing a second for "special" fields. - ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s - W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole - preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). - - ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a - feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so - `core.user_tables._clean_field` — the single validator every field created through the ordinary - door passes — has never judged a single field this engine spawned. The difference was exactly - one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` - for every automation column in the product. - ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an - absent key was already not that, so the bypass was fail-closed and therefore silent. It is - stated now, and `verify_automation` asserts the equality field-by-field, so the next key - `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). - """ - description = " ".join(str(extra.pop( - "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) - out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", - "editRole": "admins"} - if description: - out["description"] = description - out.update(extra) - return out - - -def tt_field_def(text_key, label, ftype="text", **extra): - """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2). - - ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can - never be reached from here. An explicit empty string makes `field_def` omit the key, which is - the same shape an undescribed column already has — the point is that the sentence a TikTok - column carries is one somebody wrote about TikTok, or none at all. - """ - extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, "")) - return field_def(text_key, label, ftype, **extra) - - -# --------------------------------------------------------------------------------------------- -# DEFINITIONS — validation and the bucket's read/write half -# --------------------------------------------------------------------------------------------- - -def _s(v, n=200): - return str(v if v is not None else "")[:n] - - -#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. -#: -#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 -#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config -#: disagree with what the run can possibly do. -#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect -#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve -#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is -#: worse than one that refuses it, because the refusal is the only place the ceiling can be -#: taught. So this REFUSES, and the sentence names the cap and who set it. -#: -#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the -#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the -#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask -#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` -#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a -#: property of the call we happened to make. -#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of -#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE -#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to -#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it -#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. -MAX_POSTS_PER_PULL = 30 -DEFAULT_POSTS_PER_PULL = 10 - -#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately -#: NOT `MAX_POSTS_PER_PULL`. -#: -#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap -#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so -#: raising the capture cap to 30 silently made every one of them average thirty posts under a -#: label that says twelve. Nobody would have looked at those columns again to check. -#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different -#: decisions, and only the first one was made. To widen the average, change this constant AND the -#: four keys and labels together — a column whose meaning changes underneath its own name is -#: worse than one that is merely narrow. -AVG_WINDOW_POSTS = 12 - - -#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a -#: filter cannot ask for a category that can never match a stored row. -POST_TYPES = ("video", "image", "carousel") -#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason -#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video` -#: is the stored key: a client that translated it locally would be a second copy of this -#: vocabulary, free to drift the day a fourth kind appears or a name changes. -#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a -#: label naming only reels would over-promise on a plain video post. -POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"} - - -def clean_post_groups(raw, max_posts): - """`(groups, error)` for `config.postGroups` — "last N reels, last M images". - - ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: - `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. - - ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today - carries no such key, and inventing a filter for it would silently start DROPPING posts those - automations have always captured. Empty list and missing are the same answer. - - ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the - difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them - would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken - one, so the only way to see one is a person typing it, and a filter that silently ignores the - type you asked for is how you end up paying for reels and storing carousels. - """ - if raw in (None, "", [], {}): - return [], None - if not isinstance(raw, list): - return None, "the post groups have to be a list of {type, limit} entries" - out, seen = [], set() - for item in raw: - if not isinstance(item, dict): - return None, "each post group has to be a {type, limit} entry" - t = str(item.get("type") or "").strip().lower() - if t not in POST_TYPES: - return None, (f"'{t or 'blank'}' is not a post type — use one of " - f"{', '.join(POST_TYPES)}") - if t in seen: - return None, f"the post groups name '{t}' twice — one limit per type" - seen.add(t) - try: - lim = int(item.get("limit")) - except (TypeError, ValueError): - return None, f"how many {t} posts to keep has to be a whole number" - if lim < 1: - return None, f"a {t} group has to keep at least one post" - # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures - # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is - # refused HERE, where the number they typed is still on the screen in front of them. - if lim > max_posts: - return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " - f"group cannot keep {lim} — raise the post count first") - out.append({"type": t, "limit": lim}) - return out, None - - -def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): - """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. - - ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this - takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old - clamp's default, which this cap now forbids. If an inherited value were refused the same way - a typed one is, every one of those automations would 400 on its next Save **forever**, for a - number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator - that refuses stored data does not protect the user from it, it locks them out of their own - automation. - So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely - INHERITED gets silently brought inside the cap it was already effectively subject to at the - vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default, None - try: - n = int(raw) - except (TypeError, ValueError): - if not submitted: - return default, None - return None, "the post limit must be a whole number" - if not submitted: - return max(1, min(n, MAX_POSTS_PER_PULL)), None - if n < 1: - return None, "keep at least one post per profile, or turn the post capture off" - if n > MAX_POSTS_PER_PULL: - return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts — that is the " - f"vendor's cap, not ours, so {n} would store the same " - f"{MAX_POSTS_PER_PULL} and read as more") - return n, None - - -def clean_config(kind, raw, previous=None): - """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. - - ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail - rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not - edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" - from a panel that never knew about them would silently turn off the dry run, or turn ON a - per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; - present ⇒ take it, including `false`. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - - def flag(name): - return bool(raw[name]) if name in raw else bool(prev.get(name)) - if kind == "plain": - # A plain automation has no machine step, so it only needs the database its records walk. - # - # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on - # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- - # trigger — so requiring a second copy of that fact here would be a mandatory field with - # exactly one legal answer, and a Save that refuses until you retype what you just picked. - # There is no `dryRun`: dry-run means "run the machine step but do not write", and there - # is no machine step to run. - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None - if kind == "scrape_db": - url = _s(raw.get("url"), 2000).strip() - if not url: - return None, "a source URL is required" - try: - guard(url) - except Refused as e: - return None, str(e) - extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" - fmap = {} - for k, v in list((raw.get("fieldMap") or {}).items())[:60]: - tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") - if tk: - fmap[_s(k, 120)] = tk[:60] - if not fmap: - return None, "map at least one source column to a field" - key_field = _s(raw.get("keyField"), 60).strip() - if key_field not in fmap.values(): - return None, "the key field must be one of the mapped fields" - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"url": url, "extract": extract, - "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), - "fieldMap": fmap, "keyField": key_field, - "targetTable": target, - "dryRun": flag("dryRun"), - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None - if kind == "field_instagram": - table = _s(raw.get("targetTable"), 60).strip() - if not table.startswith(UT_PREFIX): - return None, "an Instagram automation runs against a blank database (ut_*)" - fkey = _s(raw.get("fieldKey"), 80).strip() - if not fkey: - return None, "pick the automation column this run writes into" - # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). - # They are read from nothing and written to nothing: a stored definition carrying either - # still SAVES — it simply loses them on the next write — and neither is ever a reason to - # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown - # key would 400 every automation a tenant stored before this wave, forever, on a screen - # that gives them no way to remove it. Dropping a retired key is a migration; refusing it - # is an outage. - # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary - # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired - # USER concept — renaming them would be a second, unrelated change wearing this one's - # justification. - # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel - # that does not edit the post count must not reset it to the default on Save. And an - # INHERITED value is clamped rather than refused; see `clean_max_posts`. - sent = "maxPosts" in raw - max_posts, perr = clean_max_posts( - raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) - if perr: - return None, perr - return {"targetTable": table, "fieldKey": fkey, - "urlField": _s(raw.get("urlField"), 80).strip(), - # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked - # for: the Profiles row carries post IDENTITY for free but NO engagement - # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. - "postMetrics": flag("postMetrics"), - # The full Comments dataset can bill many rows per post. It is an explicit, - # independent opt-in and never follows the per-post switch automatically. - "commentMetrics": flag("commentMetrics"), - "dryRun": flag("dryRun"), - "maxPosts": max_posts}, None - # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor - # the same question of two different corpora: a record ceiling, a predicate list, a join word - # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the - # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write - # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists - # to prevent, and it would do it silently. - if kind in DISCOVERY_KINDS: - # ⭐ WAVE 30 · T05 — the inline tuple became the named one, and the two defaults below now - # come from `discovery_facts` rather than being spelled out here. They were correct; they - # were also the FIFTH copy of "which table does this kind write to", and the other four - # were the ones wave 29 forgot to update. - _, _disc_table, _disc_label, _ = discovery_facts(kind) - limit = int(raw.get("recordsLimit") or 0) - if limit < 1: - return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the " - "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") - if limit > BD_MAX_RECORDS: - return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what - # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading - # the operator afterwards is how the OR hole survived: the guard was handed the branches - # and never told they were a union. - join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" - # ⭐ WAVE 32 · T46 (D-167) — AND THE KIND GOES WITH THEM. `clean_config` already knows which - # corpus this automation searches; passing it is what stops a `discover_tiktok` being built - # on the 16 Instagram fields TikTok's dataset does not carry — a search that does not error, - # returns nothing, and reads as "no such creators exist" after the money is spent. - preds, perr = clean_predicates(raw.get("predicates"), join, kind) - if perr: - return None, perr - target = _s(raw.get("targetTable"), 60).strip() or _disc_table - if not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) - if serr: - return None, serr - return {"recordsLimit": limit, "predicates": preds, - "operator": join, - "targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or _disc_label, - # C6/R5: WHERE the conditions above came from, when they were derived. Stored so - # the surface can say "these were filled in from the 'Florists' view, over 12 - # records" instead of presenting them as if somebody typed them. - "seed": seed, - "dryRun": flag("dryRun")}, None - return None, f"unknown automation kind {kind!r}" - - -def clean_seed(raw): - """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter - somebody typed by hand has no seed, and that is the ordinary case. - - ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that - the derived conditions are "written into `config.predicates` as ordinary conditions — it is a - filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, - like every other search. Keeping it means the surface can say where those rows came from, and - a user editing them freely is exactly what is supposed to happen. - """ - if raw in (None, "", {}): - return {}, None - if not isinstance(raw, dict): - return None, "the seed must be an object" - source = _s(raw.get("source"), 12).strip().lower() - if source not in SEED_SOURCES: - return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) - table = _s(raw.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, f"a seed reads a blank database (ut_*) — {table!r} is not one" - basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} - derived, _err = clean_predicates(raw.get("derived") or [], "and") - return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), - # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what - # was suggested; the conditions that matter are already in `predicates` and were - # validated there. Refusing a Save because a stored provenance note aged badly would - # block the user from editing the very filter it describes. - "derived": derived or [], - "basis": {"rows": max(0, int(basis.get("rows") or 0)), - "fields": [f for f in (basis.get("fields") or []) - if isinstance(f, dict)][:SEED_MAX_PREDICATES], - "related": basis.get("related") - if isinstance(basis.get("related"), dict) else {}, - "note": _s(basis.get("note"), 200)}}, None - - -def clean_schedule(raw, previous=None): - """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` - measures from — without it, enabling a daily job fires it immediately).""" - raw = raw if isinstance(raw, dict) else {} - cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" - parse_cron(cron) # raises -> the route answers 400 - enabled = bool(raw.get("enabled")) - prev = previous or {} - out = {"cron": cron, "enabled": enabled} - if enabled: - out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() - return out - - -# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── -# The owner's words: "we can make this automation a loop, so the ending should always be defined, -# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, -# or the user have to click a button to reset, or after a certain amount of time it can -# automatically reset to first cycle." -# -# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition -# that predates this field must not start moving records on its own the day the code ships. A -# loop is a thing somebody turns on. -def clean_flow(raw, previous=None, notes=None): - """Validate the builder's ordered action list. `(flow, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions( - raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes) - if err: - return None, err - return {"actions": actions}, None - - -#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the -#: definition carries none. -#: -#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at -#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the -#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly -#: created automation. Two numbers for one fact, in the panel, before anybody had typed -#: anything. **Read off the screenshot; every assertion in the battery was green.** -#: -#: This is the same species as the default-versus-guard split fixed the same day: a value -#: decided in one file and a value decided in another, describing the same thing. Aligning the -#: constants closes the only window in which they can disagree — the seed — because every later -#: state comes from the stored config. -DISCOVER_SEED_RECORDS = 25 - - -def discovery_seed_spec(kind): - """-> (default table, enrich action kind) for a discovery KIND. - - ⭐⭐ WAVE 30, OWNER REPORT 2026-08-12, verbatim: *"'When a Tiktok profile fits a criteria' - should ALWAYS have a 'Create record' EXACTLY like the Instagram one … THE ONLY DIFFERENCE IS - THE COLUMNS AND DATA SCHEMA"*. They were right, and the seeding path was the one family of - sites this wave widened everywhere ELSE: `clean_definition` planted step 1 and step 2 only when - the trigger was `ig_profile_match`, so a TikTok search stored `flow.actions = []` and had - nowhere to put what it found — MEASURED live against `auto_1` (Instagram: create_record + - enrich) versus a fresh TikTok discovery (empty). - - ⭐ THIS IS A LOOKUP, NOT A SECOND SEEDER, and that is the whole point of the ruling. One - builder plants both platforms' steps; the only things that vary are the table the record lands - in and which enrich action reads it — literally "the columns and data schema". A forked - `_ensure_tt_action` would start identical and drift, which is the failure `ENRICH_KINDS` and - `DISCOVERY_KINDS` were introduced to prevent one screen up. - - ⚠ Resolved in a FUNCTION rather than a module-level dict because `DISCOVER_TABLE` is defined - far below this line; a dict here would raise at import. - """ - if kind == "discover_tiktok": - return TT_PROFILE_TABLE, "enrich_tiktok" - return DISCOVER_TABLE, "enrich_instagram" - - -def _ensure_ig_action(flow_raw, table, enrich_kind="enrich_instagram"): - """Create record is PERMANENT step 1 on a DISCOVERY trigger — it cannot be deleted or moved. - - ⚠ THE NAME IS INSTAGRAM'S AND THE BEHAVIOUR IS BOTH PLATFORMS' (wave 30). Renaming it would - churn six gate references for no behaviour change; `enrich_kind` is what makes it general, and - it defaults to Instagram's so every pre-existing caller keeps its exact meaning. - - ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, - on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a - comment warning that re-seeding on every clean would be "a control that will not take no for - an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be - deleted, because the nature of that automation is that it needs to first create a record in a - database somewhere from the list of profiles to fetch."* - - Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER - produces rows has nowhere to put them until something writes them, so an Instagram search - with no Create record is not a customised automation — it is a search whose results are - discarded. That is not a preference to respect. - - ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and - the values it maps are theirs. An existing `create_record` further down is MOVED to the front - rather than duplicated — re-seeding a second one would quietly double every run's writes. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if actions and actions[0].get("kind") == "create_record": - return _pin_unique(_ensure_enrich_step( - flow_raw if isinstance(flow_raw, dict) else flow, enrich_kind)) - at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) - if at > 0: - actions.insert(0, actions.pop(at)) - else: - # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner - # and no card: the wave-23 header records four kinds that shipped with illegal seeds and - # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. - actions.insert(0, { - "id": "act_1", "kind": "create_record", "enabled": True, "when": None, - # `config.label` follows the `review` action's precedent in `_clean_action_config` — - # an action naming itself, inside the untyped config bag, so no shared TS type changes. - "config": {"table": str(table or DISCOVER_TABLE), - "label": "Save the profile", - # C5: the picture and the real write finally agree — see `_pin_unique`. - "uniqueOn": "handle", - "values": {"handle": "{{handle}}"}}, - }) - flow["actions"] = actions - return _pin_unique(_ensure_enrich_step(flow, enrich_kind)) - - -def _ensure_enrich_step(flow_raw, enrich_kind="enrich_instagram"): - """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON A DISCOVERY SEARCH. - - ⚠ WAVE 30: `enrich_kind` selects the network. Instagram's is the default so every pre-existing - caller means exactly what it meant before; TikTok passes `enrich_tiktok` and gets the identical - step shape, which is the owner's *"only the columns and data schema differ"*. - - Owner: *"make it default that this enrichment action is Step 2 always, and under Config of - Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for - the same reason: a search that finds profiles and never reads them has done half a job. The - difference is the control — step 1 is permanent because a flow without it discards its - results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and - disabling would be two ways to say one thing, and only one of them survives a re-save. - - ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains - this step by upgrading does not quietly start spending; switching the Source to the paid - provider is an explicit choice a person makes in front of the sentence that names the cost. - ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — - a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow - re-buying the same profile nightly. Off-by-default would make the expensive behaviour the - accident. - - ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an - If/then branch has one; seeding a second at the top level would enrich twice and bill twice, - which is the duplication `apply_actions` already paid for once with the pinned step 1. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if any(a.get("kind") == enrich_kind for a in walk_actions(actions)): - return flow_raw if isinstance(flow_raw, dict) else flow - seeded = list(actions) - # Index 1 — after the pinned Create record, because there is nothing to enrich until the - # profiles have been written as records. `insert` past the end is a plain append, so a flow - # with only step 1 lands this at the end, which IS step 2. - seeded.insert(1, { - "id": "act_enrich", "kind": enrich_kind, "enabled": True, "when": None, - "config": {"postMetrics": False, "commentMetrics": False, - "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, - "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", - "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, - "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, - }) - return {**flow, "actions": seeded} - - -#: The key the discovery writer really upserts on. Both discovery runners key candidates on -#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's -#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is -#: supplied by the runner, never typed by a person. -#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that -#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is -#: the pair above and the TENANT is the unit, while `created_by` survives as an informational -#: "Found by" stamp that no run may branch on. -#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note -#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate -#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a -#: bug with a rationale attached. -IG_PINNED_UNIQUE = "handle" - - -def _pin_unique(flow): - """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. - - ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — - the builder posts the whole action list on every save — so a client that omitted the key would - silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's - own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming - "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this - module refuses everywhere else. - - ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason - this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action - does not write, so the stamp is applied ONLY when `handle` is among the action's own values. - A user who remaps the card to write different columns keeps their edit and still saves; the - alternative — stamping unconditionally — is a product-seeded default that its own validator - would 400, which is precisely the post-W24 hotfix this rule exists because of. - - ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes - `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED - definition's own dict, so writing into it would edit the live store object in memory, before - (and regardless of) any commit. Every touched level is copied instead. - """ - if not isinstance(flow, dict): - return flow - acts = flow.get("actions") - if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): - return flow - first = acts[0] - if first.get("kind") != "create_record": - return flow - cfg = first.get("config") - if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): - return flow - if str(cfg.get("uniqueOn") or "").strip(): - return flow - return {**flow, - "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} - - -#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so -#: the seeder and the un-seeder cannot disagree about what "seeded" means. -#: ⚠ WAVE 30 — `enrich_tiktok` shares `act_enrich`. The map is keyed by KIND, and only one enrich -#: step is ever seeded per flow (the network follows the trigger), so the id cannot collide. -_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich", - "enrich_tiktok": "act_enrich"} - - -def _is_untouched_ig_seed(action, table, enrich_kind="enrich_instagram"): - """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) - - ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and - that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build - the same dicts one screen above, so a step gaining a config key next wave gains it here too. - A remembered key list would quietly start calling every seeded action "edited". - - ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The - stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the - kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW - dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", - and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only - version where the two sides are the same kind of object. - - ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the - TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about - a step they still want, and an off seed is still a seed. - """ - if not isinstance(action, dict): - return False - kind = str(action.get("kind") or "") - if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): - return False - seeded, _seed_err = clean_actions( - (_ensure_ig_action({"actions": []}, table, enrich_kind).get("actions") or [])) - fresh = {a.get("kind"): a for a in (seeded or [])} - seed = fresh.get(kind) - if not seed: - return False - cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) - if kind in ENRICH_KINDS: - cfg.pop("enabled", None) - seed_cfg.pop("enabled", None) - return cfg == seed_cfg and (action.get("when") or None) is None - - -def _drop_ig_seeds(flow_raw, table, enrich_kind="enrich_instagram"): - """Strip the trigger's own seeded steps, keeping any the person has since made theirs. - - ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT - THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it - does not delete the seeded actions … a create_record the person has since re-pointed at their - own table with their own values is THEIR action now"*. That reasoning is still correct and is - exactly what this function preserves — but it was applied to EVERY seeded action, including - the ones nobody had ever opened, and the result is what the owner reported: switch an - Instagram search to Manual and you are left holding a "Save the profile" step writing - `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not - somebody's work being protected; it is the machine's own leftover, pointed at a table the - automation has nothing to do with any more. - - ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as - `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a - branch that destroys something, so the conservative half is the load-bearing half. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - kept = [a for a in actions if not _is_untouched_ig_seed(a, table, enrich_kind)] - if len(kept) == len(actions): - return flow_raw - return {**flow, "actions": kept} - - -def ig_action_pinned(defn, index): - """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` - flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" - # ⭐ WAVE 30 — BOTH discovery kinds. It read `== "discover_instagram"`, so TikTok's step 1 (once - # seeded) would have rendered as an ordinary draggable, deletable card: the same rule the owner - # asked for "EXACTLY", applied to only one network. - return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 - - -def clean_definition(raw, previous=None, username="", notes=None): - """Whole-definition validation. Returns `(defn, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous or {} - # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing - # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second - # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys - # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload - # invalid in BOTH its trigger and its config now answers with the trigger's sentence. - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr - # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this - # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a - # dead end the moment the wizard was deleted and the client's create body became {name}. - kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND - drop_ig_seeds = False - if (trigger or {}).get("key") == "ig_profile_match": - # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather - # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and - # whose kind says something else is not a preference to respect, it is two halves of one - # answer disagreeing, and the trigger is the half the person actually picked. - kind = "discover_instagram" - elif (trigger or {}).get("key") == "tiktok_profile_match": - # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how - # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram - # arm is — a definition whose trigger says TikTok discovery and whose kind says something - # else is two halves of one answer disagreeing. - kind = "discover_tiktok" - elif (kind == "discover_tiktok" - and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match" - and (trigger or {}).get("key") != "tiktok_profile_match"): - # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm - # below): with nothing to flip the kind BACK, switching the trigger to Manual left - # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus - # search on a flow the person had just made manual.** Shipped here in the same change as - # law 1 rather than discovered the same way twice. - # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the - # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because - # `patch()` builds its raw as `dict(prev)` plus the caller's keys. - kind = DEFAULT_KIND - # ⭐ WAVE 30 — un-seed on the way out, exactly as the Instagram arm below does. This line - # was ABSENT and harmless for as long as TikTok had no seeds to leave behind; the moment - # the seeder above learned TikTok, its absence became wave-27 item 15's defect on the other - # network — a flow switched to Manual still carrying a "Save the profile" step nobody - # planted deliberately. Found by widening the seeder and asking what else assumed only - # Instagram could have seeds. - drop_ig_seeds = True - elif (kind == "discover_instagram" - and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" - and (trigger or {}).get("key") != "ig_profile_match"): - # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. - # - # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram - # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch - # the trigger to Manual and the automation stayed `discover_instagram`, which means - # - # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow - # the person had just made MANUAL fired a PAID Bright Data corpus search; - # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed - # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the - # instagram trigger, the 'always first' create record just stuck there"*. The server - # would have accepted the delete; the CLIENT hid the control, because both halves ask - # the kind and the kind was lying. - # - # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in - # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as - # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY - # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. - # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` - # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest - # question: did this automation stop being an Instagram search? - # - # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram - # trigger — a `discover_instagram` created with an explicit kind and some other trigger is - # somebody's deliberate state, not a mistake to correct. And only FROM - # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose - # triggers are their own business, and clobbering them would retire them by accident. - # - # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this - # replaces argued the other way, so here is why it was half right. It read: *"it does not - # delete the seeded actions … a create_record the person has since re-pointed at their own - # table with their own values is THEIR action now, and quietly destroying it is the silent - # data loss this module refuses everywhere else."* Every word of that is still true and is - # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the - # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to - # Manual still carrying a "Save the profile" step writing `{{handle}}` into - # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces - # handles, unpinned but still there, still runnable, and still pointed at a table that has - # nothing to do with this automation. - # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. - kind = DEFAULT_KIND - drop_ig_seeds = True - if kind not in KINDS: - return None, f"unknown automation kind {kind!r}" - # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. - # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` - # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. - # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way - # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two - # live automations save through this function every time their owner edits them. The gate's - # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved - # from `create` into `clean_definition` (the tempting simplification), this helper would go - # red across a dozen sections, which is the alarm that change deserves."* It did, and the - # alarm worked. - # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every - # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody - # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an - # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored - # ones are rewritten under their owner — and that is a ruling, not a refactor. - name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) - if not name: - return None, "name the automation" - cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - if kind in DISCOVERY_KINDS and prev.get("kind") != kind: - # ⭐⭐ WAVE 30 · T04 — WIDENED FROM `discover_instagram` TO BOTH DISCOVERY KINDS, AND THIS - # ONE LINE WAS THE WHOLE OF THE OWNER'S ITEM 3. Picking "When a TikTok profile fits a - # criteria" 400'd on the very first Save with *"say how many profiles to fetch — an - # UNBOUNDED discovery query is the one shape the vendor refuses outright - # (NOT_ENOUGH_FUNDS)"*, and the trigger was therefore never STORED, which is what the - # owner saw as "the Trigger won't load". - # ⛔ INSTAGRAM WAS NEVER SURVIVING ON ITS OWN MERITS: `AutomationDetail.buildConfig` sends - # no `recordsLimit` for EITHER platform on a fresh automation (its `kind` is still `plain` - # at that moment, so it falls through to `return { targetTable }`). IG worked only because - # this seed caught it. So the defect was never "TikTok is missing something Instagram has" - # — it was one hard-coded string in the single line that rescues both. - # ⚠ `prev.get("kind") != kind` is the EXACT generalisation of the old - # `prev.get("kind") != "discover_instagram"`, not a loosening: it still fires only on a - # kind FLIP, so a later save that carries a real limit is not re-seeded (asserted). - # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very - # first Save after picking this trigger 400s — against the stored-inert-with- - # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). - # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard - # exists because an unbounded query is the one shape the vendor refuses outright, and - # coercing a blank into a number would be exactly the silent widening it protects against. - # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until - # a person presses Run now or arms the schedule. - cfg_raw = dict(cfg_raw or {}) - if not _ig_int(cfg_raw.get("recordsLimit")): - cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS - config, err = clean_config(kind, cfg_raw, prev.get("config")) - if err: - return None, err - if trigger: - # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own - # configuration IS the config, and `clean_trigger` could not see it. - trigger["configured"] = _trigger_configured(trigger, config) - try: - schedule = clean_schedule( - raw.get("schedule") if isinstance(raw.get("schedule"), dict) - else prev.get("schedule"), prev.get("schedule")) - except ValueError as e: - return None, str(e) - # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) - flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") - # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only - # call was what made the action deletable; running it on every save is what makes step 1 - # permanent, and it is deliberate rather than a widened condition nobody noticed. - # ⭐⭐ WAVE 30 (owner, 2026-08-12) — BOTH DISCOVERY TRIGGERS SEED, not just Instagram's. This - # single `==` was the whole of *"why is it not copied EXACTLY"*: a TikTok search stored an - # EMPTY flow, so the product's own rule — a search that finds profiles must have somewhere to - # put them — held for one network and not the other. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - _seed_table, _seed_enrich = discovery_seed_spec(kind) - flow_raw = _ensure_ig_action( - flow_raw, config.get("targetTable") or _seed_table, _seed_enrich) - elif drop_ig_seeds: - # ITEM 15: the trigger just STOPPED being a discovery one. Un-seed what the trigger - # planted and nobody has since made theirs — measured against the table the seeds were - # planted for, which is the PREVIOUS config's, not the one being saved. - # ⚠ And against the PREVIOUS kind's enrich action, for the same reason: the seeds to strip - # are TikTok's if that is what was planted, and asking "is this Instagram's seed?" of a - # TikTok flow answers no and silently leaves the leftover behind. - _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) - flow_raw = _drop_ig_seeds( - flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) - flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes) - if ferr: - return None, ferr - # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that - # its `config.table` is authoritative has to reach it, or the two fields that name one database - # would disagree on exactly the network that just gained the card. - if (trigger or {}).get("key") in DISCOVERY_TRIGGER_KIND: - # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and - # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since - # wave 24: the pinned step 1 says where profiles are written, and the config says where - # the automation's records live — and for a discovery flow those are the same database by - # construction. When they disagreed, every surface picked a different winner (the canvas - # Write node read the config, the actual write read the action), which is a bug you can - # only see by comparing two panels. - # - # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the - # action's own configuration, so the table named there is the one they were looking at. - # - # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: - # a stored definition carrying two different values is reconciled on its next clean, - # silently and exactly once, because nothing writes the losing value back. A refusal here - # would 400 every Save of a definition that was legal yesterday. - # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the - # TIE-BREAK rather than the whole rule. The owner's report: change the database on the - # TRIGGER, save, and the picker springs back to the old one with no message. The cause is - # the branch below read unconditionally — the pinned action still named yesterday's table, - # so it overwrote a change the person had just made in front of it, silently, every time. - # - # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled - # "when the two disagree, whose value is real?" — the action's, because R2b puts the - # preset list on the action's own panel, so that is the one they were looking at. That - # answers a definition at REST. A save is different: it carries an INTENT, and the honest - # question is which side this particular save changed. So: - # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to - # follow it (leaving the action behind would just re-revert on the next save); - # · only the action moved, or neither did and they were already inconsistent -> R2's - # migration, unchanged, reconciling a stored definition exactly once. - # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is - # also the only reading that cannot lose a value: the action's table is the one with - # the field mapping attached to it. - first = (flow.get("actions") or [{}])[0] - pinned = str((first.get("config") or {}).get("table") or "") \ - if first.get("kind") == "create_record" else "" - prev_target = str((prev.get("config") or {}).get("targetTable") or "") - prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] - ).get("config") or {}).get("table") or "") - target_moved = bool(prev_target) and config.get("targetTable") != prev_target - action_moved = bool(prev_pinned) and pinned != prev_pinned - if pinned and target_moved and not action_moved: - # The person changed the trigger's database. Carry it INTO the pinned step so the two - # agree, instead of throwing their edit away to keep a stale copy consistent. - cfg1 = dict(first.get("config") or {}) - cfg1["table"] = str(config.get("targetTable") or "") - flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] - elif pinned and pinned != config.get("targetTable"): - config["targetTable"] = pinned - # The stored label described a DIFFERENT database, so keeping it would caption the new - # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` - # never relabels a table that already exists — so an adopted hand-made database keeps - # the name its owner gave it. - config["targetLabel"] = "" - return { - "id": _s(prev.get("id") or raw.get("id"), 40), - "name": name, "kind": kind, "config": config, "schedule": schedule, - "trigger": trigger, - # The Builder's ordered actions. An absent flow is simply an automation with no actions. - "flow": flow, - "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, - "runs": list(prev.get("runs") or [])[:MAX_RUNS], - # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 - # minutes to build, so a run persists its id here and the NEXT run collects it. It is - # carried through `patch` untouched because a user pressing Save must not silently orphan - # a set the vendor is already building (and already counting against the funds gate). - "state": dict(prev.get("state") or {}), - "created": prev.get("created") or _iso(), - "createdBy": prev.get("createdBy") or username, - }, None - - -def set_state(rt, auto_id, patch_state): - """Merge into ONE automation's engine state. A key set to None is removed. - - Its own tiny store write rather than a field on `_commit_run`, because the handoff must - survive a run that ends in `error` — a snapshot the vendor is building does not stop existing - because the run that started it failed afterwards. - """ - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - st = dict(d.get("state") or {}) - for k, v in (patch_state or {}).items(): - st.pop(k, None) if v is None else st.__setitem__(k, v) - d["state"] = st - return cur - - # ⚠ W31-T35 — THE ONE `keeps_defs=True` IN THIS MODULE. `set_state` writes `state` and never a - # trigger, and `grid_hook` calls it once per CREATED RECORD; clearing the definitions memo here - # would re-read the whole bucket one row later than before and undo D-134 entirely. Safe - # because `grid_hook` reads exactly one state key (`rcHighwater`) and mirrors its own write - # into the memo in the same statement. Read `_store_update`'s note before adding a second. - _store_update(rt, _up, flush="sync", keeps_defs=True) - - -def _without_retired_board(definition): - """Return a definition with retired Board-only state removed, without mutating the store. - - This protects scheduled/manual execution before the next UI list request persists the - migration. It removes only the former Board configuration, actions, and audit trail; normal - actions and all user data remain intact. - """ - if not isinstance(definition, dict): - return definition, False - out, changed = dict(definition), False - cfg = out.get("config") - if isinstance(cfg, dict) and "lanes" in cfg: - cfg = dict(cfg) - cfg.pop("lanes", None) - out["config"] = cfg - changed = True - - def _actions(actions): - nonlocal changed - clean = [] - for action in actions if isinstance(actions, list) else []: - if not isinstance(action, dict): - clean.append(action) - continue - if action.get("kind") == "review": - changed = True - continue - item = dict(action) - if item.get("kind") == "group" and isinstance(item.get("config"), dict): - group_cfg = dict(item["config"]) - branches = group_cfg.get("branches") - if isinstance(branches, list): - kept = [] - for branch in branches: - if not isinstance(branch, dict): - changed = True - continue - next_actions = _actions(branch.get("actions")) - if not next_actions: - changed = True - continue - kept.append({**branch, "actions": next_actions}) - if not kept: - changed = True - continue - if kept != branches: - changed = True - group_cfg["branches"] = kept - item["config"] = group_cfg - elif isinstance(group_cfg.get("actions"), list): - nested = _actions(group_cfg["actions"]) - if not nested: - changed = True - continue - if nested != group_cfg["actions"]: - group_cfg["actions"] = nested - item["config"] = group_cfg - clean.append(item) - return clean - - flow = out.get("flow") - if isinstance(flow, dict): - next_flow = dict(flow) - actions = _actions(flow.get("actions")) - if actions != flow.get("actions"): - next_flow["actions"] = actions - if "ending" in next_flow: - next_flow.pop("ending", None) - changed = True - out["flow"] = next_flow - if "reviews" in out: - out.pop("reviews", None) - changed = True - return out, changed - - -def retire_automation_board_state(rt, tables=None): - """Persist the idempotent Board retirement, then remove its generated table fields. - - ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches - the stage scan and nothing else. - """ - fields = retire_automation_stage_fields(rt, tables=tables) - try: - stored = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {"definitions": 0, **fields} - plan = {str(aid): cleaned for aid, raw in stored.items() - for cleaned, changed in [_without_retired_board(raw)] if changed} - if not plan: - return {"definitions": 0, **fields} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for aid, cleaned in plan.items(): - if aid in cur: - cur[aid] = cleaned - return cur - - _store_update(rt, _up, flush="sync") - return {"definitions": len(plan), **fields} - - -#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe. -#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which -#: exactly one caller uses (`grid_hook`). -_DEFS_MEMO = {} -#: Deliberately SHORT. This exists to collapse one burst of row events into one read, not to be a -#: cache — an import of 20,000 rows arrives in far less than this, and a stale trigger for two -#: seconds is bounded and recoverable where a stale one for a minute is a mystery. -_DEFS_TTL = 2.0 - - -def _store_update(rt, fn, flush="sync", keeps_defs=False): - """THE one door every write to the automations bucket goes through — and the ONLY reason it - exists is that the memo's invalidation must be DERIVED rather than remembered. - - ⚠ `keeps_defs=True` IS AN EXPLICIT, SINGLE-SITE OPT-OUT, and it is here because the safe - default would otherwise make the memo useless on the exact path it was built for. `set_state` - writes `state` and NEVER a trigger, and it is called by `grid_hook` itself once per created - record — so clearing on it would mean a 20,000-row import re-read the bucket 20,000 times - anyway, one row later than before. The opt-out is safe because `grid_hook` reads exactly one - state key (`rcHighwater`) and mirrors its own write into the memo in the same statement. - ⛔ The DEFAULT is the safe one, so a thirteenth writer added next wave gets invalidation - without knowing this exists; only a caller that has read this paragraph can opt out. - - ⛔ THE ALTERNATIVE WAS TWELVE CALL SITES. `rt.update(STORE_KEY, …)` appeared twelve times in - this module; hanging an invalidation on each would be a hand-maintained list, and the way that - fails is silent: a thirteenth writer added next wave leaves `grid_hook` firing triggers off a - definition set that no longer exists, with nothing red. Same defect class as `patch`'s - hand-maintained patchable-key list, which this module already documents as *"the silent-drop - seat"* [[a-constant-two-features-share]]. - - ⚠ IT CLEARS THE WHOLE MEMO, not this tenant's row, on purpose: identifying the tenant here - would mean trusting `getattr(rt, 'key')` at a WRITE, and a wrong answer there is a stale - trigger set for another tenant. The memo holds at most a handful of entries and the correct, - boring thing costs nothing. - """ - if not keeps_defs: - _DEFS_MEMO.clear() - return rt.update(STORE_KEY, fn, flush=flush) - - -def all_definitions(rt, cached=False): - """Every automation definition for this tenant. - - ⭐⭐ `cached=True` IS D-134's FIX, AND IT IS OPT-IN FOR A REASON. `grid_hook` runs ONCE PER ROW - EVENT and called this unconditionally, so a 20,000-row import performed 20,000 whole-document - deep copies of the automations bucket — `Store.get` re-serialises on every call, hit or miss, - under the store lock. Every other caller is a request handler that reads it once, so widening - the memo to all of them would trade a real guarantee (a request sees the current store) for - nothing measurable. - - ⛔ THE MEMO IS THE SAME OBJECT ON A HIT, and `grid_hook` MUTATES it deliberately — see the - `rcHighwater` write there. That is not a leak of an implementation detail, it is the fix to the - hazard a naive memo creates: `grid_hook` both READS `state.rcHighwater` and WRITES it through - `set_state`, so a memo that went stale against its own write would re-fire `record_created` - for records it had already handled, breaking A2(4)'s *"once per record EVER — undo-proof"*. - Writes through `_store_update` drop the memo; the one write `grid_hook` makes to its OWN copy - is mirrored into it in the same statement. - """ - if cached: - key = str(getattr(rt, "key", "") or "") - hit = _DEFS_MEMO.get(key) - if hit is not None and (time.monotonic() - hit[0]) < _DEFS_TTL: - return hit[1] - try: - raw = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {} - out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()} - if cached: - _DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out) - return out - - -def _new_id(existing): - n = 1 - while f"auto_{n}" in existing: - n += 1 - return f"auto_{n}" - - -#: C2's three answers to "which database does this automation work on?" — the FIRST question the -#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). -TARGET_MODES = ("existing", "new", "automated") - - -def resolve_target(rt, raw, username=""): - """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. - - - `existing` — bind a database that is already there. - - `new` — mint a blank one now, so the automation has somewhere to write before its first - run instead of conjuring a table the person never agreed to. - - `automated` — leave it to the runner: a scraping automation MINTS its target on first run - (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. - Nothing is created here, deliberately — an empty table created up front for a scrape that - never runs is litter nobody can explain. - """ - if not isinstance(raw, dict) or not raw: - return {}, None # no target block = the pre-wizard shape - mode = _s(raw.get("mode"), 20).strip() or "existing" - if mode not in TARGET_MODES: - return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) - if mode == "automated": - label = " ".join(_s(raw.get("label"), 60).split()) - return ({"targetLabel": label} if label else {}), None - if mode == "existing": - key = _s(raw.get("table"), 60).strip() - if not key: - return None, "choose the database this automation works on" - t = ut_get(rt, key) - if t is None: - return None, f"{key!r} is not a database in this workspace" - return {"targetTable": key, "targetLabel": t.get("label") or key}, None - label = " ".join(_s(raw.get("label"), 60).split()) - if not label: - return None, "name the new database" - import core.user_tables as _ut_new - key = _ut_new.create(label, username or "automation", st=rt) - if not key: - return None, ("the database could not be created — this workspace may be at its table " - "limit") - return {"targetTable": key, "targetLabel": label}, None - - -def _unmint(rt, key): - """Delete a database this call minted moments ago, because the call is refusing (D-50). - - ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user - needs to read; a rollback that raised would replace that sentence with a 500 and lose the - reason the create failed in the first place. The worst case of a swallowed failure here is - the orphan we had before — strictly no worse, and the refusal still reaches the person. - """ - if not key: - return - try: - import core.user_tables as _ut_del - _ut_del.delete(key, st=rt) - except Exception: # noqa: BLE001 - pass - - -def create(rt, raw, username="", notes=None): - existing = all_definitions(rt) - if len(existing) >= MAX_AUTOMATIONS: - return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" - raw = dict(raw or {}) - # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation - # happens after it, so a correct refusal ("a source URL is required") used to leave an - # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: - # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form - # twice silently accumulated empty databases in their nav. - # - # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It - # does not test for `mode == "new"`; it asks "did the table this call's target resolved to - # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and - # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target - # rather than "any table that appeared", so a concurrent create in another request is never - # collateral. - before = set(ut_all(rt) or {}) - patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) - if terr: - return None, terr - minted = str((patch_cfg or {}).get("targetTable") or "") - minted = minted if minted and minted not in before else "" - if patch_cfg: - raw["config"] = {**(raw.get("config") or {}), **patch_cfg} - # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY - # has, instead of letting the pure validator mint a second empty one. See - # `discover_default_table`; a target the caller named is never rewritten. - raw = _apply_discover_default(rt, raw) - defn, err = clean_definition(raw, None, username, notes=notes) - if err: - _unmint(rt, minted) - return None, err - # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations - # that already use them and NO NEW ONE CAN BE CREATED." - # - # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live - # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing - # the kind there would make both of them unsaveable — the ruling retires the door, not the - # automations behind it. Enforced on the server rather than left to the deleted wizard: a - # wall that holds only because the client stopped asking is a convention, not a wall. - if defn["kind"] in RETIRED_KINDS: - _unmint(rt, minted) - return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " - f"longer created — the ones already using it keep working. " - f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") - defn["id"] = _new_id(existing) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - # `async` for the reason spelled out at `remove` and applied at `patch` — a create is the same - # blocking commit inside the same request, and the person is waiting on it in the same way. - # ⚠ NO PRESET GUARD HERE, deliberately: `patch` may skip the spawn because it can compare - # against a PREVIOUS definition, and a create has none — R10's columns must exist before a run, - # so this call stays unconditional. - _store_update(rt, _up, flush="async") - _presets_after_write(rt, defn, username) # R10: the columns exist before a run - if defn.get("trigger"): - _seed_event_state(rt, defn) - return defn, None - - -def patch(rt, auto_id, raw, username="", notes=None): - existing = all_definitions(rt) - prev = existing.get(str(auto_id)) - if prev is None: - return None, "no such automation" - merged = dict(prev) - # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a - # key missing here is accepted by the route, validated by `clean_definition`, and then - # DISCARDED — the client shows a saved flow that the store never received, and nothing goes - # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only - # ordering that never has a window where the bug exists. - for k in ("name", "config", "schedule", "kind", "trigger", "flow"): - if k in (raw or {}): - merged[k] = raw[k] - # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the - # Instagram trigger on an existing automation flips the kind, and the very next Save runs a - # config that has never carried a target through the pure validator, which mints the empty - # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. - merged = _apply_discover_default(rt, merged) - defn, err = clean_definition(merged, prev, username, notes=notes) - if err: - return None, err - defn["id"] = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - # ⛔ WHY THIS IS `async` — owner report 2026-08-12 (wave 30), MEASURED on the deployed build. - # Owner, verbatim: *"debug Automation module, its extremely slow, even renaming an Automation - # takes forever"*. A rename paid a BLOCKING HF commit inside the request, and `Store.update`'s - # sync branch holds `self._lock` across a strict DOWNLOAD **and** the upload — the same lock - # `Store.get` takes — so one rename stalls every OTHER reader of that tenant's store for its - # whole duration. That is the "module is slow" half a per-route fix never reaches. - # The reasoning, the read-your-writes contract and the `user_tables.add_row` precedent are - # written out in full at `remove` above (same wave, same owner report); this is that ruling - # applied to the save path rather than a second argument for it. - _store_update(rt, _up, flush="async") - # ⛔ THE PRESET SPAWN RUNS ONLY WHEN ITS OWN INPUTS MOVED — the other half of the same report. - # MEASURED as `nurilab-admin` on `4258a93`: **7,912 ms** for one rename against a **288 ms** - # warm `GET /automations` on the same connection, i.e. 27x the read this wave just made fast - # (T12/T13). A rename changes `name`; `_presets_after_write` reads `config`, `flow` and - # `trigger` and NOTHING else, then pays a full `user_tables` read (`ut_get`/`ut_ensure`; - # `rt.get` deep-copies by design, documented ceiling 35.8 MB / ~1.4 s) to re-derive columns - # that cannot have changed. ⭐ The generalisable half: T12/T13 took `ut_all` off the automation - # READ path this wave and every sibling WRITE path still carried it — a performance fix scoped - # to "the slow page" leaves the same call in every route beside it - # [[read-a-gates-predicate-for-what-it-excludes]]. - # ⛔ THE DANGEROUS DIRECTION IS SKIPPING, NEVER RUNNING, so the predicate is the INPUT SET of - # the function being skipped — not a hand-listed set of "cheap" edits, which is how a guard - # silently un-ships W25/R10 the next time the spawn learns to read a fourth key. When none of - # them moved the call is a no-op by construction (it re-derives from those same keys), and the - # run-time `ut_ensure` R10 deliberately kept is still the backstop if the TABLE drifted - # underneath — which is not a thing a rename should be repairing. Shaped after the trigger - # comparison directly below, which already compares `prev` against the cleaned definition. - # ⭐⭐ WAVE 31 · T30 — AND THE PREDICATE IS NOW THE SPAWN'S OWN ARGUMENT, which is the fix. - # The four-whole-objects test above was right in principle and inert in practice: `persist` - # sends a REBUILT `config`, so `prev["config"] != defn["config"]` on essentially every save and - # the guard fired every time — sparing only *rename*, the one edit the owner stopped - # complaining about after wave 30. `preset_inputs` is the canonical projection of exactly what - # `_spawn_presets` reads, and `_spawn_presets` is handed nothing else, so this comparison - # cannot go stale behind a future input the way a hand-listed key set would. The full argument, - # including why this is STRONGER than what it replaces rather than a relaxation, is in - # `preset_inputs`' own docstring — read that before widening or narrowing this line. - if preset_inputs(prev) != preset_inputs(defn): - _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS - # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching - # DISARMED — enabling never fires for records already in the state. - if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): - _seed_event_state(rt, defn) - return defn, None - - -def _presets_after_write(rt, defn, username): - """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run. - - Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset - columns immediately, "so R2b's list is real and the database is visible before anything is - spent". That last clause is the point — a person can look at the database, see the sixteen - columns, and decide whether to arm the schedule, instead of paying a vendor to find out what - they are agreeing to. - - ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs - nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first - run of a saved automation finds the table already correct and spends no commit. - - ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the - search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the - preset set, and spawning a subset here would mean the columns changed between Save and the - first run — two different answers to "what does this database look like", which is the exact - disagreement R10 exists to remove. - - ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure` - returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an - automation because the workspace is full would be a worse answer than saving one whose table - arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the - honest failure still has exactly one home. - """ - _spawn_presets(rt, preset_inputs(defn), username) - - -#: The enrich actions whose OWN config decides which preset columns and child databases a save -#: spawns, and the exact keys read off each. DECLARED here rather than spelled inside -#: `preset_inputs`, so `_spawn_presets` and the save-path guard cannot drift apart by one key. -PRESET_ACTION_INPUTS = { - "enrich_instagram": ("profileField",), - "enrich_tiktok": ("profileField", "postMetrics", "commentMetrics"), -} - - -def preset_inputs(defn): - """⭐⭐ WAVE 31 · T30 — EXACTLY the values `_spawn_presets` consumes off a definition, canonical. - - ⛔ THIS IS THE WHOLE POINT OF THE TICKET, so it is worth being precise about what changed and - why it is SAFER than what it replaces, not weaker. Owner item 7, verbatim: *"It still takes - forever to change the Config for automation as well. It just say Saving... and takes a long time - for me to change options and configs etc."* - - Wave 30 guarded the spawn on `any(prev[k] != defn[k] for k in ("config","flow","trigger", - "kind"))` and reasoned — correctly — that *"the predicate is the INPUT SET of the function being - skipped, not a hand-listed set of cheap edits"*. The flaw was not the reasoning; it was that the - input set was stated as four WHOLE OBJECTS while the spawn reads a handful of leaves out of - them. `AutomationDetail.tsx::persist` sends `config: buildConfig()` — a value RECONSTRUCTED from - React state, not the stored object — so the comparison is against something rebuilt on every - save and the guard never fires on the one edit the owner is complaining about. W30's fix spared - *rename* (the client omits `flow`/`trigger` entirely, and `kind`/`name` are unchanged), which is - exactly the edit the owner stopped complaining about after wave 30. - - ⭐ WHY NARROWING IS SAFE HERE AND WAS NOT SAFE THEN: this projection is not a hand-listed - allow-list that a future wave can silently outgrow. `_presets_after_write` hands this dict to - `_spawn_presets` and `_spawn_presets` **never receives `defn` at all** — so it is structurally - incapable of reading a fifth key that this function does not carry. Teaching the spawn to read - something new REQUIRES adding it here, and the guard then follows for free. That is a stronger - guarantee than wave 30's, which held only as long as somebody remembered the comment. - - ⚠ CANONICAL, because the input is a rebuild: each leaf is normalised exactly the way the spawn - itself normalises it (`str(...).strip()` for a field name, `bool(...)` for a switch). Without - that, `postMetrics: false` and a missing `postMetrics` would compare unequal while the spawn - treats them identically — a guard that fires on a difference its consumer cannot see is the - same defect in a smaller font. - - ⚠ IT DELIBERATELY COVERS THE DISCOVERY BRANCH **AND** THE ENRICH BRANCH IN ONE PROJECTION, even - though today's control flow returns before the second can run for a discovery automation. That - early return is D-178 (W31-T34) and it is going away; a projection built around today's branch - would silently narrow the guard the moment it does. - """ - defn = defn or {} - cfg = defn.get("config") or {} - actions = ((defn.get("flow") or {}).get("actions") or []) - return { - "target": str(_flow_table(defn) or cfg.get("targetTable") or ""), - "label": str(cfg.get("targetLabel") or ""), - "flowId": str(defn.get("id") or ""), - "discoveryKind": DISCOVERY_TRIGGER_KIND.get( - (defn.get("trigger") or {}).get("key") or ""), - "actions": { - kind: [{k: (_norm_switch(a, k) if k != "profileField" - else str((a.get("config") or {}).get(k) or "").strip()) - for k in keys} - for a in _actions_of_kind(actions, kind)] - for kind, keys in PRESET_ACTION_INPUTS.items() - }, - } - - -def _norm_switch(action, key): - """A capture switch as the spawn reads it — `bool`, so absent and False are ONE value.""" - return bool((action.get("config") or {}).get(key)) - - -def _spawn_presets(rt, inputs, username): - """The spawn itself. Takes `preset_inputs(defn)` — never the definition — see that docstring. - - ⭐⭐ WAVE 31 · T30, THE SECOND HALF: **ONE `user_tables` read for the whole pass, not five.** - Every `ut_ensure` below used to take its own copy of the tenant document through - `ut_get` → `ut_all` → `rt.get`, which deep-copies unconditionally (ceiling 35.8 MB / ~1.4 s), and - the TikTok arm reaches FOUR of them plus the `ut_get` above — measured **7,912 ms** on - `4258a93`. Lending one snapshot is the same fix W30-T13 made on the automation READ path - (`routes_automation.py::_LentTables`); this is deliberately NOT a third copy of that class, but - the `tables=` parameter `retire_automation_stage_fields` in this very module already - established, because here every consumer is a function we own and pass to directly. - """ - target = inputs["target"] - if not target: - return - tag = inputs["flowId"] - ig_actions = inputs["actions"]["enrich_instagram"] - tt_actions = inputs["actions"]["enrich_tiktok"] - try: - # ⭐ THE ONE READ. Everything below answers out of this snapshot. - tables = ut_all(rt) - # ⭐⭐ WAVE 30 · T06 — BOTH DISCOVERY KINDS SPAWN ON SAVE, and TikTok's absence here was - # R10 simply not holding for the second platform: `TT_PROFILE_FIELDS` reached `ut_ensure` - # at exactly ONE site — inside `run_discover_tiktok` — so the database did not exist until - # money had already been spent finding out what was in it. That is the precise pre-R10 - # behaviour `verify_automation`'s NC39 forbids on the Instagram side. - # ⛔⛔ THE DISCRIMINATOR IS THE **TRIGGER**, NOT THE KIND, AND THE GATE PAID FOR THAT - # SENTENCE. Widening this to `kind in DISCOVERY_KINDS` looks equivalent — law 1 makes a - # discovery trigger choose its kind — but the implication only runs ONE WAY. A definition - # may carry `kind: "discover_instagram"` with **no trigger at all**: law 1 sets the kind - # from the trigger and never the reverse, so any direct API create can do it, and one of - # this repo's own fixtures does. MEASURED: the kind test spawned a preset database under a - # dry-run fixture whose check asserts that no table exists — a red on correct-looking code, - # caught only because that check happened to exist. - # ⇒ R10 is a rule about what somebody PICKED IN THE PICKER, so it reads the picker's own - # answer. `DISCOVERY_TRIGGER_KIND` is that map, and a gate asserts it agrees with law 1. - # ⚠ TRACKED because it is the ONE key several `ut_ensure` calls in this pass can share: the - # discovery arm and both enrich arms all ensure `target`. A later one must not answer out of - # a snapshot an earlier one invalidated — see the `tables=` arguments below. - wrote_target = False - _dkind = inputs["discoveryKind"] - if _dkind: - _, _, _spawn_label, _spawn_fields = discovery_facts(_dkind) - ut_ensure(rt, inputs["label"] or _spawn_label, _spawn_fields, username, - key=target, flow_tag=tag, lock_fields=True, tables=tables) - # ⭐⭐ WAVE 31 · T34 (D-178) — AND THE `return` THAT USED TO BE HERE IS GONE. - # - # ⛔ WHAT IT COST: a DISCOVERY automation could never spawn its child databases. The - # capture switches below are the ones that promise `ut_tt_posts` / `ut_tt_comments` / - # `ut_tt_post_snapshots` at SAVE, before any money is spent (W30-T10) — and only a - # `plain` flow ever reached them, because a discovery flow left this function three - # lines earlier. So the exact automation the owner would build for TikTok discovery — - # find profiles, then capture their posts — silently got the profile table and nothing - # else, and the toggles it showed were promises the save path never kept. - # ⚠ IT IS A TRAP, NOT TODAY'S ONLY SYMPTOM: nurilab's child tables are absent for a - # CONFIGURATION reason as well, so fixing this alone will not make them appear there. - # - # ⚠ THE SNAPSHOT IS NOW STALE FOR `target`, and the arms below resolve their profile - # binding out of that very table (`bound` reads its `fields`). One re-read, paid only on - # a save that actually reached the discovery spawn — correctness before the read count, - # and it is one read against the 41 this function used to take. `wrote_target` stays - # False precisely BECAUSE we refreshed: it tracks staleness, not "did somebody write". - tables = ut_all(rt) - enrich_actions = ig_actions - table = tables.get(target) or {} - named = next((a["profileField"] for a in enrich_actions if a["profileField"]), "") - bound = next((str(f.get("key") or "") for f in table.get("fields") or [] - if isinstance(f.get("profile"), dict)), "") or named - if enrich_actions and bound: - ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username, - key=target, flow_tag=tag, lock_fields=True, tables=tables) - wrote_target = True - # ⭐ WAVE 30 · T08 — the TikTok half of W25/R10: the columns an `enrich_tiktok` step will - # fill appear when the automation is SAVED, not when money is first spent. Resolved - # through `profile_field_key(..., source=PROFILE_SOURCE_TT)` so it cannot adopt an - # Instagram binding, and through the SAME `_tt_profile_schema_for` the runner's own top-up - # uses — R10's rule is that the column set does not change between Save and the first run, - # and one shared resolver is what makes that structural instead of a convention. - if tt_actions: - tt_named = next((a["profileField"] for a in tt_actions if a["profileField"]), "") - tt_bound = profile_field_key(table, tt_named, source=PROFILE_SOURCE_TT) - if tt_bound: - ut_ensure(rt, table.get("label") or target, _tt_profile_schema_for(tt_bound), - username, key=target, flow_tag=tag, lock_fields=True, - tables=None if wrote_target else tables) - # ⭐ WAVE 30 · T10 — and the CHILD databases the step's own switches promise, at - # SAVE, before any money. R10's rule is "the columns a step will fill appear when - # you press Save"; a `postMetrics` toggle whose database only exists after a paid - # run is the same complaint one level up. - # ⛔ GATED ON THE SWITCHES, never spawned unconditionally: a database that exists - # because somebody looked at a toggle, and then never fills, is the "SECOND, empty - # database" the discovery default was rewritten to stop producing. - for _flag, _child in (("postMetrics", TT_POSTS_TABLE), - ("postMetrics", TT_POST_SNAPSHOTS_TABLE), - ("commentMetrics", TT_COMMENTS_TABLE)): - if any(a[_flag] for a in tt_actions): - # R9 (W31-T32): the child arrives LOCKED, from its own declaration. - ut_ensure(rt, TT_TABLE_LABELS[_child], TT_TABLE_FIELDS[_child], username, - key=_child, flow_tag=tag, lock_fields=True, tables=tables, - record_mode=tt_record_mode(_child)) - except Exception as e: # noqa: BLE001 - # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so - # raising here would answer 500 for an automation that IS stored — the caller would retry - # and create a second one. The columns then arrive on the first run, which is precisely - # the behaviour this function is an improvement on rather than a replacement for. - print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}") - - -def remove(rt, auto_id): - aid = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur.pop(aid, None) - return cur - # ⛔ WHY THIS IS `async` AND NOT `sync` (owner report 2026-08-12, wave 30, MEASURED). - # Owner, verbatim: *"it takes forever to delete an automation"*. A delete was paying up to - # TWO BLOCKING UPLOADS inside the request — this one, plus the whole-document rewrite below, - # which fires for every DISCOVERY automation because those always tag the columns they - # spawned (deleting one marked all 30 columns of `ut_tt_profile` disabled). Measured 977 ms - # for the CHEAPEST case (a `plain` flow, no tagged columns) against a 757 ms `GET /automations` - # baseline on the same connection; the discovery case is strictly worse by a full-document - # read plus a full-document commit. - # - # `flush="async"` is not a weakening: `Store.update`'s async branch applies the mutation to - # the in-process cache, marks it owned+dirty and schedules a COALESCING worker, and the - # class's read-your-writes contract means the very next `GET /automations` already sees the - # deletion. This is the same trade `user_tables.add_row` makes for ROW creation — strictly - # more valuable data than an automation definition — so a delete is not the place to be - # stricter than a row insert. [[sync-write-eats-pending-async-write]] is the hazard in the - # OTHER direction (a sync writer discarding a pending async write) and `Store.update`'s dirty - # branch already handles it. - _store_update(rt, _up, flush="async") - # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned - # silently: the column keeps its values and its config, and says why it stopped. Scanned - # first so a delete with no tagged fields costs no store commit. - # ⚠ HONEST RESIDUAL, stated rather than smoothed: this scan still costs ONE full read of the - # tenant's `user_tables` document on EVERY delete (`rt.get` deep-copies by design), and the - # update below reads it a second time. Removing that needs a tagged-field index, which is a - # feature and not a wave-tail edit — booked rather than improvised. What it no longer costs - # is the part the owner could feel: the blocking network commits. - # ⭐ WAVE 31 · T39(a) — D-177(a): ONE WALK, AND THE PLAN CARRIES WHAT IT FOUND. - # This used to scan the whole document to answer a boolean (`hit`), then walk the WHOLE - # document again inside `_disable` to re-derive the same matches. The scan now records the - # `(table, field)` pairs it found and the mutation applies exactly those — the shape - # `bind_unbound_fields` in this module already uses, so a delete costs one walk instead of two - # and the write touches only the fields it named. - # ⭐⭐ WAVE 33 (D-179) — THE SCAN NO LONGER READS THE ROWS, and that is where the cost was. - # - # This walk asks ONE question — "which fields did this flow tag?" — and it has never looked at - # a row to answer it. It was nevertheless taking `ut_all`, a whole deep copy of the tenant's - # `user_tables` document, of which **99.9% of the bytes are `rows` nothing here reads** - # (measured on tenant #0: 28.6 MB, 81,330 rows, 703 ms warm). A's C5 projection makes that - # ~0.1% of the bytes. - # ⛔ HANDED TO A READ AND NOTHING ELSE. `lend_defs` serves `_Projected` values, so reaching - # `defn['rows']` raises a `KeyError` NAMING the projection — by design — and `plan` carries - # only `(table_key, field_key)` STRINGS out of this scope. The mutation below takes its own - # strict document from `rt.update` and never sees a projected value, which is C5's second - # clause: a projected snapshot is never handed to a post-write read-back. - # ⚠ HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): D-179's exit condition - # asks for ZERO full-document reads and this is ONE — `rt.update` below takes a strict read - # that belongs to the WRITE and cannot be removed from here. What is gone is the expensive one. - # ⚠ AND THE FALLBACK IS DELIBERATE: `all_defs` degrades to the whole read on any failure, so a - # store that cannot project still deletes correctly, only slower. - try: - import core.user_tables as _ut_defs # noqa: PLC0415 - _scan = dict(_ut_defs.all_defs(rt) or {}) - except Exception: # noqa: BLE001 - _scan = ut_all(rt) - plan = [(tk, str(f.get("key") or "")) - for tk, t in _scan.items() - for f in ((t or {}).get("fields") or []) - if str((f.get("automation") or {}).get("flowId") or "") == aid] - if not plan: - return - - def _disable(cur): - cur = cur if isinstance(cur, dict) else {} - note = f"its automation was deleted {_stamp()}" - for tk, fk in plan: - for f in ((cur.get(tk) or {}).get("fields") or []): - a = f.get("automation") - if f.get("key") == fk and isinstance(a, dict): - a["disabled"] = True - a["statusNote"] = note - return cur - # The expensive half of the owner's complaint: a whole-document rewrite, committed while the - # person waits. Async for the reason given above — the disable is visible to the next read - # immediately; only the upload is deferred, and it coalesces with any other pending write to - # the same key instead of racing it. - rt.update(UT_STORE_KEY, _disable, flush="async") - - -# --------------------------------------------------------------------------------------------- -# RUN STATE — process memory only (see the module header for why this may never be persisted) -# --------------------------------------------------------------------------------------------- - -_RUN_LOCK = threading.RLock() -_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} - - -def running(tenant, auto_id): - with _RUN_LOCK: - return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None - - -def _claim(tenant, auto_id): - with _RUN_LOCK: - if (tenant, str(auto_id)) in _RUNNING: - return False - _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} - return True - - -def _step(tenant, auto_id, text): - with _RUN_LOCK: - cur = _RUNNING.get((tenant, str(auto_id))) - if cur is not None: - cur["step"] = text - - -def _no_step(_text): - """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner - directly). A runner must never REQUIRE the slot — the slot is process state, and the runner - is the part a test is allowed to call on its own.""" - - -def _release(tenant, auto_id): - with _RUN_LOCK: - _RUNNING.pop((tenant, str(auto_id)), None) - - -def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None): - """THE ONE store write a run performs on the `automations` bucket. - - `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here - because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to - invent a node status when this map is absent (every run stored before W19-C has no `steps`, - and painting those nodes green from the run's overall state would be a fabrication). - - ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the - comprehension below drops everything else — so the per-record sentence a vendor gave us had - literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the - product could say. MEASURED on nurilab: the same record blocked on three separate runs and - the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by - calling the vendors by hand. The note is the most valuable thing a run produces — a dead - handle, a vendor hiccup and an exhausted key are three different actions — and it was the one - thing thrown away. - ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it - did not ask for, and W25/R4 already retired writing status STRINGS into people's grids. - """ - entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), - "counts": {k: int(v) for k, v in (counts or {}).items() - if isinstance(v, (int, float))}, - "steps": {str(k): str(v) for k, v in (steps or {}).items()}, - # Bounded on both axes: a 100-profile run must not put 100 sentences into a store - # entry that is kept 20 deep per automation. - "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25], - "affected": list(affected or [])[:200]} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - d["status"] = {"state": state, "lastRunAt": entry["ts"], - "lastSummary": entry["summary"]} - d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] - # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with - # the reason on it — a dead credential must not burn quota (or a paid vendor's records) - # 96 times a day while a green toggle sits over a buried error log. Errors only: - # `partial` is the honest-progress state and pausing on it would punish honesty. - runs = d.get("runs") or [] - if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( - not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): - sch = d.get("schedule") - if isinstance(sch, dict) and sch.get("enabled"): - sch["enabled"] = False - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " - f"failures — fix the cause, then re-enable: " - f"{entry['summary'][:120]}") - return cur - - _store_update(rt, _up, flush="sync") - _notify_outcome(rt, auto_id, state, entry) - return entry - - -def _notify_outcome(rt, auto_id, state, entry): - """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7). - - Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is - written, so it is the only place this can go without a second definition of "what happened". - - ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED. - 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the - MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a - background thread with no session, so an omitted `st` files nurilab's automation failures - in Royal Imports' inbox — D-16's class, silently, forever. - 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler - has no session at all, so the alternative is not "the wrong person" but "nobody", and - `notify` returns None on a blank owner — the notification would simply never exist, on - exactly the runs nobody was watching. - ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only - view-alert CREATION; `notify()` writes straight into the notifications bucket and - `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell. - ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that - SUCCEEDED into one that raised — the run entry is already committed one line above, and the - bell is a courtesy on top of it. - """ - try: - defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {} - owner = str(defn.get("createdBy") or "").strip() - if not owner: - return - from core import alerts as _alerts - _alerts.notify( - owner=owner, - label=_s(defn.get("name") or "Automation", 80), - topic="automation", - key=str(auto_id), - detail=f"{state}: {entry.get('summary') or ''}"[:300], - st=rt) - except Exception: # noqa: BLE001 - pass - - -# --------------------------------------------------------------------------------------------- -# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN -# --------------------------------------------------------------------------------------------- -# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and -# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) -# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: -# -# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, -# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the -# supplier takes the scraping risk; it is never an Instagram credential and this module must -# never be given one. -# -# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). -# -# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress -# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been -# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and -# the cell says which. A pull that quietly wrote zero posts and reported success would be worse -# than no automation at all: the table would look maintained and be empty. -# -# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, -# so a run history answers "what still works anonymously?" from data rather than from memory. - -PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) - -#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's -#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number -#: is a NOW value, so the series starts the day capture starts and a field we do not capture -#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows -#: has exactly the same cost as delaying capture — for that column — and is justified by exactly -#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to -#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. -#: Every returned provider field is retained in Source data. Promoted columns make commonly used -#: values filterable; the raw source document prevents a new or uncommon field from being lost. -#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose -#: almost none of them, which is what the `source` column is for. -SNAPSHOT_FIELDS = [ - field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"), - field_def("following", "Following", "int"), - field_def("posts_count", "Post count", "int"), - field_def("full_name", "Name"), field_def("bio", "Bio"), - field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"), - field_def("approx", "Counts are approximate", "checkbox"), - # The link in bio — the paid rung's field, and commercially the most useful single string an - # influencer row can carry (it is the shop/affiliate destination). - field_def("external_url", "Link in bio", "url"), - # --- the rest of the vendor's Profiles schema. - field_def("ig_id", "Instagram id"), - field_def("profile_url", "Profile", "url"), - field_def("avg_engagement", "Avg engagement", "pct"), - field_def("category", "Category"), - field_def("business_category", "Business category"), - field_def("is_business", "Business account", "checkbox"), - field_def("is_professional", "Professional account", "checkbox"), - field_def("is_private", "Private", "checkbox"), - field_def("highlights_count", "Story highlight count", "int"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("pronouns", "Pronouns"), - # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot - # row maps the WHOLE schema by design, so these belong here the moment the map reads them; - # leaving them out would be the "captured but unrecorded" half of the same loss the note at - # the top of this list is about. - field_def("profile_name", "Profile name"), - field_def("is_joined_recently", "Joined recently", "checkbox"), - field_def("has_channel", "Has channel", "checkbox"), - field_def("partner_id", "Partner id"), - field_def("external_url_title", "Link title"), - field_def("fbid", "Facebook id"), - field_def("related_accounts", "Related accounts"), - field_def("country_code", "Country"), - field_def("source_payload", "Source data", "json"), -] -POST_FIELDS = [ - field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), - field_def("posted_at", "Posted at", "date"), - field_def("type", "Type", "select", options=["image", "video", "carousel"]), - field_def("caption", "Caption"), field_def("url", "URL", "url"), - # A tagged place is useful content context and can be filtered as ordinary text. It is never - # presented as the creator's location: a creator can tag a holiday, venue or brand location. - field_def("tagged_location", "Tagged location"), - # The per-field map below gives operational fields first; this locked JSON document retains - # every other value Bright Data supplied, so a vendor schema addition is preserved - # instead of being silently discarded while the canonical field model catches up. - field_def("source_payload", "Source data", "json"), - # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous - # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). - field_def("paid_partnership", "Paid partnership", "checkbox"), - field_def("partner", "Partner brand"), - field_def("hashtags", "Hashtags"), - field_def("alt_text", "Alt text"), - # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT - # ONE HOP. - # - # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's - # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a - # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated - # transitively). So the post row carries its own LATEST value and the rollup reads it - # directly. - # - # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile - # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in - # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that - # was just appended — they are a projection of that store, never a second copy of it, and - # deleting them would cost a convenience rather than a measurement. - # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is - # off by default (it buys one vendor record per post), so on most pulls this stays empty, and - # empty means "not read", which is what makes an honest average possible at all. A 0 here - # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or - # COMMENT count is a real measurement and must survive. - # - # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08, - # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought). - # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct - # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in - # the very same rows. Mapping it here is what put one number on many unrelated posts. We - # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers; - # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays - # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained - # whole, so if the vendor ever populates it per-reel the history is re-derivable. - # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30 - # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the - # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named - # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10 - # (folded into a single "Views"), not because we are reading the wrong key. - # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED. - # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the - # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` — - # measured against a browser-read ground truth to the digit. Same column name, same type, same - # rollup; the engine underneath is swappable and the preset database never moved. That is the - # owner's schema ruling working exactly as intended. - # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung. - field_def("views", "Views", "int"), - field_def("plays", "Plays", "int"), - field_def("likes", "Likes", "int"), - field_def("comments", "Comments", "int"), - # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data - # further"*). Two fields the primary source does not return AT ALL, already paid for inside - # the same engagement response — so capturing them costs nothing extra. - # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on - # every write door, so a normaliser that emits `video_duration` without this line writes - # nothing and reports success — the wave-28 defect that swallowed nine preset cells. - field_def("video_duration", "Video length (s)", "int"), - field_def("comments_disabled", "Comments off", "checkbox"), - field_def("measured_at", "Engagement read at", "date"), - # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED": - # a post reaches its own comment rows the same way a profile reaches its posts. Derived from - # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance. - # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state - # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when - # `postMetrics` is off. - field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this post.", - link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), - field_def("post_snapshots_link", "Measurement rows", "link", - description="Engagement measurements linked to this post.", - link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), - field_def("measurements_captured", "Measurements captured", "rollup", - rollup={"link": "post_snapshots_link", "fn": "countall"}), -] - -#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE. -#: -#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to -#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default -#: is the capture. -#: -#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED. -#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE: -#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of -#: `instagram-capture.md`). They are the single most expensive thing this product can buy. -#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has -#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment -#: thread ingests identifiable people who never entered anybody's influencer list. -#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code -#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments` -#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on -#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is -#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md` -#: is written against. -#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as -#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs -#: nothing; the spend and the third-party ingest are a click somebody makes knowingly. -#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because -#: they are part of a record already paid for. The separate full Comments dataset remains OFF -#: unless `commentMetrics` is explicitly enabled on the enrichment action. -COMMENT_FIELDS = [ - field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"), - field_def("influencer_key", "Influencer"), - # ⭐⭐ OWNER RULING 2026-08-12 — the same one that put `text` on the TikTok comment schema, and - # it lands on BOTH networks in the same change on purpose: the two comment tables are read side - # by side, and one carrying the content while the other does not is the divergence this repo - # keeps paying for. ⛔ TEXT ONLY — `comment_user` / `comment_user_url` are vendor-FLAGGED PII - # and stay in `source_payload`, uncolumned. - field_def("text", "Comment"), - field_def("commented_at", "Commented at", "date"), - field_def("likes", "Likes", "int"), - field_def("replies", "Replies", "int"), - field_def("source_payload", "Source data", "json"), - field_def("post_link", "Post", "link", - description="Post record linked to this comment.", - link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), - # Author/text and every provider-specific value stay intact in Source data. The promoted - # columns intentionally keep the grid concise while Likes and Replies make comment engagement - # directly filterable and rollup-ready. -] -POST_SNAPSHOT_FIELDS = [ - field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), - field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"), - field_def("comments", "Comments", "int"), - # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits - # before any rollup argument is made. - # - # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with - # only the first can describe a measurement but not its AGE, so the single most useful thing - # this series can compute — views at N days old, i.e. velocity — is not expressible over it at - # all. `type` is the same shape one column over: "reels only" is the default lens on this data - # and without it every question has to go back through `ut_ig_posts` to ask what kind of post - # this was. - # - # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row - # from the same vendor record), so this costs one dict key each and no extra call. - # - # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24 - # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the - # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied - # from it deliberately rather than shared, because these are two different tables' columns that - # happen to agree today, and `verify_automation` asserts the agreement. - # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO - # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`, - # and `ut_ig_posts` already performs the first of them. See the verdict in - # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for - # moving the window here. - field_def("posted_at", "Posted at", "date"), - field_def("type", "Type", "select", options=["image", "video", "carousel"]), - # Plays move independently of likes on video, so it is its own series rather than a thing to - # derive. Paid rung only; blank means not read. - # The series regains Views alongside the latest-value projection on the Post row — one store - # for one series (R3) is untouched; this IS that store. - field_def("views", "Views", "int"), - field_def("plays", "Plays", "int"), - field_def("source_payload", "Source data", "json"), - field_def("post_link", "Post", "link", - description="Post record linked to this measurement.", - link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - -#: ⭐⭐ WAVE 32 · T41 — THE INSTAGRAM TWIN OF `TT_TABLE_FIELDS`, WHICH DID NOT EXIST. -#: -#: `TT_TABLE_FIELDS`' own note said so and named the consequence: *"The Instagram side has no -#: equivalent map, which is exactly why its table names are scattered across the module."* It is a -#: map now, for a concrete reason rather than symmetry: A's boot-time delivery sweep (`W32-T07`) has -#: to call `ut_ensure` for every locked child of BOTH platforms, and a sweep that can ask TikTok for -#: its table set and must hand-type Instagram's is one platform away from the drift this pair of -#: maps exists to stop. -#: -#: ⛔ THESE FOUR LISTS ARE THE *CANONICAL* DECLARATION — the per-tenant BACKLINK fields -#: `ensure_ig_graph` appends are NOT here, and must not be. A backlink is derived per profile -#: database (`_profile_backlink_field`), so folding it into a module constant would make one -#: tenant's relation a global fact. `ensure_ig_graph` reads this map and adds its own backlinks on -#: top, which is why that function still owns the graph write and this only owns the schema. -IG_TABLE_FIELDS = { - IG_SNAPSHOTS_TABLE: SNAPSHOT_FIELDS, - IG_POSTS_TABLE: POST_FIELDS, - IG_POST_SNAPSHOTS_TABLE: POST_SNAPSHOT_FIELDS, - IG_COMMENTS_TABLE: COMMENT_FIELDS, -} -#: The labels those four wear in the database list. ⚠ LIFTED VERBATIM from `ensure_ig_graph`'s own -#: `graph` literal, which is now derived from this map — a renamed label here renames the table -#: everywhere rather than leaving two spellings of one database. -IG_TABLE_LABELS = { - IG_SNAPSHOTS_TABLE: "IG snapshots", - IG_POSTS_TABLE: "IG posts", - IG_POST_SNAPSHOTS_TABLE: "IG post snapshots", - IG_COMMENTS_TABLE: "IG comments", -} - - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ TIKTOK — THE FIVE `ut_tt_*` SCHEMAS (wave 29 · item 7 · D-9 · rulings R1 + R2) -# --------------------------------------------------------------------------------------------- -# ⛔ EVERY VENDOR FIELD NAME BELOW IS FROM ONE PROBED SOURCE — `waves/wave29/proto/tiktok-schema.md`, -# 40 profile / 43 post / 17 comment fields read live from `GET /datasets/{id}/metadata` for $0.00, -# each carrying the vendor's own type, description and `pii` flag. NOTHING HERE IS GUESSED, and the -# names live in `connectors_tt.py` (the map), not here (the schema). This file declares what a -# COLUMN is; the connector declares what the wire calls it. -# -# ⭐ THE SAME LAW AS THE INSTAGRAM LISTS ABOVE: map the schema, not just what was populated on the -# probe. HISTORY IS UNBUYABLE — no vendor sells "followers on 1 January" — so a field we do not -# capture today is permanently lost for today, and an occasionally-empty column costs a blank cell -# while a missing one costs the months before somebody notices. -# -# ⚠ TWO DELIBERATE DIVERGENCES FROM THE INSTAGRAM SCHEMA, both licensed by R2 ("the two schemas may -# diverge where the vendors do") and both recorded so neither reads as an omission: -# -# 1. NO `plays` COLUMN. TikTok returns ONE number, `play_count`, and it is the count TikTok -# itself displays under a video — i.e. our `views`. Instagram has two columns because Meta -# once had two metrics (`plays` retired 2025-04-10). Writing one vendor number into two of our -# columns would manufacture a second measurement that a rollup could average or double-count, -# and blank-vs-zero discipline says an unmeasured column must be blank, not a copy. -# ⇒ `play_count` -> `views`, and `plays` does not exist on this family. -# 2. `tt_id`, NOT `ig_id`. The probe doc flags this: our profile key is literally NAMED `ig_id`. -# A TikTok row carrying a column labelled "Instagram id" is a header that lies, and this is a -# NEW table family with no stored rows to migrate — so the honest name costs nothing. -# -# ⚠ THE COLUMNS THAT STAY BLANK ARE NAMED RATHER THAN DROPPED SILENTLY. TikTok has no equivalent of -# `category`, `business_category`, `is_professional`, `highlights_count`, `bio_hashtags`, -# `pronouns`, `profile_name`, `is_joined_recently`, `has_channel`, `partner_id`, -# `external_url_title`, `fbid` or `related_accounts` (profile), nor of `alt_text`, -# `comments_disabled`, `paid_partnership` or `partner` (post) — `commerce_info` is a LOCATION, not -# a paid-partnership flag, and TikTok's `comment_setting` is profile-level rather than per-post. -# Those columns are ABSENT here rather than declared and permanently blank — a column nothing can -# ever write is a promise the grid keeps making that the vendor cannot keep. - -#: ⭐ WAVE 29 (contract C2) — the value the `profile` FLAG carries on a TikTok handle column: the -#: twin of `PROFILE_SOURCE_IG`, which is declared further down beside the `PLATFORM_*` vocabulary -#: with the note on why those two families of string are NOT the same thing. It sits here rather -#: than there because the lists below are evaluated at IMPORT and would raise a NameError otherwise. -#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT, AND THE OTHER HALF HAS LANDED (verified 2026-08-12, -#: W30-T08): `platform/core/user_tables.py:PROFILE_SOURCES` now reads `('instagram', 'tiktok')`. -#: Before it did, `_clean_profile` REFUSED this flag and the handle column of a `ut_tt_profile` -#: spawn was dropped — fail-closed and loud by design, because the alternative is a profile -#: database whose profile column silently is not one. Left written down rather than deleted: the -#: refusal is what a TikTok binding looks like on any deployment where that line is missing. -PROFILE_SOURCE_TT = "tiktok" - -#: The profile row: LATEST values plus `enriched_at`. The SERIES lives in `ut_tt_snapshots` — -#: R3's "one store for one series" carries over unchanged, because the reason for it does (no -#: vendor sells history, so the append table is the only history there will ever be). -TT_PROFILE_FIELDS = [ - # `platform` FIRST, exactly as the Instagram preset set has it — and on all five tables here, - # not just this one. The IG family carries it on the profile table alone, which is why a TikTok - # POST could never have lived beside an Instagram post; declaring it everywhere is what makes a - # future cross-platform union view a UNION rather than a guess. - tt_field_def("platform", "Platform"), - tt_field_def("handle", "Handle", pinned=True, - profile={"source": PROFILE_SOURCE_TT}), - tt_field_def("full_name", "Name"), - tt_field_def("tt_id", "TikTok id"), - tt_field_def("profile_url", "Profile", "url"), - tt_field_def("bio", "Bio"), - tt_field_def("external_url", "Link in bio", "url"), - tt_field_def("verified", "Verified", "checkbox"), - tt_field_def("is_private", "Private", "checkbox"), - tt_field_def("is_business", "Business account", "checkbox"), - tt_field_def("followers", "Followers", "int"), - tt_field_def("following", "Following", "int"), - tt_field_def("posts_count", "Video count", "int"), - # TikTok gives THREE engagement rates where Instagram gives one. All three are stored ×100 for - # the same reason `avg_engagement` is on the IG side (C1-a): the vendor sends a 0–1 fraction and - # our `pct` renderer appends the sign to the stored number, so the raw fraction would print a - # 6.6% creator as `0.0%`. - tt_field_def("avg_engagement", "Avg engagement", "pct"), - tt_field_def("like_engagement", "Like engagement", "pct"), - tt_field_def("comment_engagement", "Comment engagement", "pct"), - # Total likes RECEIVED across the account's videos — a TikTok-only number with no Instagram - # equivalent, and one of the few profile-level engagement facts a vendor gives away. - tt_field_def("likes_received", "Likes received", "int"), - tt_field_def("country_code", "Country"), - tt_field_def("region", "Region"), - tt_field_def("predicted_lang", "Language"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP. The vendor's `create_time` on a profile is when the - # ACCOUNT was created; it is emphatically not "when `followers` was true". TikTok carries no - # measurement timestamp at all, exactly like Instagram — which is why the append law below - # (`@`) is the only thing that can date a number. - tt_field_def("account_created_at", "Account created", "date"), - tt_field_def("first_found", "First found", "date"), - tt_field_def("last_found", "Last found", "date"), - tt_field_def("found_count", "Times found", "int"), - tt_field_def("created_by", "Found by"), - tt_field_def("enriched_at", "Enriched at", "date"), - tt_field_def("source", "Read via"), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("profile_snapshots_link", "Measurement rows", "link", - description="Profile measurements linked to this account.", - link={"table": TT_SNAPSHOTS_TABLE, "on": "influencer_key", "from": "handle"}), - tt_field_def("posts_link", "Post rows", "link", - description="Post records linked to this account.", - link={"table": TT_POSTS_TABLE, "on": "influencer_key", "from": "handle"}), -] - -#: The profile SERIES. One row per profile per pull, keyed `@` — the append law -#: from `instagram-capture.md` §3, unchanged, because its cause is unchanged: the vendor stamps -#: nothing, so the only honest date a number can carry is the moment WE read it. -TT_SNAPSHOT_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("snapshot_key", "Snapshot"), - tt_field_def("influencer_key", "Influencer"), - tt_field_def("pulled_at", "Pulled at", "date"), - tt_field_def("followers", "Followers", "int"), - tt_field_def("following", "Following", "int"), - tt_field_def("posts_count", "Video count", "int"), - tt_field_def("likes_received", "Likes received", "int"), - tt_field_def("full_name", "Name"), - tt_field_def("bio", "Bio"), - tt_field_def("verified", "Verified", "checkbox"), - tt_field_def("is_private", "Private", "checkbox"), - tt_field_def("is_business", "Business account", "checkbox"), - tt_field_def("avg_engagement", "Avg engagement", "pct"), - tt_field_def("like_engagement", "Like engagement", "pct"), - tt_field_def("comment_engagement", "Comment engagement", "pct"), - tt_field_def("external_url", "Link in bio", "url"), - tt_field_def("tt_id", "TikTok id"), - tt_field_def("profile_url", "Profile", "url"), - tt_field_def("country_code", "Country"), - tt_field_def("region", "Region"), - tt_field_def("predicted_lang", "Language"), - tt_field_def("account_created_at", "Account created", "date"), - tt_field_def("source", "Read via"), - tt_field_def("approx", "Counts are approximate", "checkbox", - description="Checked when the counts on this row are rounded, not exact."), - tt_field_def("source_payload", "Source data", "json"), -] - -#: One row per TikTok post. Identity is `shortcode`, and on TikTok that is a 19-digit numeric id — -#: ✅ the SAME shape as the Comments dataset's `post_id`, so the comments→posts link joins on -#: equality with NO normaliser (measured in the probe doc; Instagram needed one). -TT_POST_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("shortcode", "Post id"), - tt_field_def("influencer_key", "Influencer"), - tt_field_def("posted_at", "Posted at", "date"), - # ⛔ THE VENDOR'S VOCABULARY IS `"video"` / `"content"` AND OURS HAS NO `"content"`. Declaring - # the vendor's word would put an untranslated API token in front of a user; declaring an option - # the mapper can emit but the select does not list would fail `_clean_field`. So the OPTIONS - # stay this product's words and `connectors_tt.normalize_post` does the translation — the same - # posture every other vendor value here takes. `carousel` is listed because a TikTok photo post - # carrying more than one image IS one, and the mapper decides from `carousel_images` rather - # than from the type token, which cannot express it. - tt_field_def("type", "Type", "select", options=["image", "video", "carousel"]), - tt_field_def("caption", "Caption"), - tt_field_def("url", "URL", "url"), - tt_field_def("hashtags", "Hashtags"), - tt_field_def("tagged_location", "Commerce location"), - # ⛔ `views` AND NO `plays` — see the divergence note at the top of this section. One vendor - # number, one column. - tt_field_def("views", "Views", "int"), - tt_field_def("likes", "Likes", "int"), - tt_field_def("comments", "Comments", "int"), - tt_field_def("shares", "Shares", "int"), - tt_field_def("saves", "Saves", "int"), - tt_field_def("video_duration", "Video length (s)", "int"), - tt_field_def("measured_at", "Engagement read at", "date", - description="When the engagement numbers on this row were read."), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this post.", - link={"table": TT_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), - tt_field_def("post_snapshots_link", "Measurement rows", "link", - description="Engagement measurements linked to this post.", - link={"table": TT_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), - tt_field_def("measurements_captured", "Measurements captured", "rollup", - description="How many measurements this post has.", - rollup={"link": "post_snapshots_link", "fn": "countall"}), -] - -#: The POST series. ⚠ D-117 is live on the Instagram twin — `posted_at`/`type` are denormalised onto -#: snapshot rows there and never reconcile with the post row afterwards. That pattern is NOT -#: reproduced: this table carries the measurement and its identity, and asks `ut_tt_posts` for what -#: kind of post it was. A fact that can disagree with its own dimension table is a fact nobody can -#: trust, and the convenience it buys (one fewer hop in a rollup) is not worth a column that can be -#: wrong. -TT_POST_SNAPSHOT_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("post_snapshot_key", "Snapshot"), - tt_field_def("shortcode", "Post id"), - tt_field_def("influencer_key", "Influencer"), - tt_field_def("pulled_at", "Pulled at", "date"), - tt_field_def("views", "Views", "int"), - tt_field_def("likes", "Likes", "int"), - tt_field_def("comments", "Comments", "int"), - tt_field_def("shares", "Shares", "int"), - tt_field_def("saves", "Saves", "int"), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("post_link", "Post", "link", - description="Post record linked to this measurement.", - link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - -#: Comments. Same posture as Instagram's: the SCHEMA and the LINK ship, the CAPTURE is opt-in and -#: defaults OFF (`commentMetrics`), because comments are the single most expensive thing this -#: product can buy and they ingest identifiable third parties who never entered anybody's list -#: (D-24). ⚠ TikTok's `replies` is an ARRAY where ours is an INT count — a name collision, resolved -#: in the mapper by storing `num_replies` and leaving the array in Source data. -TT_COMMENT_FIELDS = [ - tt_field_def("platform", "Platform"), - tt_field_def("comment_key", "Comment"), - tt_field_def("shortcode", "Post id"), - tt_field_def("influencer_key", "Influencer"), - # ⭐⭐ OWNER RULING 2026-08-12, verbatim: *"why isn't any of the comments column actually HAS - # the comments content, fix it."* The text was bought, stored and INVISIBLE — retained whole in - # `source_payload` under the 2026-08-07 superseding ruling (`instagram-capture.md` §4b), but - # promoted to no column, so a person paying per comment record saw only Likes and Replies. - # ⛔ THE COMMENT TEXT ONLY — the commenter's IDENTITY (`commenter_user_name`, `commenter_id`, - # `commenter_url`; the vendor FLAGS the first as PII) stays in `source_payload` and gets no - # column. The ruling names the comments' CONTENT, and promoting a third party's name and - # profile URL into a filterable, exportable column is a different decision that nobody made. - tt_field_def("text", "Comment"), - tt_field_def("commented_at", "Commented at", "date"), - tt_field_def("likes", "Likes", "int"), - tt_field_def("replies", "Replies", "int"), - tt_field_def("source_payload", "Source data", "json"), - tt_field_def("post_link", "Post", "link", - description="Post record linked to this comment.", - link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - -#: table key -> the field list that defines it. ⭐ DERIVED CONSUMPTION IS THE POINT: `ut_ensure`, -#: the gates and every future runner read THIS rather than naming five constants, so adding a sixth -#: `ut_tt_*` table is one entry instead of a sweep. The Instagram side has no equivalent map, which -#: is exactly why its table names are scattered across the module. -TT_TABLE_FIELDS = { - TT_PROFILE_TABLE: TT_PROFILE_FIELDS, - TT_SNAPSHOTS_TABLE: TT_SNAPSHOT_FIELDS, - TT_POSTS_TABLE: TT_POST_FIELDS, - TT_POST_SNAPSHOTS_TABLE: TT_POST_SNAPSHOT_FIELDS, - TT_COMMENTS_TABLE: TT_COMMENT_FIELDS, -} -#: The human labels a spawned `ut_tt_*` table wears in the database list. -TT_TABLE_LABELS = { - TT_PROFILE_TABLE: "TikTok profiles", - TT_SNAPSHOTS_TABLE: "TikTok profile measurements", - TT_POSTS_TABLE: "TikTok posts", - TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements", - TT_COMMENTS_TABLE: "TikTok comments", -} -#: ⭐⭐ WAVE 31 · OWNER RULING R9 — the `ut_tt_*` children a person may not type records into. -#: Owner, verbatim: *"Tiktok database for post and comments and their snapshots should also be a -#: locked database with the lock icon, exactly like how instagram"*. -#: -#: ⛔ DECLARED ONCE BECAUSE THE DEFECT WAS TWO CALL SITES DISAGREEING WITH A THIRD. `ensure_ig_graph` -#: passes `record_mode=AUTOMATION_RECORD_MODE` for all four Instagram children; TikTok's two spawn -#: sites — `_spawn_presets`' child loop (SAVE) and `_tt_write_tables` (RUN) — each passed -#: `lock_fields=True` and no `record_mode` at all. That is the whole bug: not a missing feature, one -#: argument missing at two places, which is exactly the shape `_tt_write_tables`' own docstring warns -#: about (*"one function, two callers, so the inline and deferred paths cannot answer differently"*). -#: A SET plus a resolver means a sixth `ut_tt_*` table joins by declaration, not by remembering. -#: -#: ⚠ FOUR, NOT THE THREE R9 ENUMERATES, and the fourth is argued rather than assumed. -#: `ut_tt_snapshots` is the profile measurement series — the exact twin of `ut_ig_snapshots`, which -#: IS locked. R9's own comparison is *"exactly like how instagram"*, and the alternative is to leave -#: a machine-append time series that a person can type into, one table away from three that they -#: cannot. That is the fix-applied-to-one-platform-and-not-its-twin shape this very wave is closing -#: elsewhere (D-177(c)). `ut_tt_profile` is deliberately ABSENT: it is the database a human adds -#: handles to, and its Instagram counterpart is not locked either. -#: -#: ⚠ LOCKED DATABASE, NOT READ-ONLY (DESIGN.md §4): `records_mutable` goes False, so records cannot -#: be added/edited/deleted — **adding a FIELD must still work**, and a UI hiding add-field here is a -#: defect, not the intent. -TT_LOCKED_TABLES = frozenset({TT_SNAPSHOTS_TABLE, TT_POSTS_TABLE, TT_POST_SNAPSHOTS_TABLE, - TT_COMMENTS_TABLE}) -#: ⭐ THE INSTAGRAM HALF, NAMED. It was only ever the four keys `ensure_ig_graph`'s loop happens to -#: iterate, which is a set that exists but cannot be ASKED — so nothing could assert that IG and -#: TikTok lock the same shape, and the QA sweep below could not be written at all. -IG_LOCKED_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, IG_POST_SNAPSHOTS_TABLE, - IG_COMMENTS_TABLE}) -#: ⭐⭐ W31 QA — OWNER, VERBATIM (2026-08-13): *"just like Instagram Post database (which is -#: locked), only the IG Profile and TT Profile should be editable."* That is this set plus its -#: complement: every `ut_ig_*`/`ut_tt_*` child is locked, and `ut_ig_profile` / `ut_tt_profile` are -#: the two a person types handles into. Registered into `core.user_tables` at import so the lock is -#: a DECLARATION rather than a flag some past automation run happened to stamp — see that module's -#: `_LOCKED_RECORD_KEYS` for why the stored flag alone left production unlocked. -#: ⛔ THE REGISTRATION ITSELF IS IN `main.py`, NOT HERE, and that is this module's own rule rather -#: than a preference: it imports NOTHING from `core` at module level (see `MAX_UT_ROWS` and -#: `MACHINE_OWNERS`, both local literals "so this module stays dependency-light for the API's boot -#: path"). Adding `import core.user_tables` at line 3570 would put `core` on the import path of -#: every process that touches the engine, to run one line. The composition root wires it, and -#: `verify_automation` asserts that main.py CARRIES that call — a declaration whose registrar is -#: missing is [[flag-shipped-without-its-writer]], which is the defect class this whole fix is in. -LOCKED_CHILD_TABLES = IG_LOCKED_TABLES | TT_LOCKED_TABLES - - -def tt_record_mode(table_key): - """R9's answer for one `ut_tt_*` key — the `record_mode` its `ut_ensure` must carry. - - `""` for anything outside the locked set, which is what `ut_ensure` already treats as "say - nothing about record mode", so an unlocked table is untouched rather than explicitly opened. - """ - return AUTOMATION_RECORD_MODE if str(table_key) in TT_LOCKED_TABLES else "" - - -# --------------------------------------------------------------------------------------------- -# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`. -# --------------------------------------------------------------------------------------------- -# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE. -# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation -# runtime needs that, and the runtime is what this file is for. -# -# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation -# runtime still reaches for after the split — twenty, down from the seventy-one that used to be -# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor -# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and -# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that -# true without those files having to learn where the wire went. -# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked -# that would be green whether or not anything moved. `verify_automation`'s `section_split` -# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING -# system instead of trusting this import line ([[gate-answers-the-wrong-question]]). -# -# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back -# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily, -# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail -# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a -# module-level import, the names it wants already exist and the cycle still resolves. -# ⭐⭐ WAVE 30 · T09 (D-128) — TWO IMPORT BLOCKS NOW, AND THE SPLIT BETWEEN THEM IS THE POINT. -# `connectors_bd` is the SUPPLIER's wire — it serves both platforms and knows neither. `connectors_ig` -# is INSTAGRAM's vocabulary: its dataset ids, its row mappers, its free rungs. Anything that would -# have to change to serve a third platform belongs in the second block, not the first. -# ⚠ `bd_ready` is re-exported through this module ON PURPOSE — `routes_automation` and -# `routes_connectors` both reach `engine.bd_ready()` on the module object, and those files belong to -# other sessions. Dropping it here would 500 two surfaces that never mention a vendor. -from connectors_bd import ( # noqa: E402 - BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, BD_RECORD_PRICE_SPEC, - _bd_deferral, _bd_first_url, _bd_rows, - _first, _ig_int, - bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, - bd_snapshot_progress, depth_refusal, -) -from connectors_ig import ( # noqa: E402 - BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, - _bd_comment, _bd_post_metrics, _bd_profile, - _bd_tagged_location, - DEFERRED_MARK, - apify_profile, - bd_profiles_batch, - ig_handle, public_source, pull_profile, select_post_groups, top_up_views, -) - - -# --------------------------------------------------------------------------------------------- -# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) -# --------------------------------------------------------------------------------------------- -# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's -# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns -# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND -# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower -# Georgia/Florida floral-design company — Fisch Floral's actual market. -# -# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: -# -# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` -# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is -# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the -# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. -# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, -# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, -# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still -# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a -# snapshot that is not ready when it looks. -# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery -# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` -# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is -# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the -# next run collects it. A `partial` that says "still building, the next run collects it" is -# the honest report of exactly what happened. -# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and -# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into -# a wider profile set would silently multiply the enrichment bill AND pollute the -# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich. -# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important -# test died on it. -# -# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE -# the API returns a number, `price: 0` means "not priced" rather than "free", and -# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to -# report. Every estimate below is derived from the PRICING PAGE and says so. - -#: The table a discovery run writes when the tenant has no Instagram profile database yet — the -#: FIRST one, never a second (`discover_default_table` elects an existing one before this is -#: reached). Upsert by handle, so re-finding a profile is free. -#: -#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG -#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said -#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`, -#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept -#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it -#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on -#: any tenant who renamed the table, which is exactly the mismatch the owner hit. -#: -#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three -#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the -#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key -#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant -#: would keep working; there simply is not one. -#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration -#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family. -DISCOVER_TABLE = "ut_ig_profile" -#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels, -#: so changing this can rename nothing that already exists. One constant because it was three -#: copies of the same string literal, and a default spelled three times is one edit from drifting. -DISCOVER_LABEL = "IG profile" -#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at -#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed -#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real -#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. -BD_MAX_RECORDS = 500 -#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). -BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) -BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) - -#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error -#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. -BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", - "array_includes", "not_array_includes", "is_null", "is_not_null") -#: Operators that take NO value (everything else requires one). -BD_NULLARY_OPS = ("is_null", "is_not_null") - -#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** -#: by the API and is therefore absent rather than offered-and-broken). -#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY -#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier -#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the -#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by -#: accident; adding them back is a decision, not a typo. -BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", - "category_name", "business_category_name", "is_business_account", - "is_professional_account", "is_verified", "account", "full_name", - "external_url", "bio_hashtags", "post_hashtags", "profile_url", - "related_accounts", "profile_name", "id", "fbid", "highlights_count") -#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: -#: these are the fields seen carrying values on real CORPUS rows. `category_name` and -#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter -#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". -BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") - -#: ⭐⭐ WAVE 32 · T46 / DEBT D-167 — THE FILTER VOCABULARY HAS A PLATFORM NOW, AND IT COST MONEY -#: NOT TO. `BD_FILTER_FIELDS` above was measured against INSTAGRAM in wave 22; waves 29 and 30 hung -#: a second platform on the same tuple, so a `discover_tiktok` could be built on any of the **16 -#: fields Bright Data's TikTok Profiles dataset does not have** — and `BD_FILTER_LEAD`'s third -#: field, `avg_engagement`, is one of the three the Find panel LEADS with and is absent there. -#: A search on a field the corpus does not carry does not error: it returns nothing, and reads -#: exactly like *"no such creators exist"* after the money has been spent. -#: -#: ⛔ MEASURED, NOT REASONED — W30-B20, against TikTok's 40 declared dataset fields: **5 of the 21 -#: names survive.** They are these. Anything else is refused at the door with the field named, -#: which is `clean_predicates`' existing posture (*"refuses rather than coerces … the alternative -#: turns 'find me verified accounts in Georgia' into 'find me any account' and bills for the -#: difference"*) — now applied per platform rather than per product. -#: ⚠ NOT DERIVED FROM `connectors_tt.normalize_profile`, TEMPTING AS THAT IS. That mapper declares -#: the vendor's READ names on a returned row; these are the names its FILTER validator accepts, and -#: the two vocabularies are not the same thing on either platform (`awg_engagement_rate` reads, -#: `avg_engagement` filters). Deriving one from the other would look rigorous and be wrong. -TT_FILTER_FIELDS = ("followers", "following", "biography", "is_verified", "id") -#: The two of `BD_FILTER_LEAD`'s three that TikTok actually carries. `avg_engagement` is dropped -#: for the reason above — leading with a field the corpus cannot answer is worse than leading with -#: two. -TT_FILTER_LEAD = ("followers", "biography") - - -def filter_fields(kind=""): - """The searchable field names for one discovery kind — `(fields, lead)`. - - ⛔ ONE ACCESSOR, so the ROUTE that publishes the vocabulary and the VALIDATOR that enforces it - cannot come to disagree about it. That divergence is D-167 itself: the route served 21 names - with no platform dimension and `clean_predicates` validated against the same tuple with no kind - parameter, so both halves were consistently wrong together and nothing could notice. - """ - if str(kind or "") == "discover_tiktok": - return TT_FILTER_FIELDS, TT_FILTER_LEAD - return BD_FILTER_FIELDS, BD_FILTER_LEAD - -#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. -#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still -#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not -#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a -#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the -#: scanner still has to walk, so they refine a search and cannot BE one. -BD_NARROWING_FIELDS = frozenset({ - "biography", "account", "full_name", "profile_name", "category_name", - "business_category_name", "external_url", "bio_hashtags", "post_hashtags", - "related_accounts", "id", "fbid", "profile_url"}) -#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the -#: whole corpus, which is breadth wearing a condition's clothes. -BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) -#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed -#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% -#: populated: below it a filter misses more corpus than it matches, and the surface should -#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, -#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never -#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. -#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding -#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. -BD_POPULATED_FIELDS = frozenset({ - "account", "followers", "following", "posts_count", "avg_engagement", "biography", - "external_url", "is_business_account", "is_professional_account", "is_verified", - "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) -BD_MIN_NARROWING = 1 - -# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ────────────────── -# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY -# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what -# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner: -# *"look at each damn schema and only provide toggles operator that make sense"*. -# -# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx` -# defended raw names on the grounds that a refusal names the field exactly as the row does — and -# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI -# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio". -BD_FIELD_LABELS = { - "followers": "Followers", "following": "Following", "posts_count": "Post count", - "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category", - "business_category_name": "Business category", "is_business_account": "Business account", - "is_professional_account": "Professional account", "is_verified": "Verified", - "account": "Handle", "full_name": "Name", "profile_name": "Profile name", - "external_url": "Link in bio", "profile_url": "Profile link", - "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts", - "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID", - "highlights_count": "Story highlight count", -} -#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident; -#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson). -BD_FIELD_HINTS = { - "account": "The @username", - "avg_engagement": "Likes and comments as a share of followers", - "id": "Instagram's internal number for the account — for matching a list you already have", - "fbid": "The linked Facebook id — for matching a list you already have", - "profile_name": "The name shown above the bio, when it differs from the account name", - "related_accounts": "Accounts Instagram suggests alongside this one", -} -#: The KIND decides the comparisons offered and the control drawn for the value. -BD_FIELD_KINDS = { - **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement", - "highlights_count")}, - **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")}, - **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")}, - **{n: "choice" for n in ("category_name", "business_category_name")}, - **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url", - "profile_url", "id", "fbid")}, -} -#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of -#: each list is what `default_operator` must agree with. -BD_OPS_BY_KIND = { - # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second - # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one - # idea — and the second one is the one written in vendor grammar. - "text": ("includes", "not_includes", "=", "is_not_null", "is_null"), - "choice": ("=", "!=", "is_not_null", "is_null"), - "number": (">=", "<=", "=", ">", "<"), - # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean - # is `=` with the other value, written the confusing way. - "boolean": ("=",), - "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"), -} -#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire -#: (it is what the API accepts); nobody has to read it. -BD_OP_LABELS = { - "includes": "contains", "not_includes": "does not contain", - "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of", - ">=": "at least", "<=": "at most", ">": "more than", "<": "less than", - "array_includes": "has any of", "not_array_includes": "does not have", - "is_null": "is empty", "is_not_null": "is not empty", -} -#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into. -BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}] - -#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many -#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start` -#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing -#: the whole search to "match any" (which would also drag Followers into the union — the hole the -#: OR guard closed the same day). -#: -#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not -#: contain floral OR does not contain flower" matches nearly every account alive — a negative over -#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a -#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs -#: them enough to write the AND branch. -BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"}) -MAX_PREDICATE_VALUES = 12 - - -def field_label(name): - """The human name for a searchable field — used by the SURFACE and by every REFUSAL.""" - return BD_FIELD_LABELS.get(name) or str(name or "that field") - - -def field_kind(name): - return BD_FIELD_KINDS.get(name, "text") - - -def ops_for(name): - """The comparisons this field may be asked. A tuple, in offer order.""" - return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"]) - -#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing -#: law, because the two drifted apart and the drift cost a user their first automation. -#: -#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no -#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its -#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS` -#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door -#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in — -#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one -#: sentence of drift, and nothing red anywhere at any point. -#: -#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field -#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks -#: one itself — a client-side default is a second copy of this table, free to disagree with the -#: guard that judges it, which is exactly what happened. -BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"}) -BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"}) -#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a -#: worse question than an equality, and both narrow. -BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"}) - - -def default_operator(name): - """The operator a NEW condition on `name` starts with. - - Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from - saveable rather than two (change the comparison, then type a value) — with the second step - being one the surface never told anyone to take. - """ - kind = field_kind(name) - if kind == "tags": - return "array_includes" - if kind in ("boolean", "choice"): - # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through - # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so - # the default would have been an operator the same module refuses to offer — the exact - # default-versus-guard split this function was written to close, reintroduced one commit - # later by adding a kind. Asserted per field in `verify_automation.py`. - return "=" - if kind == "number": - # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person - # reaching for Followers means, and the guard's sentence is the honest answer when it is - # the ONLY condition present. - return ">=" - return "=" if name in BD_EXACT_FIELDS else "includes" - - -def predicate_narrows(p): - """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" - return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS - and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) - - -def narrowing_refusal(preds, operator="and", kind=""): - """C4's server law: the sentence when a predicate set does not narrow, else ''. One function - so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law) - refuse in the same words. - - ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY - predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a - UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR - followers >= 10000` saved clean and asked the vendor for every account over ten thousand - followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop. - A money wall that a dropdown three rows above it can walk through is not a wall. - - So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows - either way — `all([])` is True, which would have made "no conditions at all" the widest legal - query of the lot. - """ - preds = list(preds or []) - if str(operator or "").lower() == "or": - if preds and all(predicate_narrows(p) for p in preds): - return "" - return ("with Match set to any, every condition has to describe the account itself " - "(Bio, Handle, Name, a hashtag). Matching any means the results are added " - "together, so one follower or yes/no condition widens the whole search. Narrow " - "every condition, or set Match to all") - if any(predicate_narrows(p) for p in preds): - return "" - # ⭐ WAVE 32 · T46 (D-167's neighbour) — THE NETWORK'S OWN NAME. This sentence said - # "Instagram" to somebody building a TIKTOK search, on the refusal they are most likely to - # read, because the string predates the second platform. `discovery_facts` already carries the - # network's name for exactly this class of string, so there is no second literal. - _net = discovery_facts(kind)[0] if kind else PLATFORM_INSTAGRAM - return ("add a condition that describes the account itself — Bio contains, Handle is, a " - f"hashtag. Follower counts and yes/no conditions alone match too much of {_net} " - "to search") - - -def filter_meta(): - """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]` - over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, - R3). - - `defaultOperator` rides here rather than being a client constant for the reason written over - `default_operator`: the client's own default was `is_not_null` for every field, which the - narrowing guard refuses on every field. - """ - return [{"name": n, - "label": field_label(n), - "hint": BD_FIELD_HINTS.get(n, ""), - "kind": field_kind(n), - "populated": n in BD_POPULATED_FIELDS, - "narrowing": n in BD_NARROWING_FIELDS, - "defaultOperator": default_operator(n), - # The comparisons THIS field may be asked, already in plain English and already in - # offer order. A client that filtered a global list by field kind would be a second - # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale. - "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o), - "nullary": o in BD_NULLARY_OPS, - # `in`/`is any of` and the tag operators take a LIST — the control - # has to know that, and deriving it from the token in the client is - # the same second-copy mistake one level down. - "multi": o in BD_MULTI_VALUE_OPS} - for o in ops_for(n)], - "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []} - for n in BD_FILTER_FIELDS] - -#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET. -#: -#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what -#: "unified across tenant" means and what makes the pooled master series joinable at all — two -#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query. -#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET -#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat. -#: -#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in -#: `ut_ig_snapshots`. ONE STORE FOR ONE SERIES: no per-record `json` history column, no second copy -#: of a number that already has a home. -#: ⛔ W29-T34 — THIS LINE USED TO ADD *"which the metric fields already read"*, AND THAT WAS WRONG. -#: A `metric` field reads the PLATFORM-WIDE MASTER, not this tenant's snapshot table: -#: `compute_metric_cells` imports `ig_master` and calls `ig_master.series_for(...)` — a -#: cross-tenant pooled repo in a different HF dataset, which is the whole point of C6's pooling and -#: which no rollup kind can reach. The metric header below says exactly that, so the module was -#: contradicting itself on which store answers the question. -#: ⚠ And the practical finding behind the correction, recorded because it is surprising: a live -#: census of all four tenants (13 tables / 228 fields / 64 store files) found **ZERO** metric fields -#: in existence, while 47 rollups are live. The vocabulary stays by owner ruling; nothing uses it. -#: -#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION, -#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields -#: available to us from Bright Data to be pre-set Fields for us and populated."* -#: -#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule -#: and the next reader will otherwise re-derive it and prune these columns back. It said: a -#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on -#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles, -#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is -#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were -#: captured into the snapshots and were NOT preset columns. -#: -#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be -#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the -#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus; -#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does -#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and -#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a -#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make. -#: Source data retains every field the provider returns. The only separate decision is whether to -#: request the paid full Comments dataset (`commentMetrics`, default OFF). -#: -#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is -#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`). -#: -#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person -#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and -#: written down here rather than inferred from a runner's module name, because the day a second -#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the -#: profile simply appears twice. -#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it. -#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals -#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as -#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names -#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things, -#: which is exactly why both are written down instead of inferred. -PROFILE_SOURCE_IG = "instagram" -#: ⚠ Its TikTok twin, `PROFILE_SOURCE_TT`, is declared UP with the `ut_tt_*` schemas (wave 29): the -#: field lists there are evaluated at import and would not see a constant defined here. - -PLATFORM_INSTAGRAM = "Instagram" -#: ⭐ WAVE 29 — the `ut_tt_*` family stamps this on every row of all five of its tables. (It really -#: was written by nothing until D-9 landed; the note that said so is kept in the log, not here.) -PLATFORM_TIKTOK = "TikTok" -PLATFORM_FACEBOOK = "Facebook" -PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK) - - -def discovery_facts(kind): - """⭐⭐ WAVE 30 · T05 — THE THREE THINGS THAT GENUINELY DIFFER BETWEEN THE DISCOVERY KINDS: - the network's NAME, the database a run writes to when the config names none, and that - database's label. `(platform, table, label)`. - - ⛔ WHY A FUNCTION AND NOT A DICT LITERAL: `DISCOVER_TABLE` and `DISCOVER_LABEL` are declared - several hundred lines BELOW this point, so a dict evaluated here would NameError at import. - A function body resolves at call time and does not care. (`PROFILE_SOURCE_IG`'s note directly - above records the same import-order trap costing a constant its natural home.) - - ⛔ AND WHY IT EXISTS AT ALL. These three facts were written out inline at FIVE call sites — - `clean_config`, `graph`'s discovery arm, `graph`'s `det` dict, `_flow_table` and - `compose_sentence` — and wave 29 taught four of the five about TikTok by not touching them: - they tested the string `"discover_instagram"`, so a stored TikTok automation fell through to - a default meant for Instagram, or to no arm at all. Every one of those misses was SILENT and - every gate stayed green. `DISCOVERY_KINDS` answers *"is this a corpus search?"*; this answers - *"whose?"* — and between them a sixth platform is one tuple entry and one arm here, rather - than a hunt through the file for string comparisons somebody has to think to look for. - - ⚠ DELIBERATELY NOT A PLATFORM REGISTRY. The dataset ids, the field maps and the row mappers - stay in the connector modules that own them. This is the presentation-and-default quartet the - ENGINE needs, and widening it is how it becomes a second `SOURCES` with a longer name. - - ⭐ T06 ADDED THE FOURTH ELEMENT, `spawn_fields` — the field list `_presets_after_write` gives - the target database on SAVE. It belongs here rather than beside that function because it must - equal what the RUN-TIME `ut_ensure` uses, and wave 25's R10 exists precisely to stop those two - disagreeing: columns that change between Save and the first run are two different answers to - "what does this database look like". - ⚠ THE TWO PLATFORMS PASS DIFFERENT-LOOKING LISTS AND THAT IS CORRECT, not an oversight. - Instagram spawns `CANDIDATE_FIELDS` (= the preset set PLUS the discovery bookkeeping) because - its preset list alone is a subset of what its runner writes; TikTok's `TT_PROFILE_FIELDS` - already carries its own bookkeeping columns, so it IS the whole set. Each side is the list its - own runner ensures — which is the rule, rather than "both use the one named CANDIDATE". - """ - if kind == "discover_tiktok": - return (PLATFORM_TIKTOK, TT_PROFILE_TABLE, TT_TABLE_LABELS[TT_PROFILE_TABLE], - TT_PROFILE_FIELDS) - return PLATFORM_INSTAGRAM, DISCOVER_TABLE, DISCOVER_LABEL, CANDIDATE_FIELDS - -#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT. -#: -#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as -#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring -#: `followers` as `int` gives NEW tables a schema every table already in production does not have -#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument -#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason -#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format? -#: change this."* -#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without -#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it -#: would do it silently, because a merged-by-key field list produces no error when two tables -#: disagree about what a column IS. -#: -#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor -#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this -#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction -#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the -#: mapping and in the migration both. See `_pct100`. -PRESET_PROFILE_FIELDS = [ - # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now. - # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different - # people as often as not, so the dedup key is (platform, handle) and never handle alone - # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung - # answered* (`field_def("source", "Read via")`), and one word meaning two things across two - # tables in one product is the drift this whole list exists to prevent. - field_def("platform", "Platform"), - # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING. - # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be - # REPLACED or made as the Handle. in any case the user can add their own record right and edit - # those records."* Two declarations, one field, and they are the same sentence read twice: - # - # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT - # this the primary is an accident of creation order. It is a DECLARATION, not a - # reorder: the stored field order is left alone (saved view orders keep working) - # and the client pins the column to position 0 wherever it sits. That is why this - # fixes tables that already exist without moving a single stored field. - # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to - # (`profile_field_key` step 2), which is what turns "I added a profile to my - # database, how do I enrich it?" from an unanswerable question into a step. It - # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle - # all normalise to the bare handle) and routes a typed value to the SHARED - # definition row rather than one user's overlay (C3-A1) — which is precisely what - # makes the owner's second clause true: a record you add BY HAND is a record the - # engine can read and enrich. - # - # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag - # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on - # anything but `text`. - field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}), - field_def("profile_url", "Profile", "url"), - field_def("full_name", "Name"), field_def("followers", "Followers", "int"), - field_def("following", "Following", "int"), - field_def("avg_engagement", "Avg engagement", "pct"), - field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), - field_def("verified", "Verified", "checkbox"), field_def("category", "Category"), - # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields - # MEASURED ≥60% populated (see the note above). - field_def("posts_count", "Post count", "int"), - field_def("highlights_count", "Story highlight count", "int"), - field_def("is_business", "Business account", "checkbox"), - field_def("is_professional", "Professional account", "checkbox"), - # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would - # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it. - field_def("ig_id", "Instagram id"), - # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction - # (see the block comment above for what that reverses). Types are HONEST from birth, which - # costs nothing here: no table has ever carried these columns, so there is no stored `text` - # definition for the migration to convert — R3's conversion rule applies to the columns that - # already exist, and these are new everywhere. - field_def("business_category", "Business category"), - field_def("is_private", "Private account", "checkbox"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("pronouns", "Pronouns"), - field_def("profile_name", "Profile name"), - field_def("is_joined_recently", "Joined recently", "checkbox"), - field_def("has_channel", "Has channel", "checkbox"), - field_def("partner_id", "Partner id"), - field_def("external_url_title", "Link title"), - # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity. - field_def("fbid", "Facebook id"), - field_def("related_accounts", "Related accounts"), - field_def("country_code", "Country"), - field_def("source_payload", "Source data", "json"), - # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT. - # - # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is - # linked to the profile automatically… and this rollup needs to have formula that we can use - # to calculate things like average Views over last N posts."* Both halves are these five - # columns, and they are PRESETS rather than something a person assembles, because "linked - # automatically" is the instruction — a relation you have to wire up by hand is the feature - # not existing. - # - # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged - # column and then the pinned one, both of which are `handle` on every preset table — so this - # links correctly on a database where the handle column was renamed, and on one where the - # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject - # [[gate-answers-the-wrong-question]] warns about, one layer down. - # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total. - # Two count-like columns with the same label would be unreadable. Caught by the label-collision - # gate rather than on screen, which is what that gate is for. - field_def("posts_link", "Post rows", "link", - description="Post records linked to this profile.", - link={"table": IG_POSTS_TABLE, "on": "influencer_key"}), - field_def("profile_snapshots_link", "Profile history", "link", - description="Profile snapshots linked to this profile.", - link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}), - field_def("post_snapshots_link", "Post measurement rows", "link", - description="Post engagement measurements linked to this profile.", - link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}), - field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this profile.", - link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}), - # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup - # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further - # out in `ut_ig_post_snapshots`. - # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be - # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30 - # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant - # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a - # `limit`, and this is why — "the last 12" has to name what makes one post later than another. - # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours - # because its input was an account-grain constant; with the input now a real per-reel - # measurement the average means what its label says again. - field_def("avg_views_12", "Avg views · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "views", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "plays", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "likes", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "comments", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO - # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after - # a paid enrich. This counts the post rows actually captured — a different number and a true - # one, which is why it gets its own column and its own label rather than quietly filling - # `posts_count` with something that is not what that field means. - field_def("posts_captured", "Posts captured", "rollup", - rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}), - field_def("profile_reads", "Profile reads", "rollup", - rollup={"link": "profile_snapshots_link", "fn": "countall", - "distinctBy": "snapshot_key"}), - field_def("post_measurements_captured", "Post measurements captured", "rollup", - rollup={"link": "post_snapshots_link", "fn": "countall", - "distinctBy": "post_snapshot_key"}), - field_def("comments_captured", "Comments captured", "rollup", - rollup={"link": "comments_link", "fn": "countall", - "distinctBy": "comment_key"}), - # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING. - # - # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we - # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one - # thing a buyer most wants and the one thing the profile row cannot answer. The competitor - # teardown found the same gap solved by GUESSING from bio words, two buckets deep - # ([[janney-ai-teardown]]). - # - # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place - # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the - # post row, plus the vendor's fuller `location`/`location_details` inside the paid - # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a - # creator's own posts is a far better residency signal than a word in a bio, and it costs a - # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1. - # - # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator - # posting from twelve cities gets a low confidence rather than a confident wrong answer, and - # `location_confidence` is the number a person filters on before trusting the guess. A single - # geotagged post produces NO guess at all — n=1 dressed as a pattern is what - # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain. - field_def("location_guess", "Location (guess)"), - field_def("location_confidence", "Location confidence", "pct"), - # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never - # enriched" and a stale one means "enriched in March", and no cell on the row can tell them - # apart. It is the single field that turns the other fifteen from numbers into measurements. - field_def("enriched_at", "Enriched at", "date"), - # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE. - # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the - # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the - # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying - # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two, - # rewritten each run, so a person reading the profile row can see the recent posts without a - # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2. -] -#: The keys the preset set owns — derived, so a field added above cannot be forgotten here. -PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS) - -#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile -#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what -#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the -#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now -#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to -#: enforce a rule nobody declares any more. -#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" — -#: never as "field number zero". -PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "") -PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "") - -#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the -#: alternative is two lists that describe the same columns and drift one label at a time, which is -#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found -#: them, who for, and a human's decision) rather than about the profile, so they are discovery's -#: and not part of the cross-tenant set. -CANDIDATE_FIELDS = [ - *PRESET_PROFILE_FIELDS, - # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell. - # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner - # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a - # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its - # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what - # changed is that "when did we first see this account" is a DAY, and a day is all anybody - # asks it for. `migrate_ig_field_types()` converts the stored strings. - field_def("found_count", "Times found", "int"), - field_def("first_found", "First found", "date"), - field_def("last_found", "Last found", "date"), - # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it - # describes, which is the first place anybody looks up what this column means. It read: *"the - # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two - # people discovering the same profile each get their own row, their own found_count and their - # own review card."* Wave 26 · R4/R5 retired every clause of that. - # - # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two - # people in one workspace who discover the same profile share ONE row, one `found_count` and - # one history — because they are looking at one company's leads, not two private lists. - # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it - # first. It is no part of the dedup key, and a run must never branch on it. - # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged, - # and the one clause of the old note that was still true. - field_def("created_by", "Found by"), - # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have - # a Field called stage to track the progress of the Automation per Record. We don't need - # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage - # column: every candidate carried BOTH a `stage_` select saying where it was AND a - # boolean saying whether it was kept — two progress fields that could disagree, with no rule - # about which one won. The stage field is the one progress column. Nothing replaces this. -] - - -def _profile_backlink_field(table_key, table_label, profile_key): - """A deterministic reciprocal link from one canonical IG table to one profile database. - - The key includes the target table identity, so ten separate Profile databases can all point - into the same canonical Posts/Comments/history tables without one relation overwriting the - next. The join is derived in both directions and therefore needs no cross-table fan-out write. - """ - digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10] - return field_def( - f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link", - description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.", - link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"}, - ) - - -def _profile_schema_for(bound_key): - """Preset fields for an existing Profile table without inventing a second identity column.""" - out = [] - for field in PRESET_PROFILE_FIELDS: - item = dict(field) - if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY: - item.pop("profile", None) - out.append(item) - return out - - -def _tt_profile_schema_for(bound_key): - """⭐ WAVE 30 · T08 — TikTok preset columns for a database whose profile column is `bound_key`. - - ⛔ NOT `_profile_schema_for` WITH A LIST ARGUMENT, and the difference is not stylistic. That - function walks `PRESET_PROFILE_FIELDS` and only ever removes the flag from `PRESET_FLAG_KEY`, - because on the Instagram side the flag lives on exactly one known column. TikTok's binding may - be ANY column a person named (`profile_field_key` step 1), so the rule here has to be stated - the other way round: **every field that is not the bound one arrives as DATA**, stripped of - both the identity flag and `pinned`. - ⚠ Otherwise a database whose TikTok column is `creator` would gain a rival `handle` carrying - `profile: {source: "tiktok"}` — a SECOND profile column, which `user_tables` refuses at both - write doors, so the whole top-up would be rejected and every preset cell would then be dropped - for want of a column. One shared helper for the save-time and run-time paths, so those two - cannot answer "which columns does a TikTok enrich need" differently. - """ - out = [] - for field in TT_PROFILE_FIELDS: - item = dict(field) - if str(item.get("key") or "") != str(bound_key or ""): - item.pop("profile", None) - item.pop("pinned", None) - out.append(item) - return out - - -def _locked_ig_field(field): - """One canonical Instagram field with the immutable preset declaration attached.""" - item = dict(field) - automation = dict(item.get("automation") or {}) - automation["preset"] = True - item["automation"] = automation - return item - - -def _profile_binding(table): - """The declared Profile identity field, falling back only to the canonical handle.""" - fields = list((table or {}).get("fields") or []) - flagged = next((str(f.get("key") or "") for f in fields - if isinstance(f.get("profile"), dict)), "") - if flagged: - return flagged - return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else "" - - -def _ig_schema_contract(rt, profile_tables=None): - """The complete per-tenant Instagram graph contract, including every Profile backlink. - - The fixed child tables are shared. Profile tables are discovered structurally, so an older - user-named target and the built-in candidate database receive the same preset columns, Links, - Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted. - """ - tables = ut_all(rt) - # ⭐ WAVE 32 · T42 — `_ig_contract_tables`, NOT `_ig_profile_tables`: this function decides - # which databases RECEIVE Instagram's 48 columns, and the owner's rule is that a database gets - # them only if it is used for Instagram. See that function for why the detector stays wider. - profile_keys = sorted(set(profile_tables if profile_tables is not None - else _ig_contract_tables(rt))) - backlinks, wanted, drops = [], {}, { - IG_SNAPSHOTS_TABLE: {"post_hashtags"}, - # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today - # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS - # and fed by the `ig_post_views` capability. Leaving it here would have made the migration - # delete, on every authenticated read, the column the same release just added — the - # drop set and the field list are two halves of ONE contract and must move together. - } - for table_key in profile_keys: - table = tables.get(table_key) or {} - bound = _profile_binding(table) - if not bound: - continue - label = str(table.get("label") or table_key) - backlinks.append(_profile_backlink_field(table_key, label, bound)) - schema = _profile_schema_for(bound) - # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram - # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT - # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION. - # - # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every - # contract key it owns — `rollup` included, by its own note — so a column converted beside - # the contract is re-typed back to `int` on the next reconcile, silently, hours later. - # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the - # schema instead of two that disagree. - # - # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a - # column only where this tenant's own rows show the fold already equals the stored value — - # measured, because a named list would have blanked 221 live cells on nurilab alone - # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts - # fewer columns and loses nothing; as its history fills in, later passes convert more. - derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {})) - wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f - for f in schema] - # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states - # to disagree, so it is retired by key just like the old nested Posts JSON. - drops[table_key] = {"posts", "post_hashtags", "tracked"} - wanted.update({ - IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks], - IG_POSTS_TABLE: [*POST_FIELDS, *backlinks], - IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks], - IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks], - }) - return wanted, drops - - -# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use -# the scalar strings their renderers expect; engagement is the exception because Bright Data's -# 0-1 fraction becomes this product's 0-100 percentage. -_IG_RETYPE_CONVERTERS = { - (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100, -} - - -def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): - """Repair machine-owned IG field declarations in one coalesced store update. - - `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not - sufficient for a canonical relation: a stale `link.table`, rollup function, or type would - survive forever. This pass overwrites the contract keys for the fields we own, preserves - unrelated/user fields, and removes only explicitly retired preset columns plus their cells. - """ - drop_by_table = drop_by_table or {} - wanted_by_table = { - key: [_locked_ig_field(f) for f in fields] - for key, fields in wanted_by_table.items() - } - changed = False - - def _up(cur): - nonlocal changed - cur = cur if isinstance(cur, dict) else {} - for table_key, wanted_fields in wanted_by_table.items(): - table = cur.get(table_key) - if table is None: - continue - wanted = {str(f.get("key")): dict(f) for f in wanted_fields} - drops = set(drop_by_table.get(table_key) or ()) - fields, seen = [], set() - drops = _guarded_drops(table, drops) - for stored in table.get("fields") or []: - key = str(stored.get("key") or "") - if key in drops: - changed = True - continue - desired = wanted.get(key) - if desired is None: - fields.append(stored) - continue - # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A - # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten - # from the shipped contract, `rollup` included, so without this the newly - # unlocked "edit a preset rollup" would save, render, recompute and then revert - # at the next enrichment run — an edit that looks like it worked and undoes - # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp. - if _ut().user_edited(stored): - fields.append(stored) - seen.add(key) - continue - repaired = dict(stored) - for dkey, value in desired.items(): - if dkey != "automation": - repaired[dkey] = value - automation = dict(stored.get("automation") or {}) - automation.update(desired.get("automation") or {}) - repaired["automation"] = automation - # These bags define behaviour, not decoration. If the desired field does not use - # one, a stale bag from an earlier type must not remain attached to it. - for bag in ("link", "rollup", "profile", "pinned"): - if bag not in desired: - repaired.pop(bag, None) - if desired.get("type") not in ("select", "multiselect"): - repaired.pop("options", None) - converter = _IG_RETYPE_CONVERTERS.get((table_key, key)) - if (converter and stored.get("type") in ("text", "", None) - and repaired.get("type") != stored.get("type")): - for row in (table.get("rows") or {}).values(): - if str(row.get(key) or "").strip(): - converted = converter(row.get(key)) - if converted: - row[key] = converted - changed = True - if repaired != stored: - changed = True - fields.append(repaired) - seen.add(key) - for key, desired in wanted.items(): - if key not in seen and not any(str(f.get("key") or "") == key for f in fields): - fields.append(desired) - changed = True - if drops: - for row in (table.get("rows") or {}).values(): - for key in drops: - if key in row: - row.pop(key, None) - changed = True - table["fields"] = fields - return cur - - # Avoid a commit when the declarations are already exact. - snapshot = ut_all(rt) - needs = False - for table_key, wanted_fields in wanted_by_table.items(): - if table_key not in snapshot: - continue - table = snapshot.get(table_key) or {} - by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []} - # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip - # the commit when nothing would change; a spared `tracked` column left in the drop set - # answers "yes, work to do" on every single call, forever, and the schema pass would - # rewrite the table on every authenticated read without ever changing a byte. - drops = _guarded_drops(table, drop_by_table.get(table_key) or ()) - if drops & set(by_key): - needs = True - break - for desired in wanted_fields: - stored = by_key.get(str(desired.get("key") or "")) - mismatch = stored is None - if stored is not None: - for key, value in desired.items(): - if key == "automation": - if any((stored.get("automation") or {}).get(k) != v - for k, v in value.items()): - mismatch = True - break - elif stored.get(key) != value: - mismatch = True - break - if not mismatch: - stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired) - mismatch = any(k in stored for k in stale_bags) - if (not mismatch and desired.get("type") not in ("select", "multiselect") - and "options" in stored): - mismatch = True - if mismatch: - needs = True - break - if needs: - break - if needs: - rt.update(UT_STORE_KEY, _up, flush="sync") - return changed - - -def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""): - """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys. - - Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new - Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect - each Profile database to the same Posts, Comments, profile-history, and post-history stores. - """ - profile = ut_get(rt, profile_table) if profile_table else None - bound = str(profile_field or "").strip() - if profile is not None and not bound: - bound = next((str(f.get("key") or "") for f in profile.get("fields") or [] - if isinstance(f.get("profile"), dict)), "") - profile_label = str((profile or {}).get("label") or profile_table) - backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \ - if profile_table and profile is not None and bound else [] - - # ⭐ WAVE 32 · T41 — DERIVED from `IG_TABLE_FIELDS`/`IG_TABLE_LABELS`, which used to be this - # literal. The backlinks stay HERE because they are per-tenant, per-profile-database facts; the - # schema is a module constant. Splitting it that way is what let A's delivery sweep ask this - # module for Instagram's child set instead of hand-typing a fifth copy of these four names. - graph = {key: (IG_TABLE_LABELS[key], [*fields, *backlink]) - for key, fields in IG_TABLE_FIELDS.items()} - keys = {} - for key, (label, fields) in graph.items(): - keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag, - record_mode=AUTOMATION_RECORD_MODE, lock_fields=True) - - if profile_table and profile is not None and bound: - profile_fields = _profile_schema_for(bound) - ut_ensure(rt, profile_label or profile_table, profile_fields, username, - key=profile_table, flow_tag=flow_tag, lock_fields=True) - # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older - # Profile database and IG candidates on the same universal schema while all of them point to - # the same four children. Existing flow provenance is merged, never replaced. - wanted, drops = _ig_schema_contract(rt) - _reconcile_ig_graph_fields(rt, wanted, drops) - return keys - - -# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ─────────────── -# -# Two changes land on data that already exists, and neither is optional once the field defs move: -# -# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a -# `date` column and a "20872" string in an `int` column are the split-schema the old -# "types are text on purpose" note correctly feared. Re-typing without converting is worse -# than not re-typing. -# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as -# `(handle, created_by)` DUPLICATES must be merged rather than left to collide. -# -# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a -# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would -# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a -# column is converted only while its STORED definition still says `text`; the moment it says `pct` -# the work is done and re-running is a no-op. That makes the migration safe to call on every boot, -# which is what it is for. -# -# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice. -# `observed_categories` named its two source tables and went silently empty the day wave 25 let a -# user point an automation somewhere else. Naming tables here would leave exactly those user-named -# databases on the old schema, which is the same bug wearing a migration's clothes. - -#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset -#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told. -MIGRATE_MIN_PRESET_FIELDS = 3 -#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/''). -_MIGRATE_CONVERT = { - "first_found": _day, "last_found": _day, "enriched_at": _day, - "avg_engagement": _pct100, -} - - -#: ⭐⭐ WAVE 30 · T08 — THE COLUMNS THAT CAN ONLY BE TIKTOK'S, DERIVED rather than typed out, so -#: that adding a column to either declaration keeps this set correct instead of quietly emptying -#: it. Eight today: `tt_id`, `like_engagement`, `comment_engagement`, `likes_received`, `region`, -#: `predicted_lang`, `account_created_at`, `source`. -TT_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in TT_PROFILE_FIELDS) - - frozenset(PRESET_PROFILE_KEYS) - - frozenset(f["key"] for f in CANDIDATE_FIELDS)) - - -def _is_tt_profile_table(tbl): - """⭐⭐ WAVE 30 · T08 — is this database a TIKTOK profile table? - - ⛔ THE DEFECT THIS EXISTS TO CLOSE WAS LIVE AND SHIPPED, and it converted a customer's TikTok - database into an Instagram one on the SECOND write to it. MEASURED: a `ut_tt_profile` spawned - by W30-T06 comes out correct — `handle` carrying `profile: {source: "tiktok"}` — and one more - `ut_ensure` on that table (a re-save, the discovery runner's own ensure, the next run) leaves - it carrying `profile: {source: "instagram"}` and the description *"Instagram username without - the @ symbol"*. `profile_field_key(..., source="tiktok")` then answers `""` forever, so every - TikTok enrich step on that database reports UNBOUND, and every row gets stamped - `platform: Instagram` — which is half of W26/R4's `(platform, handle)` identity, so two - different people's accounts merge under one key with nothing red anywhere. - ⚠ It hid because the migration SKIPS a table that does not exist yet, so the spawn itself is - always clean and only the second write is not. A gate that creates and asserts once cannot see - it; the assertion has to survive a second `ut_ensure`. - - ⭐ TWO LEGS, and the second is the one that still works after the first has been eaten: - 1. the table SAYS SO — a column declaring `profile: {source: "tiktok"}` is the database's own - statement of which network it is about, and it is the same declaration `profile_field_key` - reads, so there is one answer to "whose profile table is this"; - 2. it declares a column only TikTok has. Needed because leg 1 is exactly what the defect - destroys: on a table already corrupted in production, the flag now says Instagram, and a - detector resting on it alone would agree with the corruption and keep re-applying it. - """ - fields = (tbl or {}).get("fields") or [] - for f in fields: - p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_TT: - return True - return bool({f.get("key") for f in fields} & TT_ONLY_PROFILE_KEYS) - - -#: ⭐⭐ WAVE 32 · T40 — THE MIRROR OF `TT_ONLY_PROFILE_KEYS`, and it is the subject of owner item 1: -#: the profile columns that can only ever be INSTAGRAM's. 26 today. Derived by the same subtraction -#: in the other direction, so a column moved between the two declarations changes both sets at once -#: rather than leaving one of them quietly asserting a stale fence. -#: ⚠ IT IS DERIVED FROM `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS` — discovery's bookkeeping -#: columns (`found_count`, `first_found`, `last_found`, `created_by`) are declared on BOTH platforms, -#: so subtracting the preset list alone would report four keys as Instagram-only that TikTok's own -#: table has carried since wave 29. -IG_ONLY_PROFILE_KEYS = (frozenset(f["key"] for f in CANDIDATE_FIELDS) - - frozenset(f["key"] for f in TT_PROFILE_FIELDS)) - -#: ⭐⭐ WAVE 32 · T42 — the preset columns NOBODY TYPES: every `link` and `rollup` in the Instagram -#: preset set. Derived from the declaration, so adding a rollup to the contract widens this for -#: free. `_carries_ig_presets` uses it as the leg that survives when the `automation.preset` stamp -#: is absent — a hand-made creator list has `handle` and `followers`; it does not have `posts_link`. -PRESET_MACHINE_ONLY_KEYS = frozenset( - f["key"] for f in PRESET_PROFILE_FIELDS if f.get("type") in ("link", "rollup")) - - -def platform_schema(platform): - """⭐⭐ WAVE 32 · T41 — ONE ASKABLE DECLARATION OF WHAT A PLATFORM'S DATABASES ARE. - - Returns, for `PLATFORM_INSTAGRAM` or `PLATFORM_TIKTOK`: - - {"platform", "profile_table", "profile_label", "profile_fields", "preset_keys", - "only_keys", "locked_tables", "children": {key: {"label", "fields", "record_mode"}}} - - ⛔ **WHY IT IS A FUNCTION AND NOT A DICT LITERAL — the same import-order trap `discovery_facts` - records twenty lines from its own declaration.** `CANDIDATE_FIELDS` and `DISCOVER_TABLE` are - declared HUNDREDS of lines below `TT_TABLE_FIELDS`, so any module-level dict spanning both - platforms NameErrors at import. A function body resolves at call time and does not care. That is - not a style preference here; it is the reason two earlier attempts at a platform registry in this - file ended up as five inline copies instead. - - ⛔⛔ **AND IT IMPORTS NOTHING FROM `core`, WHICH IS THE ACTUAL CONTRACT WITH `main.py`.** This - module is deliberately dependency-light on the API's boot path (`MAX_UT_ROWS`, `MACHINE_OWNERS` - and `LOCKED_CHILD_TABLES` all say so in their own notes), and the REGISTRAR — the thing that - turns these names into `core.user_tables` state — lives in the composition root. So this returns - plain lists and dicts: a caller can `register_locked_records(...)`, `ut_ensure(...)` or diff it - against a live tenant without `core` ever appearing on the engine's import path. `verify_ - automation` asserts that, by importing this module in a CLEAN interpreter and checking - `core.user_tables` is absent from `sys.modules` afterwards — [[artifact-with-no-importer]] in - reverse: a declaration whose registrar cannot reach it is the same defect as a registrar with - nothing to register. - - ⚠ FIELD DICTS ARE COPIED ONE LEVEL. Every existing call site passes the module lists straight - into `ut_ensure`, which is safe only because nobody has yet appended to one; a boot sweep looping - over both platforms is exactly the caller that would. The copy costs a few hundred dict - constructions once per boot and removes the whole question. - - ⚠ `profile_fields` is the platform's PRESET set as spawned — Instagram's is `CANDIDATE_FIELDS` - (the preset set PLUS discovery's bookkeeping), not `PRESET_PROFILE_FIELDS`, because that is what - `discovery_facts` actually hands the spawn. Two answers to "Instagram's field set" is the drift - this function exists to end, so it gives the one the product uses. - """ - name = str(platform or "") - if name == PLATFORM_TIKTOK: - children = {k: {"label": TT_TABLE_LABELS[k], "fields": [dict(f) for f in TT_TABLE_FIELDS[k]], - "record_mode": tt_record_mode(k)} - for k in sorted(TT_TABLE_FIELDS) if k != TT_PROFILE_TABLE} - return { - "platform": PLATFORM_TIKTOK, - "profile_table": TT_PROFILE_TABLE, - "profile_label": TT_TABLE_LABELS[TT_PROFILE_TABLE], - "profile_fields": [dict(f) for f in TT_PROFILE_FIELDS], - "preset_keys": tuple(f["key"] for f in TT_PROFILE_FIELDS), - "only_keys": TT_ONLY_PROFILE_KEYS, - "locked_tables": TT_LOCKED_TABLES, - "children": children, - } - if name == PLATFORM_INSTAGRAM: - children = {k: {"label": IG_TABLE_LABELS[k], "fields": [dict(f) for f in fields], - # ⚠ NOT `tt_record_mode`'s twin by accident: `ensure_ig_graph` passes - # `AUTOMATION_RECORD_MODE` for all four unconditionally, and - # `IG_LOCKED_TABLES` is exactly those four. Derived from the SET so that - # unlocking one there unlocks it here, rather than from the constant. - "record_mode": (AUTOMATION_RECORD_MODE if k in IG_LOCKED_TABLES else "")} - for k, fields in sorted(IG_TABLE_FIELDS.items())} - return { - "platform": PLATFORM_INSTAGRAM, - "profile_table": DISCOVER_TABLE, - "profile_label": DISCOVER_LABEL, - "profile_fields": [dict(f) for f in CANDIDATE_FIELDS], - "preset_keys": tuple(f["key"] for f in CANDIDATE_FIELDS), - "only_keys": IG_ONLY_PROFILE_KEYS, - "locked_tables": IG_LOCKED_TABLES, - "children": children, - } - raise ValueError(f"no schema is declared for platform {platform!r}") - - -def definition_platforms(defn): - """Which networks ONE automation uses — `{"Instagram"}`, `{"TikTok"}`, both, or empty. - - ⭐ WAVE 32 · T42. Three ways a definition names a network, and all three count, because the - owner's rule is about what a database is USED FOR rather than about how the automation was - built: its discovery KIND (a corpus search is a network by construction), its enrich ACTIONS - (a `plain` flow that enriches is using that network), and the retired `field_instagram` kind, - which is uncreatable but alive on stored definitions (D-65) and still runs Instagram. - """ - defn = defn or {} - kind = str(defn.get("kind") or "") - out = set() - if kind in DISCOVERY_KINDS: - out.add(discovery_facts(kind)[0]) - if kind == "field_instagram": - out.add(PLATFORM_INSTAGRAM) - actions = (defn.get("flow") or {}).get("actions") or [] - if _actions_of_kind(actions, "enrich_instagram"): - out.add(PLATFORM_INSTAGRAM) - if _actions_of_kind(actions, "enrich_tiktok"): - out.add(PLATFORM_TIKTOK) - return out - - -def automation_platforms_by_table(rt, definitions=None): - """⭐⭐ WAVE 32 · T42 (owner item 1, third clause) — WHICH NETWORKS EACH DATABASE IS USED FOR. - - `{table_key: {"Instagram", "TikTok"}}`, over every stored automation. The owner's words are the - whole specification: *"if a user decide to use one database for both Tiktok and Instagram - scraping, only then can the pre-set Fields can exist in the same database."* — so the question a - schema pass has to be able to ask is *"which networks target THIS database"*, and until now - nothing in this module could answer it. - - ⛔ ASKABLE AND `core`-FREE, for `platform_schema`'s reason one function up: it reads only the - automations bucket through `all_definitions`, returns plain sets, and the T41 subprocess probe - covers it — a second askable declaration that quietly needed `core` would make the import-purity - guarantee mean "true for the things we remembered". - - ⚠ `definitions` is a LENT list, same contract as `ut_ensure`'s `tables=`: a caller already - holding the bucket must not pay for a second deep copy of it. - """ - # ⚠ `all_definitions` answers a DICT keyed by id, not a list — and iterating it directly hands - # every consumer a STRING that looks like a definition until the first `.get`. Normalised here - # so a caller lending its own list gets the same treatment. - defs = all_definitions(rt) if definitions is None else definitions - defs = list(defs.values()) if isinstance(defs, dict) else list(defs or []) - out = {} - for defn in defs: - if not isinstance(defn, dict): - continue - table = str(_flow_table(defn) or (defn.get("config") or {}).get("targetTable") or "") - if not table: - continue - found = definition_platforms(defn) - if found: - out.setdefault(table, set()).update(found) - return out - - -def platform_schemas(): - """Both declarations, in `PLATFORMS` order — what a boot sweep loops over. - - ⚠ `PLATFORM_FACEBOOK` is in `PLATFORMS` (it is a value the `platform` COLUMN may hold) and has - no schema, so this filters rather than raising: a caller sweeping every declared platform must - not be broken by a vocabulary entry that names no databases. `platform_schema` still raises for - it, which is the right answer to somebody ASKING for a schema that does not exist. - """ - out = [] - for name in PLATFORMS: - try: - out.append(platform_schema(name)) - except ValueError: - continue - return out - - -def _ig_profile_tables(rt, tables=None): - """Every `ut_*` table in this tenant that looks like an Instagram profile table. - - ``tables`` is the optional lent snapshot (W29-T01): read-only walk, so a caller that already - holds the bucket must not pay for a second deep copy of it. - - ⛔ WAVE 30 · T08 — AND THE TEST IS NOW EXCLUSIVE, because "looks like" was network-blind and a - TikTok profile table looks EXACTLY like one: 18 of `ut_tt_profile`'s 30 columns are Instagram - preset keys, and the bar here is `handle` plus three. Every caller of this function then treats - the match as an Instagram table — `migrate_ig_tables` rewrites its declarations to Instagram's - contract, and `discover_default_table` offers it as the default target for an Instagram search. - """ - out = [] - for key, tbl in sorted(((ut_all(rt) if tables is None else tables) or {}).items()): - keys = {f.get("key") for f in (tbl or {}).get("fields") or []} - if "handle" not in keys or len(keys & set(PRESET_PROFILE_KEYS)) < MIGRATE_MIN_PRESET_FIELDS: - continue - if _is_tt_profile_table(tbl): - continue - out.append(key) - return out - - -def _ig_contract_tables(rt, tables=None): - """⭐⭐ WAVE 32 · T42 — the tables Instagram's preset CONTRACT may be applied to. - - ⛔⛔ THIS IS A DIFFERENT QUESTION FROM `_ig_profile_tables` AND CONFLATING THEM COST FIVE RED - CHECKS. That function answers *"does this database LOOK like an Instagram profile table?"* and - two callers need exactly that: `discover_default_table`, which elects an existing table so a new - automation does not mint a second empty one — and it necessarily runs BEFORE any automation - targets that table, so a targeted-only test makes it elect nothing forever — and the detector's - own negative control. THIS function answers *"may we write Instagram's 48 columns into it?"*, - which is the owner's rule and a strictly narrower set. One normalizer answering two questions is - how a fix in one becomes a silent regression in the other ([[one-question-two-normalizers]]). - - A table qualifies when it looks like one AND any of: - - it is `ut_ig_profile`, Instagram's canonical database by declaration; - - it ALREADY carries the contract (`_carries_ig_presets`) — narrowing must never orphan a - table that has the columns, or its links and rollups silently stop being repaired; - - an INSTAGRAM automation targets it, which is the owner's rule stated in code. - - ⚠ The automations bucket is read LAZILY, on the first candidate that needs the question asked — - a tenant whose tables all already carry the contract pays nothing, on a path this wave is - otherwise trying to make faster. - """ - targets = None - out = [] - for key in _ig_profile_tables(rt, tables=tables): - tbl = (ut_all(rt) if tables is None else tables).get(key) or {} - if key != DISCOVER_TABLE and not _carries_ig_presets(tbl): - if targets is None: - targets = automation_platforms_by_table(rt) - if PLATFORM_INSTAGRAM not in targets.get(key, set()): - continue - out.append(key) - return out - - -def _preset_owned(field): - """Is this column MACHINE-authored, i.e. may `retract_foreign_presets` delete it? - - ⛔ MODULE-LEVEL, NOT A CLOSURE, AND THAT IS WHY IT MOVED. It was a nested `_machine` and the - negative control written against it came out BLIND: nothing outside the function could break - the one guard standing between a boot-time repair and a customer's own column, so the control - ended up patching the key SETS instead and proved something else. A guard a control cannot - reach is a guard nobody has tested [[gate-negative-control]]. - - Two legs, the second for tables that predate the stamp: the `automation.preset` mark that - `ut_ensure(lock_fields=True)` and `_locked_ig_field` both write, or a key that is a preset - LINK/ROLLUP — nobody types `posts_link` by hand. - """ - automation = (field or {}).get("automation") - if isinstance(automation, dict) and automation.get("preset") is True: - return True - return str((field or {}).get("key") or "") in PRESET_MACHINE_ONLY_KEYS - - -def _filled_count(rows, key): - """How many rows carry a non-blank value in `key` — the number that decides REPORT vs DELETE. - - ⛔ ALSO MODULE-LEVEL FOR ITS CONTROL'S SAKE. This one number is the whole of W30/R6's second - sentence inside `retract_foreign_presets`: a foreign column with cells in it is reported, never - deleted. A control that cannot make this lie cannot prove the report is doing anything. - """ - return sum(1 for r in (rows or {}).values() - if isinstance(r, dict) and str(r.get(key) or "").strip()) - - -def retract_foreign_presets(rt, log=print): - """⭐⭐ WAVE 32 · T46 / DEBT D-152 — REMOVE THE PRESET COLUMNS OF A NETWORK A DATABASE IS NOT - USED FOR. The RETRACTION half of `_ig_contract_tables`' recruitment rule, and A's boot sweep - (`W32-T07`) calls it once per tenant. - - ⛔ WHY IT HAS TO EXIST AT ALL, and this is the wave's own thesis one layer down. W30-T08 fixed - the DETECTOR, so the corruption cannot recur — and nothing undoes it. MEASURED on a fixture: a - `ut_tt_profile` corrupted before that fix keeps ALL 26 Instagram-only columns and its - `profile: {source: "instagram"}` flag through `ut_ensure`, `migrate_ig_tables` AND the discovery - spawn. Three write paths, zero repairs. *"The corruption cannot recur"* and *"the corruption is - gone"* are different claims [[a-migration-that-runs-on-the-next-write]]. - - Returns `{"tables", "columns", "cells", "flags", "kept"}`. **`kept` is the important one** — - see guarantee 4. - - THE FOUR GUARANTEES, in the order they matter: - 1. **Only machine-authored columns go.** A field must be stamped `automation.preset: True` - (or be a link/rollup this contract owns) to be eligible. A column a PERSON typed is never - touched, whatever it is named — the D-152 row calls this *"a VALUE BACKFILL over live - customer rows"*, and that is the line it must not cross. - 2. **Only the EXCLUSIVE sets.** `IG_ONLY_PROFILE_KEYS` (26) and `TT_ONLY_PROFILE_KEYS` (8), - both derived. A key both platforms declare — `platform`, `handle`, `followers`, the four - discovery bookkeeping columns — is shared vocabulary and is never a foreign column. - 3. **Idempotent.** A second call finds nothing and writes nothing, so it is safe on every - boot, which is what makes it callable from `main.py` rather than being a script somebody - has to remember to run (the whole reason D-201 is still open). - 4. ⛔ **A FOREIGN COLUMN THAT HOLDS DATA IS REPORTED, NOT DELETED.** W30/R6's second sentence: - a limit that cannot be removed is reported with its cause. Deleting a customer's populated - cells to make a grid look clean is not a repair, and this is precisely the *"deserves its - own supervision"* clause of D-152. `kept` names every such column with its row count, so - the caller can print it and a person can decide. - - ⚠ THE FLAG IS REPAIRED TOO, and it is the half D-152 names explicitly: a TikTok profile table - whose `handle` was re-declared `profile: {source: "instagram"}` answers `""` to - `profile_field_key(..., source="tiktok")` forever, so every TikTok enrich on it reports UNBOUND. - Repaired only where `_is_tt_profile_table` recognises the table by its OWN TikTok-only columns, - i.e. by the leg the corruption cannot have eaten. - """ - stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} - targets = automation_platforms_by_table(rt) - exclusive = {PLATFORM_INSTAGRAM: IG_ONLY_PROFILE_KEYS, - PLATFORM_TIKTOK: TT_ONLY_PROFILE_KEYS} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for key, tbl in sorted(cur.items()): - if not isinstance(tbl, dict): - continue - fields = tbl.get("fields") or [] - keys = {str(f.get("key") or "") for f in fields} - # Only PROFILE databases are in scope: `handle` plus a real slice of either preset - # vocabulary. A posts or comments table shares no profile columns and is never a - # candidate; a database with nothing in common with either is not one either. - if "handle" not in keys: - continue - tt = _is_tt_profile_table(tbl) - used = set(targets.get(key) or ()) - if not used: - # Nothing targets it. Its own declaration is then the only statement of what it is - # for — and a table that says TikTok is not an Instagram table, whatever columns a - # past migration wrote into it. - used = {PLATFORM_TIKTOK} if tt else {PLATFORM_INSTAGRAM} - foreign = set() - for platform, own in exclusive.items(): - if platform not in used: - foreign |= set(own) - if not foreign: - continue - rows = tbl.get("rows") or {} - drop, touched = [], False - for f in fields: - fkey = str(f.get("key") or "") - if fkey not in foreign or not _preset_owned(f): - continue - filled = _filled_count(rows, fkey) - if filled: - # Guarantee 4 — REPORTED, never deleted. - stats["kept"].append({"table": key, "column": fkey, "rows": filled}) - continue - drop.append(fkey) - if drop: - tbl["fields"] = [f for f in fields if str(f.get("key") or "") not in drop] - for r in rows.values(): - if not isinstance(r, dict): - continue - for fkey in drop: - if fkey in r: - r.pop(fkey, None) - stats["cells"] += 1 - stats["columns"] += len(drop) - touched = True - # ⚠ THE FLAG, and only on a table TikTok's own columns still identify. - if tt: - for f in tbl.get("fields") or []: - p = f.get("profile") - if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG: - f["profile"] = {**p, "source": PROFILE_SOURCE_TT} - stats["flags"] += 1 - touched = True - if touched: - stats["tables"] += 1 - return cur - - # ⚠ The precheck is the WHOLE mutation, run against a snapshot, so the common case (nothing to - # do) spends no commit — the same posture `_reconcile_ig_graph_fields` takes, and the reason - # this is safe to call on every boot. - probe = copy.deepcopy(ut_all(rt)) - _up(probe) - if not (stats["columns"] or stats["flags"]): - if stats["kept"]: - log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they " - f"hold data — " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" - for k in stats["kept"])) - return stats - stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} - rt.update(UT_STORE_KEY, _up, flush="sync") - log(f"[aios-auto] retract: {stats['columns']} foreign column(s), {stats['cells']} cell(s) and " - f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)") - if stats["kept"]: - log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they hold " - f"data — " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" - for k in stats["kept"])) - return stats - - -def _carries_ig_presets(tbl): - """Has this table ALREADY been given Instagram's preset contract? - - ⭐⭐ WAVE 32 · T42 — THE RECRUIT/REPAIR LINE, AND IT RESTS ON A STAMP THE CODE ALREADY WRITES. - `ut_ensure(lock_fields=True)` and `_locked_ig_field` both set `automation: {preset: True}` on - every preset column, and `migrate_ig_tables` BACKFILLS it on a table that predates the stamp - (MEASURED: an unstamped 44-column fixture comes out 44/44 stamped after one pass). So "already - carries the contract" is a declaration to read, not a threshold to invent — and inventing one - was the alternative, because `MIGRATE_MIN_PRESET_FIELDS` is 3 and cannot tell a recruited table - from a hand-made creator list that happens to have `handle`, `followers` and `bio`. - - ⛔ WHY THE LINE EXISTS AT ALL (owner item 1's third clause, MEASURED): a hand-made 5-column - database that NO automation targets went to **45 columns** the moment an Instagram automation - aimed at a DIFFERENT database was SAVED — because `_ig_schema_contract` walks every - structurally-matching table in the tenant and hands each one the full 48-column contract. The - owner's rule is that the preset fields may live in a database only if it is used for that - scraping; recruitment by resemblance is the opposite of that rule. - - ⚠ NARROWING NEVER DROPS A COLUMN. `_reconcile_ig_graph_fields` deletes only the fixed retired-key - `drops` set, so a de-recruited table keeps every field and cell it has — it stops being ADDED to - and REPAIRED, which is the whole of the change. Asserted on a fixture rather than reasoned. - ⚠ AND THE COST IS PAID ONLY WHEN IT HAS TO BE: `_ig_profile_tables` reads the automations bucket - lazily, on the first candidate that is neither stamped nor the canonical `ut_ig_profile` — so a - tenant whose tables are all genuinely Instagram's pays nothing on a path this wave is otherwise - trying to make faster. - - ⛔⛔ TWO LEGS, AND THE SECOND IS THE ONE THAT KEEPS THIS FROM BREAKING A LIVE TENANT. The stamp - is written by the very passes this predicate now gates, so a table recruited BEFORE the stamp - existed and targeted by no surviving automation would be de-recruited and never stamped — - silently losing its rollup/link repair, which is the slowest and worst failure available here - (MEASURED on a stripped fixture: 44 columns, 0 stamps, 0 repaired). So a table ALSO counts as - already-carrying when it declares one of the preset set's LINK or ROLLUP columns. Those are - machine-authored by construction — nobody types `posts_link` or `avg_views_12` into a hand-made - creator list — so the second leg is derived from the contract itself, not a threshold somebody - picked. [[gate-and-nc-must-not-share-a-binding]]'s cousin: a discriminator written by the thing - it discriminates needs an independent second leg, exactly as `_is_tt_profile_table` needed one. - """ - fields = (tbl or {}).get("fields") or [] - for f in fields: - if str(f.get("key") or "") not in PRESET_PROFILE_KEYS: - continue - automation = f.get("automation") - if isinstance(automation, dict) and automation.get("preset") is True: - return True - return bool({str(f.get("key") or "") for f in fields} & PRESET_MACHINE_ONLY_KEYS) - - -def discover_default_table(rt, tables=None): - """Which database should a discovery automation write to when nobody has said? (2026-08-10) - - Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there. - We only need ONE IG profile so it's not confusing."* - - ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_ - leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the - label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own - note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is - a constant rather than a question, a tenant that already had an Instagram profile database got - a SECOND, empty one the first time a discovery automation was saved without a target. - MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside - `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them - pure confusion. - - So the default becomes a question asked of the tenant: - - exactly one profile database (excluding the fallback) -> that one - several, exactly one of which holds rows -> the one in use - several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE) - - ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and - picking one silently would write a paid discovery run into a database the user did not name — - worse than the empty table this exists to prevent. "" means "no opinion", and the caller's - existing fallback stands, which is exactly today's behaviour for that case. - ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset - schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the - empty table would keep re-electing it and nothing would ever change. - - ⭐ WAVE 29 (W29-T01) — ONE READ, NOT TWO. This function read the whole `user_tables` bucket - here and `_ig_profile_tables` read it AGAIN one line below: two 35.8 MB-ceiling deep copies to - answer one question, on a route the automation surface calls on every open. The election is - now computed from a single snapshot, which is also the only way it can be internally - consistent — the two reads could disagree with each other under a concurrent write. - ``tables`` lets `GET /automations` lend the copy it already holds; the answer is still derived - fresh on every call, so there is no memo to invalidate when a database is created or deleted. - """ - try: - tables = (ut_all(rt) if tables is None else tables) or {} - found = [k for k in _ig_profile_tables(rt, tables=tables) if k != DISCOVER_TABLE] - except Exception: # noqa: BLE001 - return "" - if len(found) == 1: - return found[0] - if len(found) > 1: - used = [k for k in found if (tables.get(k) or {}).get("rows")] - if len(used) == 1: - return used[0] - return "" - - -def _apply_discover_default(rt, raw): - """Fill in a discovery automation's target BEFORE the pure validator invents one. - - ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that - turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure - validator with a two-argument contract every gate and route depends on). Resolving only at the - RUN sites would be cosmetic: the literal is written into the STORED config at save time, so - `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted. - `create`/`patch` are the one pair that both know the tenant and sit above that validator. - ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save - stored — is the user's choice and is never rewritten. - """ - cfg = raw.get("config") - if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip(): - return raw - default = discover_default_table(rt) - if not default: - return raw - return {**raw, "config": {**cfg, "targetTable": default}} - - -def _vestigial_name_field(fields, rows): - """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half. - - `user_tables.create()` mints EXACTLY ONE column on a hand-made database — - `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the - automation's columns after it, never reordering. So on every Instagram database somebody - created before pointing an automation at it, column 1 is a `Name` nothing will ever write. - - ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative - because this is the one branch in the migration that DESTROYS something. A column carrying any - declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with - a value in any row is somebody's data. Either way it survives as an ordinary column and merely - stops being the locked primary, which the `pinned` half achieves on its own. - - ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted - `custom__` (`buildOverlayField`) and every machine one comes from a `field_def` - list, so nothing but `create()`'s default can hold this key. The label and type are checked - anyway — a renamed or retyped column is a column someone touched on purpose. - """ - f = next((f for f in fields if f.get("key") == "name"), None) - if f is None or f.get("label") != "Name" or f.get("type") != "text": - return None - if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")): - return None - if any(str(r.get("name") or "").strip() for r in rows.values()): - return None - return f - - -def _merge_candidates(rows): - """C3's merge rule, applied to rows now colliding on `(platform, handle)`. - - Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell - takes the first non-blank, preferring the most recently enriched row. - - ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and - board cards; minting a new one would orphan all of them, so a merge picks a survivor rather - than creating one. - """ - groups, keepers = {}, {} - for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))): - r = rows[rid] - h = str(r.get("handle") or "").strip() - if not h: - # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a - # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a - # manual handle in my 'Beauty influencer leads' database, after an automation run, it - # gets deleted?"* - # - # A row you add by hand is born blank and stays blank until you type into it, so the - # window is not narrow — it is every row, from creation until the cell is committed. - # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S - # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was - # dropped even after it looked filled in on screen. - # - # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is - # not a candidate — it is somebody's row. It cannot collide with anything (there is - # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A - # merge pass that deletes what it cannot classify is not a merge, it is a filter. - keepers[rid] = r - continue - groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h), - []).append((rid, r)) - out, merged = dict(keepers), 0 - for _key, members in groups.items(): - if len(members) == 1: - out[members[0][0]] = members[0][1] - continue - merged += len(members) - 1 - # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement. - ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True) - keep_id = sorted(rid for rid, _r in members)[0] - acc = {} - for _rid, r in ordered: - for k, v in r.items(): - if k not in acc and str(v or "").strip(): - acc[k] = v - firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")] - lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")] - if firsts: - acc["first_found"] = min(firsts) - if lasts: - acc["last_found"] = max(lasts) - total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members) - if total: - acc["found_count"] = str(total) - out[keep_id] = acc - return out, merged - - -#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER -#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites -#: *"we already have a Field called stage… we don't need another checkbox for this"* — and -#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still -#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing -#: writes, nothing reads, and anybody can still click. -#: -#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the -#: record of who approved what before the wave, or leave them and let the next hand-made `tracked` -#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving -#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT -#: as a column nobody declared is the worse half. -TRACKED_KEY = "tracked" - - -def _retired_tracked_field(fields): - """The retired `tracked` column when it is still the MACHINE's own, else None. - - ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that - DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a - column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED, - or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as - an ordinary column and merely stops being fed by anything, which was already true. - - ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are - dropped unguarded: both are machine-derived cells with no history of anyone editing them. - `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test. - """ - f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None) - if f is None or f.get("type") not in ("checkbox", "", None): - return None - if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")): - return None - return f - - -def _guarded_drops(table, drops): - """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81). - - ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check. - `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks - `_retired_tracked_field` whether the column is still the machine's own, and the schema - reconciliation, which drops by KEY with no question asked. Guarding only the first was - decorative — the schema pass runs immediately afterwards on the same table and deleted the - retyped column the guard had just spared. Two paths to one deletion with one of them checked - is the shape where a control looks present and enforces nothing - ([[defects-that-mask-each-other]]). - """ - drops = set(drops or ()) - if TRACKED_KEY in drops: - stored = [f for f in (table or {}).get("fields") or [] - if str(f.get("key") or "") == TRACKED_KEY] - if stored and _retired_tracked_field(stored) is None: - drops.discard(TRACKED_KEY) - return drops - - -def _orphan_tracked_rows(fields, rows): - """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject. - - ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE - BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose - declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every - surface that renders from `fields` — and a migration that only looks at `fields` finds nothing - to do and `continue`s past the table. Storage, export and any future re-declaration of the key - all still see them. - """ - if any(str(f.get("key") or "") == TRACKED_KEY for f in fields): - return [] # the column above owns its own cells - return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})] - - -def backfill_corpus_snapshots(rt, log=print): - """The BACKFILL half: every profile row holding a measurement with no series behind it gets - its corpus observation. Returns how many were written. - - ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows - that arrived before the forward fix existed. A second row-shape here would be two ideas of - what a corpus observation is, and they would disagree on the next promoted column. - - ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is - deliberate. A profile that HAS been enriched already owns real observations; synthesising a - corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and - replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind - it, which is the only case where a corpus observation can only improve matters. - - ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on - every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has - a series, so it stops matching and the scan costs a dict comparison. - """ - snaps = ut_get(rt, IG_SNAPSHOTS_TABLE) - if snaps is None: - return 0 # no series store — see `append_ig_snapshots` on why we never mint one - known = {str((r or {}).get("influencer_key") or "").strip().lower() - for r in (snaps.get("rows") or {}).values()} - known.discard("") - #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived - #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a - #: named list goes silently empty the day the schema moves). - measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS - if fd["key"] not in _SNAPSHOT_OWN_KEYS - and fd.get("type") in ("int", "pct")) - incoming = [] - for table_key in _ig_profile_tables(rt): - for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): - handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if not handle or handle in known: - continue - if not any(str((row or {}).get(k) or "").strip() for k in measured): - continue # nothing was ever read about this row — there is no observation to record - built = corpus_snapshot_row(row) - if built: - incoming.append(built) - known.add(handle) # two profile tables holding one handle write ONE observation - return append_ig_snapshots(rt, incoming, log=log) - - -def backfill_post_snapshot_grain(rt, log=print): - """Copy `posted_at`/`type` off the POST row onto every measurement missing them. - - ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's - is not, so a pass that has already run matches nothing and a post that genuinely has no date - is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration - around it. - - ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN - THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own - record of the moment is the one to keep — a fact table that lets a dimension rewrite its - history is not a fact table. - """ - psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE) - posts = ut_get(rt, IG_POSTS_TABLE) - if psnaps is None or posts is None: - return 0 - have = {f.get("key") for f in (psnaps.get("fields") or [])} - carried = [k for k in ("posted_at", "type") if k in have] - if not carried: - return 0 # the columns are not declared yet — `ut_ensure` adds them first - by_shortcode = {} - for row in (posts.get("rows") or {}).values(): - code = str((row or {}).get("shortcode") or "").strip() - if code: - by_shortcode[code] = row or {} - changes = {} - for rid, row in (psnaps.get("rows") or {}).items(): - post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip()) - if not post: - continue - for key in carried: - want = str(post.get(key) or "").strip() - if want and not str((row or {}).get(key) or "").strip(): - changes.setdefault(str(rid), {})[key] = _s(want, 200) - if not changes: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - table = cur.get(IG_POST_SNAPSHOTS_TABLE) - if table is not None: - for rid, values in changes.items(): - table.setdefault("rows", {}).setdefault(rid, {}).update(values) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind") - return len(changes) - - -#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work -#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS. -#: -#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same -#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset -#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already -#: relational (the 12 links and rollups), or have no observation behind them at all -#: (`location_guess`/`location_confidence`, which WE infer rather than read). -#: -#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST. -#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells — -#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those -#: values were never recorded as observations (the hole this same wave closed going forward, in its -#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three -#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a -#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author -#: assumed. -#: -#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that -#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`. -#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not -#: safe today is not refused forever, it is simply not converted yet. -IG_DERIVABLE_KEYS = tuple( - fd["key"] for fd in PRESET_PROFILE_FIELDS - if fd.get("type") not in ("link", "rollup") - and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found", - "found_count", "created_by") - and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS}) - - -def _derived_profile_bag(key): - """The rollup that replaces the mapper's write of ONE profile column. - - ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's - value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an - expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the - scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest - observation that ACTUALLY READ this field", which is what the materialised cell meant. - MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the - observation itself, which is why the guard below is a proof and not a hope. - """ - return {"link": "profile_snapshots_link", "field": key, "fn": "latest", - "sortBy": "pulled_at", "sortDir": "desc", - "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"} - - -def _derived_profile_columns(table, snapshots): - """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table. - - = the ones already converted, PLUS the ones whose derived value equals the stored value on - every row. Returns keys. - - ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with - a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns — - `rollup` included, by design — so a converted column that this function stopped naming would be - re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written - by the mapper. The edit would look applied and undo itself later, which is exactly - [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore - included unconditionally rather than re-proved: its cells ARE the derived values, so a proof - would be comparing the fold against itself. - - ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody - measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere - else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest - authority is the data in front of the migration. - ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an - oversight: there is nothing to lose, and leaving it materialised would mean the schema differs - between two tenants for no reason a reader could discover. - """ - fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []} - rows = (table or {}).get("rows") or {} - by_subject = {} - for snap in ((snapshots or {}).get("rows") or {}).values(): - key = str((snap or {}).get("influencer_key") or "").strip().lower() - if key: - by_subject.setdefault(key, []).append(snap or {}) - # The engine's own ordering, once per subject — newest first, unrankable rows last. - ordered = {} - for subject, observations in by_subject.items(): - keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations] - rankable = [(o, k) for o, k in keyed if k is not None] - rankable.sort(key=lambda ok: ok[1], reverse=True) - ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None] - # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph - # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose - # link resolves to nothing — a blank cell wearing a configured column's clothes. - if "profile_snapshots_link" not in fields: - return [] - out = [] - for key in IG_DERIVABLE_KEYS: - field = fields.get(key) - if not field: - continue - if field.get("type") == "rollup": - out.append(key) # already converted — see the note above on why - continue - declared = str(field.get("type") or "text") - safe = True - for row in rows.values(): - stored = str((row or {}).get(key) or "").strip() - window = [o for o in ordered.get( - str((row or {}).get("handle") or "").strip().lower(), []) - if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here - derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared) - or "").strip() - if stored != derived: - safe = False - break - if safe: - out.append(key) - return out - - -def _as_derived_field(field): - """One preset scalar declaration → the same column declared as a rollup over the series.""" - key = str(field.get("key") or "") - return {**{k: v for k, v in field.items() if k not in ("agg",)}, - "type": "rollup", "rollup": _derived_profile_bag(key)} - - -def migrate_ig_tables(rt, log=print, only=""): - """Bring every Instagram Profile and canonical child table onto one current contract. - - Returns counted changes rather than a reassuring line. Profile cells are converted before - their type declarations change; then the graph reconciliation adds any missing preset - Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes - only retired machine fields. - - `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides - the WRITE PATH, so a table is brought forward immediately before anything appends to it and no - caller has to remember to run anything. - """ - stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0, - # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from - # `stamped` because they answer different questions: that one is "how many ROWS got a - # platform", these are "how many TABLES changed shape". - "pinned": 0, "flagged": 0, "droppedName": 0, - "droppedRecentPosts": 0, "droppedPostHashtags": 0, - "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0} - want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS} - # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red - # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a - # first run the table does not exist yet: with the detection bypassed it fell through to the - # writer and stamped a stub table with empty fields, clobbering the creation that was the whole - # point of the call (the owner stamp and the flow tag went with it). A table that does not - # exist, or that is not an Instagram profile table, has nothing to migrate. - # ⭐ WAVE 32 · T42 — the CONTRACT set, not the detector's. This loop retypes columns, stamps - # `platform` and drops retired keys, i.e. it edits the schema — so it answers to the owner's - # rule about which databases Instagram may write into, exactly like `_ig_schema_contract` at - # the bottom of this function. `_ig_profile_tables` stays the wider question and keeps its own - # callers (`discover_default_table`, `backfill_corpus_snapshots`, the detector's NC). - detected = set(_ig_contract_tables(rt)) - targets = ([only] if only in detected else []) if only else sorted(detected) - # ⭐⭐ WAVE 31 · T30 — `only` NOW NARROWS THE WHOLE FUNCTION, AND IT DID NOT. - # - # ⛔ THE BUG WAS THAT `only` NARROWED THE LOOP ABOVE AND NOTHING ELSE. The tenant-wide passes - # at the bottom — two full `_ig_schema_contract` + `_reconcile_ig_graph_fields` rounds and two - # backfills — ignore `only` entirely, so `migrate_ig_tables(only=X)` paid a whole-tenant - # Instagram reconciliation no matter what X was. And `ut_ensure` calls it that way on EVERY - # ensure whose field set intersects `PRESET_PROFILE_KEYS` — which TikTok's profile schema does, - # because the two networks share column names. - # MEASURED on the wave-30 fixture, `only="ut_tt_profile"`: `_ig_profile_tables` returns `[]`, - # `stats` comes back completely empty and ZERO store writes happen — at a cost of **10 - # whole-document reads**, each a `Store.get` deep copy (documented ceiling 35.8 MB / ~1.4 s). - # Across one save's four `ut_ensure` calls that was **40 of the 41** reads behind owner item 7's - # *"it just say Saving... and takes a long time"*. The migration was not slow; it was running at - # all. - # ⚠ THE COMMENT ONE SCREEN DOWN IS WHY THIS WENT UNSEEN FOR THREE WAVES: *"Both are guarded on - # 'is there anything to do' and cost a dict scan in the steady state, which is what makes them - # safe on a function that rides every `ut_ensure`."* That guard is on the WRITE. The read was - # never guarded, and a scan of a freshly deep-copied 35 MB document is not a dict scan - # [[read-a-gates-predicate-for-what-it-excludes]]. - # ⛔ SCOPED TO THE PROVEN-EMPTY CASE ONLY, deliberately. When `only` IS a detected profile table - # the function is unchanged, tenant-wide passes included — that is the Instagram path every - # migration check in `verify_automation` exercises. And a caller passing no `only` still gets - # the full sweep. The claim being made here is narrow and checkable: *a table that is not in the - # Instagram graph has no Instagram migration*, which is the same thing the `detected` test above - # already decided one line earlier and then threw away. - # ⛔ AND IT IS NOT A LEND. Lending one snapshot through the rest of this function would be the - # obvious next fix and it is WRONG: the second contract pass exists precisely because it must - # read what the first pass WROTE (see its own note below), so a read-write-read sequence cannot - # answer out of one snapshot without silently un-fixing that idempotency. - if only and not targets: - return stats - for table_key in targets: - tbl = ut_get(rt, table_key) or {} - fields = [dict(f) for f in tbl.get("fields") or []] - rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} - # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, - # so the cell conversions below can never run twice on the same value. - stale = [f for f in fields - if f.get("key") in want and f.get("type") != want[f["key"]] - and f.get("type") in ("text", "", None)] - stale_keys = {f["key"] for f in stale} - has_platform = any(f.get("key") == "platform" for f in fields) - needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] - # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. - # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the - # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in - # production needs them stamped here, and that is the whole reason this branch exists. - # - # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, - # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which - # is exactly why these are safe to run on every write where a value conversion would not be. - pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ - if PRESET_PINNED_KEY else None - flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ - if PRESET_FLAG_KEY else None - # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write - # doors, and a migration is not exempt from it. If somebody has already flagged another - # column on this table, stamping the handle too would leave the table in a state its own - # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would - # silently bind to whichever came first in the field list. Leave it alone and pin only. - other_profile = next((f for f in fields if f is not flag_f - and isinstance(f.get("profile"), dict)), None) - want_pin = pin_f is not None and pin_f.get("pinned") is not True - want_flag = (flag_f is not None and other_profile is None - and not isinstance(flag_f.get("profile"), dict)) - vestigial = _vestigial_name_field(fields, rows) - retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} - if _retired_tracked_field(fields) is not None: - retired.add(TRACKED_KEY) - orphan_tracked = _orphan_tracked_rows(fields, rows) - if (not stale_keys and has_platform and not needs_stamp - and not want_pin and not want_flag and vestigial is None and not retired - and not orphan_tracked): - continue - stats["tables"] += 1 - if want_pin: - pin_f["pinned"] = True - stats["pinned"] += 1 - if want_flag: - flag_f["profile"] = {"source": PROFILE_SOURCE_IG} - stats["flagged"] += 1 - if vestigial is not None: - # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer - # exists is invisible everywhere except the next export, where it reappears as a column - # nobody declared. `_vestigial_name_field` has already proven every one of them blank. - fields = [f for f in fields if f is not vestigial] - for r in rows.values(): - r.pop("name", None) - stats["droppedName"] += 1 - if retired: - fields = [f for f in fields if f.get("key") not in retired] - for r in rows.values(): - for key in retired: - r.pop(key, None) - stats["droppedRecentPosts"] += int("posts" in retired) - stats["droppedPostHashtags"] += int("post_hashtags" in retired) - stats["droppedTracked"] += int(TRACKED_KEY in retired) - for rid in orphan_tracked: - # D-74: a tick whose column was already gone. Counted apart from the column drop - # because they answer different questions — "how many tables still declared it" and - # "how many rows were still carrying it after the declaration went". - rows[rid].pop(TRACKED_KEY, None) - stats["droppedTrackedCells"] += 1 - - for key in stale_keys: - conv = _MIGRATE_CONVERT.get(key) - if not conv: - continue # int/checkbox already store the right text shape - for r in rows.values(): - if str(r.get(key) or "").strip(): - new = conv(r[key]) - # "" from a converter means UNREADABLE, and the cell keeps its original text. - # A migration that empties a cell is indistinguishable from one that moved it. - if new: - r[key], stats["converted"] = new, stats["converted"] + 1 - for f in stale: - f["type"] = want[f["key"]] - stats["retyped"] += 1 - for rid in needs_stamp: - rows[rid]["platform"] = PLATFORM_INSTAGRAM - stats["stamped"] += 1 - if not has_platform: - fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields - rows, merged = _merge_candidates(rows) - stats["merged"] += merged - - def _up(cur, _f=fields, _r=rows): - cur = cur or {} - t = dict(cur.get(table_key) or {}) - t["fields"], t["rows"] = _f, _r - cur[table_key] = t - return cur - - rt.update(UT_STORE_KEY, _up) - # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION. - # - # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early - # (`if only and not targets` above). What it left is the case the row is actually about — `only` - # naming a table that IS in the graph, i.e. **every save an Instagram tenant makes**. The four - # passes below are tenant-wide by construction: two `_ig_schema_contract` + - # `_reconcile_ig_graph_fields` rounds and two backfills, each walking the whole document, none - # of them reading `only`. So one `ut_ensure` on one table paid a whole-tenant Instagram - # reconciliation, and `ut_ensure` runs several times per save. - # - # ⛔ THE ROW'S EXIT OFFERS TWO BRANCHES AND THIS TAKES THE FIRST ONE — *"the tenant-wide passes - # are reachable from a scheduled/boot caller"* — because they ARE, and were before this change: - # `routes_tables.py:882` (`_ig_forward`) calls `migrate_ig_tables(rt)` with NO `only`, once per - # tenant per process, guarded by `_IG_FORWARDED`. That is the right home for a whole-tenant - # repair: once, off the save path, rather than on every ensure. - # ⚠ THE SECOND BRANCH — "bound them by an anything-to-do test guarded on the READ" — is NOT - # available here and the comment above says why: the second contract pass exists to read what - # the first pass WROTE, so it cannot be answered out of one snapshot without silently - # un-fixing the idempotency everything else depends on. - # ⛔ NARROW AND CHECKABLE, like its W31-T30 sibling: a call that NAMES ONE TABLE does that - # table's migration. A call that names none still does the whole sweep, unchanged, and every - # tenant-wide check in `verify_automation` goes through that path. - if only: - return stats - # The profile loop owns value conversions and deduplication. The schema pass owns the - # declarations across ALL related databases and is safe only after those conversions, because - # the stored old type is the idempotency key for unit-changing values such as engagement. - wanted, drops = _ig_schema_contract(rt) - stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops)) - # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT - # IDEMPOTENT, which is the one property everything else here is built on. - # - # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the - # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs - # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our - # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired - # profile cell against an UNREPAIRED observation, finds them different, and declines to convert - # a column that is in fact perfectly derivable. The next call then converts it — and - # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that - # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1. - # - # ⚠ The module already knew this shape one level up — the profile loop's own note says the - # schema pass "is safe only after those conversions, because the stored old type is the - # idempotency key". Same argument, one table further down. - # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same - # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is - # what moves the data, and it has already run. - wanted2, drops2 = _ig_schema_contract(rt) - stats["schema"] = int(bool(stats["schema"] - | int(_reconcile_ig_graph_fields(rt, wanted2, drops2)))) - # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose: - # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running - # it first would find nothing to fill and report a truthful zero about the wrong moment. - # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which - # is what makes them safe on a function that rides every `ut_ensure`. - stats["series"] = backfill_corpus_snapshots(rt, log=log) - stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log) - if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]: - log(f"[ig-migrate] {stats}") - return stats - - -# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ────────────────── -# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their -# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never -# hidden (R5). -# -# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN: -# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the -# corpus barely populates produces a filter that returns nothing and looks exactly like -# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about). -# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The -# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`, -# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them -# narrows under its own `default_operator`, asserted in the gate rather than assumed. -# -# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`, -# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and -# nothing else — a search that returns what you gave it, at full price. So the ranker reads only -# the three that describe a KIND of account: shared words in the bio, a shared domain in the link, -# and shared words in the name. -SEED_SOURCES = ("view", "cohort") -SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N] -SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter -SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared -SEED_MIN_ROWS = 2 # one row has nothing to share WITH -#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two -#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every -#: bio word in a two-row seed qualified as "shared", and the surface would have offered a -#: characteristic derived from a single record as the thing those records have in common. A -#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is -#: the fabrication this module refuses everywhere else. -SEED_MIN_ROWS_SHARING = 2 -#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor -#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down. -SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"), - ("full_name", "full_name")) -#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT — -#: an aggressive list would silently drop a real signal, and the coverage bar already removes most -#: noise. Anything ≤2 characters is dropped by length instead of by enumeration. -SEED_STOPWORDS = frozenset({ - "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from", - "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more", - "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"}) - - -def _seed_tokens(text): - """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars.""" - words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower()) - return {w for w in words if w not in SEED_STOPWORDS} - - -def _seed_domain(url): - """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a - domain and that IS the shared characteristic; the path is the thing that differs.""" - raw = str(url or "").strip() - if not raw: - return "" - host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower() - return host[4:] if host.startswith("www.") else host - - -def seed_predicates(rows): - """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network. - - `basis` is what makes the offer honest rather than magic: how many rows were read, which - characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads - as weak instead of being presented with the same confidence as a strong one. - - ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED - (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24 - default-versus-guard defect rebuilt — and it would be worse here, because the user did not - type it: they would be told their own product's suggestion is invalid. - - ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage - bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks - plausible, returns zero rows, and reads as "no such accounts exist". - """ - rows = [r for r in (rows or []) if isinstance(r, dict)] - total = len(rows) - basis = {"rows": total, "fields": [], - # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature. - # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1 - # preset set, so on our own seed rows it is usually simply absent — which the surface - # must SAY, or a lane that adds nothing reads as a lane that failed. - "related": {"tried": total, - "found": sum(1 for r in rows - if str(r.get("related_accounts") or "").strip())}, - "note": ""} - if total < SEED_MIN_ROWS: - basis["note"] = ("a seed needs at least two records — one record has nothing to share " - "with anything") - return [], basis - ranked = [] - for row_key, vendor_field in SEED_FROM_ROW: - counts = {} - for r in rows: - vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url" - else _seed_tokens(r.get(row_key))) - for v in vals: - if v: - counts[v] = counts.get(v, 0) + 1 - for value, n in counts.items(): - cov = n / total - if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING: - ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3), - "rows": n}) - # Strongest first; ties broken by the LONGER value, which is the more specific one. - ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"])) - derived, seen_fields = [], set() - for f in ranked: - if len(derived) >= SEED_MAX_PREDICATES: - break - # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the - # accounts carrying both words, which is a much smaller search than the seed implies. - if f["name"] in seen_fields: - continue - p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]} - if not predicate_narrows(p): - continue # cannot be offered; it would be refused at the save door - seen_fields.add(f["name"]) - derived.append(p) - basis["fields"].append({"name": f["name"], "label": field_label(f["name"]), - "value": f["value"], "coverage": f["coverage"]}) - if derived and narrowing_refusal(derived, "and"): - # Belt AND braces: the per-predicate check above should make this unreachable, and it is - # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest - # empty rather than offering something the save door will reject. - basis["note"] = ("the shared characteristics found are not specific enough to search on " - "— add a condition of your own") - return [], basis - if not derived: - basis["note"] = basis["note"] or ( - "these records share no bio words, link domain or name in common — nothing could be " - "derived, so the conditions are yours to write") - return derived, basis - - -def seed_rows(rt, table_key, view_id=""): - """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by - `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]]).""" - t = ut_get(rt, str(table_key or "")) - if t is None: - return [], f"{table_key!r} is not a database in this workspace" - rows = list((t.get("rows") or {}).values()) - if str(view_id or "").strip(): - # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a - # different row set from the view it names would be describing a view nobody has. - tree, _fields, problem = view_filter(rt, table_key, view_id) - if problem: - return [], problem - rows = [r for r in rows if lane_match(tree, r)] - return rows[:SEED_MAX_ROWS], "" - - -def automation_tables(defn): - """Every `ut_*` table THIS automation writes into — its config target plus every - `create_record` action's table, deduped, order preserved. - - ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition. - Split out rather than parameterised because the two questions are different: that one asks - "where could a Category value be, anywhere in this tenant" and is deliberately generous; this - one asks "which rows has THIS automation already found" and must not reach into a table it - does not write, or a nightly search would start excluding another automation's finds. - """ - keys, seen = [], set() - cfg = (defn or {}).get("config") or {} - for raw in [cfg.get("targetTable"), - *(((a or {}).get("config") or {}).get("table") - for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]: - k = str(raw or "").strip() - if k.startswith(UT_PREFIX) and k not in seen: - seen.add(k) - keys.append(k) - return keys - - -def candidate_key(platform, handle): - """The discovery upsert identity: `(platform, handle)`, as one string. - - ⭐ WAVE 29 — LIFTED OUT OF `run_discover_instagram`, where it was a closure, because a second - runner now needs the same identity. Two closures computing "the same" key is how one of them - grows a different default for a blank `platform` and the two networks quietly start sharing - rows — which is the exact data-loss shape wave 26's R4 added `platform` to prevent. - - ⚠ A BLANK PLATFORM DEFAULTS TO INSTAGRAM, and that is a FACT about the stored data rather than - an assumption: every row written before wave 26 came from the Instagram runner. Blank-keyed - rows would fail to match their own re-find and duplicate the whole table on the next run. - """ - return f"{str(platform or PLATFORM_INSTAGRAM).strip()}\n{str(handle or '').strip()}" - - -def already_found_handles(rt, defn, cap=None): - """The handles this automation has already written, for the vendor-side exclusion (item 7). - - ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a - profile has always been harmless — and never free: the vendor bills for every record it - returns, so a nightly search over stable keywords pays again, in full, for the same accounts - every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty - scout re-found its whole result set on every run. This is the list that stops it, sent as one - `not_in` the vendor evaluates BEFORE billing. - - ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would - hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords - are two questions, and one must not answer with "somebody already looked at that". - """ - cap = BD_EXCLUDE_MAX if cap is None else int(cap) - out, seen = [], set() - for table_key in automation_tables(defn): - for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): - h = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if h and h not in seen: - seen.add(h) - out.append(h) - if len(out) >= cap: - return out - return out - - -def observed_category_tables(rt): - """Every table a Category value could have been written into, DERIVED from the automations - this tenant actually has — the default discovery table and the snapshot table are the FLOOR, - not the list. - - ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared: - `config.targetTable` (the discovery runner's own write) and any `create_record` action's - `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker - writes). Both are read, deduped, order preserved so the defaults come first. - """ - keys, seen = [], set() - - def _add(k): - k = str(k or "").strip() - if k and k.startswith("ut_") and k not in seen: - seen.add(k) - keys.append(k) - - _add(DISCOVER_TABLE) - _add("ut_ig_snapshots") - try: - for defn in (all_definitions(rt) or {}).values(): - cfg = (defn or {}).get("config") or {} - _add(cfg.get("targetTable")) - for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")): - _add(((act or {}).get("config") or {}).get("table")) - except Exception: # noqa: BLE001 - # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults — - # the same posture as the master lane below: degrade to less vocabulary, never to an error. - pass - return keys - - -def observed_categories(rt, limit=60): - """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts - free text. - - D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen - (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that - still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**: - transcribe a published taxonomy and hope it lines up."* This is the first branch. - - ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the - corpus does not use returns zero rows and looks exactly like an honest "no such accounts - exist" — so a picker built from Instagram's own category list would mislead precisely when it - looked most authoritative. Every option here has been observed on a real row, and each carries - its COUNT so a value seen once reads differently from one seen forty times. - - ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a - control that refused anything else would be a second, quieter version of the same lie. - """ - seen = {} - - def _eat(rows, key): - for r in rows or []: - v = " ".join(str((r or {}).get(key) or "").split())[:60] - if v: - seen[v] = seen.get(v, 0) + 1 - - # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped - # green. The two constants below were the whole list, which was correct until **wave 25 R2 - # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a - # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely - # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows - # while the tenant's two real target tables held 20 each, so this function honestly reported - # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that - # names its sources goes quietly empty the moment the product lets a user choose one. - for key in observed_category_tables(rt): - _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category") - try: - # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three - # candidates still gets a vocabulary drawn from every profile the platform has captured. - import ig_master - if ig_master.configured(): - # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which - # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers - # None, so the master lane would have degraded to nothing INSIDE the except below and - # this would have shipped as a silent no-op with the gate green. Verified against - # `ig_master.series_for`, which reads the same bucket the same way. - _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category") - except Exception: # noqa: BLE001 - # A master that cannot be read costs the tenant its own values and nothing else. - pass - return [{"value": v, "count": n} - for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]] - - -def preset_plan(rt, table_key): - """C1 / R2b: the preset set diffed against ONE database's CURRENT fields. - - `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` - - ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns - an automation will USE and which it will CREATE — and the only thing that can answer it is - whatever holds both lists. A client that diffed the preset set against a table's fields would - be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that - actually runs; it would be wrong precisely when the two lists differ, which is the only case - anybody is asking about. - - ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save - moment, and the honest answer is "all of them will be created". `present` is a fact about the - table, so it reads False for every field rather than the call refusing. - - ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG - (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the - relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY. - That is CORRECT for this payload — it answers "which columns will be used vs created", where - a name and a type are the whole question — and it is safe today because nothing RENDERS a - column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes - `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from - `PRESET_PROFILE_FIELDS`, which carries the bag. - ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would - render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or - widen this projection deliberately; do not assume a bag is here because the field has one. - """ - have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields") - or [])} - fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text", - "present": f["key"] in have} - for f in PRESET_PROFILE_FIELDS] - return {"table": str(table_key or ""), - "exists": ut_get(rt, str(table_key or "")) is not None, - "fields": fields, - "willUse": [f for f in fields if f["present"]], - "willCreate": [f for f in fields if not f["present"]]} - - -def clean_predicates(raw, operator="and", kind=""): - """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. - - Refuses rather than coerces: a predicate naming a field the API rejects comes back as a - 400 with the field named, because the alternative — dropping it — turns "find me verified - accounts in Georgia" into "find me any account" and bills for the difference. - - ⭐ WAVE 32 · T46 (D-167) — `kind` NARROWS THE VOCABULARY TO THE PLATFORM'S. Default `""` keeps - Instagram's 21 names, so every existing caller and every stored Instagram automation is - unchanged; a `discover_tiktok` is validated against the 5 names its corpus actually has. The - refusal it produces is the same sentence, listing the platform's own fields — which is the - difference between a search that says why it cannot be built and one that is built, paid for, - and comes back empty. - """ - fields, _lead = filter_fields(kind) - out = [] - for p in (raw or [])[:12]: - if not isinstance(p, dict): - continue - name = _s(p.get("name") or p.get("field"), 60).strip() - op = _s(p.get("operator") or p.get("op"), 20).strip() - if name not in fields: - return None, ("that field cannot be searched — the searchable ones are: " - + ", ".join(field_label(f) for f in fields)) - # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly - # because every operator was legal on every field; a yes/no column offering "at least" - # is a question with no meaning that the vendor is nevertheless asked. - allowed = ops_for(name) - if op not in allowed: - return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r} — " - "it takes: " - + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) - entry = {"name": name, "operator": op} - if op not in BD_NULLARY_OPS: - raw_v = p.get("value") - # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and - # it is stored as a list rather than as N sibling predicates so the row a person sees - # and the row that is stored are the same thing. - vals = raw_v if isinstance(raw_v, list) else [raw_v] - vals = [v.strip() if isinstance(v, str) else v for v in vals] - vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))] - if not vals: - return None, f"give {field_label(name)} something to compare against" - if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: - return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value — " - f"{field_label(name)} can only be given a list with " - + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) - if o in BD_MULTI_VALUE_OPS)) - if len(vals) > MAX_PREDICATE_VALUES: - return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values " - "in one condition") - if field_kind(name) == "boolean": - ok = {o["value"] for o in BD_BOOLEAN_OPTIONS} - bad = [v for v in vals if str(v).lower() not in ok] - if bad: - return None, f"{field_label(name)} is answered Yes or No" - # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers - # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and - # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field - # was a free-text box until this wave, so the Yes/No dropdown that makes it easy - # to ask is also the thing that would have made every boolean condition fail. - # The dropdown's option VALUES stay "true"/"false" (a carries strings); + # the conversion belongs here, at the edge that talks to the vendor. + vals = [str(v).lower() == "true" for v in vals] + if field_kind(name) == "number": + try: + vals = [float(v) if "." in str(v) else int(v) for v in vals] + except (TypeError, ValueError): + return None, f"{field_label(name)} takes a number" + # A single value stays a SCALAR on the wire. The vendor has only ever been sent + # scalars for these operators; the list form is expanded at send time and the + # one-value case must not quietly start exercising an untested shape. + entry["value"] = vals[0] if len(vals) == 1 else vals + out.append(entry) + if not out: + # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES. + # This used to refuse, which was right while a wizard collected the filters before the + # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger + # CREATES the automation, and the filters are typed afterwards — so a save-time refusal + # here meant the trigger could never be picked at all. It is the A3 + # stored-inert-with-`configured:false` pattern this module already runs on. + # + # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient: + # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a + # search, deliberately, because "a stored config can predate the law, and the vendor + # bills for breadth whether the filter was saved yesterday or last month". The RUN is the + # wall. Every MALFORMED predicate above is still refused here, and the narrowing law + # below still refuses a non-empty filter that narrows nothing — only EMPTY changed. + return [], None + # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the + # measured hang this guards against. Same sentence at create/patch and at run. + guard_err = narrowing_refusal(out, operator, kind) or depth_refusal(out, operator) + if guard_err: + return None, guard_err + return out, None + + +def discover_estimate(records_limit): + """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the + section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" + n = max(0, int(records_limit or 0)) + return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), + "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", + "note": "estimated from the published rate — an exact price is only known after a run" + "a price before a run, and this account's token cannot read a balance"} + + +def _candidate_row(row, stamp): + """One corpus row → a `ut_ig_candidates` row. None when it has no handle. + + ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already + stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted + `tracked` either; R6 deleted that column in wave 26.) + """ + if not isinstance(row, dict): + return None + handle = str(_first(row, "account", "username", default="") or "").strip() + if not handle: + return None + return { + # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads + # Instagram, so it is a constant here rather than something derived from the row; the + # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart. + "platform": PLATFORM_INSTAGRAM, + "handle": handle, + "profile_url": str(_first(row, "profile_url", "url", + default=f"https://www.instagram.com/{handle}/")), + "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), + "followers": _s(_ig_int(_first(row, "followers"))), + "following": _s(_ig_int(_first(row, "following"))), + # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is + # the path where it carries values (measured on all five of the 2026-08-04 result set). + # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`. + "avg_engagement": _pct100(_first(row, "avg_engagement", default="")), + "bio": _s(_first(row, "biography", "bio", default=""), 500), + "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), + "verified": "1" if _first(row, "is_verified", default=False) else "", + "category": _s(_first(row, "category_name", "business_category_name", default="")), + # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp + # still exists on the snapshot series, which is where a time axis belongs. + "last_found": _day(stamp), + } + + +# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ────────────────────────────────────────── +# +# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a +# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a +# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via +# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers, +# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at +# all, so discovery was the one rung in this module that read numbers and recorded no observation. +# +# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold +# real corpus numbers that are already on screen and already filterable. The gap is that the +# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law +# exists to forbid one level up. +# +# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make +# it honest already existed: `source` says **Discovery** (this was read off the vendor's +# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not +# read this as an exact count taken at `pulled_at`" — which is the whole difference between +# recording what we know and inventing what we do not. +# +# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it +# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as +# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over +# the series therefore prefers the exact read over the corpus read whenever both happened on one +# day — the ordering you would have to hand-write, falling out of the grain. + +#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell. +IG_VIA_DISCOVERY = "brightdata:discovery" + +#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement +#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather +#: than typed out — a hand-list is what silently stops carrying the next promoted column. +_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") + + +def corpus_snapshot_row(row, day=""): + """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable. + + `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's + only date is `first_found`. + + ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is + `@`, so re-running this over the same rows on the same day rewrites the same key + and `upsert_rows` merges it — the migration is safe to call on every write, which is the only + way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys + idempotency on the stored TYPE). + + ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty + ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed. + Here the merge is the hazard instead: a second discovery on the same day returning a thinner + corpus record would otherwise ERASE what the first one learned. An absent key renders exactly + as an empty cell, so nothing is lost by leaving it out. + """ + handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if not handle: + return None + when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "") + if not when: + return None + out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when, + "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"} + for fd in SNAPSHOT_FIELDS: + key = fd["key"] + if key in _SNAPSHOT_OWN_KEYS: + continue + value = (row or {}).get(key) + if str(value or "").strip(): + out[key] = str(value) if key == "source_payload" else _s(value, 400) + return out + + +def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print): + """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written. + + ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only + honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls + `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the + creator from there is unbounded recursion. A tenant with no series store has nothing to + back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this + function then has somewhere to write. + """ + rows_in = [r for r in (incoming or []) if r] + if not rows_in: + return 0 + existing = ut_get(rt, snap_key) + if existing is None: + return 0 + merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key", + cap=row_cap(snap_key)) + if counts["capped"]: + # D-11's law: a full append table means the SERIES has stopped growing, which is the one + # failure a chart cannot show you. + log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by " + f"{snap_key}'s row cap") + written = counts["inserted"] + counts["updated"] + if not written: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if cur.get(snap_key) is not None: + cur[snap_key]["rows"] = merged + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return written + + +# --------------------------------------------------------------------------------------------- +# RUNNERS +# --------------------------------------------------------------------------------------------- + +def cap_note(capped, missing=()): + """The sentence(s) a cap breach adds to a run summary. + + `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. + `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not + be created at all. + + ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the + ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that + does not say WHERE is only marginally better than a silent one. + + ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused + SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the + runners' `if tgt is not None` write guard then skipped that bucket, and the run reported + success over a database that does not exist. Exactly the silent-stop failure the row cap + already had, one level up — and harder to spot, because the table is not there to look at. + """ + out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} " + f"from this run were NOT written" for k, n in (capped or []) if n] + out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its " + f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" + for k in (missing or [])] + return "; ".join(out) + + +def ut_missing(rt, *keys): + """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" + have = ut_all(rt) + return [k for k in keys if k and k not in have] + + +def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" + cfg = defn.get("config") or {} + fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") + dry = bool(cfg.get("dryRun")) + # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from + # the rollup — inferring is exactly how a green dot ended up over an empty table (see the + # rollup comment in `run_field_instagram`). + steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} + step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}") + status, final, body = fetch(cfg["url"]) + if not (200 <= status < 300): + steps["fetch"] = "error" + return ("error", f"{cfg['url']} answered {status} — nothing was written", + {"status": status}, [], steps) + steps["fetch"] = "ok" + soup = _soup(body) + if cfg.get("extract") == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 + raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] + if not raw_rows: + steps["extract"] = "error" + return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], + steps) + steps["extract"] = "ok" + step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}") + + incoming = [] + for r in raw_rows: + mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} + if any(v for v in mapped.values()): + incoming.append(mapped) + + fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) + for src, tgt in fmap.items()] + # the key column leads, so the table opens on the identity it is upserted by + fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) + label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" + if dry: + # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, + # which would create the table: a dry run that leaves a new empty database behind is not + # a dry run. + table_key = ut_key_for(label, cfg.get("targetTable") or None) + else: + table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) + t = ut_get(rt, table_key) or {} + rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, + cap=row_cap(table_key)) + touched = [rid for rid, row in rows.items() + if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] + if not dry: + ut_write_rows(rt, table_key, rows) + counts["source_rows"] = len(incoming) + affected = touched[:200] + summary = (f"{counts['inserted']} new, {counts['updated']} updated, " + f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " + f"(kept)") + capped = [(table_key, counts.get("capped", 0))] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + note = cap_note(capped, missing) + if note: + summary += f" — {note}" + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + steps["write"] = "skipped" + elif not note: + steps["write"] = "ok" + state = "partial" if (counts.get("capped") or counts.get("skipped") or missing + or dry) else "ok" + log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def capture_rows(res, pulled): + """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`. + + ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT + RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path — + do not fork them." A second copy of this mapping is the shape that goes wrong invisibly, + because the two copies would each be *plausible* and would disagree only on the rows a + particular rung happened to return. + + ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE: + * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every + time and a blank is unambiguous — it means THIS PULL did not read it. + * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are + readable on some rungs and not others (the Bright Data PROFILE row carries post identity + with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and + `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run + ERASE what an expensive one learned. + * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is + the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement + rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k + ceiling and drawing a chart of nothing. A row in a time series should mean "this was true + then"; a row meaning "nobody looked" belongs nowhere. + """ + prof = (res or {}).get("profile") or {} + snap_row = { + "snapshot_key": f"{prof.get('username')}@{pulled}", + "influencer_key": prof.get("username"), "pulled_at": pulled, + # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called + # "Read via". `res["via"]` keeps the real key for the call graph and the server log. + "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")), + "bio": _s(prof.get("bio"), 500), + "external_url": _s(prof.get("external_url"), 300), + } + for fd in SNAPSHOT_FIELDS: + k = fd["key"] + if k not in snap_row: + value = prof.get(k) + # Profile history declares the same percentage dialect as the current Profile row. + # Converting only the latest projection left the append series at 0-1 while its field + # now said pct (0-100), so the two views of one measurement disagreed by 100x. + if k == "source_payload": + snap_row[k] = str(value or "") + else: + snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400) + idents, metrics_rows, comment_rows = [], [], [] + for p in (res or {}).get("posts") or []: + ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"), + "url": _s(p.get("url"), 300)} + for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), + ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), + ("alt_text", 400), ("tagged_location", 400)): + if p.get(k): + ident[k] = _s(p.get(k), n) + if p.get("source_payload"): + ident["source_payload"] = str(p["source_payload"]) + metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")} + # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF. + # + # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average + # views over the last 12 posts" would have to walk profile → posts → each post's most + # recent snapshot, and a two-hop rollup is a much larger feature with a much worse + # invalidation story. The post row now carries its own latest, exactly as the profile row + # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3). + # + # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the + # authoritative engagement series and still gets its appended row. These cells are a + # projection of the row being appended in the same breath, never a second source. + # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent + # key leaves the previous pull's value standing — which is the correct behaviour for a + # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing + # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the + # same failure the snapshot-append rule above is written against. + measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""} + if measured: + ident.update({k: _s(v, 40) for k, v in measured.items()}) + # The stamp is what makes the numbers readable: a blank `plays` beside a + # `measured_at` of last week means "we looked and the vendor had nothing", and with + # no stamp it means that AND "we never looked", indistinguishably. + ident["measured_at"] = _day(pulled) + idents.append(ident) + if any(v is not None for v in metrics.values()): + metrics_rows.append({ + "post_snapshot_key": f"{p['shortcode']}@{pulled}", + "shortcode": p["shortcode"], "influencer_key": prof.get("username"), + "pulled_at": pulled, + # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the + # SAME vendor record the identity row above is built from, so they cannot disagree + # with it on this pull. + # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split` + # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull + # that learns a post's engagement often does not learn its date in the same + # breath. `backfill_post_snapshot_grain` fills those from the post row, which by + # then may have learned it from a different rung. + "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200), + "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), + "views": _s(metrics["views"]), "plays": _s(metrics["plays"]), + "source_payload": str(p.get("source_payload") or "")}) + for comment in p.get("embedded_comments") or []: + # The embedded shape often names only the commenter. Its parent Post is the + # authoritative owner, so bind every preview to this profile and post explicitly. + comment_rows.append({**comment, "shortcode": p["shortcode"], + "influencer_key": prof.get("username") or ""}) + comment_rows.extend((res or {}).get("comments") or []) + # The same comment can be returned as both `latest_comments` and `top_comments`, or by the + # optional full endpoint after an embedded preview. The canonical key is the one row truth. + unique_comments = {} + for row in comment_rows: + key = str((row or {}).get("comment_key") or "") + if key: + unique_comments[key] = {**(unique_comments.get(key) or {}), **row} + return snap_row, idents, metrics_rows, list(unique_comments.values()) + + +#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched. +#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of +#: `row[...] = prof.get(...)` that a future field addition can silently miss. +#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did +#: not read it", and `enriched_at` is what distinguishes that from "never enriched"). +PRESET_FROM_PROFILE = { + "username": "handle", "profile_url": "profile_url", "full_name": "full_name", + "followers": "followers", "following": "following", "avg_engagement": "avg_engagement", + "bio": "bio", "external_url": "external_url", "verified": "verified", + "category": "category", "posts_count": "posts_count", + "highlights_count": "highlights_count", "is_business": "is_business", + "is_professional": "is_professional", "ig_id": "ig_id", + # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column + # that can only ever be blank, so "make the preset fields populated" is THIS half of the + # owner's instruction and the field list is only the other half. + # ⚠ The two lists are held in step by a derived gate rather than by care: every + # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared + # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`). + "business_category": "business_category", "is_private": "is_private", + "bio_hashtags": "bio_hashtags", + "pronouns": "pronouns", "profile_name": "profile_name", + "is_joined_recently": "is_joined_recently", "has_channel": "has_channel", + "partner_id": "partner_id", "external_url_title": "external_url_title", + "fbid": "fbid", "related_accounts": "related_accounts", + "country_code": "country_code", "source_payload": "source_payload", +} + +#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a +#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate +#: could only be written as "these three are fine" — a hard-coded exception list, which is the +#: shape [[gate-answers-the-wrong-question]] warns about. +#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an +#: enrichment run — which is the whole point of them: they stay true when the LINKED table +#: changes, and a pull that touched no profile still updates a profile's post count. +PRESET_WRITTEN_ELSEWHERE = ( + "platform", "handle", "enriched_at", + "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link", + "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12", + "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured", +) + + +#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a +#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING` +#: refuses on the discovery side for exactly the same reason. +LOCATION_MIN_POSTS = 2 + + +def _post_place(post): + """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload + second. + + ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the + vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the + `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details` + shape. The complete vendor row is retained in `source_payload` on the same record, already + paid for, so a post whose normalised field came back blank can still be read here. + ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region, + country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to + "where is this person"; "Indonesia" mostly is not. + """ + if not isinstance(post, dict): + return "" + raw = str(post.get("tagged_location") or "").strip() + if not raw: + try: + payload = json.loads(post.get("source_payload") or "{}") + except Exception: # noqa: BLE001 + payload = {} + raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else "" + return raw.split(",")[0].strip() + + +def location_guess(posts): + """`(city, confidence_pct)` from the places a creator's own posts were tagged in. + + `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not + the share of ALL posts, because a creator who geotags three of twelve is telling us about + three, and dividing by twelve would report a real signal as weak. How thin the evidence is + stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two. + """ + places = [p for p in (_post_place(x) for x in posts or []) if p] + if len(places) < LOCATION_MIN_POSTS: + return "", None + counts = {} + for p in places: + counts[p] = counts.get(p, 0) + 1 + # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be + # resolved alphabetically, which is an arbitrary answer wearing a confident number. + best = max(places, key=lambda p: (counts[p], -places.index(p))) + return best, round(100.0 * counts[best] / len(places), 1) + + +def preset_cells(res, pulled): + """C4/R3: one pull → the LATEST-value cells written onto the enriched record. + + ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the + other fifteen readable: without it a blank `followers` means both "never enriched" and + "enriched in March and the vendor had nothing", and no cell on the row can tell them apart. + """ + prof = (res or {}).get("profile") or {} + cells = {} + for src, dest in PRESET_FROM_PROFILE.items(): + v = prof.get(src) + if v is not None and v != "": + # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The + # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a + # straight-through copy here would write the same wrong number the discovery path + # used to write. Both writers convert; neither is the exception. + cells[dest] = (_pct100(v) if dest == "avg_engagement" else + (str(v) if dest == "source_payload" else _s(v, 500))) + # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an + # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have + # arrived with no platform at all — and a blank half of the dedup key is how one account + # becomes two rows. + cells["platform"] = PLATFORM_INSTAGRAM + # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when + # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose + # twelve posts happened to carry no geotag would make the column worse the more it ran. + place, confidence = location_guess((res or {}).get("posts")) + if place: + cells["location_guess"] = _s(place, 120) + cells["location_confidence"] = str(confidence) + # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a + # question in days; the full stamp keeps its precision on the snapshot series. + cells["enriched_at"] = _day(pulled) + return cells + + +#: Keys the TikTok SNAPSHOT row owns itself, so the copy loop below never overwrites them from the +#: profile map. Mirrors `_SNAPSHOT_OWN_KEYS` on the Instagram side. +_TT_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") + + +def tt_preset_cells(res, pulled): + """⭐⭐ WAVE 30 · T08 — one TikTok pull → the LATEST-value cells written onto the record. + + ⛔ THIS IS NOT `preset_cells` WITH A FLAG, AND THAT IS THE WHOLE DESIGN DECISION. Instagram's + `preset_cells` walks `PRESET_FROM_PROFILE` (an Instagram map), stamps + `cells["platform"] = PLATFORM_INSTAGRAM` UNCONDITIONALLY, and computes a location guess from + posts. Passing a TikTok row through it would stamp every TikTok creator as Instagram — which is + the identity half of W26's `(platform, handle)` key, so it would not merely mislabel a row, it + would MERGE two different people's accounts under one key. + ⭐ AND THE TIKTOK ROW DOES NOT NEED A MAP AT ALL. `connectors_tt.normalize_profile` already + emits our column names — it is the SAME function the discovery runner uses — so a scraped row + and a corpus row cannot disagree about which vendor key became which column. All this adds is + what the FETCH knows and the map cannot: when it was read, and by which route. + + ⚠ FILTERED TO THE DECLARED SCHEMA. `ut_write_rows`/`_clean_field` DROP an undeclared key with + no error and a successful-looking run (the defect `section_w29_tiktok_schema`'s emit-vs-declare + sweep exists for), so an unknown key is refused here where it is visible rather than swallowed + three layers down. + """ + prof = (res or {}).get("profile") or {} + declared = {f["key"] for f in TT_PROFILE_FIELDS} + cells = {k: v for k, v in prof.items() if k in declared} + # `source` is the ROUTE that answered, not the vendor's name — the same thing `via` carries on + # the Instagram snapshot, and the reason a stored measurement can always say how it was read. + if (res or {}).get("via"): + cells["source"] = _s((res or {}).get("via"), 60) + # R3, as on the Instagram side: a `date`, because "how stale is this number" is a question in + # days. Without it a blank `followers` means both "never enriched" and "enriched and empty". + cells["enriched_at"] = _day(pulled) + return cells + + +#: The metric columns a TikTok post SNAPSHOT carries. ⛔ DERIVED from the declaration, minus the +#: keys the snapshot owns itself — so adding a metric to `TT_POST_SNAPSHOT_FIELDS` starts being +#: captured, and adding an IDENTITY column to it never does. +_TT_PSNAP_OWN_KEYS = ("platform", "post_snapshot_key", "shortcode", "influencer_key", "pulled_at", + "post_link") +TT_PSNAP_METRIC_KEYS = tuple(f["key"] for f in TT_POST_SNAPSHOT_FIELDS + if f["key"] not in _TT_PSNAP_OWN_KEYS) + + +def tt_capture_rows(res, pulled): + """⭐⭐ WAVE 30 · T10 — one TikTok pull → `(post_identity_rows, post_metric_rows, comment_rows)`. + + The TikTok twin of `capture_rows`, and separate for the same reason `tt_preset_cells` is: that + function walks Instagram's `SNAPSHOT_FIELDS`/`PRESET_FROM_PROFILE` and stamps Instagram's + platform. The mappers have already done the vendor→our-keys work here + (`connectors_tt.normalize_post` / `normalize_comment`), so this adds only what the FETCH knows. + + ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE, exactly as on the Instagram side: + * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED — `upsert_rows` writes the keys it + is handed, so sending "" would let a thin run ERASE what a full one learned. + * A METRIC SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. A row in a time series + should mean "this was true then"; a row meaning "nobody looked" eats the cap and draws a + chart of nothing. + + ⛔⛔ D-117 IS NOT REPRODUCED HERE, AND THAT IS THE INSTRUCTION THE TICKET LEADS WITH. + Instagram denormalises `posted_at` and `type` onto its post SNAPSHOT rows, where they never + reconcile against the identity table and quietly become a second, ageing answer to a question + the posts table already answers. This snapshot carries the METRICS and the join key and nothing + else — `TT_PSNAP_METRIC_KEYS` is derived from the declaration minus the keys the snapshot owns, + so the exclusion is structural rather than a list somebody has to remember to keep short. + """ + prof = (res or {}).get("profile") or {} + who = str(prof.get("handle") or "").strip().lstrip("@").lower() + idents, metric_rows, comment_rows = [], [], [] + for p in (res or {}).get("posts") or []: + shortcode = str((p or {}).get("shortcode") or "").strip() + if not shortcode: + continue + ident = {k: v for k, v in p.items() if v not in (None, "")} + # ⚠ The Posts dataset carries `profile_username`, but a row that omits it must still join: + # the profile whose `top_videos` we scraped IS the influencer, and that is a fact of the + # call rather than of the row. + ident.setdefault("influencer_key", who) + ident["platform"] = PLATFORM_TIKTOK + idents.append(ident) + measured = {k: p.get(k) for k in TT_PSNAP_METRIC_KEYS + if k != "source_payload" and p.get(k) is not None} + if not measured: + continue + row = {"platform": PLATFORM_TIKTOK, "shortcode": shortcode, + "influencer_key": str(p.get("influencer_key") or who), + "post_snapshot_key": f"{shortcode}@{pulled}", "pulled_at": pulled} + for k, v in measured.items(): + row[k] = _s(v, 400) + metric_rows.append(row) + for c in (res or {}).get("comments") or []: + if not str((c or {}).get("comment_key") or "").strip(): + continue + row = {k: v for k, v in c.items() if v not in (None, "")} + # Same fact-of-the-call argument: the Comments dataset has no influencer field at all, so + # without this the comments table could never be filtered by creator. + row.setdefault("influencer_key", who) + row["platform"] = PLATFORM_TIKTOK + comment_rows.append(row) + return idents, metric_rows, comment_rows + + +def tt_snapshot_row(res, pulled): + """One TikTok pull → its `ut_tt_snapshots` observation, or None when there is no handle. + + ⛔ THE APPEND LAW, UNCHANGED FROM INSTAGRAM AND FOR THE SAME MEASURED REASON: TikTok's Profiles + dataset carries **no measurement timestamp** (`create_time` is when the ACCOUNT was made, not + when `followers` was true — probed, `tiktok-capture.md`). So the series is dated by when WE + read it, and the key is `@` — idempotent by its KEY rather than by a flag, + so re-running over the same rows rewrites the same row instead of growing the table. + ⚠ `pulled_at` keeps the FULL stamp while the record's `enriched_at` is a day: the record answers + "how stale", the series answers "when exactly", and collapsing the second into the first would + make two reads on one day indistinguishable. + """ + prof = (res or {}).get("profile") or {} + handle = str(prof.get("handle") or "").strip().lstrip("@").lower() + if not handle: + return None + out = {"platform": PLATFORM_TIKTOK, "snapshot_key": f"{handle}@{pulled}", + "influencer_key": handle, "pulled_at": pulled, + "source": _s((res or {}).get("via") or "brightdata", 60)} + for fd in TT_SNAPSHOT_FIELDS: + key = fd["key"] + if key in _TT_SNAPSHOT_OWN_KEYS or key == "platform": + continue + value = prof.get(key) + # ⚠ A ZERO IS A MEASUREMENT AND SURVIVES — the test is `is not None`, never truthiness. + # `str(value or "").strip()` would drop a genuine 0 follower count, and this is a series + # whose whole purpose is that a number moved. + if value is not None and str(value).strip() != "": + out[key] = str(value) if key == "source_payload" else _s(value, 400) + return out + + +def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, + rows=None): + """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public + profile through the paid capability chain, write a status string into the automation column, + and append a timestamped row to each of the three IG tables. + + ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per + CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two + "which rung answered" counters this function used to keep have nothing left to distinguish.""" + cfg = defn.get("config") or {} + table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") + post_metrics = bool(cfg.get("postMetrics")) + comment_metrics = bool(cfg.get("commentMetrics")) + dry = bool(cfg.get("dryRun")) + # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a + # status written under a node id `graph()` no longer emits is a dot nothing renders, which is + # indistinguishable from a step that never ran. + steps = {"trigger": "ok", "source": "idle", + "capture_posts": "idle" if post_metrics else "skipped", + "capture_comments": "idle" if comment_metrics else "skipped", + "write": "idle"} + t = ut_get(rt, table_key) + if not t: + steps["source"] = "error" + return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) + url_field = cfg.get("urlField") or _auto_url_field(t, fkey) + if not url_field: + steps["source"] = "error" + return ("error", "the automation column has no URL field bound to it", {}, [], steps) + steps["source"] = "ok" + rows = dict(t.get("rows") or {}) + # the relational tables the pull lands in (R7): every row timestamped for time-range filters + if dry: + # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) + snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, + IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE) + else: + graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=table_key) + snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], + graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) + missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key) + snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) + posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) + psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) + comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {}) + + counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, + "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0, + "missing_tables": len(missing)} + cells, affected, notes = {}, [], [] + pending_metrics = [] + # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` + # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. + # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation + # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) + in_snaps, in_posts, in_psnaps, in_comments = [], [], [], [] + targets = [(rid, str(r.get(url_field, "") or "").strip()) + for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] + for i, (rid, url) in enumerate(targets): + if i: + time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) + # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile + # column automation blocks for minutes by design. A counting step is the difference + # between "working through them" and "stuck". + step(f"Capturing profile {i + 1} of {len(targets)}") + counts["profiles"] += 1 + res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log, + post_metrics=post_metrics, comment_metrics=comment_metrics, + pending_metrics=(pending_metrics if not dry else None)) + pulled = _iso() + via = res.get("via") or "" + read_ok = res["state"] in ("ok", "partial") + # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID + # vendor actually answered, which is the run's spend report — the thing the owner reads to + # reconcile a bill. What died is the free rung it used to be contrasted with, so the test + # is now simply "did a vendor answer" rather than "did the vendor we were told to try". + if read_ok and via in ("brightdata", "apify"): + counts["paid"] += 1 + if read_ok: + counts["ok" if res["state"] == "ok" else "partial"] += 1 + prof = res["profile"] + snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled) + in_snaps.append(snap_row) + in_posts.extend(ident_rows) + in_psnaps.extend(metric_rows) + in_comments.extend(captured_comments) + counts["metrics"] += len(metric_rows) + counts["comment_rows"] += len(captured_comments) + counts["posts"] += len(res.get("posts") or []) + detail = f"{prof.get('followers') or '?'} followers" + if prof.get("approx"): + detail += " (approx)" + if res.get("posts"): + detail += f", {len(res['posts'])} posts" + elif res["state"] == "partial": + detail += ", posts not readable on the rung that answered" + cells[rid] = f"{res['state']} · {_stamp()} · {detail}" + elif res["state"] == "blocked": + counts["blocked"] += 1 + cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "blocked") + else: + counts["error"] += 1 + cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "error") + affected.append(rid) + + # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. + snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) + posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) + comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key") + comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + counts["new_posts"] = c_post["inserted"] + capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), + (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])] + counts["capped"] = sum(n for _k, n in capped) + + if not dry: + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, val in cells.items(): + tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), + (comment_key, comments)): + tgt = cur.get(k) + if tgt is not None: + tgt["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics) + if queued: + counts["metric_batches_pending"] = queued + + # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: + # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), + # `error` a LOUD partial — a pooled history with silent holes is worse than none. + master_note = "" + if not dry and (in_snaps or in_posts or in_psnaps): + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), + in_snaps, in_posts, in_psnaps) + if m_status == "error": + master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \ + f"is complete and the next run re-appends" + counts["master_failed"] = 1 + elif m_status == "ok": + counts["master"] = len(in_snaps) + len(in_psnaps) + + # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while + # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail + # showed a green dot over a table with no posts in it. A summary that disagrees with the + # cells it summarises is the failure this whole module's honest-status rule exists to + # prevent, so `partial` now propagates. Measured 2026-08-04. + read = counts["ok"] + counts["partial"] + if not read: + state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" + elif counts["blocked"] or counts["error"] or counts["partial"]: + state = "partial" + else: + state = "ok" + if counts.get("metric_batches_pending"): + state = "partial" + # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) + if not targets: + cap_state = "idle" + elif not read: + cap_state = "error" if counts["error"] else "blocked" + else: + cap_state = "partial" if (counts["blocked"] or counts["error"] + or counts["partial"]) else "ok" + # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to + # read a profile now, and the node that names the profile set is the honest owner of "did + # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch + # that no longer exists in behaviour OR on screen. + steps["source"] = cap_state if targets else steps["source"] + # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually + # appended. "It was switched on" is not the same fact as "it answered", and painting the + # second from the first is the fabrication `node_status` refuses to make. + # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A + # run that found no posts to enrich did not have a rung refuse it — nothing was requested — + # so that reads `skipped`, the same word an off switch earns. + steps["capture_posts"] = ("ok" if counts["metrics"] + else "partial" if counts.get("metric_batches_pending") + else "blocked" if (post_metrics and counts["posts"]) + else "skipped") + steps["capture_comments"] = ("ok" if counts.get("comments") + else "blocked" if (comment_metrics and counts["posts"]) + else "skipped") + + summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " + f"({counts['new_posts']} new)") + if counts.get("metric_batches_pending"): + summary += (f", {counts['metric_batches_pending']} post-engagement batch" + f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building " + "(collected automatically)") + if counts["paid"]: + summary += f", {counts['paid']} with exact counts" + if counts["metrics"]: + summary += f", {counts['metrics']} post engagement snapshots" + elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"): + # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there + # was no engagement", which is a claim about Instagram rather than about our run. + summary += ", no post engagement was readable" + if counts["comment_rows"]: + summary += f", {counts['comment_rows']} comment rows captured" + elif comment_metrics and counts["posts"]: + summary += ", no comment engagement was readable" + if counts["partial"]: + summary += f", {counts['partial']} profile-only (posts not readable)" + if counts["blocked"]: + summary += f", {counts['blocked']} blocked" + note = cap_note(capped, missing) + if note: + summary += f" — {note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + elif not dry: + steps["write"] = "ok" + if master_note: + summary += f" — {master_note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + steps["write"] = "skipped" + state = "partial" if state != "error" else state + if notes: + summary += f" — {notes[0][:120]}" + if not dry: + # C7: the fresh master rows are exactly what the table's metric cells summarise. + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + return (state, summary, counts, affected, steps) + + +def _auto_url_field(table, fkey): + """The URL field an automation column is bound to: its own `automation.urlField`, else the + first url-typed column on the table. Never guessed silently — the caller reports which.""" + for f in table.get("fields") or []: + if f.get("key") == fkey: + bound = ((f.get("automation") or {}).get("urlField") or "").strip() + if bound: + return bound + for f in table.get("fields") or []: + if f.get("type") == "url": + return f.get("key") + return "" + + +#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge +#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a +#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own +#: cheaper approximation. +TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", + "first_found", "last_found", "source", "source_payload"}) + + +def _tt_candidate_row(row, stamp): + """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. + + ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so + writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as + `_candidate_row` on the Instagram side. + ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what + the enrich path will use too, so a corpus row and a scrape row cannot disagree about which + vendor key becomes which column. + """ + import connectors_tt as _tt + cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) + if not str(cells.get("handle") or "").strip(): + return None + return {**cells, "last_found": stamp, "source": "corpus"} + + + +def _ig_discovery_series(rt, table_key, incoming, stamp, username, auto_id, counts, log): + """⭐⭐ THE FORWARD HALF (2026-08-10), Instagram only. Every candidate a run wrote also leaves + an OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the next + discovery run re-opens exactly the gap the backfill just closed, which is how a data defect + becomes a recurring one. + + ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change worth + owning: a tenant that has never enriched gets the four canonical child databases the first time + discovery runs (four of `MAX_UT_TABLES = 40`). The alternative writes a series into a store the + profile table has no LINK to — the numbers would be recorded and unreachable, which is half a + feature wearing a whole one's clothes. + ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not to every + row the table happens to hold. + """ + try: + graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) + counts["series"] = append_ig_snapshots( + rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], + graph[IG_SNAPSHOTS_TABLE], log=log) + except Exception as e: # noqa: BLE001 + # The candidates already landed. A series write that fails must not throw away the rows we + # just paid the vendor for — the same posture the preset top-up takes. + log(f"[aios-auto] discover {auto_id}: corpus series write failed: " + f"{type(e).__name__}: {e}") + + +def _ig_discovery_metrics(rt, table_key, auto_id, log): + """C7 — the metric FIELDS computed off the master series. See the field runner.""" + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + + +#: ⭐⭐ WAVE 30 · T09 (DEBT D-129) — ONE DISCOVERY RUNNER, TWO CORPORA, AND EVERY DIFFERENCE BETWEEN +#: THEM IS A ROW IN THIS TABLE. +#: +#: `run_discover_tiktok` was written in wave 29 as a deliberate SIBLING of `run_discover_instagram` +#: — its own docstring said so and gave the reason: *"it would rewrite the one code path this +#: product bills money through"*. That was the right call with one platform's worth of evidence and +#: it stopped being right when the duplication reached ~210 lines, because a sibling does not +#: inherit fixes. The proof is already in the copy below: the TikTok runner carries the **D-116** +#: precedence rule (a corpus re-find must not overwrite an ENRICHED measurement with the corpus's +#: cheaper approximation) and the Instagram one, which is the one with paying rows in it, does not. +#: +#: ⛔ **THE COLLAPSE IS BEHAVIOUR-PRESERVING, DELIBERATELY, AND `discovery_keys` IS WHERE YOU CAN +#: SEE THAT DECISION RATHER THAN INFER IT.** Setting Instagram's to `IG_DISCOVERY_KEYS` would close +#: D-116 in one line — and it would change what a live, billed, nightly automation writes to rows +#: the owner reads, inside a refactor whose whole claim is that nothing moved. It is REPORTED +#: instead (see D-116's row), which is the standing rule for a limit that is not being removed +#: today. ⚠ The one-line fix is now visible in a table instead of buried 300 lines apart, which is +#: most of the value of doing this at all. +#: +#: ⚠ `dataset` IS A CALLABLE, not a string: `connectors_tt` imports this module at import time, so +#: the engine may only reach it from INSIDE a function. `mapper` has the same shape for the same +#: reason on the TikTok side. +#: ⚠ `handle_field` is the column the engine-added `not_in` exclusion names, and it is per platform +#: because B-20 MEASURED that Bright Data's TikTok Profiles dataset calls Instagram's `account` +#: `account_id`. The exclusion never appears as a condition row, so nothing a person types can +#: correct it. +DISCOVERY_SPECS = { + "discover_instagram": { + "noun": "profile", + "log_tag": "discover", + "platform": PLATFORM_INSTAGRAM, + "dataset": lambda: BD_DS_PROFILES, + "handle_field": "account", + "mapper": lambda row, stamp: _candidate_row(row, stamp), + "table": DISCOVER_TABLE, + "label": DISCOVER_LABEL, + "fields": CANDIDATE_FIELDS, + # ⛔ D-116 IS OPEN ON THIS SIDE. `None` = discovery overwrites whatever it re-finds. + "discovery_keys": None, + "on_write": _ig_discovery_series, + "after_write": _ig_discovery_metrics, + }, + "discover_tiktok": { + "noun": "TikTok profile", + "log_tag": "discover-tt", + "platform": PLATFORM_TIKTOK, + "dataset": lambda: _tt_module().TT_DS_PROFILES, + "handle_field": "account_id", + "mapper": lambda row, stamp: _tt_candidate_row(row, stamp), + "table": TT_PROFILE_TABLE, + "label": TT_TABLE_LABELS[TT_PROFILE_TABLE], + "fields": TT_PROFILE_FIELDS, + "discovery_keys": TT_DISCOVERY_KEYS, + # ⚠ NO series and NO metric fields on this side, and that is not an oversight: both hooks + # write into Instagram's own snapshot store, through Instagram's own vocabulary. TikTok's + # series is W29-T06's `ut_tt_post_snapshots`, which the ENRICH path owns. + "on_write": None, + "after_write": None, + }, +} + + +def _tt_module(): + """`connectors_tt`, imported LAZILY. It imports this module at module level, so a top-level + import here would close the cycle.""" + import connectors_tt as _tt + return _tt + + +def run_discovery(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #3 (owner ruling R7 / D-23) and DEBT D-9: FIND handles nobody here has typed in, + on whichever corpus this automation's KIND names. + + Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus + query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every + outcome is a sentence about what actually happened; none of them is a green zero. + + ⛔ THE PLATFORM COMES FROM `defn["kind"]`, WHICH IS THE ONLY THING THAT DECIDES IT NOW. Before + wave 30 the choice was made by WHICH FUNCTION `RUNNERS` pointed at, so a fixture (or a stored + definition) with a stale kind still ran the corpus its caller had in mind. It cannot any more: + an unknown kind is refused rather than defaulted, because defaulting here means billing one + platform's corpus for another platform's search. + """ + kind = str(defn.get("kind") or "") + spec = DISCOVERY_SPECS.get(kind) + cfg = defn.get("config") or {} + dry = bool(cfg.get("dryRun")) + limit = int(cfg.get("recordsLimit") or 5) + preds = list(cfg.get("predicates") or []) + auto_id = str(defn.get("id") or "") + steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} + counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, + "missing_tables": 0} + if spec is None: + # ⛔ NEVER A DEFAULT. `RUNNERS` maps exactly the kinds in `DISCOVERY_SPECS` onto this + # function, so reaching here means somebody widened one of the two without the other. + steps["find"] = "blocked" + return ("error", f"this automation's kind ({kind or 'blank'}) has no profile corpus, so " + f"nothing was searched and nothing was charged", counts, [], steps) + noun = spec["noun"] + + if not bd_ready(): + # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is + # the vendor's — so this is where the run stops, saying which env var is missing. + steps["find"] = "blocked" + return ("partial", "Profile search is not set up yet — nothing was searched and nothing " + "was charged", counts, [], steps) + + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + if not pending: + # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and + # the vendor bills for breadth whether the filter was saved yesterday or last month. + # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config + # stored before this check existed must not reach the vendor either. + # ⭐ W32-T46: the RUN-time twin of the save-time guard says the right network's name too — + # `defn["kind"]` is what decides the corpus everywhere else in this function. + guard_err = (narrowing_refusal(preds, cfg.get("operator"), str(defn.get("kind") or "")) + or depth_refusal(preds, cfg.get("operator"))) + if guard_err: + steps["find"] = "blocked" + return ("error", f"the search was not started — {guard_err}", counts, [], steps) + if dry: + est = discover_estimate(limit) + steps["find"] = steps["write"] = "skipped" + return ("partial", + f"Test run — nothing saved. Would look for up to {limit} " + f"{noun}{'' if limit == 1 else 's'} matching " + f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " + f"${est['usd']}", counts, [], steps) + step("Starting the search") + # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never + # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the + # vendor refuses. Reported either way, because "why did this run find fewer" and "why did + # this run cost the same as last night" are both questions this line answers. + # ⚠ `already_found_handles` NEEDS NO PER-PLATFORM BRANCH — it is scoped to THIS definition's + # own tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` + # excludes exactly the handles it has itself found. Verified rather than assumed: the + # function reads `config.targetTable` plus each `create_record` action's table, and names + # no IG constant. + excl = {} + sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, + dataset_id=spec["dataset"](), + exclude_handles=already_found_handles(rt, defn), + applied=excl, handle_field=spec["handle_field"]) + if excl.get("excluded"): + step(f"excluded {excl['excluded']} already-found " + f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") + elif excl.get("dropped"): + step(f"searching without the already-found list — {excl['dropped']}") + if note: + steps["find"] = "blocked" + return ("error", f"the search was not started — {note}", counts, [], steps) + pending = sid + # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes + # through `rt.update`, which can RAISE on an unavailable store — and that exception + # becomes an error run, so a `log()` placed after it never executes. The snapshot would + # then exist, be billed, and have its id in neither the store nor the logs. Logging first + # means the worst case is still recoverable by a human reading the Space output. + log(f"[aios-auto] {spec['log_tag']} {auto_id}: started {sid}") + # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop + # existing because this process dies thirty seconds later, and a lost id is a set we + # paid attention to and can never collect. + set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) + steps["find"] = "ok" + + waited, status, size, note = 0.0, "", 0, "" + while True: + # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to + # BD_FILTER_WAIT at the vendor, and D measured that this wait IS the whole of "Run once now + # is laggy / looks stuck". A counter here is what makes a legitimate wait distinguishable + # from a hung thread; without it the rendered step reads the same word for both, which is a + # progress indicator that cannot indicate progress. + step(f"Searching — {int(waited)}s of " + f"{int(BD_FILTER_WAIT)}s") + status, size, note = bd_filter_status(pending) + if note: + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", note, counts, [], steps) + if status != "building" or waited >= BD_FILTER_WAIT: + break + time.sleep(BD_FILTER_POLL) + waited += BD_FILTER_POLL + step("Reading the profiles the search found") + + if status == "building": + # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 + # minutes; holding a worker thread that long on a free-tier container to poll would be + # the wrong shape. Say plainly that it is still running and who collects it. + # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made + # this read as "stuck": *"The next run picks up the results"* is a promise about a run + # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick + # finish it, so the promise is now kept by something rather than by the reader. + # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by + # naming the corpus size. It was TRUE, and it was answering a question the person had not + # asked. It was also the one place a vendor's catalogue size was quoted to a customer, so + # it would have gone stale the day the vendor grew. THE SHAPE OF THE ANSWER SURVIVES and is + # the part that mattered: the wait does not shrink when you ask for fewer records, and a + # person who does not know that reads "20 minutes for 10 rows" as a fault. + mins = 0 + try: + since = (defn.get("state") or {}).get("pendingSince") + if since: + mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) + - _dt.datetime.fromisoformat(str(since))).total_seconds() + // 60)) + except Exception: # noqa: BLE001 + mins = 0 + been = f" — {mins} min so far" if mins else "" + steps["collect"] = "partial" + return ("partial", + f"Still searching at the provider{been}. A corpus search takes about 20 minutes " + f"however few records you asked for. Nothing is lost, you are not charged twice, " + f"and the results are collected automatically as soon as they are ready", + counts, [], steps) + if status == "empty": + # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword + # that is too specific, so it says what to do about it instead of reporting a fault. + steps["collect"] = "ok" + steps["write"] = "skipped" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("partial", "No profiles matched. Try a shorter or more common keyword — " + "'floral' finds more accounts than 'floral design studio'", + counts, [], steps) + if status != "ready": + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", + counts, [], steps) + + rows, dnote = bd_filter_rows(pending) + if dnote: + # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. + steps["collect"] = "partial" + return ("partial", f"the search finished with {size} match" + f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + steps["collect"] = "ok" + + stamp = _iso() + _map = spec["mapper"] + incoming = [c for c in (_map(r, stamp) for r in rows) if c] + counts["found"] = len(incoming) + # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and + # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to + # "the vendor delivered 3". It also catches the shape where an error document comes back + # down the rows path and parses as one unusable row instead of an error. + counts["dropped"] = max(0, len(rows) - len(incoming)) + label = cfg.get("targetLabel") or spec["label"] + table_key = (ut_key_for(label, cfg.get("targetTable") or spec["table"]) if dry + else ut_ensure(rt, label, spec["fields"], username, + key=cfg.get("targetTable") or spec["table"], + flow_tag=auto_id, lock_fields=True)) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`. + # + # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning + # that built it is worth stating before it is discarded rather than after: per-user rows gave + # each person their own `found_count` and their own review card, so two colleagues scouting the + # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The + # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no + # longer part of the identity. + # + # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN + # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on + # TikTok are routinely different people, so keying on the handle ALONE would have silently + # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with + # no error, discovered months later by someone wondering why a creator's followers halved. + # Hence `platform`, and hence it being part of the key rather than a label beside it. + _ck = candidate_key + + # ⚠ A ROW WITH NO `platform` CELL IS STAMPED WITH **THIS CORPUS'S** PLATFORM, and the default is + # per spec rather than global. On the Instagram table every legacy row came from the Instagram + # runner, so `Instagram` is a FACT about the stored data; in a `ut_tt_*` table the same blank + # means TikTok for exactly the same reason. Blank-keyed rows would fail to match their own + # re-find and duplicate the whole table on the next run. + for r in existing.values(): + if not str(r.get("platform") or "").strip(): + r["platform"] = spec["platform"] + existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))} + for rid, r in existing.items()} + # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding + # a profile is the signal that it keeps matching, so the counter grows instead of + # resetting. `first_found` is written only when the pair is new, so it never moves. + seen = {r["_ckey"]: r for r in existing2.values()} + _dkeys = spec["discovery_keys"] + for c in incoming: + # "Found by" — the first finder is recorded and later finders do not overwrite them, which + # is what the column means now that it is no longer part of the identity. + c["created_by"] = username + c["_ckey"] = _ck(c.get("platform"), c["handle"]) + prev_row = seen.get(c["_ckey"]) + if prev_row and str(prev_row.get("created_by") or "").strip(): + c["created_by"] = prev_row["created_by"] + if prev_row: + counts["seen_again"] += 1 + c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) + # ⛔⛔ DEBT D-116 — THE ENRICHED-MEASUREMENT PRECEDENCE RULE, AND IT RUNS ONLY WHERE THE + # SPEC DECLARES A `discovery_keys` SET. A discovery re-find OVERWRITES an enriched + # `followers` with the corpus number — the corpus is cheaper, rounder and older than a + # scrape, so the row silently gets WORSE every night while the automation reports + # success. The fix is a precedence rule, not a blank check: a row that has been ENRICHED + # (`enriched_at` is stamped only by the enrich path) keeps its exact measurements, and + # discovery may only fill what is genuinely empty and update its own bookkeeping. + # ⚠ The declared set is what discovery OWNS; everything else on a corpus row is a + # measurement, and an approximation must never replace an exact one. + if _dkeys and str(prev_row.get("enriched_at") or "").strip(): + for k in list(c): + if (k not in _dkeys and k != "_ckey" + and str(prev_row.get(k) or "").strip()): + c.pop(k) + else: + counts["new"] += 1 + c["found_count"] = "1" + c["first_found"] = stamp + # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF + # "is this row new to the TABLE". + # + # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's + # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true, + # and then drew nothing, because this stamp lived inside the `else` above. `skey` is + # `stage_`; the branch it sat in asks whether some OTHER automation had + # already created the row. So the first automation to reach a handle claimed it, and every + # later automation sharing that database silently had no cards — with a summary saying it + # had found them. + # + # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an + # unconditional write: a re-find must never pull a card somebody has already moved back + # into Review. So the question is "has THIS automation placed this record yet?" — not "is + # this record new?" A blank stage cell for this automation means unplaced, which is + # exactly the state a new card is in. + merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) + merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} + for rid, r in merged.items()} + counts["capped"] = mc["capped"] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + if not dry and not missing: + ut_write_rows(rt, table_key, merged) + steps["write"] = "ok" + if spec["on_write"]: + spec["on_write"](rt, table_key, incoming, stamp, username, auto_id, counts, log) + elif dry: + steps["write"] = "skipped" + + affected = [rid for rid, r in merged.items() + if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] + est = discover_estimate(counts["found"]) + summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found — " + f"{counts['new']} new, {counts['seen_again']} seen before. About " + f"${est['usd']}") + if counts["dropped"]: + summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " + f"username and could not be saved") + cnote = cap_note([(table_key, counts["capped"])], missing) + if cnote: + summary += f" — {cnote}" + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" + if not dry and not missing and spec["after_write"]: + spec["after_write"](rt, table_key, auto_id, log) + log(f"[aios-auto] {spec['log_tag']} {auto_id} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def _predicate_sentence(preds, operator="and"): + """A predicate list as something a person can read back. Used in summaries and the canvas. + + Human words on BOTH halves — it used to render `biography includes floral`, the vendor's + column name beside the vendor's operator token, in a sentence shown to a customer. + """ + joiner = " or " if str(operator or "").lower() == "or" else " and " + parts = [] + for p in preds or []: + v = p.get("value") + shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v + parts.append(f"{field_label(p.get('name'))} " + f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}" + + ("" if shown is None else f" {shown}")) + return joiner.join(parts) or "no conditions" + + +#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather +#: than a second, smaller number nobody could explain: a plain automation's records ARE its +#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a +#: flow could be pointed at and which would not finish. +#: ⚠ W31-T36 (D-143 / R6) — THIS IS NOW THE FALLBACK ONLY: what a flow may walk when the table's +#: own limit cannot be resolved (no runtime, or a key `core.user_tables` does not know). The real +#: answer is per TABLE and comes from `core.user_tables.row_limit`; see `_flow_record_cap`. +#: ⛔ Do not re-point a caller at this constant — a private copy of "is this database bounded" is +#: exactly the second evaluator D-143 was. +MAX_FLOW_RECORDS = MAX_UT_ROWS + + +def _flow_record_cap(table_key, rt=None): + """How many records may a flow walk in this database? `None` means NO CAP (R6). + + ⛔ DELEGATES, NEVER DECIDES. `core.user_tables.row_limit` is the one evaluator for "is this + database bounded", and it distinguishes three states this module must not re-derive: `0` + (read-through — its rows are not in the document at all), `None` (connected and UNCAPPED, which + is R6's whole point), and `MAX_ROWS` (the editable substrate, genuinely bounded by the shared + document). A second copy here is exactly the defect D-143 IS. + + ⚠ `0` MEANS "NO ROWS LIVE HERE", NOT "WALK NOTHING". A read-through grid's rows arrive through + the mirror route, so a flow that reached this function already holds ids from somewhere else; + capping it at zero would refuse a walk whose records are in hand. Treated as uncapped, and the + honest place to fix a read-through walk is the walker. + """ + try: + import core.user_tables as _ut_cap + cap = _ut_cap.row_limit(str(table_key or ""), st=rt) + except Exception: # noqa: BLE001 + return MAX_FLOW_RECORDS # the fallback, and it is the conservative direction + return None if not cap else int(cap) + + +def _flow_cap_reason(table_key, rt=None): + """R6's SECOND sentence, appended to the run's own note: the cause and the recommended fix. + + ⛔ R6 IS TWO SENTENCES AND THE SECOND IS THE ONE THAT GETS DROPPED — *"if there is lag or it + can't be done, you need to explicitly tell me why and recommend a fix."* A run that says only + *"walked the first 5000"* has disclosed the number and hidden everything a person could act on. + `core.user_tables.limit_report` already carries both, so this READS them rather than writing a + second wording that would drift from the one the grid shows. + """ + try: + import core.user_tables as _ut_cap + rep = _ut_cap.limit_report(str(table_key or ""), st=rt) or {} + except Exception: # noqa: BLE001 + return "" + cause, fix = str(rep.get("cause") or ""), str(rep.get("recommendation") or "") + if not cause and not fix: + return "" + return f" — {cause}." + (f" To walk them all: {fix}." if fix else "") + + +def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole + automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special + case and `apply_actions` still runs at the ONE call site that already exists. + + Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed + `error: unknown automation kind 'plain'` — on the kind every new automation now has. + + ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here. + `apply_actions` returns immediately when there is no table, so a flow whose only action is + `create_record` into some other database does NOTHING — and a green dot over nothing is the + exact defect this module names in three other places. It reports the honest state and says + which fact is missing. + """ + table = _flow_table(defn) + if not table: + return ("partial", + "no database is bound yet — a plain automation walks the records of its target " + "database, and this one names none (pick one on the trigger, or in Properties)", + {}, [], {}) + all_rows = (ut_get(rt, table) or {}).get("rows") or {} + # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug + # on the kind every new automation now has. `plain` has no machine step, so it has nothing of + # its own to call "the records this run touched" — and walking the WHOLE table would mean + # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the + # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them + # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what + # those genuinely mean. + ids = ([r for r in (rows or []) if str(r) in all_rows] if rows + else sorted(all_rows, key=_rid_num)) + + # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its + # saved View and quota are not merely how the action decides whether to spend; they ARE the + # set of records the automation was asked to process. Walking every row first meant a run + # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows. + # Besides being misleading, that shape made the runtime scale with unrelated historical rows. + # + # Keep this deliberately narrow: any flow with another action must still hand that action the + # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record + # semantics to preserve, so it can begin at its selected records safely. + flow_actions = list((defn.get("flow") or {}).get("actions") or []) + enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)] + enrich_scope_note = "" + if rows is None and len(enabled_direct) == 1 \ + and enabled_direct[0].get("kind") == "enrich_instagram": + enrich_cfg = enabled_direct[0].get("config") or {} + profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField")) + if profile_key: + # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES. + # `enrich_selection` is called TWICE per run — here, to decide which records the + # runner walks and announces, and again inside `_walk` to decide which the action + # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so + # the run announced *"(1 of the 30 asked for)"* in its summary while its own note + # from the second call said *"0 of the 30"* — one run, two answers, both printed. + # ⚠ `gone` is read from the definition here rather than passed in, because this + # function has the definition and `_walk` does the same read; a third source of the + # same fact is how the two would drift again ([[one-evaluator-per-question]]). + selected, enrich_scope_note = enrich_selection( + rt, table, enrich_cfg, profile_key, + gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})) + if enrich_scope_note.startswith("the enrich step names a view it cannot read"): + return ("partial", f"nothing walked in {table} — {enrich_scope_note}", {}, [], {}) + ids = selected + step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") + counts = {"records": len(ids)} + # ⭐⭐ WAVE 31 · T36 (D-143, owner ruling R6) — THE CAP IS THE TABLE'S, NOT THIS MODULE'S. + # + # ⛔ WHAT WAS WRONG, and it was a SECOND EVALUATOR rather than a wrong number. This line read + # `MAX_FLOW_RECORDS = MAX_UT_ROWS = 5000` — a constant mirrored from the EDITABLE substrate's + # ceiling — and applied it to every database alike. `ut_odoo_orders` is a CONNECTED source + # (32,826 rows live), and R6 is explicit that connected-source data has NO CAP: *"I thought we + # decided there is no cap in how many data from the API source … can be pulled into the app."* + # So a flow over it silently walked 5,000 of 32,826 — 15% of the records — and every total it + # produced understated the book while looking exactly like a complete run. + # + # ⭐ `core.user_tables.row_limit` IS THE ONE EVALUATOR for this question and it already answers + # the three cases (0 = read-through, None = connected and UNCAPPED, MAX_ROWS = editable). Using + # it here retires this module's private copy instead of correcting it — one question, one + # normaliser [[one-question-two-normalizers]]. `limit_report` is R6's SECOND sentence already + # expressed as data, so the sentence a person reads carries the cause and the recommended fix + # rather than just a number. + _cap = _flow_record_cap(table, rt) + if _cap is not None and len(ids) > _cap: + # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names + # both numbers so "it only processed some of them" is readable rather than deducible — and + # since T36, WHY it applies and what to do about it. + total = len(ids) + ids = ids[:_cap] + return ("partial", + f"{total} records in {table} — this run walked the first {_cap}" + + _flow_cap_reason(table, rt), counts, ids, {}) + n = len(ids) + # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK. + # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)? + # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours: + # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product + # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key. + # · `(all)` meant "every record, because a manual run has no trigger records to narrow to", + # which is unguessable from the word. It reads as a filter nobody chose. + # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole + # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that + # this sentence never claimed to report. What was wrong was that nothing said so. + label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table + scope = ("the records that fired the trigger" if rows + else ("the records selected for Instagram enrichment" + if len(enabled_direct) == 1 + and enabled_direct[0].get("kind") == "enrich_instagram" + else "every record — this run was started by hand, so nothing narrowed it")) + if enrich_scope_note: + scope += f" ({enrich_scope_note})" + # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400 + # characters by `_commit_run`, and this note is the longest thing a run says — it names + # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING + # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy + # of this sentence is never produced. Without this line the explanation exists only in a + # string that is about to be truncated, on exactly the runs that look like the automation + # has stopped working. + counts[RUN_NOTES_KEY] = [enrich_scope_note] + return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}", + counts, ids, {}) + + +RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, + "field_instagram": run_field_instagram, + # ⭐ WAVE 30 · T09 (D-129) — BOTH DISCOVERY KINDS MAP ONTO THE SAME FUNCTION. The kind is + # no longer chosen by which callable this dict holds; `run_discovery` reads it off the + # definition and looks the corpus up in `DISCOVERY_SPECS`. The two dicts are asserted + # key-for-key by the gate, because a kind in one and not the other is either an + # unroutable automation or a refused run. + "discover_instagram": run_discovery, + "discover_tiktok": run_discovery} + + +# --------------------------------------------------------------------------------------------- +# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) +# --------------------------------------------------------------------------------------------- +# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → +# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness +# boolean on the wire, four client strings and two gate sections. That is what a hard-coded +# vendor costs, and TikTok (D-9) would have paid it again from scratch. +# +# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the +# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: +# +# probe "is this configured?" -> the honest readiness bit the UI shows +# capture enrich a handle we know -> `pull_profile`-shaped +# discover find handles we do not -> the corpus query +# +# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in +# later; it is the honest statement that this source has no discovery route, and a surface that +# offers one anyway is offering a button that must refuse. +SOURCES = { + "instagram": { + # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing + # copy, and which provider serves a capability is a routing decision the product owns — + # `providers.py` still holds the real keys, chains and costs. + "key": "instagram", "label": "Instagram", "vendor": "Scraper", + "probe": bd_ready, + "capture": pull_profile, + "discover": bd_filter_start, + "captureKind": "field_instagram", + "discoverKind": "discover_instagram", + "tiers": TIERS, + }, + # ⭐⭐ WAVE 29 — D-9 LANDS, AND IT LANDED AS THE ONE ENTRY THIS REGISTRY PREDICTED IT WOULD. + # The canvas, the append tables, the caps and the honest-status contract were already in place; + # what TikTok added was a field map, a discovery runner and three dataset ids. + "tiktok": { + # `vendor` NAMES NO COMPANY (owner instruction 2026-08-09) — it reaches operator-facing + # copy, and which provider serves a capability is a routing decision `providers.py` owns. + "key": "tiktok", "label": "TikTok", "vendor": "Scraper", + "probe": bd_ready, + # ⭐ WAVE 30 · T08 — CAPTURE IS WIRED, and it is a LAZY reference rather than the function + # object every other row here holds. `connectors_tt` imports THIS module at module level, + # so naming `connectors_tt.pull_profile_tt` at import time is a circular import — which is + # why every existing site does `import connectors_tt` inside the function body. A + # zero-argument callable keeps the registry's shape (a reader gets a callable, not a + # string) without moving the import. + # ⚠ AND SETTING IT SWITCHES NOTHING ON. `SOURCES` is never subscripted anywhere; its one + # reader is `source_status()`, which reads `key/label/vendor/probe/discover`. An action + # kind is gated in three other places entirely (`ACTION_CATALOG`, `clean_actions`, + # `apply_actions`). This row is documentation that must not lie, not wiring. + "capture": lambda *a, **k: __import__("connectors_tt").pull_profile_tt(*a, **k), + "discover": bd_filter_start, + "captureKind": "enrich_tiktok", + "discoverKind": "discover_tiktok", + # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never + # had one. An empty tuple is the honest statement, not a gap to fill in later. + "tiers": (), + }, +} + + +def source_status(): + """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. + + ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface + needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the + credential to a browser would put a billable secret in every user's devtools. + """ + # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which + # supplier this rung actually uses — that history is worth keeping in the code — but a + # customer has no use for it, and a server-supplied string rendered by a generic component + # is precisely the half a client-file scan cannot see. + return [{"key": s["key"], "label": s["label"], + "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} + for s in SOURCES.values()] + + +# --------------------------------------------------------------------------------------------- +# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client +# --------------------------------------------------------------------------------------------- +# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are +# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — +# a canvas that still shows a step the engine stopped running looks completely fine. So the node +# list is derived from the DEFINITION by the same module that executes it, rides on the payload +# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. +# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. +# +# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is +# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine +# walks it linearly. Nothing here should be read as a general branching runtime. + +#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be +#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw +#: away the only information the dot is there to carry. +NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") + +#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a +#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is +#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones +#: here are all REAL — each changes what the next run does: +#: schedule the trigger fires on its cron, or only by hand +#: postMetrics likes/comments per post are bought, or the engagement series does not grow — +#: it costs a vendor record PER POST rather than per profile (measured: the +#: profile row carries post identity and no engagement) +#: commentMetrics the Comments dataset is bought, which can bill many rows per post +#: write DRY RUN — read everything, compute the counts, write nothing anywhere +#: +#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is +#: the point of the ruling: the money switches used to be "which rung do we try" (a question about +#: our plumbing), and they are now "what do you want captured" (a question about the user's data). +#: Profile is always captured and has no switch — an automation that fetches nothing is not an +#: automation with a step turned off, it is a broken one with a lie on it. +#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and +#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the +#: three toggles" names the CAPTURE ladder it is reshaping. +NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics", + "capture_comments": "commentMetrics", "write": "write"} + + +def _cron_label(cron): + return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") + + +def node_status(steps, nid): + """A node's dot — from the last run's MEASURED step outcome, or `idle`. + + ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely + so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, + and painting those nodes green because the run said `ok` would manufacture a measurement — the + same defect as the green dot over the empty posts table recorded in `run_field_instagram`. + No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. + """ + v = str((steps or {}).get(nid) or "") + return v if v in NODE_STATES else "idle" + + +def graph(defn): + """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" + defn = defn if isinstance(defn, dict) else {} + cfg = defn.get("config") or {} + sched = defn.get("schedule") or {} + runs = defn.get("runs") or [] + last = runs[0] if runs and isinstance(runs[0], dict) else {} + steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} + + def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): + return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, + "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), + "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} + + on = bool(sched.get("enabled")) + trg = defn.get("trigger") or {} + if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): + # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail + # read the same node, so an event-triggered flow must not read "Schedule". The membership + # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old + # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so + # the else-branch was reachable) — green, compiling, and wrong on screen. + t_on = bool(trg.get("enabled", True)) and not trg.get("paused") + sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) + watched = ", ".join(trg.get("fields") or []) or "any field" + # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the + # whole of it) and `record_updated` no longer names a condition (its watched fields are). + # These two lines are the migration's visible half: a node still describing a `.field` + # the validator has stopped storing would be the surface and the store disagreeing. + det = {"event_field": (f"{trg.get('table', '')}" + + (f" — {_lane_sentence(trg.get('when'))}" + if trg.get("when") else "")), + "record_updated": f"{trg.get('table', '')} — {watched}", + "record_created": str(trg.get("table") or ""), + "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}", + "form_submitted": str(trg.get("table") or ""), + "webhook": "POST the hook URL to fire it", + # ⭐ WAVE 30 · T05 — BOTH corpus triggers, and the value is identical because the + # question is: a discovery trigger's detail line IS its filter. Without the TikTok + # key this fell to `.get(..., "")` and the trigger node rendered with a blank + # subtitle, which reads as "nothing configured" on a fully configured automation. + "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet — nothing to search for"), + "tiktok_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet — nothing to search for"), + "email": str(trg.get("query") or "")}.get(trg["key"], "") + if not trg.get("configured", True): + det = "Finish setting this trigger up before it can fire" + if trg.get("paused"): + det = str(defn.get("statusNote") or "paused") + nodes = [node("trigger", "trigger", sub, + "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] + else: + # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off + # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with + # the user's own pick — the same class of defect as the Database picker that could not be + # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the + # disabled-schedule gate asserts, and it was never the wrong half. + # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires + # TODAY, not that it may never be scheduled — the cron has to stay one click away, and + # this is the only node that offers it. + nodes = [node("trigger", "trigger", "Schedule" if on else "Manual", + _cron_label(sched.get("cron")) if on else "Manual only", 0, + panel="schedule", on=on, + detail="" if on else "It still runs when you press Run now")] + edges = [] + + if defn.get("kind") == "field_instagram": + # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED. + # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and + # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5 + # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the + # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners + # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas. + # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an + # implementation detail: the profile is always captured (R6), Posts and Comments are the + # opt-ins, and both are OFF until asked for. + metrics_on = bool(cfg.get("postMetrics")) + comments_on = bool(cfg.get("commentMetrics")) + max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL + nodes += [ + node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, + panel="source", + detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") + else "URL column: the one the field is bound to"), + # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY + # and no engagement (measured), so likes/comments are a SECOND vendor call per post. + # A switch that multiplies a run's cost by the post count deserves to be visible on + # the canvas rather than buried in a config panel. + node("capture_posts", "capture", "Post data", + "Likes and comments per post" if metrics_on else "Off", + 2, panel="capture", on=metrics_on, + detail=(f"Up to {max_posts} posts per profile" if metrics_on else + "ut_ig_post_snapshots only grows while this is on")), + node("capture_comments", "capture", "Comment data", + "Comments on those posts" if comments_on else "Off", + 3, panel="capture", on=comments_on, + detail=("The full comments dataset — many rows per post" if comments_on else + "Comments already embedded in a paid post row are kept either way")), + node("write", "write", "Write", + cfg.get("targetTable") or "No database", 4, panel="write", + on=not cfg.get("dryRun"), + detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), + ] + edges = [{"from": "trigger", "to": "source", "label": ""}, + {"from": "source", "to": "capture_posts", "label": ""}, + {"from": "capture_posts", "to": "capture_comments", "label": ""}, + {"from": "capture_comments", "to": "write", "label": ""}] + elif defn.get("kind") in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — WIDENED FROM `== "discover_instagram"`, AND THIS IS THE FAILURE T04 + # UNMASKS. With the seed fixed but this arm still Instagram-only, a stored TikTok search + # fell past every arm to the no-bare-`else` promise below and returned THE TRIGGER NODE + # ALONE — so no node carried `panel="find"`, and `AutomationFind` (which mounts on + # `panelKey === "find"`) could never appear. The person would have picked the trigger + # successfully and then found nowhere to type a filter: a second, stranger bug, arriving + # as the reward for fixing the first one. + _platform, _default_table, _, _ = discovery_facts(defn.get("kind")) + limit = int(cfg.get("recordsLimit") or 0) + est = discover_estimate(limit) + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + nodes += [ + node("find", "source", f"Find {_platform} profiles", + _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, + panel="find", + detail=(f"Up to {limit} profiles · about ${est['usd']}" + if bd_ready() + else "Profile search is not set up yet")), + node("collect", "capture", "Collect results", + "Searching…" if pending else "Takes about 20 minutes", 2, + panel="find", + detail=""), + node("write", "write", "Save results", + cfg.get("targetTable") or _default_table, 3, panel="write", + on=not cfg.get("dryRun"), + detail=""), + ] + edges = [{"from": "trigger", "to": "find", "label": ""}, + {"from": "find", "to": "collect", "label": ""}, + {"from": "collect", "to": "write", "label": ""}] + elif defn.get("kind") == "scrape_db": + url = str(cfg.get("url") or "") + host = (urlparse(url).hostname or url or "No URL") if url else "No URL" + which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" + else f"HTML table #{int(cfg.get('tableIndex') or 0)}") + mapped = len(cfg.get("fieldMap") or {}) + nodes += [ + node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), + node("extract", "capture", "Extract", which, 2, panel="columns", + detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " + f"{cfg.get('keyField') or '—'}"), + node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, + panel="write", on=not cfg.get("dryRun"), + detail=cfg.get("targetTable") or "a new database"), + ] + edges = [{"from": "trigger", "to": "fetch", "label": ""}, + {"from": "fetch", "to": "extract", "label": ""}, + {"from": "extract", "to": "write", "label": ""}] + # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point. + # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught + # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML + # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A + # surface that draws nothing is obviously incomplete; one that draws a fetch step that does + # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at + # once because all three read this. + # + # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE + # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node, + # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on + # that promise, so it is stated rather than left to be inferred from the control flow. + return {"nodes": nodes, "edges": edges} + + +def toggle_node(rt, auto_id, node_id): + """Flip ONE node's switch. Returns `(defn, error)`. + + The node → config mapping lives on the SERVER for the reason the whole graph does: a client + that knew which field each node writes would be a second copy of that knowledge, free to + drift. The canvas just says "this node was clicked". + """ + prev = all_definitions(rt).get(str(auto_id)) + if prev is None: + return None, "no such automation" + target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) + if target is None: + return None, f"there is no {node_id!r} step in this automation" + if not target.get("toggle"): + return None, (f"the {target['title']} step cannot be turned off — it is what this " + f"automation is") + cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) + which = target["toggle"] + trg_key = str((prev.get("trigger") or {}).get("key") or "") + if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS: + # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping + # the cron under a node labelled "When a field changes" would be a switch that lies. + # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that + # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so + # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so + # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half. + trg = dict(prev.get("trigger") or {}) + trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) + trg["paused"] = False # a deliberate flip clears an auto-pause: human wins + return patch(rt, auto_id, {"trigger": trg}) + if which == "schedule": + sched["enabled"] = not sched.get("enabled") + elif which == "postMetrics": + cfg["postMetrics"] = not cfg.get("postMetrics") + elif which == "commentMetrics": + cfg["commentMetrics"] = not cfg.get("commentMetrics") + elif which == "write": + cfg["dryRun"] = not cfg.get("dryRun") + return patch(rt, auto_id, {"config": cfg, "schedule": sched}) + + +# --------------------------------------------------------------------------------------------- +# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed +# --------------------------------------------------------------------------------------------- +# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were +# records of the target database, and each record's position lived in a single-select STAGE FIELD +# the engine created on the customer's own database. Wave 26 deleted the board's built-in +# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`, +# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review +# branch of the action walk, and the endings that sent a terminal card back round. +# +# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote +# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles` +# counter per automation — to render a view of state the RUN LOG already holds. The customer paid +# for that in columns they did not ask for, in a store commit on every scheduled run, and in a +# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we +# had added. `migrate_ig_tables` now drops those columns wherever an automation created them. +# +# ⛔⛔ `notify_review` IS GONE FROM HERE AND ITS CONSUMER IS STILL ALIVE — DEBT D-101, and this +# paragraph is its TOMBSTONE, placed where the next reader of the review machinery will look. +# +# `notify_review` was the producer of the `automation_review` notification: a card reaching a +# review stage told a human to come and look. It had ZERO callers even before R3 (wave-27 session +# C found it and correctly left it alone as another lane's subject), and R3's deletion took the +# function with the rest of the board. `verify_automation`'s W23-B section asserts its absence by +# name, so it cannot come back by accident. +# +# ⚠ THE SWEEP FOR ITS READERS, RUN 2026-08-12 (W31-T37) AND REPORTED RATHER THAN ASSUMED EMPTY: +# · SERVER — `notify_review` appears in exactly ONE file, `verify_automation.py`, inside the +# list of names asserted GONE. **No `.py` file anywhere produces an `automation_review` +# notification**; `core.alerts.notify` defaults `topic='automation'`, which is a different, +# live shape (the run-outcome alert). +# · CLIENT — the branch is FULLY ALIVE and is NOT in this fence: +# `web/src/alerts/alertsModel.ts` (`AUTOMATION_REVIEW_KIND`, `isAutomationReview`, the +# `autoId` guard) and `web/src/alerts/AlertsPane.tsx` (two call sites), plus legs in +# `web/verify_alerts.py`. All four are session B's files. +# ⇒ **The producer is deleted; the consumer guards an event that can never arrive.** D-101's own +# exit condition says these are one defect seen from two ends and must be closed in ONE change — +# so this half is the tombstone plus the sweep, and the client half is routed to B rather than +# reached across a fence. If a future wave gives review stages a real producer, THIS is the note +# that says the client already knows how to render it. +# +# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE: +# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the +# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was +# one of four callers. The name is the last thing the board left behind here. +# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board` +# — the MIGRATION. It must outlive the thing it retires, or a definition written before the +# retirement walks into a runtime that no longer understands it. +# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are +# parked with no caller and say so at their own definitions. + +LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", + "is_empty", "is_not_empty") +LANE_NULLARY_OPS = ("is_empty", "is_not_empty") + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` +# +# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why +# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches +# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc +# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the +# same function applied one level up, so the cheaper honest version is to evaluate the leaf where +# it lies and never touch the bytes.) +# +# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An +# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and +# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. +# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an +# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that +# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine +# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. +MAX_COND_DEPTH = 3 +MAX_COND_CHILDREN = 12 +COND_GROUP_KEYS = ("all", "any") + + +def clean_cond(raw, depth=0, where=""): + """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no + condition", which is legal everywhere a condition is optional (the catch-all lane, an + unconditioned trigger, an action group that always runs).""" + if raw in (None, "", {}): + return None, None + at = f" on {where}" if where else "" + if not isinstance(raw, dict): + return None, f"the condition{at} must be an object" + conj = [k for k in COND_GROUP_KEYS if k in raw] + if len(conj) > 1: + return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the " + f"other") + if conj: + key = conj[0] + if depth + 1 > MAX_COND_DEPTH: + return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — " + f"the group{at} is deeper") + kids_raw = raw.get(key) + if not isinstance(kids_raw, list) or not kids_raw: + return None, f"the '{key}' group{at} needs at least one condition inside it" + if len(kids_raw) > MAX_COND_CHILDREN: + return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — " + f"the '{key}' group{at} has {len(kids_raw)}") + kids = [] + for child in kids_raw: + c, err = clean_cond(child, depth + 1, where) + if err: + return None, err + if c is None: + return None, (f"an empty condition sits inside the '{key}' group{at} — " + f"finish it or remove it") + kids.append(c) + return {key: kids}, None + field = _s(raw.get("field"), 80).strip() + op = _s(raw.get("op") or raw.get("operator"), 20).strip() + if not field: + # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could + # act on. It states a fact about the stored tree; it never said what to DO, and the usual + # cause is not a typo but an empty field picker (the flow had no walking record to offer + # columns from). The fix belongs in the message. + return None, (f"the condition{at} names no field — pick a column, or remove the " + f"condition") + if op not in LANE_OPS: + return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) + cond = {"field": field, "op": op} + if op not in LANE_NULLARY_OPS: + v = raw.get("value") + if v is None or (isinstance(v, str) and not v.strip()): + return None, f"give a value to compare {field!r} against{at}" + cond["value"] = v.strip() if isinstance(v, str) else v + return cond, None + + +def cond_fields(cond): + """Every field name a tree reads — the set a caller must have on the row image before the + answer means anything.""" + if not isinstance(cond, dict): + return set() + for key in COND_GROUP_KEYS: + if key in cond: + out = set() + for child in cond.get(key) or []: + out |= cond_fields(child) + return out + f = cond.get("field") + return {f} if f else set() +#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would +#: collide with the stage the label is a choice FOR, and the board could no longer tell a +#: routed card from a gated one. +# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the +# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a +# legacy stamped cell would read as the same lane while behaving as a different one. +def _lane_num(v): + try: + return float(str(v).replace(",", "")) + except (TypeError, ValueError): + return None + + +def lane_match(cond, row): + """Does `row` satisfy this condition TREE? None matches everything (the catch-all). + + Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 + caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping + this name rather than growing a second entry point that could disagree with it. + + ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse + as a number answers False — the record simply does not enter the lane — never "treat blank + as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank + cell is an unknown, and an unknown cannot be less than 10,000. + """ + if cond is None: + return True + if isinstance(cond, dict): + if "all" in cond: + return all(lane_match(c, row) for c in cond.get("all") or []) + if "any" in cond: + return any(lane_match(c, row) for c in cond.get("any") or []) + raw = (row or {}).get(cond.get("field")) + op = cond.get("op") + if op == "is_empty": + return raw in (None, "") + if op == "is_not_empty": + return raw not in (None, "") + want = cond.get("value") + if op in (">", ">=", "<", "<="): + a, b = _lane_num(raw), _lane_num(want) + if a is None or b is None: + return False + return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] + have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") + if op == "includes": + return want_s.lower() in have_s.lower() + if op == "not_includes": + return want_s.lower() not in have_s.lower() + a, b = _lane_num(raw), _lane_num(want) + same = (a is not None and b is not None and a == b) or have_s == want_s + return same if op == "=" else (not same) if op == "!=" else False + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. +# +# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. +# A `group` holds nested actions behind a condition — Airtable's "conditional action group", +# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's +# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this +# builder is a column, not a canvas). +# +# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a +# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / +# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty +# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready +# kind with a sentence, so the faded state is a wall and not a styling choice. +MAX_ACTIONS = 20 # per automation, counting nested ones +MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow +MAX_BRANCHES = 6 # per If / then +MAX_ACTION_VALUES = 20 # cells one update/create action may write +FIND_LIMIT_MAX = 100 +#: ⭐⭐ W31-T38 / E-4 — HOW MANY BROWSER JOBS ONE RUN MAY SUBMIT, and it belongs HERE because this +#: is the only layer that knows a record WALK is happening. `web_agent.MAX_STEPS` bounds steps per +#: JOB; nothing bounded jobs per RUN, and the web arm submits one job PER RECORD. A `web_read` on a +#: flow over the Customer grid is therefore thousands of ~10 s submissions against HF's +#: 6-concurrent cap — a run whose wall-clock is `records x 10 s` and a burst the `/hf-jobs` rules +#: exist to prevent. Session E raised it and could not fix it: the seam sees one step and cannot +#: know a walk is happening. +#: ⚠ 50 IS A STATED CHOICE, NOT A MEASUREMENT: at E's measured ~9-32 s per job that is roughly +#: 8-27 minutes of serial browsing, which is a long automation run and not a runaway one. It is +#: DISCLOSED when it binds (R6's second sentence — see the web arm), never silently applied. +#: ⛔ THE REAL FIX IS BATCHING, NOT A BIGGER NUMBER: `web_agent.run_plan(steps, ctx)` already takes +#: a list and the cost is per JOB, so a flow's web steps collected into ONE call would make this +#: ceiling nearly unreachable. That is E's own PENDING row; this bound is what stops the damage +#: until it lands. +MAX_WEB_JOBS_PER_RUN = 50 +#: ⭐⭐ W31 QA — THE FIVE WEB KINDS THE ENGINE DISPATCHES, and this is the THIRD list that names +#: them (`web_agent.RUNNABLE_KINDS` is the seam's, `jobs/web_agent_job.py::RUNNABLE_KINDS` is the +#: runner's). ⛔ IT IS DELIBERATELY NOT AN IMPORT: `_web_agent()` resolves the seam LAZILY so an +#: absent module cannot break the engine, and a module-level `from web_agent import RUNNABLE_KINDS` +#: would throw that property away for a tuple of five strings. The three lists are held in step by +#: `verify_web_agent.py::section_one_kind_only`, which reds if any pair disagrees — a per-kind +#: allow-list in three places that CAN disagree is the defect, whichever way it points. +WEB_KINDS = ("web_read", "web_goto", "web_fill", "web_click", "web_repair") +#: What each kind cannot run WITHOUT. Absence is never refused at SAVE (a step is addable before it +#: is configured — see `_clean_action_config`'s web arm); it is refused at RUN with a sentence +#: naming what is missing. `url` is required only by `web_goto`: every other kind acts on the page +#: the flow is already on, and the runner supplies its own refusal when it genuinely needs one. +WEB_REQUIRED = { + "web_read": (("a URL", "url"), ("a CSS selector", "selector"), + ("a column to write into", "field")), + "web_goto": (("a URL", "url"),), + "web_click": (("a CSS selector", "selector"),), + "web_repair": (("a CSS selector", "selector"),), + "web_fill": (("a CSS selector", "selector"), ("a value to type", "value")), +} +#: ⭐⭐ WAVE 32 · T45 (owner item 10) — WHAT EACH ACTION KIND CANNOT RUN WITHOUT, for every kind. +#: +#: `WEB_REQUIRED` above already WAS this table for five kinds, complete with the human phrase each +#: refusal says out loud, so this is its widening rather than a second opinion — spread in, never +#: re-typed, or a wave that adds a web key would teach the run refusal and not the label. +#: +#: ⛔ MOST KINDS ARE ABSENT, AND THE ABSENCES ARE THE INTERESTING PART. `create_record`, +#: `update_record`, `find_records` and `group` are REFUSED AT SAVE when their config is incomplete +#: (`_clean_action_config`: *"the create record action names no database"*, *"…writes no values"*, +#: *"a branch with no actions inside it does nothing"*), so a stored one is configured by +#: construction and a row here would be a second, weaker copy of a wall that already holds. +#: +#: ⛔⛔ AND `enrich_instagram` / `enrich_tiktok` ARE DELIBERATELY ABSENT DESPITE BEING THE OBVIOUS +#: CANDIDATE. Their `profileField` may be empty on purpose: the C3 profile FLAG on the target +#: database resolves the binding at run time (`profile_field_key`), so an empty one is a working +#: action on any flagged database and marking it Unconfigured would put a red label on the +#: commonest correct configuration there is. The genuinely unbound case still fails closed at RUN +#: with its own sentence — which is the honest division of labour: this table holds what can be +#: answered from the ACTION ALONE, and anything needing the target database's schema stays a +#: run-time refusal rather than becoming a store read on the automations LIST path (W30-T12 took +#: that read off this route; putting it back to draw a label would undo the wave before this one). +ACTION_REQUIRED = dict(WEB_REQUIRED) +#: ⭐ W31-T38 (C5) — the web step's own clamps. `20000` matches the default E's seam documents; +#: the ceiling exists because this blocks the record walk (~9-32 s of cold start ALREADY), and an +#: automation that can be configured to wait ten minutes per record is a stalled run, not a slow one. +WEB_READ_TIMEOUT_MS = 20000 +WEB_READ_TIMEOUT_MAX_MS = 120000 +#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder` +#: and never by a literal list of group names: a client-side ordering is a second copy of this +#: table, and the way it fails is that a group added here renders last, or not at all, with +#: nothing red anywhere. +ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} +ACTION_CATALOG = [ + # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ──────── + # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row + # is a WALL, not a styling choice — and each `detail` says the true reason rather than a + # placeholder, because "coming soon" on five rows is how a menu stops being believed. + # + # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24 + # scheduled this build for wave 25 and wave 25's six owner items do not include it, so + # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its + # own promises has to be re-read by whoever misses the date, and a stale date is worse than + # none because it reads as a commitment somebody already broke. The replacement names the + # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule. + # ⭐⭐ WAVE 31 · T38 + QA (C5) — ALL FIVE ROWS ARE READY, AND THE RULING IS WHY. + # + # ⛔ THIS REVERSES T38's ORIGINAL `how:`, ON THE OWNER'S OWN WORDS. T38 shipped `web_read` + # alone and held the other four at `ready:False`, citing PRD R10 ("the ones that WRITE to a + # third party do not flip without R5's approval gate existing"). **R5 was REVOKED the same + # day, and the revocation is recorded as an amendment on this wave's board** + # (`TICKETS.md:1418`, `mailbox/E.md:278`, `web_agent.py:103`) — owner, verbatim: + # *"make sure we unblock all web actions, we don't need approval step first wtf, I never ask + # for that."* Session C declined E's ask against the superseded half of R10, so four actions + # the owner asked for shipped unusable, and `verify_web_agent` was RED on correct-by-ruling + # code for the whole wave. Confirmed at QA against four independent recordings of the quote. + # ⚠ THE FADED ROW IS A WALL, NOT A STYLE: `clean_actions` refuses an unready kind with a + # sentence, so this flag is what makes the action storable at all — which is exactly why a + # row left `False` against a ruling is a feature that does not exist. + # ⚠ THE `detail` STRINGS CHANGED WITH THE FLAGS, deliberately. They said *"Needs the browser + # job"* and *"Needs the recorder that captures a selector"* — both false since E's runner + # landed. A menu that dates its own promises is this module's own recorded complaint one + # comment up; a row that advertises a missing prerequisite it already has is the same defect. + {"kind": "web_goto", "label": "Open a page", "group": "Web action", "ready": True, + "detail": "Opens a page in a browser job and reports the title it landed on"}, + {"kind": "web_fill", "label": "Fill a field", "group": "Web action", "ready": True, + "detail": "Types a value into a field — mark it secret and the value is masked in the log"}, + {"kind": "web_click", "label": "Click something", "group": "Web action", "ready": True, + "detail": "Clicks the element a selector names, and reports where it landed"}, + {"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": True, + "detail": "Reads one value off a live page in a browser job — expect ~10-30 s per step"}, + {"kind": "web_repair", "label": "Repair a broken step", "group": "Web action", + "ready": True, + "detail": "Follows a label to its control when a selector has gone stale, and proposes one"}, + # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── + {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, + "detail": "Write values onto the record walking the flow"}, + {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True, + "detail": "Add a row to another database"}, + {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True, + "detail": "Look rows up by condition; the run log opens them"}, + # ── 3. CONNECTED ───────────────────────────────────────────────────────────────────────── + # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND. + # `field_instagram` was an automation you created to fill one column; this is a step any flow + # can take, which is the shape it should always have had — enriching a profile is something + # you do TO a record, not a species of automation. + # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu, + # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`. + {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected", + "ready": True, "connector": "scraper", + "detail": "Fill this record's Instagram columns from its profile, and add a point to its " + "history"}, + # ⭐⭐ WAVE 30 · T08 / CONTRACT C3 — THE SECOND NETWORK, AND THIS ROW IS THE LAST SWITCH THAT + # LANDS, deliberately. `apply_actions._walk`'s kind dispatch has NO terminal `else` (measured + # from the AST, not read): an unknown kind is walked, counted, reports the run `ok`, and + # writes nothing. So a catalog entry ahead of its runner arm would ship an action that is + # addable, clickable, storable and silently inert — strictly worse than the state before it, + # where `clean_actions` refuses the kind with a sentence. + # ⚠ `connector: "scraper"` NESTS IT UNDER THE SAME BUCKET AS INSTAGRAM (R3), which is the same + # value T07 gave the trigger — one word, both menus. + {"kind": "enrich_tiktok", "label": "Enrich TikTok profile", "group": "Connected", + "ready": True, "connector": "scraper", + "detail": "Fill this record's TikTok columns from its profile, and add a point to its " + "history"}, + {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, + "detail": "Needs a send scope on the Gmail connection"}, + {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, + "detail": "Needs the Slack connector"}, + # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── + # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism, + # and R8 made it a FORK with lettered branches, which is a thing people already have a name + # for. The KIND is untouched — renaming it would orphan every stored action for a caption. + {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True, + "detail": "Send the record down one of several branches, by condition"}, + {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", + "ready": False, "detail": "Run the same actions on every item in a list"}, + {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, + "detail": "Not built — a sandbox is its own decision"}, + {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", + "ready": False, "detail": "Needs an AI action implementation"}, +] + + +#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can +#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an +#: "Integrations" header, one row per connector, a chevron into that connector's own submenu). +#: +#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are +#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the +#: key and render the label, never join it against the directory. +#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key. +#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one +#: connector wearing two names on two screens somebody sees within a second of each other, and a +#: second literal is how that happens. This tuple says only WHICH connectors an action may name; +#: what they are CALLED has exactly one source. +ACTION_CONNECTORS = ("scraper",) + + +def _connector_meta(key): + """`{key, label}` for a connector key, from the one place either picker declares it.""" + key = str(key or "") + if key not in ACTION_CONNECTORS: + return None + return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) + + +def action_catalog(): + """The catalog as the wire carries it — a copy, because a caller that mutated the module + constant would change every later reader's answer. + + ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from + `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two + different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST + (not first) — a new group appearing above "Web action" because its order defaulted to 0 is + the failure that would look deliberate. + + ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client + nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two + rows in one nest cannot disagree about what that nest is called. A row naming an unknown + connector loses the stamp instead of inventing a nest with a raw slug for a title. + """ + last = max(ACTION_GROUP_ORDER.values()) + 1 + out = [] + for a in ACTION_CATALOG: + row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)} + conn = _connector_meta(a.get("connector")) + if conn: + row["connector"] = dict(conn) + else: + row.pop("connector", None) + out.append(row) + return out + + +def action_needs(action): + """⭐⭐ WAVE 32 · T45 (owner item 10) — what THIS action is still missing, in the words a person + reads. `[]` means Configured. + + Pure over `(kind, config)` — no runtime, no store read. That is what lets `_wire` stamp every + action on the automations LIST without putting the `user_tables` document back on a route + W30-T12 just took it off. + """ + action = action or {} + cfg = action.get("config") if isinstance(action.get("config"), dict) else {} + return [name for name, key in ACTION_REQUIRED.get(str(action.get("kind") or ""), ()) + if not str(cfg.get(key) or "").strip()] + + +def _walk_actions(actions): + """Every action in a flow, INCLUDING the ones nested inside If / then branches. + + ⛔ A FLAT `for a in actions` MISSES HALF A FLOW. `group.config.branches[].actions` is where a + conditional puts its real work, and a configured-check that only saw the top level would + report a flow ready to run while the step inside branch B had never been filled in — which is + the exact class of defect this ticket exists to surface, hiding inside the ticket's own fix. + """ + for action in actions or []: + if not isinstance(action, dict): + continue + yield action + for branch in ((action.get("config") or {}).get("branches") or []): + if isinstance(branch, dict): + yield from _walk_actions(branch.get("actions")) + + +def unconfigured_actions(defn): + """Every ENABLED action of this automation that cannot run, as `[{id, kind, label, needs}]`. + + ⚠ DISABLED ACTIONS ARE SKIPPED, and that is the point of being able to disable one: a step + somebody switched off is not a step blocking the run. `apply_actions` already ignores them. + """ + out = [] + for action in _walk_actions(((defn or {}).get("flow") or {}).get("actions")): + if not action.get("enabled", True): + continue + needs = action_needs(action) + if needs: + out.append({"id": str(action.get("id") or ""), "kind": str(action.get("kind") or ""), + "label": _action_label(action), "needs": needs}) + return out + + +def run_refusal(defn): + """⛔ WAVE 32 · T45 — the sentence a run is refused with, or `""`. + + Owner item 10: *"running an automation with ANY unconfigured action is REFUSED with a message + naming which action"*. Naming it is half the requirement and the half that is easy to drop — a + bare *"an action is not configured"* on a twelve-step flow is a hunt, not a message. + + ⛔ IT LIVES ON THE ENGINE, NOT ON THE ROUTE, BECAUSE THE ROUTE IS NOT THE ONLY DOOR. The tick + runs automations on a schedule, the webhook door runs them, and a check mounted in + `POST /automations/{id}/run` alone would refuse the button and let the cron sail past it — + [[seal-the-transport-not-the-rung]], and D-112's own shape (an action bound to a deleted view + walked zero records and reported `ok`). `run_now` refuses for every caller; the route asks + first only so the person clicking gets a 400 with the sentence instead of a silent no-op. + + ⚠ THIS IS A BEHAVIOUR CHANGE FOR STORED AUTOMATIONS AND IT IS THE RULING. A flow with one + unconfigured web step used to run, skip that step with a note, and report `ok`; it now does not + run at all. That is what "blocks the run" means, and the alternative — running everything else + and reporting success — is precisely what the owner is asking to stop. + """ + missing = unconfigured_actions(defn) + if not missing: + return "" + return "; ".join(f"{m['label']} still needs " + ", ".join(m["needs"]) for m in missing) + + +def _action_label(act): + for row in ACTION_CATALOG: + if row["kind"] == act.get("kind"): + return row["label"] + return str(act.get("kind") or "Action") + + +def clean_actions(raw, depth=0, _seen=None, _count=None): + """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. + + Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder + survives a Save. A missing or colliding one is minted `act_`; minting on every clean would + move the selection under the person editing it. + """ + if raw in (None, ""): + return [], None + if not isinstance(raw, list): + return None, "actions must be a list" + seen = _seen if _seen is not None else set() + count = _count if _count is not None else [0] + out = [] + for entry in raw: + if not isinstance(entry, dict): + return None, "each action must be an object with a kind" + kind = _s(entry.get("kind"), 30).strip() + row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) + if row is None: + return None, (f"{kind or 'that action'!r} is not one of: " + + ", ".join(a["kind"] for a in ACTION_CATALOG)) + if not row["ready"]: + return None, (f"{row['label']!r} is on the menu but not built yet — " + f"{row['detail'][0].lower()}{row['detail'][1:]}") + count[0] += 1 + if count[0] > MAX_ACTIONS: + return None, f"an automation runs at most {MAX_ACTIONS} actions" + aid = _s(entry.get("id"), 40).strip() + if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: + n = 1 + while f"act_{n}" in seen: + n += 1 + aid = f"act_{n}" + seen.add(aid) + # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER. + # + # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't + # think you can even add Conditions at all. That's not how the Create Record works."* + # Airtable agrees, and so does the shape: every other action operates ON the record the + # flow is walking, so "run this only when …" is a question about something + # that exists. Create record MAKES one. There is nothing to test yet, which is why the + # picker on this panel was empty and why every condition saved against it named no field — + # the owner's *"the condition on action act_1 names no field"*. The condition belongs on + # the trigger, or on a conditional group wrapping the action. + # + # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored + # automation that already carries one — forever, with no way to edit it out, because the + # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a + # locked door. + # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26 + # removes the condition editor from this action kind entirely (C4), so there is no control + # whose value could appear to be ignored. If a `create_record` condition editor ever comes + # back, this needs a disclosure channel — `clean_actions` has none today. + if kind == "create_record": + when = None + else: + when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") + if cerr: + return None, cerr + cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} + cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count) + if cerr: + return None, cerr + out.append({"id": aid, "kind": kind, + "enabled": bool(entry.get("enabled", True)), + "when": when, "config": cfg}) + return out, None + + +def _clean_action_config(kind, cfg, depth, seen, count): + """One action's `config`, per kind. Returns `(config, error)`.""" + if kind == "group": + # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, + # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an + # if with no else, so expressing "otherwise" meant a second group carrying the negation + # by hand — two conditions a later edit could put out of step with each other. + if depth + 1 > MAX_GROUP_DEPTH: + return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — " + f"past that a flow is a flowchart and belongs on the board") + raw_branches = cfg.get("branches") + if not isinstance(raw_branches, list) or not raw_branches: + # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and + # the live automations carry it — refusing it would 400 every Save of a flow that + # was legal yesterday. Read-side, so nothing has to be rewritten in the store. + raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions")}] + if len(raw_branches) > MAX_BRANCHES: + return None, (f"an If / then has at most {MAX_BRANCHES} branches — past that the " + f"record is being routed, which is what the board's lanes are for") + out_branches, bseen = [], set() + for i, br in enumerate(raw_branches): + if not isinstance(br, dict): + return None, "each branch of an If / then is an object" + cond, cerr = clean_cond(br.get("cond"), where="the branch") + if cerr: + return None, cerr + if cond is None and i != len(raw_branches) - 1: + # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches + # and a null condition matches everything, so a catch-all above another branch + # would make every branch below it dead code — silently, and only for the + # records that reached it. Refused with the reason rather than reordered: this + # module does not quietly rewrite what somebody built. + return None, ("only the LAST branch of an If / then can be the Otherwise leg — " + "a catch-all above another branch makes the ones below it " + "unreachable") + kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count) + if kerr: + return None, kerr + if not kids: + return None, "a branch with no actions inside it does nothing" + bid = _s(br.get("id"), 40).strip() + if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen: + bid = f"b{i + 1}" + bseen.add(bid) + label = " ".join(_s(br.get("label"), 40).split()) or ( + "Otherwise" if cond is None else _branch_letter(i)) + out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids}) + return {"branches": out_branches}, None + if kind in ("update_record", "create_record"): + values = cfg.get("values") + if not isinstance(values, dict) or not values: + return None, f"the {kind.replace('_', ' ')} action writes no values" + if len(values) > MAX_ACTION_VALUES: + return None, f"one action writes at most {MAX_ACTION_VALUES} cells" + clean_vals = {} + for k, v in values.items(): + key = _s(k, 80).strip() + if not key: + return None, "a value is written to a field with no name" + if v is not None and not isinstance(v, (str, int, float, bool)): + return None, (f"the value for {key!r} must be text or a number — " + f"an automation writes cells, not objects") + clean_vals[key] = _s(v, 500) if isinstance(v, str) else v + out = {"values": clean_vals} + # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is + # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic + # "Create record". Follows the `review` arm below, which has carried `config.label` since + # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config` + # bag so no shared client type has to change for it. + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + if kind == "create_record": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the create record action names no database" + if not table.startswith(UT_PREFIX): + return None, (f"actions write to blank databases (ut_*) this wave — " + f"{table!r} is not one") + out["table"] = table + # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS + # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a + # Create record duplicates a row per run — silently, and worse every day. + # + # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a + # guess: a stored automation must not change meaning because this key arrived. The + # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`. + # + # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the + # action never sets means every incoming row carries a BLANK key — `upsert_rows` + # counts those as `skipped` and writes nothing at all. That is a control that reads + # as "no duplicates" and delivers "no rows", which is the worse failure by a distance. + # Same shape as `scrape_db`'s "the key field must be one of the mapped fields" + # (`clean_config`, above) — one precedent, not a second mechanism. + unique = _s(cfg.get("uniqueOn"), 80).strip() + if unique and unique not in clean_vals: + return None, (f"this action does not write {unique!r}, so it cannot keep records " + f"unique on it — it writes: " + ", ".join(sorted(clean_vals))) + out["uniqueOn"] = unique + return out, None + if kind in ENRICH_KINDS: + # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the + # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for + # TikTok would drift until one network accepted a limit the other refused. + # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid + # rung, its fallback, the per-post engagement buy and the dry run — because they are the + # things that decide what a run COSTS and R4 moves the capability, not the controls. + # + # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than + # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a + # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a + # sentence at run (`profile_field_key`), which is the honest half — refusing the save + # would make the action unaddable on a database that has not been flagged yet. + # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent + # clamps on the same key is how the action and the legacy kind come to disagree about + # what "10 posts" means. + # + # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference + # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH, + # so a key being there is a person having typed it. This function gets the whole action + # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a + # `maxPosts` here is usually just the client echoing back what was already stored — and + # every action written before this wave stored the old default of 24. Refusing would + # therefore 400 those automations forever on a number nobody chose, which is D-65. + # The ceiling is still taught, in the place where a person can act on it: the panel's + # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more). + max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False) + if perr: + return None, perr + # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a + # working default. That is D-65's lesson applied ahead of time rather than after: an enrich + # action stored before this wave carries none of these, and it must keep saving and keep + # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT; + # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching. + # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door + # sees a person's typed number and can refuse politely, while the selection sees whatever + # is STORED — including configs written before the cap existed. Clamping only here would + # let an old 5000 through at run time. + try: + e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) + except (TypeError, ValueError): + return None, "how many records to enrich has to be a number" + if e_limit < 1: + return None, "an enrich step has to be allowed at least one record" + try: + e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) + except (TypeError, ValueError): + return None, "the number of days has to be a number" + groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts) + if gerr: + return None, gerr + return {"profileField": _s(cfg.get("profileField"), 80).strip(), + "fromView": _s(cfg.get("fromView"), 80).strip(), + "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT, + # Anything that is not "asc" is newest-first, so a typo cannot invert the order a + # person is spending against — it lands on the documented default instead. + "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc" + else "desc", + "limit": min(e_limit, MAX_ENRICH_PER_RUN), + "skipRecent": bool(cfg.get("skipRecent")), + "skipRecentDays": max(1, e_days), + # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in + # `clean_config` (R5 / C2): a stored action carrying them still saves and simply + # loses them, because refusing a retired key 400s every definition written before + # this wave (D-65). + # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for. + # ⭐ WAVE 30 · T10 — THE TIKTOK FORCE-OFF IS GONE, because the capability it was + # standing in for now exists. T08 shipped `... and kind != "enrich_tiktok"` here + # deliberately: the key was ACCEPTED (never a 400, D-65's lesson) and answered + # honestly with `False`, because storing a `True` nothing acts on is a money switch + # claiming a capability nobody built. Both networks now read the SAME two keys. + "postMetrics": bool(cfg.get("postMetrics")), + # The separate Comments dataset is more granular than a post read and stays off + # until the user intentionally enables it. + "commentMetrics": bool(cfg.get("commentMetrics")), + "dryRun": bool(cfg.get("dryRun")), + # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so + # every action stored before today keeps saving and keeps running unchanged. + "postGroups": groups, + "maxPosts": max_posts}, None + if kind == "find_records": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the find records action names no database" + cond, cerr = clean_cond(cfg.get("cond"), where="the find") + if cerr: + return None, cerr + try: + limit = int(cfg.get("limit") or 25) + except (TypeError, ValueError): + return None, "the find records limit must be a whole number" + if limit < 1 or limit > FIND_LIMIT_MAX: + return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" + return {"table": table, "cond": cond, "limit": limit}, None + if kind in WEB_KINDS: + # ⭐⭐ WAVE 31 · T38 (C5 / R10) — AND THIS ARM IS THE HALF THAT WOULD HAVE SHIPPED MISSING. + # + # ⭐⭐ W31 QA WIDENED IT FROM `web_read` TO ALL FIVE KINDS, AND THAT IS THE SAME DEFECT THIS + # ARM'S OWN COMMENT DESCRIBES, ARRIVING A SECOND TIME. When the four write kinds flipped to + # `ready:True` on the owner's revocation of R5, they became storable — and with this arm + # still testing `kind == "web_read"` they fell through to `return {}, None` below, keeping + # their KIND and losing `url` / `selector` / `value` / `hint` **silently, with no error + # anywhere**. Addable, storable, runnable, and forever unconfigured. The paragraph below + # was written about exactly this, one kind earlier; widening the test is what makes it + # true of the feature rather than of one row in it. + # + # ⛔ FOUND BY THE GATE, NOT BY REVIEW, AND IT IS THE TICKET'S OWN FAILURE MODE. Flipping + # `ready:True` and mounting the dispatch arm is not enough: with no branch here the + # function falls through to `return {}, None` below, so a stored `web_read` action keeps + # its KIND and loses its `url`, `selector` and `field` **silently, with no error anywhere**. + # The action would be addable, storable, runnable — and would fetch `""` forever. That is + # precisely the *"addable and inert"* outcome E's hand-off warned about, arriving through a + # different door than the one being watched, and the same silent-drop class as `patch`'s + # hand-maintained key list one screen up. + # + # ⚠ THE URL IS NOT `guard`ed HERE, on purpose. `guard` does DNS, and this is a validator + # that runs on every save of every automation — a save must not block on name resolution, + # and a template like `https://{Domain}/p` is not resolvable until a record supplies the + # value. The rail is enforced where the fetch happens (the job, and `web_agent`'s own + # refusal), which is the only place the FINAL url exists. What is enforced here is the + # shape: http(s) or an interpolation that could become one. + # ⛔⛔ AN EMPTY CONFIG IS STORED, NOT REFUSED — THE A3 STORED-INERT PATTERN, and the first + # draft of this arm got it wrong in a way only a gate could see. Requiring `url` / + # `selector` / `field` here makes the action UNADDABLE: `AutomationBuilder::seedFor` mints + # a new step from a seed and persists it immediately, so a validator that refuses an empty + # seed yields a red banner and no card — the wave-23 illegal-default-seed defect, caught by + # `verify_automation_ui`'s catalog-derived seed check the moment `web_read` became ready. + # ⇒ The same posture the enrich arm already takes with `profileField`: a step may be added + # to a flow before it is configured, and it FAILS CLOSED AT RUN with a sentence naming what + # is missing. What is refused here is a value somebody actually TYPED and got wrong — never + # the absence of one they have not typed yet. + url = _s(cfg.get("url"), 500).strip() + # ⚠ `{{` IS THE TEMPLATE FORM — `interpolate`'s syntax is `{{field_key}}`, NOT `{field}`. + # A url that STARTS with a placeholder is legitimate (the record supplies the host), so the + # shape test admits it; anything else must be an absolute http(s) address. + if url and not (url.startswith("http://") or url.startswith("https://") + or url.startswith("{{")): + return None, "a web address must start with http:// or https://" + selector = _s(cfg.get("selector"), 200).strip() + field = _s(cfg.get("field"), 80).strip() + try: + timeout = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) + except (TypeError, ValueError): + return None, "the web step timeout must be a whole number of milliseconds" + timeout = max(1000, min(timeout, WEB_READ_TIMEOUT_MAX_MS)) + out = {"url": url, "selector": selector, "field": field, + "attr": _s(cfg.get("attr"), 40).strip() or "text", + "all": bool(cfg.get("all")), "timeoutMs": timeout} + wait = _s(cfg.get("waitFor"), 200).strip() + if wait: + out["waitFor"] = wait + # ⭐ THE FOUR WRITE KINDS' OWN KEYS. Carried for EVERY web kind rather than switched on + # `kind`, because the cost of carrying an unused key is nothing and the cost of dropping a + # used one is a step that runs forever against a value the user typed and cannot see. + # `value` is the text `web_fill` types; `secret` masks it in the run log; `hint` is the + # human label `web_repair` follows when a selector has gone stale; `dryRun` rehearses a + # write without performing it (E measured the field reading back empty afterwards). + value = _s(cfg.get("value"), 500) + if value: + out["value"] = value + hint = _s(cfg.get("hint"), 200).strip() + if hint: + out["hint"] = hint + if cfg.get("secret"): + out["secret"] = True + if cfg.get("dryRun"): + out["dryRun"] = True + return out, None + return {}, None + + +def _branch_letter(i): + """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a + label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is + worse than an ugly name.""" + return chr(65 + i % 26) * (1 + i // 26) + + +def group_branches(act): + """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape. + + ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying + for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored + group might carry either shape, and the one that forgot would silently skip a live + automation's actions. A definition cleaned since wave 24 always carries `branches`; one read + straight out of the store (the runner reads definitions, not payloads) may not. + """ + cfg = (act or {}).get("config") or {} + brs = cfg.get("branches") + if isinstance(brs, list) and brs: + return [b for b in brs if isinstance(b, dict)] + return [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions") or []}] + + +def walk_actions(actions): + """Every action in the tree, depth-first, groups included. One walker, so "how many actions + does this flow have" and "which review stages exist" cannot answer differently. + + ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action + COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version + that still read `config.actions` would report a fork's contents as zero actions, hide every + review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing. + """ + for act in actions or []: + yield act + if act.get("kind") == "group": + for br in group_branches(act): + for kid in walk_actions(br.get("actions")): + yield kid + + +def interpolate(text, row): + """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an + action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would + put template syntax in front of a customer, and a blank is the visible failure.""" + if not isinstance(text, str) or "{{" not in text: + return text + def _sub(m): + v = (row or {}).get(m.group(1).strip()) + return "" if v is None else str(v) + return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. +# +# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits +# them once at the end (the 256-commits/hr budget every writer in this module respects). A +# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget +# on one busy flow. +# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the +# runner stamps its stage and stops walking THAT record — later actions belong to the branch a +# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. +# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the +# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write +# cannot fire an event trigger, its own or a sibling's. + +def _act_row_patch(patches, table, rid, values): + patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) + + +def profile_field_key(table, named="", source=PROFILE_SOURCE_IG): + """C3/C4: WHICH COLUMN on this database carries the profile handle. '' when none does. + + Resolution order, and the missing branch is the important one: + 1. the column the action NAMES — the person picked it, so it is not a guess; + 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`; + 3. otherwise **NOTHING**, and the run says so. + + ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url` + column" and is exactly right for `field_instagram`, whose whole configuration was a URL column + — but here it would silently bind the enrich step to whatever URL column happens to be first, + on a database where nobody has said which column is a profile. The failure would be a run that + reports success having enriched from the wrong column, which is worse than one that refuses: + a wrong number is harder to notice than a missing one, and it would be written into the + permanent history as if it had been measured. + + ⭐ WAVE 30 · T08 — `source` PARAMETERISES STEP 2 ONLY, defaulting to Instagram so every + existing caller is byte-for-byte unchanged. Step 1 (the column the person NAMED) is already + network-agnostic: a named column is a decision, not a guess, and second-guessing it against a + flag would refuse a binding somebody made on purpose. + ⛔ AND THE TWO SOURCES MUST NOT BE MERGED INTO "any profile flag". `ut_tt_profile.handle` + carries `profile: {source: "tiktok"}` and `ut_ig_profile.handle` carries `"instagram"`; a + resolver that accepted either would let a TikTok enrich step bind to an Instagram column and + spend money asking a TikTok dataset about an Instagram handle — a wrong number written into a + permanent history, which is the exact failure the no-fallback rule above exists to prevent. + """ + fields = (table or {}).get("fields") or [] + want = str(named or "").strip() + if want: + return want if any(str(f.get("key")) == want for f in fields) else "" + for f in fields: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == str(source): + return str(f.get("key") or "") + return "" + + +def _flow_table(defn): + """The database a flow's actions and ending operate on: the automation's own target, or the + table its event trigger watches when the flow has no target of its own. + + ⭐ WAVE 30 · T05 — the discovery arm covers BOTH kinds. It read `== "discover_instagram"`, so a + TikTok discovery with no stored `targetTable` fell to the trigger's table — which a corpus + search does not have — and resolved to `""`. `_presets_after_write` asks this function where to + spawn the preset columns, so an empty answer there is a database that never appears. + """ + cfg = (defn or {}).get("config") or {} + if defn.get("kind") in DISCOVERY_KINDS: + return cfg.get("targetTable") or discovery_facts(defn.get("kind"))[1] + return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") + + +#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN. +#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the +#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]` +#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own. +MAX_ENRICH_PER_RUN = 100 +DEFAULT_ENRICH_LIMIT = 25 +DEFAULT_ENRICH_COOLDOWN_DAYS = 30 +#: Which column the selection orders by when nobody has said. `first_found` newest-first is the +#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what +#: discovery just found rather than re-walking the oldest leads in the book. +DEFAULT_ENRICH_SORT = "first_found" + +#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL. +#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence +#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is +#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored. +#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with +#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by +#: accident, and `_commit_run`'s numeric filter is the second net under it. +RUN_NOTES_KEY = "_notes" +#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES +#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval. +#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence +#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so +#: the verdict should expire. But nothing about US changes on day 31 — the only new information +#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per +#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a +#: hundred stale handles pays a hundred times a month to be told the same thing. +#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors: +#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no +#: wiring at all, because the corrected handle simply is not the one we recorded; and +#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle +#: (a person saying "try it again") also re-arms. +#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered +#: them: a tombstone the owner cannot see is just a disappearance. +#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An +#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the +#: opposite mask ([[gate-answers-the-wrong-question]]). +PENDING_PROFILE_MAX_HOURS = 24 + + +def _order_key(value): + """Order one CELL. Numbers numerically, everything else as text. + + ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text — + which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting + paid for. Numbers are tried first and text is the fallback, never the other way round. + ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where + lexicographic order IS chronological order. If that format ever changes, this comment is the + thing that was relied on. + """ + s = str(value if value is not None else "").strip() + try: + return (0, float(s.replace(",", "")), "") + except ValueError: + return (1, 0.0, s.lower()) + + +def _days_since(day, today=None): + """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read. + + ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a + large number — the cooldown below reads an unreadable stamp as "never enriched", which spends + money rather than skipping. That is the right way round: a skipped record is invisible, and a + stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data. + """ + import datetime as _dt + s = str(day or "").strip()[:10] + if not s: + return None + try: + d = _dt.date.fromisoformat(s) + except ValueError: + return None + return ((today or _dt.date.today()) - d).days + + +def _hours_since(stamp): + """Whole hours since a full ISO stamp, or None when it cannot be read. + + ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date` + column and truncates to 10 characters. A pending snapshot's age is measured in hours and its + stamp carries a time, so reusing that function would read every entry as "queued at + midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid, + outstanding snapshot out of the queue. + """ + import datetime as _dt + s = str(stamp or "").strip() + if not s: + return None + try: + t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + except ValueError: + return None + now = _dt.datetime.now(_dt.timezone.utc) + if t.tzinfo is None: + t = t.replace(tzinfo=_dt.timezone.utc) + return max(0, int((now - t).total_seconds() // 3600)) + + +def _gone_key(row, handle, platform=None): + """The identity a "this account does not exist" verdict is remembered under. + + `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on + Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what + every row on a preset profile table is and what `preset_cells` stamps. + + ⭐ WAVE 30 · T08 — `platform` OVERRIDES THE ROW, and the override is what makes the key honest + on a table the row cannot speak for. The fallback above reads the row's own `platform` cell, + which the TikTok runner writes — but only AFTER the first successful pull. A hand-typed row on + an arbitrary database that a TikTok step is bound to by NAME (`profile_field_key` step 1, + which is deliberately network-blind) carries no such cell, so a TikTok "this account does not + exist" verdict would have been filed under `instagram:`. A tombstone has no expiry + (W28/R9), so that would permanently suppress an Instagram read of a DIFFERENT person's + account with the same handle — silently, and without ever spending the money that would have + shown it was wrong. The caller knows which network it asked; the row does not. + """ + plat = str(platform or (row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() + return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}" + + +def clear_gone(rt, table_key, handle, row=None): + """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle. + + Returns the number of automations whose memory was changed — 0 is the ordinary answer and is + not an error, because most cell edits are not on a dead handle. + + ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER. + The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to, + because it is a fact about what a run learned and paid for. But the thing that re-arms it is a + ROW event, and the row writer must not need to know the shape of an automation's state to + trigger it. So the seam is one call with the three things the writer already has, and every + walk of `all_definitions` stays on this side of the fence. + ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which + handle is this", so a verdict can never be recorded under a key this cannot find + ([[one-evaluator-per-question]]). + + ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first + door — correcting a typo — works with no wiring at all, because the key IS the handle and a + different handle is simply not the one we recorded. This second door only opens when the row + write path calls it, which lives in `routes_tables.patch_row` (another session's fence). + Until that line lands, re-typing the SAME dead handle stays skipped + ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later). + """ + key = _gone_key(row or {}, handle) + if not str(handle or "").strip(): + return 0 + changed = 0 + for auto_id, defn in (all_definitions(rt) or {}).items(): + if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key): + continue + known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {}) + if key not in known: + continue + known.pop(key, None) + # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what + # every other state writer here does and what keeps a definition from growing a graveyard + # of empty maps. + set_state(rt, str(auto_id), {"enrichNotFound": known or None}) + changed += 1 + return changed + + +def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch, + step=_no_step, log=print): + """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved. + + ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a + LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an + hour"); the only profile caller passed a single-element list from inside the per-record walk. + MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25 + vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand. + + ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from + `prefetch`, and the per-record rung then behaves exactly as it does today — including its + corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a + batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure + returns 0 instead of raising. + + ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now + serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they + share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the + handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch` + behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time. + """ + want, dests, handles = [], {}, [] + for rid in (chosen or []): + row = rows.get(str(rid)) or {} + h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower() + if not h or h in prefetch: + continue + want.append((h, str(rid))) + if h not in dests: + handles.append(h) + dests.setdefault(h, []).append(str(rid)) + if not handles: + return 0 + step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source") + deferrals = [] + try: + nodes, note = bd_profiles_batch(handles, deferred=deferrals) + except Exception as exc: # noqa: BLE001 — never fatal, see docstring + log(f"[aios-auto] profile batch failed, falling back to per-record reads: " + f"{type(exc).__name__}: {exc}") + return 0 + prefetch.update(nodes) + if note: + log(f"[aios-auto] profile batch note: {_s(note, 300)}") + for d in deferrals: + sid = str(d.get("snapshotId") or "") + if not sid: + continue + for u in (d.get("urls") or []): + h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower() + if not h or h in nodes: + continue + prefetch[h] = {DEFERRED_MARK: sid} + for rid in dests.get(h, []): + enrich["pendingProfiles"].append( + {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""), + "urls": [str(u)], "kind": "profile", "influencer": h, + "table": table, "rowId": rid, "requestedAt": _iso()}) + return len(nodes) + + +def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, platform=None): + """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order. + + Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did + and is surfaced in the run summary; `""` means there is nothing worth saying. + + ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's + ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is + enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every + skipped record until 30 have actually been enriched or the list runs out. The naive reading — + take the top 30, then filter — would have billed for 10 and reported success, and the number in + the box would have meant something different every run depending on how much of the top of the + list happened to be fresh. A quota is predictable; a filtered window is not. + + The order of operations, each step narrowing the one above: + 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view` + and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an + empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich + the entire database", at vendor prices. + 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions + (a blank is unknown, not smallest). + 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on. + 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`. + + ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is + nothing to enrich and it must not consume a slot somebody paid for. + """ + t = ut_get(rt, str(table_key or "")) + if t is None: + return [], f"{table_key!r} is not a database in this workspace" + rows = dict(t.get("rows") or {}) + notes = [] + + view_id = str(cfg.get("fromView") or "").strip() + if view_id: + tree, fields, problem = view_filter(rt, table_key, view_id) + if problem: + # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet + # fallback costs real money. + return [], f"the enrich step names a view it cannot read — {problem}" + # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS + # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE. + # + # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}], + # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/ + # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition — + # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on + # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`, + # got `""`, and answered **False for every row**: MEASURED on the owner's own automation, + # a view matching exactly one record selected NOTHING, reported NO error, and the run + # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual + # record... how come it says 51 records walked?"* + # + # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch + # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has + # always been correct. `_seed_event_state`'s own docstring states the law this line broke: + # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it + # is supposed to arm."* There were three. Now there are two callers of one function. + # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date + # as a date and a number as a number, which is the half `lane_match` could not have had. + import harness.filter_eval as filter_eval + rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)} + + sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT + newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc" + # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN". + # + # Owner: *"treat a missing first found as now, and manual entry should go first, instead of + # treated as last. I want to see my inayma manual entry works with automation enrichment."* + # And they are right about the semantics, not just the preference: on these tables the ONLY + # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it + # on every row it writes. So a blank is not missing data — it is a row that was first found + # today, by the person sitting in front of it, and the one they most want enriched. + # + # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position + # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one + # it never reached. + # + # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about + # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and + # treating it as the maximum would spend the budget on the rows we know least about. So a + # non-date sort keeps the old rule: unknown sorts last, in both directions. + # + # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real + # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at + # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One + # rule, both directions, no branch that can disagree with itself. + ftype = str(((next((f for f in (t.get("fields") or []) + if str(f.get("key")) == sort_field), None)) or {}).get("type") or "") + if ftype == "date": + ordered = sorted(rows.items(), + key=lambda x: (0, str(x[1].get(sort_field) or "").strip()) + if str(x[1].get(sort_field) or "").strip() else (1, ""), + reverse=newest_first) + else: + have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()] + blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()] + have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first) + blank.sort(key=lambda x: _rid_num(x[0])) + ordered = have + blank + + try: + quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) + except (TypeError, ValueError): + quota = DEFAULT_ENRICH_LIMIT + quota = max(1, min(quota, MAX_ENRICH_PER_RUN)) + if cfg.get("limit") and quota != int(cfg.get("limit") or 0): + notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run") + + cooling = bool(cfg.get("skipRecent")) + try: + days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) + except (TypeError, ValueError): + days = DEFAULT_ENRICH_COOLDOWN_DAYS + days = max(1, days) + + picked, cooled, blank_handle = [], 0, 0 + # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST. + # + # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT. + # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is + # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it + # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at + # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix + # removes that loop; only a memory of the verdict does. + # + # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the + # note goes into the run every single time, not only on the run that discovered it, because a + # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf". + skipped_gone = [] + for rid, r in ordered: + if len(picked) >= quota: + break + raw_handle = str(r.get(profile_key) or "").strip() + if not raw_handle: + blank_handle += 1 + continue + # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is + # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the + # owner reads to know when we last paid to be told this), it is simply not a clock. + # ⚠ WAVE 30 · T08 — the SAME `platform` the runner will file a new verdict under, so the + # skip and the write cannot disagree about which account a tombstone belongs to. + if isinstance((gone or {}).get(_gone_key(r, raw_handle, platform)), dict): + skipped_gone.append(raw_handle) + continue + if cooling: + since = _days_since(r.get("enriched_at"), today=today) + if since is not None and since < days: + cooled += 1 + continue + picked.append(str(rid)) + + # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the + # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and + # `run_plain` follow. Silence here would make a shrinking selection invisible. + if cooled: + notes.append(f"{cooled} skipped as enriched in the last {days} days") + if skipped_gone: + shown = ", ".join(f"@{h}" for h in skipped_gone[:5]) + # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can + # do, because nothing else will ever retry them — under R9 this note is the only path + # back from a tombstone, so a vaguer version would strand the row permanently. + notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account " + f"({shown}{', …' if len(skipped_gone) > 5 else ''}) — delete the row or " + f"correct the handle; they are not retried automatically") + if blank_handle: + notes.append(f"{blank_handle} skipped with no handle") + if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered): + notes.append(f"{len(picked)} of the {quota} asked for — the list ran out") + return picked, "; ".join(notes) + + +def _has_action(actions, kind): + """Does this flow contain `kind` ANYWHERE, forks included? + + ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under + `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an + enrich step somebody put inside an If — and the caller would then skip a schema top-up the + run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24 + recorded when four hand-rolled walks all forgot to descend. + """ + for a in (actions or []): + if not isinstance(a, dict): + continue + if a.get("kind") == kind: + return True + for br in ((a.get("config") or {}).get("branches") or []): + if _has_action((br or {}).get("actions") or [], kind): + return True + return False + + +def _actions_of_kind(actions, kind): + """Every action of `kind`, including actions nested inside If branches.""" + out = [] + for action in actions or []: + if not isinstance(action, dict): + continue + if action.get("kind") == kind: + out.append(action) + for branch in ((action.get("config") or {}).get("branches") or []): + out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind)) + return out + + +def _web_agent(): + """C5's seam, resolved LAZILY — `web_agent.run_step(step, ctx) -> (dict|None, str)`. + + ⚠ IMPORTED INSIDE THE CALL, like every `connectors_tt` site in this module, and here it also + buys a failure mode worth having: if `web_agent.py` is ever missing from a deployment, the + engine still imports and every OTHER action still runs — the web step alone reports a sentence. + A module-level import would turn one absent file into a dead automation module. + + ⛔ AND THE ABSENCE IS REPORTED, NEVER SWALLOWED (R6's second sentence). The shim below answers + the same `(None, sentence)` contract the real seam does, so the caller's `if why:` branch is + the only branch there has ever been. + """ + try: + import web_agent + return web_agent + except Exception as exc: # noqa: BLE001 + class _Absent: + @staticmethod + def run_step(_step, _ctx=None): + return None, ("The web-browsing agent is not available in this deployment " + f"({type(exc).__name__}). Nothing was read.") + return _Absent + + +def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print, + step=_no_step): + """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. + + ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery + automation's review stage comes from `stages_for`, not from actions, + so an early return on an empty action list would have silently disabled the owner's loop + feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in + review; the tests all happened to pass a non-empty action list, which is why it was green. + """ + defn, _retired = _without_retired_board(defn) + flow = (defn or {}).get("flow") or {} + actions = flow.get("actions") or [] + # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE. + # + # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the + # "Save results" node) and THEN walked the flow over the rows it had just written — where + # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on + # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner + # ruling, same day) would have made the duplication permanent and unavoidable with it. + # + # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what + # it finds in a database" is exactly what the engine does. What it must not do is do it + # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the + # server still refuses to let it be removed. + if ig_action_pinned(defn, 0) and actions: + actions = actions[1:] + # ⭐ `webRead` / `webBlocked` JOIN HERE (W31-T38, C5): declared up front like every other + # counter, so a run that read nothing and a run with no web step are DIFFERENT numbers. R6's + # second sentence is a reporting rule, and a counter that only exists once it is non-zero + # cannot report a refusal. + counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0, + "webRead": 0, "webBlocked": 0} + #: W31-T38 — the once-per-run markers the web arm uses, so an unconfigured step says so ONCE + #: rather than once per record. Same shape as the enrich accumulators' `notes`. + web_notes = [] + # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected + # across the WHOLE walk and written once at the end, for the reason the module header states + # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on + # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an + # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts. + enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor) + # and handles a vendor has said do not exist. Both are collected across the whole + # walk and written once, for the same reason every other list here is. + "pendingProfiles": [], "gone": {}, + "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} + # ⭐⭐ WAVE 30 · T08 — THE TIKTOK ACCUMULATOR IS ITS OWN, and the separation is the design + # decision rather than a convenience. `_enrich_flush` is not network-agnostic: it calls + # `ensure_ig_graph`, addresses the four `IG_*_TABLE` constants and writes through to the + # platform Instagram master (`ig_master.append_run`). Routing TikTok rows through it would + # append a TikTok creator's followers into Instagram's pooled history, which the metric + # FIELDS read (W29-T34) — a wrong number in a permanent series, i.e. the same failure class + # as `preset_cells`' unconditional `PLATFORM_INSTAGRAM` stamp that `tt_preset_cells` exists + # to avoid. + # ⚠ AND THE KEYS IT FLUSHES ARE PREFIXED `tt…` FOR A MEASURED REASON: `apply_actions` merges + # both flushes into ONE `counts` dict, so a shared `enriched`/`enrichBlocked` key would let a + # flow carrying BOTH an Instagram and a TikTok step report one network's numbers under the + # other's name, last writer winning, with no error anywhere. + # ⭐ W30 · D-156 — `pending` is the TikTok twin of Instagram's, and it is a RUN-level list for + # the same reason theirs is: a media snapshot belongs to a HANDLE, not to a row, so nothing + # about it needs the per-record sink that `pendingProfiles` needs. + tt = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} + # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them + # per record and re-reading the definition per row would be a store read per record. + known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) + # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD. + # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk + # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked + # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps + # in one flow are two independent budgets. + enrich_plan = {} + # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK. + # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a + # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was + # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever + # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are + # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth + # buying twice. + enrich_prefetch = {} + if not table_key: + return counts + table = str(table_key) + # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN. + # + # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled + # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a + # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door. + # So every preset the target table did not already happen to have was silently dropped, and + # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET + # FILLED"* — this is the half that was swallowing them. + # + # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`, + # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose + # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the + # feature worked on discovery tables purely because discovery creates them for its own reasons. + # + # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present: + # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady + # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no + # enrich step never touches the schema of the database it walks. + # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset + # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an + # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a + # step at it, and (b) add a SECOND profile column to a table that already flagged a different + # one, which `user_tables` forbids at both write doors. A table with no profile column is an + # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its + # own honest refusal, not something to paper over by inventing the binding. + _tbl_now = ut_get(rt, table) or {} + _enrich_actions = _actions_of_kind(actions, "enrich_instagram") + _bound = next((f for f in (_tbl_now.get("fields") or []) + if isinstance(f.get("profile"), dict)), None) + if _bound is None: + _named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in _enrich_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + _bound = next((f for f in (_tbl_now.get("fields") or []) + if str(f.get("key") or "") == _named), None) + if _bound and _enrich_actions: + # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists — + # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a + # table whose profile column is `ig_handle` would otherwise gain a rival `handle`. + topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")} + if f.get("key") != _bound.get("key") else dict(f)) + for f in PRESET_PROFILE_FIELDS] + try: + if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions): + ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=table, profile_field=str(_bound.get("key") or "")) + else: + # Dry run keeps its original no-create contract; only the already-established + # Profile table is topped up, and no canonical related database is spawned. + ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table, + flow_tag=str(defn.get("id") or ""), lock_fields=True) + except Exception as exc: # noqa: BLE001 + # A schema top-up that cannot run must not stop the enrichment: the cells that DO + # have columns still land, which is strictly better than the pull being thrown away. + log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}") + # ⭐⭐ WAVE 30 · T08 — THE SAME TOP-UP FOR TIKTOK, AND THE SAME NARROWING: only on a table that + # is ALREADY BOUND. Separate from the block above because every constant in it is Instagram's + # (`PRESET_PROFILE_FIELDS`, `ensure_ig_graph`) and because the binding is resolved by a + # DIFFERENT question — `profile_field_key(..., source=PROFILE_SOURCE_TT)`, so a TikTok step + # cannot adopt an Instagram column and go on to ask a TikTok dataset about an Instagram handle. + # ⚠ It runs on a dry run too, exactly as the Instagram branch does: the target's own columns + # are topped up, and the related append database is NOT spawned (that is the flush's job, and + # it returns before writing anything on a dry run). + _tt_actions = _actions_of_kind(actions, "enrich_tiktok") + if _tt_actions: + _tt_named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in _tt_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + _tt_bound = profile_field_key(_tbl_now, _tt_named, source=PROFILE_SOURCE_TT) + if _tt_bound: + try: + ut_ensure(rt, _tbl_now.get("label") or table, + _tt_profile_schema_for(_tt_bound), username, key=table, + flow_tag=str(defn.get("id") or ""), lock_fields=True) + except Exception as exc: # noqa: BLE001 + log(f"[aios-auto] TikTok preset top-up on {table} failed: " + f"{type(exc).__name__}: {exc}") + rows = dict((ut_get(rt, table) or {}).get("rows") or {}) + if not rows: + return counts + # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The + # walk used to keep a `reached` map and stamp each record with the furthest action it made — + # a board column's worth of bookkeeping written into the tenant's own table on every run. + # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a + # cell the customer has to look at and a store commit we pay for on every scheduled run. + # The RUN LOG is the operational record of what happened now, and it is the only one. + patches, creates = {}, {} + + def _walk(acts, row, rid, depth=0): + """Walk the actions in order for ONE record. + + ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean + "this record was SUSPENDED by a review gate" — the one branch that could stop a walk + early. With review retired nothing suspends anything, so every record walks its whole + flow; the signature stays so a future gate-style action has an obvious place to say so. + """ + for act in acts: + if not act.get("enabled", True): + continue + if not lane_match(act.get("when"), row): + continue + kind = act.get("kind") + cfg = act.get("config") or {} + counts["actionsRun"] += 1 + if kind == "group": + # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs. + # Declaration order is priority order, the same rule `route_record` applies to + # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch + # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions` + # has already guaranteed a null condition can only be LAST. + for br in group_branches(act): + if lane_match(br.get("cond"), row): + if _walk(br.get("actions") or [], row, rid, depth + 1): + return True + break + elif kind == "update_record": + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + row.update(vals) # later actions see the write, as they must + _act_row_patch(patches, table, rid, vals) + counts["updated"] += 1 + elif kind == "create_record": + target = str(cfg.get("table") or "") + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions + # may legitimately write to ONE database on different keys — collapsing them onto + # the table would silently apply one action's uniqueness rule to the other's rows. + creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) + counts["created"] += 1 + elif kind in WEB_KINDS: + # ⭐⭐ WAVE 31 · T38 (C5) — THE WEB-BROWSING AGENT'S LIVE ARM, ALL FIVE KINDS. + # + # ⛔ WHY THIS ARM EXISTS SEPARATELY FROM THE RUNNER: session E built + # `web_agent.run_step` and **could not verify its own mounting**. An unmounted + # runner is a whole, correct, unreachable feature — the exact class five wave-29 + # features shipped as — so the mount and its `verify_wiring` row are C's, in one + # change. + # ⭐⭐ W31 QA WIDENED THIS ARM FROM `web_read` TO ALL FIVE KINDS on the owner's + # revocation of D-51/R5 (`TICKETS.md:1418`). ⚠ IT NEEDS NO PER-KIND HANDLING, and + # that is E's design rather than an omission: the runner normalises EVERY kind to + # set `result["value"]` (read → the text · goto → the title · click → the title it + # landed on · fill → the typed value, masked when secret · repair → the proposed + # selector), precisely so this one arm does not grow a switch that would be a + # second copy of the runner's table living in another lane's file. + # + # ⛔ IT BLOCKS FOR ~9-32 s (E measured it; `proto/web-agent-job.md` §4). That is + # tolerable HERE and nowhere else: this is the automation RUNNER, already a + # background walk. It must never be called from a route a person is waiting on. + # + # ⚠ `interpolate` ON THE CALLER'S SIDE, exactly as the update/create arms do it, so + # `{Field}` works in a url or a selector. E's seam takes a plain dict and does not + # know about rows. + # ⛔ FAIL CLOSED, ONCE, WITH THE REASON — the counterpart to the validator storing + # an unconfigured step (see `_clean_action_config`'s `web_read` arm). Reported once + # per RUN and not per record: the missing config is a property of the flow, so a + # 100-record walk would otherwise print the same sentence a hundred times and bury + # everything else. The same shape the enrich arm's `unbound` note uses. + # ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and + # no selector; `web_fill` needs a value nobody else takes; only `web_read` needs a + # column to write into. One shared three-field test would have blocked every + # `web_goto` ever configured for want of a selector it does not use. + _missing = [n for n, key in WEB_REQUIRED.get(kind, ()) + if not str(cfg.get(key) or "").strip()] + if _missing: + # ⚠ THE NOTE KEY CARRIES THE KIND. It used to be the literal + # `"web_read_unconfigured"`, so a flow with an unconfigured `web_goto` AND an + # unconfigured `web_fill` would have reported the first and swallowed the + # second — once-per-RUN is the property, not once-per-FLOW. + _note = f"{kind}_unconfigured" + if _note not in web_notes: + web_notes.append(_note) + log(f"[aios-auto] {kind}: this step still needs " + + ", ".join(_missing) + " — nothing was done") + counts["webBlocked"] += 1 + continue + # ⛔⛔ THE PER-RUN JOB CEILING (E-4). One browser job per RECORD, ~9-32 s each, + # against HF's 6-concurrent cap — a flow over a few thousand rows would submit a few + # thousand jobs and run for days. Reported ONCE with its cause AND the fix, which is + # R6's second sentence: a limit that cannot be removed today must say why and what + # would remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning. + if counts["webRead"] >= MAX_WEB_JOBS_PER_RUN: + if "web_cap" not in web_notes: + web_notes.append("web_cap") + log(f"[aios-auto] web_read: this run stopped after " + f"{MAX_WEB_JOBS_PER_RUN} pages — each one is a separate browser job " + f"of about 10-30 seconds, and they do not run in parallel. To read " + f"more, narrow the flow's records, or batch the flow's web steps into " + f"one job (web_agent.run_plan).") + counts["webBlocked"] += 1 + continue + _url = interpolate(str(cfg.get("url") or ""), row) + _sel = interpolate(str(cfg.get("selector") or ""), row) + _wait = cfg.get("waitFor") + _step = {"kind": kind, "id": str(act.get("id") or ""), + "url": _url, "selector": _sel, + "attr": str(cfg.get("attr") or "text"), + "all": bool(cfg.get("all")), + "waitFor": interpolate(str(_wait), row) if _wait else None, + "timeoutMs": int(cfg.get("timeoutMs") or 20000)} + # ⚠ `value` IS INTERPOLATED LIKE THE URL AND THE SELECTOR — a `web_fill` typing + # `{{Email}}` is the whole point of running it over a record walk. ⛔ AND IT IS + # ADDED ONLY WHEN PRESENT: the seam distinguishes a key that is absent from one + # that is empty, and an empty `value` on a `web_read` would be a typed blank. + if cfg.get("value"): + _step["value"] = interpolate(str(cfg.get("value")), row) + if cfg.get("hint"): + _step["hint"] = interpolate(str(cfg.get("hint")), row) + if cfg.get("secret"): + _step["secret"] = True + if cfg.get("dryRun"): + _step["dryRun"] = True + result, why = _web_agent().run_step( + _step, + {"tenant": str(defn.get("tenant") or ""), + "automationId": str(defn.get("id") or ""), + "runId": str(defn.get("id") or ""), "log": log}) + # ⛔ EXACTLY ONE OF THE TWO IS TRUTHY — E's contract, and `if why:` is the COMPLETE + # client-side branch. There is no exception path to catch: `run_step` turns + # everything into a sentence, which is R6's second sentence built into the seam. + if why: + log(f"[aios-auto] web_read: {why}") + counts["webBlocked"] += 1 + continue # ⛔ NOTHING IS WRITTEN on a refusal. + _target = str(cfg.get("field") or "") + if _target: + vals = {_target: (result or {}).get("value")} + row.update(vals) # later actions see the write, as they must + _act_row_patch(patches, table, rid, vals) + counts["webRead"] += 1 + elif kind == "enrich_instagram": + # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions + # `run_field_instagram` calls, not a second implementation of either. + pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField")) + if not pkey: + # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a + # property of the flow, so a 100-record run would otherwise put the same + # sentence in the log a hundred times and bury everything else. + if "unbound" not in enrich["notes"]: + enrich["notes"].append("unbound") + log("[aios-auto] enrich_instagram: no profile column on " + f"{table} — name one on the action, or mark a text column as an " + "Instagram profile") + continue + # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET? + # Resolved once (see `enrich_plan`) and then a membership test. A record outside + # the selection is NOT an error and NOT a skip worth logging per row — it is simply + # not this run's work, and the selection's own note already accounts for it. + aid = str(act.get("id") or "") + if aid not in enrich_plan: + chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone) + enrich_plan[aid] = set(chosen) + if sel_note: + enrich["notes"].append(sel_note) + # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else: + # this is the only place the full chosen set and the row bodies are both in + # hand. Anything it resolves the per-record rung below reads from memory. + _prime_enrich_batch(chosen, rows, pkey, table, enrich, + enrich_prefetch, step, log) + if str(rid) not in enrich_plan[aid]: + continue + handle_raw = str(row.get(pkey, "") or "").strip() + if not handle_raw: + continue # nothing to enrich on this record, not an error + # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED. + # A handle already in `enrich_prefetch` was bought by the batch above and is read + # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the + # batching win straight back (25 records = ~62 s of pure waiting). + if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \ + not in enrich_prefetch: + time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does + enrich["profiles"] += 1 + # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with + # the row it belongs to: the collector writes preset cells back onto THAT record, + # and a run-wide list would have no way to say which handle each snapshot was for. + pend_prof = [] + # ⭐ W31-T39(c) — the DEFERRAL WATERMARK, taken before the pull. See the `partial` + # report below: a capability that was deferred is not a capability that failed, + # and `pull_profile` appends into these two sinks from inside the call. + _pend_before = len(enrich["pending"]) + res = pull_profile(handle_raw, + max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, + log=log, + post_metrics=bool(cfg.get("postMetrics")), + comment_metrics=bool(cfg.get("commentMetrics")), + pending_metrics=(enrich["pending"] + if not cfg.get("dryRun") else None), + pending_profile=(pend_prof + if not cfg.get("dryRun") else None), + prefetch=enrich_prefetch, + post_groups=cfg.get("postGroups")) + # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels / + # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is + # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped. + # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this + # costs nothing and changes nothing until somebody configures it. + # ⭐⭐ WAVE 30 · T16 — THE WINDOW IS NOW WIDENED BEFORE IT IS FILTERED, and the + # paragraph that used to sit here explaining why it could not be is gone with the + # reason. It said: *"it FILTERS the window we already bought … the views top-up + # runs INSIDE `pull_profile` against the posts it captured, so swapping a wider + # list in afterwards would hand back posts with no view counts."* Correct, and + # exactly why the widening went INTO `pull_profile` (`post_groups=`, above) rather + # than being bolted on out here — ahead of the top-up, not after it. + # ⚠ THIS LINE STAYS, and it is not redundant. A single-type ask is served by a + # native route that returns only that type; a MIXED ask has no such route, so the + # wide window comes back mixed and this is what keeps N of each. + if cfg.get("postGroups"): + res = {**res, "posts": select_post_groups(res.get("posts") or [], + cfg.get("postGroups"))} + for _p in pend_prof: + _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() + enrich["pendingProfiles"].extend(pend_prof) + pulled = _iso() + if res["state"] in ("ok", "partial"): + enrich["ok"] += 1 + snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) + enrich["snaps"].append(snap_row) + enrich["posts"].extend(idents) + enrich["psnaps"].extend(metric_rows) + enrich["comments"].extend(comment_rows) + # R3: LATEST onto the record itself. `row.update` too, so a LATER action in + # this same flow sees the enriched values — the rule `update_record` already + # follows, and without it "enrich, then route by follower count" would judge + # the record on the values it had before the pull. + # ⭐⭐ WAVE 31 · T39(c) — D-177(c): THE FIX APPLIED TO ONE PLATFORM AND NOT ITS + # TWIN, and that is the shape worth naming rather than the line. + # + # W30-T11 found this on TikTok the expensive way: a paid run enriched two + # profiles with `postMetrics` ON, wrote no posts, and said NOTHING — because + # `partial` takes THIS branch and only the `else` ever appended a note, so the + # reason `pull_profile` carries in `res["note"]` was discarded. Instagram has + # the identical branch and never got the fix; the sibling arm below has carried + # it since wave 30. One defect, two networks, one of them repaired — which is + # exactly how the last four TikTok/Instagram divergences were found. + # + # ⛔ ONLY WHEN IT WAS ASKED FOR, and ⛔ NOT WHEN THE BATCH DEFERRED — both + # conditions copied deliberately from the TikTok arm rather than re-reasoned. + # With `postMetrics` off, a `partial` is the normal answer for every profile + # and reporting it would put one useless line per record in the run entry; a + # DEFERRAL is a paid snapshot already filed for collection, and reporting it as + # a miss teaches a person to re-run and buy it twice. + # ⚠ Instagram's deferral sinks are `enrich["pending"]` (post metrics) and + # `pend_prof` (the profile snapshot) — the two lists `pull_profile` appends + # into — where TikTok reads `res["deferredMedia"]`. Different signal, same + # question: did this record's work get filed rather than lost? + if (cfg.get("postMetrics") and not (res.get("posts") or []) + and len(enrich["pending"]) == _pend_before and not pend_prof): + _why = _s(res.get("note"), 300) + if _why: + enrich["notes"].append(f"@{handle_raw}: {_why}") + cells = preset_cells(res, pulled) + if cfg.get("dryRun"): + enrich["dry"] = True + else: + row.update(cells) + _act_row_patch(patches, table, rid, cells) + else: + enrich["blocked"] += 1 + # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH. + # It used to be `_s(note, 90)` with no handle attached: three runs blocked + # the same record and the store could not say which record, let alone why. + # 90 characters also truncated the one measured sentence mid-clause. This is + # the most valuable thing the run produces and it now reaches the run entry. + enrich["notes"].append( + f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}") + # A vendor STATING that the account does not exist is remembered, so the next + # run stops paying to be told the same thing. + if res.get("gone"): + enrich["gone"][_gone_key(row, handle_raw)] = { + "at": _iso(), "handle": handle_raw, + "note": _s(res.get("note"), 300)} + elif kind == "enrich_tiktok": + # ⭐⭐ WAVE 30 · T08 (carrying wave-29's dropped T05). THE SECOND NETWORK, and it + # is a sibling of the branch above rather than a flag inside it. What is genuinely + # shared is shared by CALL — `enrich_selection`, `enrich_plan`, the pace floor, + # the tombstone memory; what differs is the four things B-9 measured as + # Instagram-hardcoded, and each has a TikTok counterpart of its own name. + pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"), + source=PROFILE_SOURCE_TT) + if not pkey: + # D-79, on the network that did not exist when D-79 was written: FAIL CLOSED, + # ONCE, WITH THE REASON — and the marker is its OWN, so a flow carrying both + # steps cannot report the Instagram sentence about the TikTok one. + if "unbound" not in tt["notes"]: + tt["notes"].append("unbound") + log("[aios-auto] enrich_tiktok: no profile column on " + f"{table} — name one on the action, or mark a text column as a " + "TikTok profile") + continue + aid = str(act.get("id") or "") + if aid not in enrich_plan: + # ⚠ ONE plan dict for both networks is correct and not an oversight: it is + # keyed by ACTION id, and an action has exactly one kind. Two steps over one + # table are two budgets whichever networks they read. + chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone, + platform=PLATFORM_TIKTOK) + enrich_plan[aid] = set(chosen) + if sel_note: + tt["notes"].append(sel_note) + if str(rid) not in enrich_plan[aid]: + continue + handle_raw = str(row.get(pkey, "") or "").strip() + if not handle_raw: + continue # nothing to enrich on this record, not an error + # ⚠ THE PACE FLOOR IS THE VENDOR'S. There is no batch prefetch on this path yet + # (`pull_profile_tt` accepts one and nothing writes it — see the mailbox), so + # every profile after the first pays it. + if tt["profiles"]: + time.sleep(PACE_SECONDS) + tt["profiles"] += 1 + import connectors_tt as _tt_conn # lazy: connectors_tt imports this module + pend_prof = [] + res = _tt_conn.pull_profile_tt( + handle_raw, log=log, + pending_profile=(pend_prof if not cfg.get("dryRun") else None), + # ⭐ W30-T10. Both INCLUDE axes default OFF (W28/R5-R7), and the same + # `config` keys the Instagram step reads — one vocabulary, two networks. + max_posts=int(cfg.get("maxPosts") or 0), + post_metrics=bool(cfg.get("postMetrics")), + comment_metrics=bool(cfg.get("commentMetrics"))) + for _p in pend_prof: + _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() + # The deferred-profile queue is the Instagram one BY DESIGN: it is a vendor + # snapshot id waiting to be collected, and `_pending_profile_tasks` keys tasks by + # table+row, not by network. Sharing it is what makes the tick finish a TikTok + # read it has already been charged for. + enrich["pendingProfiles"].extend(pend_prof) + # ⭐⭐ W30 · D-156 — A DEFERRED MEDIA BATCH IS FILED, NOT NARRATED. Before this, + # a posts or comments scrape the vendor took too long over was reported in the + # note and collected by NOTHING: paid for, and recoverable only by a human reading + # a sentence. `connectors_tt` has already filtered these to the media corpora, so + # a profile snapshot cannot arrive here; what the engine adds is the vocabulary the + # QUEUE speaks — the `kind` (from the dataset id, which is one-to-one on TikTok) + # and the HANDLE, which the transport never knew. + # ⚠ `dryRun` queues NOTHING: a dry run buys nothing, so an entry here could only + # be a fixture leaking, and filing it would make the tick collect a snapshot that + # was never paid for. + if not cfg.get("dryRun"): + for _d in (res.get("deferredMedia") or []): + _kind = tt_metric_kind(_d.get("datasetId")) + if not _kind: + continue # not a media corpus ⇒ not this queue's business + tt["pending"].append({**_d, "kind": _kind, "requestedAt": _iso(), + "influencer": str(handle_raw).strip() + .lstrip("@").lower()}) + pulled = _iso() + if res["state"] in ("ok", "partial"): + tt["ok"] += 1 + snap_row = tt_snapshot_row(res, pulled) + if snap_row: + tt["snaps"].append(snap_row) + tt_idents, tt_metrics, tt_comments = tt_capture_rows(res, pulled) + tt["posts"].extend(tt_idents) + tt["psnaps"].extend(tt_metrics) + tt["comments"].extend(tt_comments) + # ⭐⭐ WAVE 30 · T11 — A CAPABILITY THAT WAS ASKED FOR AND DID NOT ARRIVE IS + # REPORTED. This is R6's second sentence applied to a capability rather than to + # a row cap, and it was found the expensive way: the 09:14 UTC paid run on + # nurilab enriched two profiles with `postMetrics` ON, wrote no posts, and said + # NOTHING — `ok: true`, no note, no count — because `partial` takes the branch + # ABOVE and only the `else` ever appended a note. Every `partial` return in + # `pull_profile_tt` carries the reason in `note` (*"this account's row carried + # no post links"*, *"the post source returned nothing"*, or the vendor's own + # words), and all of them were being discarded. A person then sees two enriched + # rows, an empty posts database and no explanation anywhere. + # ⛔ ONLY WHEN IT WAS ASKED FOR, which is the difference between a report and + # noise: with `postMetrics` off, `pull_profile_tt` returns `partial` + *"post + # capture is off for this step"* for EVERY profile, and appending that would put + # one useless line per record into the run entry and let the summary quote it. + # ⛔ AND NOT WHEN THE BATCH DEFERRED — a deferral is not a failure to deliver, + # it is a paid snapshot already filed for collection + # (`ttEnrichMetricBatchesPending`, D-156), and reporting it as a miss would + # teach a person to re-run and buy it twice. + if (cfg.get("postMetrics") and not (res.get("posts") or []) + and not (res.get("deferredMedia") or [])): + _why = _s(res.get("note"), 300) + if _why: + tt["notes"].append(f"@{handle_raw} (TikTok): {_why}") + cells = tt_preset_cells(res, pulled) + if cfg.get("dryRun"): + tt["dry"] = True + else: + row.update(cells) + _act_row_patch(patches, table, rid, cells) + else: + tt["blocked"] += 1 + # ⚠ TAGGED, because `run_notes` now carries BOTH networks' per-record reasons + # (the two flushes are concatenated). The summary picks ONE note to quote, so an + # untagged TikTok line could be quoted under Instagram's sentence and vice versa + # — which would undo the whole point of giving each network its own sentence. + tt["notes"].append( + f"@{handle_raw} (TikTok): {_s(res.get('note'), 300) or res['state']}") + # ⛔ AND THERE IS DELIBERATELY NO TOMBSTONE WRITER HERE, which is the opposite + # of an oversight. On the Instagram side `res["gone"]` has exactly ONE source — + # Apify answering `ACCOUNT_GONE_NOTE` (`connectors_ig`); Bright Data has no + # not-found verdict at all, and `DEFAULT_CHAINS["tt_profile"]` is deliberately + # single-provider, so nothing on this chain can say "no account exists". A + # phrase match invented against unmeasured vendor output would file a + # PERMANENT verdict (W28/R9 — no expiry) on the strength of a guess. + # ⚠ The READ side is still network-scoped above (`platform=PLATFORM_TIKTOK`) + # and that half is load-bearing today: `known_gone` is shared, Apify DOES + # write `instagram:` verdicts, and without the scoping one of those + # would silently suppress a TikTok read of a different person's account. + elif kind == "find_records": + found = find_records(rt, cfg.get("table"), cfg.get("cond"), + int(cfg.get("limit") or 25)) + counts["found"] += len(found) + # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had + # already stopped being reachable one wave earlier — which is the part worth reading. + # `_without_retired_board` strips every `review` action out of a definition on the + # READ path, so `all_definitions` and this function have not seen one since the board + # was retired. What was left behind was a branch referencing `skey`, `stamp` and + # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not + # dead code that merely wasted space: it was a `NameError` held back by a migration + # rather than by a guard, and any change that let one stored `review` action through + # would have crashed the whole action walk for every record in that flow. + # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps + # review "as an AI decision without lanes", and the cheap-first provider ladder behind + # it is real, working, measured work. They are PARKED, deliberately and in writing, + # not orphaned; see their own notes. + return False + + if actions: + for rid in list(row_ids or [])[:FLOOD_LIMIT]: + row = dict(rows.get(str(rid)) or {}) + if not row: + continue + _walk(actions, row, str(rid)) + # ⭐⭐ WAVE 30 · T08 — TWO FLUSHES, ONE `counts`, AND THE NOTES ARE CONCATENATED RATHER THAN + # OVERWRITTEN. `RUN_NOTES_KEY` is the one key both flushes legitimately produce, so a plain + # `counts.update(a); counts.update(b)` would drop every Instagram per-record reason the moment + # a flow also carried a TikTok step — silently, and precisely on the mixed flows where a + # person most needs to know which half failed. Every other key is prefixed and cannot collide. + _ig_out = _enrich_flush(rt, defn, username, enrich, log) + _tt_out = _tt_enrich_flush(rt, defn, username, tt, log) + _flush_notes = (list(_ig_out.pop(RUN_NOTES_KEY, None) or []) + + list(_tt_out.pop(RUN_NOTES_KEY, None) or [])) + counts.update(_ig_out) + counts.update(_tt_out) + if _flush_notes: + counts[RUN_NOTES_KEY] = _flush_notes + # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was + # incremented once per create the flow decided to make; what landed is what the store says, + # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches + # every row it made last time — which is the whole point of the feature). + counts.update(_commit_action_writes(rt, table, patches, creates, username, log)) + return counts + + +def migrate_field_instagram(rt, defn): + """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the + `enrich_instagram` action. Returns `(definition, changed)`. + + R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live + `field_instagram` automation to a plain flow carrying it; delete the kind and its label." + + ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration + turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** — + so a live automation whose `urlField` is blank has been working off that fallback for months. + The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a + blank `urlField` verbatim would produce an automation that USED to work and now refuses. The + fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit + binding — which is also the honest outcome: the column stops being implicit. + + ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The + old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s + column; the action writes the C1 PRESET CELLS instead. The status column is left in place and + simply stops being written — deleting somebody's column as part of a migration would be data + loss, and a stale cell beside a fresh `enriched_at` is readable for what it is. + + ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can + run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN + against the real stored definition" is meant to be satisfied. + """ + if (defn or {}).get("kind") != "field_instagram": + return defn, False + cfg = dict(defn.get("config") or {}) + target = str(cfg.get("targetTable") or "") + bound = str(cfg.get("urlField") or "").strip() + if not bound and target: + bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "") + try: + max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL) + except (TypeError, ValueError): + max_posts = 24 + act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None, + "config": {"profileField": bound, + "postMetrics": bool(cfg.get("postMetrics")), + "commentMetrics": bool(cfg.get("commentMetrics")), + "dryRun": bool(cfg.get("dryRun")), + "maxPosts": max(1, min(max_posts, 200))}} + flow = dict(defn.get("flow") or {}) + out = dict(defn) + out["kind"] = DEFAULT_KIND + # The enrich step goes FIRST — it is what the automation was, and every action the owner + # added afterwards was written expecting the capture to have happened. + out["flow"] = {"actions": [act] + list(flow.get("actions") or [])} + out["config"] = {"targetTable": target, + "targetLabel": str(cfg.get("targetLabel") or "")} + return out, True + + +def _enrich_flush(rt, defn, username, acc, log): + """C4: the enrich action's canonical IG tables, written ONCE for the whole run. + + Returns the counts the run reports. Empty when no enrich action ran, so a flow without one + pays nothing and its run history is unchanged (`run_now` merges only non-zero keys). + + ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three + tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then + the write-through to the platform master. Forking any of those would give the enrich action a + history that the metric fields could not see — and it is the LAST step, the write-through to + the platform master, that they read (W29-T34: `compute_metric_cells` → `ig_master.series_for`, + never this tenant's `ut_ig_snapshots`), so dropping that one line is the version of this fork + that would look harmless. + """ + if "unbound" in (acc.get("notes") or []): + # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step + # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so + # this function used to return `{}`, `run_now` merged only non-zero keys, and the run + # committed **`ok` — "N records walked"** having written nothing at all. The sentence + # naming the fix went to `log()`, i.e. the server console, which no customer reads. That + # is "green over nothing" on the one path where somebody is waiting for data. + # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a + # special case — and so the run entry itself records that this happened. + return {"enrichUnbound": 1} + # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail. + # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate + # was skipped as a known-dead handle — so the sentence explaining why the automation appears + # to do nothing would have been dropped on precisely the runs that most need it, and the + # owner would be back at "0 records walked, wtf". The selection note is produced before any + # profile is read and must outlive that early return. + if not acc.get("profiles"): + notes = list(acc.get("notes") or []) + return {RUN_NOTES_KEY: notes} if notes else {} + out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]} + if acc.get("notes"): + out[RUN_NOTES_KEY] = list(acc["notes"]) + # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so + # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel. + if not acc.get("dry"): + # The vendor's not-found verdicts, merged into engine state (never a tenant column and + # never a status string — W25/R4 retired those). Merged rather than replaced: a run that + # walked one record must not forget what earlier runs learned about the others. + if acc.get("gone"): + merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) + merged.update(acc["gone"]) + set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged}) + pending_profiles = queue_pending_profile_snapshots( + rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or []) + if pending_profiles: + out["enrichProfileBatchesPending"] = pending_profiles + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), + acc.get("pending") or []) + if queued: + out["enrichMetricBatchesPending"] = queued + if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]): + # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. + if acc.get("dry"): + out["enrichDryRun"] = acc["profiles"] + return out + tag = str(defn.get("id") or "") + profile_table = str(_flow_table(defn) or "") + graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table) + snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], + graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) + missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key) + snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}), + acc["snaps"], "snapshot_key", cap=row_cap(snap_key)) + old_posts, collapsed_posts = dedupe_canonical_rows( + dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), + acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key)) + old_comments, collapsed_comments = dedupe_canonical_rows( + dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") + comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key", + cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), + (comment_key, comments)): + if cur.get(k) is not None: + cur[k]["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables + # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which + # is the failure a chart cannot show you. + capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"] + if capped: + out["enrichCapped"] = capped + log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap") + if missing: + out["enrichMissingTables"] = len(missing) + log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}") + out["enrichPoints"] = len(acc["snaps"]) + out["enrichPosts"] = c_post["inserted"] + out["enrichComments"] = c_comments["inserted"] + # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same + # contract `run_field_instagram` follows, because a pooled history with silent holes is worse + # than none. + try: + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"], + acc["posts"], acc["psnaps"]) + if m_status == "error": + out["enrichMasterFailed"] = 1 + log(f"[aios-auto] enrich_instagram: the platform master copy FAILED — {m_note}; " + f"the tenant copy is complete and the next run re-appends") + elif m_status == "ok": + out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"]) + except Exception as e: # noqa: BLE001 + out["enrichMasterFailed"] = 1 + log(f"[aios-auto] enrich_instagram: master write-through raised " + f"{type(e).__name__}: {e}") + return out + + +#: How many candidate columns an unbound-enrich sentence names before it stops. Three, because the +#: sentence is read in a run log line: naming forty columns is the same as naming none. +UNBOUND_HINT_MAX = 3 + + +def _unbound_hint(rt, table_key): + """⭐ WAVE 30 · T14 — the *"…which column to mark"* half of D-79's sentence. + + Returns `" — this database's text columns are Handle, Who"` or `""`. Never a binding. + + ⛔ IT SUGGESTS AND DOES NOT RESOLVE, and the distinction is the whole reason this is a string + rather than a fallback. `profile_field_key`'s own docstring refuses to guess — *"the failure + would be a run that reports success having enriched from the wrong column … a wrong number is + harder to notice than a missing one"*. That argument is about BINDING. It says nothing against + telling a person, in the sentence they are already reading, which columns are even eligible: + the enrich still refuses, nothing is written, and the human makes the choice. + ⚠ `text` ONLY, matching the writer F shipped for the flag (`ColumnMenu`'s toggle is gated on + `editType === "text"`), so the sentence cannot offer a column the editor would then refuse. + """ + fields = (ut_get(rt, table_key) or {}).get("fields") or [] + # ⚠ AND A PRESET/LOCKED COLUMN IS NOT OFFERED. `Platform` is a `text` column on every preset + # profile table and carries `automation.preset` + an `editRole`, so the field editor refuses to + # retype it — offering it would send a person to a control that says no, which is exactly the + # claim the docstring above makes and did not honour on its first draft. + names = [str(f.get("label") or f.get("key") or "").strip() for f in fields + if str(f.get("type") or "text") == "text" + and not (f.get("automation") or {}).get("preset") + and not str(f.get("editRole") or "").strip() + and str(f.get("label") or f.get("key") or "").strip()] + if not names: + return "" + shown = ", ".join(names[:UNBOUND_HINT_MAX]) + more = len(names) - UNBOUND_HINT_MAX + return (f" — this database's text columns are {shown}" + + (f" (+{more} more)" if more > 0 else "")) + + +def _tt_write_tables(rt, tag, username, snaps, posts, psnaps, comments, log=print): + """The four `ut_tt_*` tables, created-if-needed and written in ONE coalesced store update. + Returns `(inserted_by_table, capped, missing)`. + + ⭐⭐ WAVE 30 · D-156 — ONE WRITER, TWO CALLERS, AND THE SECOND CALLER IS WHY IT EXISTS. + This was the tail of `_tt_enrich_flush`, i.e. reachable only from an INLINE enrich. The + deferred collector needs exactly the same write, and the repo has already paid once for the + version where it did not: `top_up_views` lived inside `pull_profile_bd`, so it ran only when + the Posts scrape answered in time, and every DEFERRED Instagram run wrote posts with a blank + Views column. The fix there was this same shape — one function, two callers, so the inline and + deferred paths cannot answer differently — and copying the block instead would reintroduce the + class rather than the bug. + + ⭐ ONE LOOP OVER `(table, rows, key)` RATHER THAN FOUR HAND-WRITTEN BLOCKS, and the field list + comes from `TT_TABLE_FIELDS` — so a fifth `ut_tt_*` table is one tuple, and no table can be + created with a field list that disagrees with its own declaration. + ⚠ NO ROWS ⇒ NO DATABASE. Spawning `ut_tt_comments` on a collect that carried none gives a + person a database to watch never fill, which is the visible half of green-over-nothing. + """ + plan = [(TT_SNAPSHOTS_TABLE, snaps or [], "snapshot_key"), + (TT_POSTS_TABLE, posts or [], "shortcode"), + (TT_POST_SNAPSHOTS_TABLE, psnaps or [], "post_snapshot_key"), + (TT_COMMENTS_TABLE, comments or [], "comment_key")] + written, capped, missing = {}, 0, [] + inserted = {} + for table_key, rows_in, key_field in plan: + if not rows_in: + continue + # ⭐ R9 (W31-T32) — THE RUN PATH IS THE SITE THAT ACTUALLY CREATED TODAY'S TABLES, and it + # is the one the ticket's `how:` does not name. `waves/wave30/proof/tiktok-e2e-ut_tt_posts + # .png` shows `ut_tt_posts` with 8 real rows offering "+ New record" — those rows arrived + # HERE, not through the save path (which, until W31-T34, returned before the child spawn for + # a discovery automation). Stamping only the save site would have left every table that + # already exists unlocked. ⭐ And an EXISTING table does come forward on the next call: + # `ut_ensure`'s skip test carries `not (record_mode and have.get("recordMode") != + # record_mode)`, so no separate migration is needed. + real_key = ut_ensure(rt, TT_TABLE_LABELS[table_key], TT_TABLE_FIELDS[table_key], username, + key=table_key, flow_tag=tag, lock_fields=True, + record_mode=tt_record_mode(table_key)) + missing.extend(ut_missing(rt, real_key)) + merged, counts_ = upsert_rows(dict((ut_get(rt, real_key) or {}).get("rows") or {}), + rows_in, key_field, cap=row_cap(real_key)) + written[real_key] = merged + inserted[table_key] = counts_["inserted"] + capped += counts_["capped"] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for k, rws in written.items(): + if cur.get(k) is not None: + cur[k]["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + if written: + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for every table + return inserted, capped, missing + + +def _tt_enrich_flush(rt, defn, username, acc, log): + """⭐⭐ WAVE 30 · T08 — the TikTok enrich action's append table, written ONCE for the whole run. + + The twin of `_enrich_flush` and deliberately NOT a call into it. That function is bound to + Instagram at four points — `ensure_ig_graph`, the four `IG_*_TABLE` keys, `PRESET_*` and the + write-through to the platform Instagram master — and the last of those is the one that would + look harmless: `ig_master.append_run` is what the metric FIELDS read (W29-T34), so a TikTok + creator's followers appended there would become an Instagram number in a pooled history that + no later run could tell apart. + + ⛔ EVERY KEY IT RETURNS IS PREFIXED `tt…`. `apply_actions` merges both flushes into one + `counts`, so `enriched`/`enrichBlocked` would be one name for two networks' numbers and a + mixed flow would report whichever flushed last. + + ⭐ WAVE 30 · T10 — FOUR TABLES NOW, AND EACH IS CREATED ONLY WHEN IT HAS A ROW. Spawning + `ut_tt_posts` on a run that captured no posts gives a person a database to watch never fill, + which is the visible half of green-over-nothing and the same complaint that produced + `discover_default_table` (*"a SECOND, empty database"*). So the create rides the rows. + """ + if "unbound" in (acc.get("notes") or []): + # D-79 on the second network: a COUNT, so `run_now`'s existing non-zero merge carries it + # and the run entry itself records that the step could not run. Its own key, so the + # summary can name TikTok rather than borrowing Instagram's sentence. + return {"ttEnrichUnbound": 1} + if not acc.get("profiles"): + # The selection's note outlives a run that read nothing — the same reason as the IG side: + # "every candidate was skipped" is exactly the run whose silence needs explaining. + notes = list(acc.get("notes") or []) + return {RUN_NOTES_KEY: notes} if notes else {} + out = {"ttEnriched": acc["ok"], "ttEnrichBlocked": acc["blocked"]} + if acc.get("notes"): + out[RUN_NOTES_KEY] = list(acc["notes"]) + # ⭐⭐ W30 · D-156 — QUEUED BEFORE THE DRY-RUN AND EMPTY-ROWS RETURNS BELOW, and the order is + # the point. A run whose media batch DEFERRED has no posts and no comments to write, so it + # takes the `not (snaps or posts or ...)` exit — the exact run that owns a paid snapshot id. + # Filing after that return would have collected nothing, forever, which is the shape the + # inline-vs-deferred split keeps producing. + # ⚠ Its own count key (`tt…`), like every other key here: `apply_actions` merges both + # networks' flushes into ONE dict, so sharing Instagram's name would let a mixed flow report + # one network's pending batches under the other's. + if not acc.get("dry"): + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), + acc.get("pending") or []) + if queued: + out["ttEnrichMetricBatchesPending"] = queued + if acc.get("dry") or not (acc.get("snaps") or acc.get("posts") or acc.get("psnaps") + or acc.get("comments")): + # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. + if acc.get("dry"): + out["ttEnrichDryRun"] = acc["profiles"] + return out + tag = str(defn.get("id") or "") + inserted, capped, missing = _tt_write_tables( + rt, tag, username, acc.get("snaps"), acc.get("posts"), acc.get("psnaps"), + acc.get("comments"), log) + # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which + # is the failure a chart cannot show you. + if capped: + out["ttEnrichCapped"] = capped + log(f"[aios-auto] enrich_tiktok: {capped} row(s) refused by a table's row cap") + if missing: + out["ttEnrichMissingTables"] = len(missing) + log(f"[aios-auto] enrich_tiktok: could not create {', '.join(missing)}") + out["ttEnrichPoints"] = len(acc.get("snaps") or []) + if inserted.get(TT_POSTS_TABLE): + out["ttEnrichPosts"] = inserted[TT_POSTS_TABLE] + if inserted.get(TT_COMMENTS_TABLE): + out["ttEnrichComments"] = inserted[TT_COMMENTS_TABLE] + # ⛔ AND NO PLATFORM-MASTER WRITE-THROUGH, which is a deliberate absence rather than a missing + # line. `ig_master` is Instagram's pooled history and there is no TikTok equivalent yet; the + # tenant's own `ut_tt_snapshots` is the complete record today, and inventing a second store + # for a series nobody reads would be the fork this function exists to avoid. + return out + + +_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS)) +_PENDING_METRIC_KINDS = frozenset(("posts", "comments")) + +#: ⭐⭐ WAVE 30 · D-156 — THE TIKTOK HALF OF THE METRIC QUEUE, BUILT LAZILY FROM `connectors_tt`'s +#: OWN CONSTANTS. Re-declaring the two ids here would be a second copy of a vendor identifier that +#: nothing compares — the drift `MAX_UT_ROWS` demonstrated at 12x — so this reads them from the one +#: module that owns them, through `_tt_module()` because `connectors_tt` imports THIS module. +#: ⚠ It is a MAP rather than a set because on TikTok one dataset is exactly one kind, which is why +#: this network needs no `_tag_metric_deferrals` twin: the id the transport already recorded says +#: whether a batch is posts or comments, so nothing downstream has to be told twice. +_TT_METRIC_KIND_BY_DATASET = None + + +def tt_metric_kind(dataset_id): + """`"posts"` / `"comments"` for a TikTok media dataset id; `""` for anything else. + + ⛔ THE `""` IS LOAD-BEARING AND IS THE PLATFORM TEST. `collect_pending_metric_snapshots` + branches on it, so a dataset this map does not know keeps Instagram's mappers — which is the + safe direction, because Instagram's are what every stored pre-wave-30 task needs. + """ + global _TT_METRIC_KIND_BY_DATASET + if _TT_METRIC_KIND_BY_DATASET is None: + _tt = _tt_module() + _TT_METRIC_KIND_BY_DATASET = {str(_tt.TT_DS_POSTS): "posts", + str(_tt.TT_DS_COMMENTS): "comments"} + return _TT_METRIC_KIND_BY_DATASET.get(str(dataset_id or ""), "") + + +def _pending_metric_tasks(defn): + """Read validated, deduplicated paid metric snapshots from automation continuation state. + + A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally + stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or + the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor + endpoint, and the collector never starts a second scrape request. + """ + raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or [] + out, seen = [], set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + sid = str(item.get("snapshotId") or "").strip() + dataset = str(item.get("datasetId") or "").strip() + kind = str(item.get("kind") or "").strip() + handle = str(item.get("influencer") or "").strip().lstrip("@").lower() + # ⭐ W30 · D-156 — BOTH NETWORKS' MEDIA CORPORA ARE COLLECTABLE NOW. The membership test + # stays a WHITELIST (a snapshot id is a vendor capability that has already been paid for; + # accepting an unknown dataset would send our key at a corpus no mapper here can read), + # and TikTok's half is asked of the map that owns it rather than listed again. + if (not sid.startswith("sd_") + or (dataset not in _PENDING_METRIC_DATASETS and not tt_metric_kind(dataset)) + or kind not in _PENDING_METRIC_KINDS or not handle): + continue + key = (sid, dataset, kind, handle) + if key in seen: + continue + seen.add(key) + out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind, + "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""), + "lastChecked": str(item.get("lastChecked") or ""), + "lastNote": _s(item.get("lastNote"), 160)}) + return out + + +def queue_pending_metric_snapshots(rt, auto_id, pending): + """Durably retain new engagement snapshots without creating another paid provider request.""" + aid = str(auto_id or "").strip() + if not aid: + return 0 + incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}}) + if not incoming: + return 0 + added = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + definition = cur.get(aid) + if not isinstance(definition, dict): + return cur + state = definition.setdefault("state", {}) + existing = _pending_metric_tasks(definition) + known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"]) + for x in existing} + for task in incoming: + key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"]) + if key not in known: + existing.append(task) + known.add(key) + added[0] += 1 + state["pendingMetricSnapshots"] = existing + return cur + + _store_update(rt, _up, flush="sync") + return added[0] + + +def _match_profile_row(rows, handle): + """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there. + + ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With + `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and + taking the first row would write ONE profile's followers onto all of them — a wrong number that + looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the + same field `_bd_profile` reads first. + + ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points + at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that + shape the task's own handle is still the best evidence. It applies only when the snapshot holds + ONE row, so it can never mis-assign inside a batch. + """ + want = str(handle or "").strip().lstrip("@").lower() + usable = [n for n in (rows or []) if isinstance(n, dict)] + for node in usable: + got = str(_first(node, "account", "username", default="") or "").strip() + if got.lstrip("@").lower() == want and want: + return node + if len(usable) == 1: + return usable[0] + return None + + +def _pending_profile_tasks(defn): + """Validated, deduplicated deferred PROFILE snapshots from automation state. + + ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather + than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and + `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to + zero by its own validator, and even if it survived, the collector would hand it to + `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile + row. Reusing the name would have shipped a green no-op of exactly the class this change + exists to remove. + + ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written + back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be + resolved from the handle alone: two databases may both hold `@x`. + """ + raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or [] + out, seen = [], set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + sid = str(item.get("snapshotId") or "").strip() + dataset = str(item.get("datasetId") or "").strip() + handle = str(item.get("influencer") or "").strip().lstrip("@").lower() + table = str(item.get("table") or "").strip() + row_id = str(item.get("rowId") or "").strip() + # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a + # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor + # endpoint at all. + if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle + or not table or not row_id): + continue + if sid in seen: + continue + seen.add(sid) + out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile", + "influencer": handle, "table": table, "rowId": row_id, + "requestedAt": str(item.get("requestedAt") or ""), + "lastChecked": str(item.get("lastChecked") or ""), + "lastNote": _s(item.get("lastNote"), 200)}) + return out + + +def queue_pending_profile_snapshots(rt, auto_id, pending): + """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many + were newly added.""" + aid = str(auto_id or "").strip() + if not aid: + return 0 + incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}}) + if not incoming: + return 0 + added = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + definition = cur.get(aid) + if not isinstance(definition, dict): + return cur + state = definition.setdefault("state", {}) + existing = _pending_profile_tasks(definition) + known = {x["snapshotId"] for x in existing} + for task in incoming: + if task["snapshotId"] not in known: + existing.append(task) + known.add(task["snapshotId"]) + added[0] += 1 + state["pendingProfileSnapshots"] = existing + return cur + + _store_update(rt, _up, flush="sync") + return added[0] + + +def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step): + """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape. + + ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred` + list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the + vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown + away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a + 180 s budget, two abandoned `sd_…` snapshots in two runs. + + ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same + forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero + records ends the task with the vendor's reason attached — and, when the vendor blames the + target rather than itself, records the not-found verdict so the handle stops being re-bought. + """ + tasks = _pending_profile_tasks(defn) + if not tasks: + return ("ok", "No pending profile reads.", {}, [], {"source": "idle"}) + step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}") + remaining, patches, affected, notes = [], {}, [], [] + acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0, + "notes": [], "dry": False} + ready = waiting = closed = 0 + # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME + # `snapshotId`, and asking the vendor about it once per task would spend the round trips the + # batching just saved. Memoised for this collection pass only — a snapshot's state must be + # re-read on the NEXT run, which is a fresh call and a fresh dict. + _progress_memo, _rows_memo = {}, {} + + def _progress_of(sid): + if sid not in _progress_memo: + _progress_memo[sid] = bd_snapshot_progress(sid) + return _progress_memo[sid] + + def _store_profile(task, profile, via): + """Write ONE collected profile — snapshot row + preset cells on the record that asked. + + ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the + two paths cannot drift into writing different things — the same rule `top_up_views` + follows for the inline/deferred split. + """ + res = {"state": "ok", "profile": profile, "posts": [], "comments": [], + "via": via, "note": ""} + pulled = _iso() + snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) + acc["snaps"].append(snap_row) + acc["posts"].extend(idents) + acc["psnaps"].extend(metric_rows) + acc["comments"].extend(comment_rows) + _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled)) + affected.append(task["rowId"]) + + def _backup_profile(handle): + """The second rung, asked ONLY once the first has finished and delivered nothing. + + ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS + CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but + a profile that DEFERRED never came back through `pull_profile` — it came back here, and + this collector had no second rung at all. So the one path where the primary is most + likely to have failed was the one path with no fallback. + + ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary + answered is never re-bought from a second vendor — the mistake the capability split exists + to prevent. A refusal returns None and the caller reports blocked exactly as before. + """ + try: + import providers as _p + if not _p.PROVIDERS["apify"].can("ig_profile"): + return None + prof, note = apify_profile(str(handle)) + except Exception as exc: # noqa: BLE001 — a backup must not raise + log(f"[aios-auto] backup profile rung failed for @{handle}: " + f"{type(exc).__name__}: {exc}") + return None + if prof and prof.get("followers") is not None: + return prof + return None + + def _rows_of(sid): + if sid not in _rows_memo: + payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) + got = _bd_rows(payload) if not err else [] + if got and (_bd_deferral(payload) or + (len(got) == 1 and str(got[0].get("status") or "") in + ("running", "building", "collecting"))): + got = [] + _rows_memo[sid] = got + return _rows_memo[sid] + + for task in tasks: + stale_h = _hours_since(task.get("requestedAt")) + state, records, empty_note = _progress_of(task["snapshotId"]) + if state in ("done", "failed") and not records: + acc["profiles"] += 1 + # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing, + # so there is nothing left to wait for and no risk of buying the same record twice. + backup = _backup_profile(task["influencer"]) + if backup is not None: + ready += 1 + acc["ok"] += 1 + _store_profile(task, backup, "apify") + note = (f"@{task['influencer']}: the primary source returned nothing, so a backup " + f"source supplied the profile ({backup.get('followers')} followers)") + notes.append(note) + acc["notes"].append(note) + continue + closed += 1 + acc["blocked"] += 1 + note = f"@{task['influencer']}: {_s(empty_note, 240)}" + notes.append(note) + acc["notes"].append(note) + # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a + # profile request that means the account could not be reached at all, so the verdict + # is remembered and the selection stops re-buying it (R9: permanently, until a + # human edits the handle cell — see `clear_gone`). + # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about + # the query, and turning it into "this account does not exist" would silently retire + # live handles. + if state == "failed": + acc["gone"][_gone_key({}, task["influencer"])] = { + "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)} + continue + rows = _rows_of(task["snapshotId"]) if state != "running" else [] + if not rows: + # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of + # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never + # silently. An unbounded queue is the forever-loop this change removes, inverted. + if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS: + closed += 1 + note = (f"@{task['influencer']}: the source never finished the profile read " + f"queued {int(stale_h)}h ago ({task['snapshotId']}) — it was dropped; " + f"the next run will ask again") + notes.append(note) + acc["notes"].append(note) + continue + waiting += 1 + remaining.append({**task, "lastChecked": _iso(), + "lastNote": _s("still building", 200)}) + continue + # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot + # holds several profiles and `rows[0]` would write one creator's numbers onto every record + # in the chunk. + node = _match_profile_row(rows, task["influencer"]) + if node is None: + closed += 1 + acc["profiles"] += 1 + acc["blocked"] += 1 + note = (f"@{task['influencer']}: the source delivered {len(rows)} profile" + f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle — " + f"it was dropped from the batch and the next run will ask again") + notes.append(note) + acc["notes"].append(note) + continue + ready += 1 + acc["profiles"] += 1 + profile = _bd_profile(node, task["influencer"]) + if profile.get("followers") is None and profile.get("following") is None: + acc["blocked"] += 1 + note = (f"@{task['influencer']}: the source delivered the profile but no " + f"follower/following counts were readable in it") + notes.append(note) + acc["notes"].append(note) + continue + acc["ok"] += 1 + _store_profile(task, profile, "brightdata:deferred") + notes.append(f"@{task['influencer']}: collected the profile the source had already been " + f"paid for ({profile.get('followers')} followers)") + + set_state(rt, str(defn.get("id") or ""), + {"pendingProfileSnapshots": remaining or None}) + counts = _enrich_flush(rt, defn, username, acc, log) + counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below + counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {}, + username, log)) + counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting, + "profileBatchesEmpty": closed}) + if notes: + counts[RUN_NOTES_KEY] = notes + head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected" + if ready else "no deferred profile read was ready") + tail = "".join([f"; {closed} finished with nothing to collect" if closed else "", + f"; {waiting} still building" if waiting else ""]) + state = "ok" if ready and not closed else "partial" + return (state, head + tail, counts, affected, + {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"}) + + +def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log): + """Write a completed metric snapshot through the canonical Post/Comment graph once.""" + if not (idents or snapshots or comments): + return {"posts": 0, "snapshots": 0, "comments": 0} + profile_table = str(_flow_table(defn) or "") + if not profile_table: + raise Refused("the pending engagement snapshot has no Profile database to link to") + graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=profile_table) + post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE], + graph[IG_COMMENTS_TABLE]) + old_posts, collapsed_posts = dedupe_canonical_rows( + dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots, + "post_snapshot_key", cap=row_cap(ps_key)) + old_comments, collapsed_comments = dedupe_canonical_rows( + dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") + comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key", + cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)): + if cur.get(key) is not None: + cur[key]["rows"] = rows + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + try: + import ig_master + status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots) + if status == "error": + log(f"[aios-auto] deferred post metrics master copy FAILED: {note}") + except Exception as exc: # noqa: BLE001 + log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}") + return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"], + "comments": c_comments["inserted"]} + + +def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step): + """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request.""" + tasks = _pending_metric_tasks(defn) + if not tasks: + return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"}) + step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}") + remaining, idents, snapshots, comments = [], [], [], [] + # ⭐ W30 · D-156 — TikTok's rows accumulate SEPARATELY and are written by TikTok's own writer. + # One queue can hold both networks' snapshots (they are keyed by dataset), but the two write + # paths address different tables and neither may touch the other's. + tt_collected = {"posts": [], "psnaps": [], "comments": []} + ready, waiting, closed, run_notes = 0, 0, 0, [] + for task in tasks: + # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be + # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued + # as "still building" on every tick — forever, because nothing about it would ever + # change. That is the same forever-loop the profile path was measured in, one dataset + # over, and it was latent here the whole time. + state, records, empty_note = bd_snapshot_progress(task["snapshotId"]) + if state in ("done", "failed") and not records: + # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect; + # keeping the entry would be a pending task that can never resolve. + closed += 1 + run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}") + continue + payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"}) + rows = _bd_rows(payload) if not note else [] + building = (state == "running" or not rows or _bd_deferral(payload) or + (len(rows) == 1 and str(rows[0].get("status") or "") in + ("running", "building", "collecting"))) + if building: + remaining.append({**task, "lastChecked": _iso(), + "lastNote": _s(note or "still building", 160)}) + waiting += 1 + continue + ready += 1 + pulled = _iso() + # ⭐⭐ WAVE 30 · D-156 — THE TIKTOK ARM. Before this, `_PENDING_METRIC_DATASETS` was + # Instagram's three, so a TikTok media batch the vendor deferred was paid for and + # collected by NOTHING — the note carried the snapshot id and a human was the only + # collector. It routes through `connectors_tt`'s own mappers and `tt_capture_rows`, + # never Instagram's, because every one of those stamps `PLATFORM_INSTAGRAM`. + # ⛔ TWO THINGS THE INSTAGRAM ARM DOES THAT THIS ONE MUST NOT, both deliberate: + # * NO `top_up_views`. TikTok's `play_count` arrives inline on the posts row, so there + # is no views capability to route to; calling it would buy an Instagram permalink. + # * NO `ig_master.append_run`. There is no TikTok master by design (`_tt_enrich_flush`), + # and a write-through would put a TikTok creator into Instagram's pooled history, + # which the metric FIELDS read — a wrong number in a permanent series. + if tt_metric_kind(task["datasetId"]): + _tt = _tt_module() + who = task["influencer"] + if task["kind"] == "posts": + mapped = [m for m in (_tt.normalize_post(r) for r in rows) if m] + for m in mapped: + # The snapshot was bought from THIS profile's own permalinks, so the backlink + # is a fact of the call even when a vendor row omits the author field. + m.setdefault("influencer_key", who) + res_tt = {"profile": {"handle": who}, "posts": mapped, "comments": []} + else: + mapped = [m for m in (_tt.normalize_comment(r) for r in rows) if m] + res_tt = {"profile": {"handle": who}, "posts": [], "comments": mapped} + tt_idents, tt_metrics, tt_comments = tt_capture_rows(res_tt, pulled) + tt_collected["posts"].extend(tt_idents) + tt_collected["psnaps"].extend(tt_metrics) + tt_collected["comments"].extend(tt_comments) + continue + if task["kind"] == "posts": + mapped_posts = [] + for raw in rows: + post = _bd_post_metrics(raw) + if post: + # The snapshot was created from this Profile's canonical post URLs. Keep + # that explicit backlink even when a vendor response omits `user_posted`. + post["influencer_key"] = task["influencer"] + mapped_posts.append(post) + if mapped_posts: + # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the + # owner's `theresalearns` run filled every column except Views. + # + # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a + # Posts row physically cannot carry a view count — MEASURED on the stored + # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them. + # Views only ever comes from the Apify capability. That top-up lived INSIDE + # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait + # budget; when the batch deferred — which is routine, and what happened here — the + # rows came back through THIS function and Apify was never asked. + # ⇒ `top_up_views` is now one function with two callers rather than a copy, so + # the inline and deferred paths cannot answer this differently again. + # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which + # is what freezes the values into the post + snapshot rows. + v_note = top_up_views(mapped_posts, log=log) + if v_note: + run_notes.append(f"@{task['influencer']}: {v_note}") + _unused, post_rows, metric_rows, embedded = capture_rows( + {"state": "ok", "profile": {"username": task["influencer"]}, + "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled) + idents.extend(post_rows) + snapshots.extend(metric_rows) + comments.extend(embedded) + else: + for raw in rows: + comment = _bd_comment(raw, influencer_key=task["influencer"]) + if comment: + comments.append(comment) + + set_state(rt, str(defn.get("id") or ""), + {"pendingMetricSnapshots": remaining or None}) + written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log) + counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting, + "metricBatchesEmpty": closed, + "postEngagementSnapshots": written["snapshots"], + "commentsCollected": written["comments"]} + # ⭐ W30 · D-156 — the TikTok write, through the SAME function the inline enrich uses, so a + # collected batch and an inline one cannot land differently. Its counts carry the `tt` prefix + # for the reason every other TikTok count does. + if any(tt_collected.values()): + tt_inserted, tt_capped, tt_missing = _tt_write_tables( + rt, str(defn.get("id") or ""), username, [], tt_collected["posts"], + tt_collected["psnaps"], tt_collected["comments"], log) + for key, name in ((TT_POSTS_TABLE, "ttPostsCollected"), + (TT_POST_SNAPSHOTS_TABLE, "ttPostSnapshotsCollected"), + (TT_COMMENTS_TABLE, "ttCommentsCollected")): + if tt_inserted.get(key): + counts[name] = tt_inserted[key] + if tt_capped: + counts["ttCollectCapped"] = tt_capped + log(f"[aios-auto] collect: {tt_capped} TikTok row(s) refused by a table's row cap") + if tt_missing: + log(f"[aios-auto] collect: could not create {', '.join(tt_missing)}") + # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved + # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four + # other call sites disagree about the shape of a run — and `_commit_run` already drops + # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather + # than to a crash. + if run_notes: + counts[RUN_NOTES_KEY] = run_notes + # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is + # a real outcome the tenant paid for and it must read as an answer, not as a disappearance. + tail = (f"; {closed} finished with nothing to collect" if closed else "") + if waiting: + return ("partial", + f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; " + f"{waiting} still building and will be collected automatically{tail}", + counts, [], {"capture_posts": "partial", "write": "ok"}) + return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}", + counts, [], {"capture_posts": "ok", "write": "ok"}) + + +def ai_decide(rt, defn, act, row, row_id="", log=print): + """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" + to leave the card for a person. + + ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed + answer, or a label the review does not offer all return "" — and "" means the card sits at + the review gate exactly as it would with no AI at all. The feature can be broken, absent or + wrong and the worst outcome is a human doing the work. + + ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN + ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board + under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely + worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter → + anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is + MISSING is only the door — an action kind that asks a question and takes an answer, without a + stage column to write it into. **Do not delete this in a dead-code sweep without reading that + sentence first**; equally, do not treat it as shipped — nothing reaches it today. + """ + cfg = act.get("config") or {} + options = list(cfg.get("next") or []) + if not options: + return "" + try: + import ai_review + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") + return "" + fields = [f.get("key") for f in + (ut_get(rt, ((defn.get("config") or {}).get("targetTable") + or (defn.get("trigger") or {}).get("table") or "")) or {} + ).get("fields") or []] + choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, + row=row, fields=[f for f in fields if f], + label=cfg.get("label") or "Review") + if not choice: + if meta.get("problem"): + log(f"[aios-auto] ai review declined to answer: {meta['problem']}") + return "" + review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", + choice, meta.get("provider") or "ai", by="ai", + note=meta.get("reason") or "", model=meta.get("model") or "") + return choice + + +def find_records(rt, table_key, cond, limit=25): + """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts + the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" + rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} + out = [] + for rid, row in rows.items(): + if lane_match(cond, row or {}): + out.append(str(rid)) + if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): + break + return out + + +def _commit_action_writes(rt, table, patches, creates, username, log): + """The ONE write. Row patches merge into the target's rows; creates land in their own tables — + APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a). + + Returns the REALIZED create counts, and returning them is the point rather than a + convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only + this function knows how many of those became rows, how many matched one that was already + there, and how many the cap refused. A run that reported the attempt as "created" would be + the summary-disagrees-with-what-happened defect this module names in three other places. + + ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE + implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand + rolling a third inside this function is that debt returning with a new name — and it would + quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED, + NEVER DELETED, and `capped` is its own count rather than folded into `skipped`). + """ + out = {"created": 0, "createUpdated": 0, "createUnchanged": 0, + "createCapped": 0, "createSkipped": 0} + if not patches and not creates: + return out + # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one + # `ut_write_rows` per patched table, and that was free while the only patcher was a review + # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the + # step it reached, so the flow's own table is now patched on essentially every run — and a + # flow that also creates into that same table would have committed it TWICE per run, against + # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around. + # Staged into `staged` and handed to the creates pass, which writes each table exactly once. + staged = {} + for tkey, rowpatch in (patches or {}).items(): + cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) + for rid, vals in rowpatch.items(): + cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} + staged[str(tkey)] = cur + # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The + # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once + # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr + # repo budget this whole module is shaped around. + by_table = {} + for (tkey, unique), new_rows in (creates or {}).items(): + by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows)) + for tkey, groups in by_table.items(): + t = ut_get(rt, tkey) + if t is None: + n = sum(len(r) for _u, r in groups) + log(f"[aios-auto] create_record: {tkey} no longer exists — {n} skipped") + out["createSkipped"] += n + continue + # The PATCHED rows when this table was also stamped this run, so the creates land on top + # of the stamp rather than on a copy of the store that predates it. + cur = staged.pop(tkey, None) + cur = dict(t.get("rows") or {}) if cur is None else cur + cap = row_cap(tkey) + for unique, new_rows in groups: + if unique: + cur, c = upsert_rows(cur, new_rows, unique, cap=cap) + out["created"] += c["inserted"] + out["createUpdated"] += c["updated"] + out["createUnchanged"] += c["unchanged"] + out["createCapped"] += c["capped"] + # ⚠ `skipped` here means "this row had no value for the unique key", which for a + # create action is a mapped value that interpolated to nothing — worth surfacing, + # because the symptom is otherwise a run that says it created less than it walked. + out["createSkipped"] += c["skipped"] + if c["capped"]: + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — " + f"{c['capped']} row(s) not written") + continue + nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 + for i, vals in enumerate(new_rows): + if len(cur) >= cap: + # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is + # named, here and in the run's counts. This used to `break` with a log line + # that said the cap was hit and never said how much was lost. + out["createCapped"] += len(new_rows) - i + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — " + f"{len(new_rows) - i} row(s) not written") + break + cur[str(nxt)] = dict(vals) + nxt += 1 + out["created"] += 1 + ut_write_rows(rt, tkey, cur) + # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and + # once each, so "one store write per table" holds whether a table was patched, created into, + # or both. + for tkey, cur in staged.items(): + ut_write_rows(rt, tkey, cur) + return out + + +def _lane_sentence(cond, top=True): + """One condition tree → the sentence a step's `detail` carries on the canvas. + + C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a + label never claims a flat comparison the tree does not make. The SERVER composes it, for the + same reason it composes every other `detail` — a client paraphrase of a structure the engine + evaluates is a second implementation of the same sentence, free to drift from it. + + ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is + live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a + rename would be churn on a working function to fix a word. + """ + if cond is None: + return "Everything else" if top else "" + if isinstance(cond, dict): + for key, joiner in (("all", " and "), ("any", " or ")): + if key in cond: + parts = [_lane_sentence(c, False) for c in cond.get(key) or []] + parts = [p for p in parts if p] + if not parts: + return "" + inner = joiner.join(parts) + return inner if top or len(parts) == 1 else f"({inner})" + v = cond.get("value") + return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") + + +def _rid_num(rid): + return int(rid) if str(rid).isdigit() else 10 ** 9 + + +#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can +#: grow a definition without limit is a serialisation cost wearing a compliance hat. +MAX_REVIEWS = 100 + + +def review_audit(rt, auto_id, row_id, from_label, to_label, username, + by="user", note="", model=""): + """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to + the definition (newest first, bounded). + + ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that + made it and the one-line reason it gave. One audit log, not two — a reader asking "who + decided this card" must not have to know there are two places to look, and the moment an + AI decision is invisible beside a human one the log stops being an audit. + + ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone: + `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote + this shape off a stage field's `flowId` — a field the migration now drops. Kept because the + SHAPE is the contract a future decision action would write, and re-deriving an audit format + is how two of them end up existing. + """ + entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), + "from": _s(from_label, 60), "to": _s(to_label, 60), + "by": "ai" if by == "ai" else "user"} + if note: + entry["note"] = _s(note, 300) + if model: + entry["model"] = _s(model, 60) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] + return cur + + _store_update(rt, _up, flush="sync") + + +# --------------------------------------------------------------------------------------------- +# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) +# --------------------------------------------------------------------------------------------- +# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — +# the value reflects every pull the platform has, not just this tenant's). Computed by the +# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files +# no session owns this wave), human-write-refused at the grid door, and every number drills to +# the exact snapshot rows behind it. +# +# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no +# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero +# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not +# readable") — the `_bd_posts_count` law, one layer up. + +METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") +METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") +#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each +#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; +#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). +PROFILE_MEASURES = ("followers", "avg_engagement") +METRIC_AGGS = ("avg", "sum", "latest") + + +def metric_value(series, measure, window, agg="", today=None): + """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. + + `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference + day; post-count windows count back from the newest post. `drill_rows` are the exact rows + the number came from, so the route can honour [[no-unverifiable-aggregates]] without + recomputing differently.""" + series = series or {} + today = today or _now() + if measure in PROFILE_MEASURES: + rows = [r for r in series.get("snapshots") or [] + if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] + if window in ("last_7d", "last_30d"): + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + rows = [r for r in rows + if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] + if not rows: + return None, [] + if window == "latest" or agg == "latest": + picked = [rows[-1]] + else: + picked = rows + vals = [_lane_num(r.get(measure)) for r in picked] + vals = [v for v in vals if v is not None] + if not vals: + return None, [] + out = vals[-1] if (window == "latest" or agg == "latest") else \ + (sum(vals) if agg == "sum" else sum(vals) / len(vals)) + if measure == "avg_engagement": + # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs + # transform-pct scar) — scaled exactly once, here. + return f"{out * 100:.2f}", picked + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked + # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. + posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] + posts.sort(key=lambda p: str(p.get("posted_at"))) + if window == "latest": + picked_posts = posts[-1:] + elif window == "last_3_posts": + picked_posts = posts[-3:] + else: + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + picked_posts = [p for p in posts + if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) + or _dt.datetime.min) >= floor] + vals, drill = [], [] + for p in picked_posts: + snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] + for snap in reversed(snaps): + raw = snap.get(measure) + v = _lane_num(raw) + if v is not None and str(raw).strip() != "": + vals.append(v) + drill.append(snap) + break + if not vals: + return None, [] + out = sum(vals) if (agg or "sum") == "sum" else \ + (vals[-1] if agg == "latest" else sum(vals) / len(vals)) + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill + + +def metric_fields_of_table(t): + return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] + + +def _table_handle(row, url_field): + return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) + or "").strip().lower() + + +def compute_metric_cells(rt, table_key, today=None): + """Recompute every metric cell on ONE table from the master series. One coalesced write, + only when something actually changed (the flush-ceiling law); zero reads when the table + has no metric fields or the master is off. Returns the number of rows touched.""" + t = ut_get(rt, table_key) + mfields = metric_fields_of_table(t) + if not mfields: + return 0 + import ig_master + if not ig_master.configured(): + return 0 + url_field = next((f.get("key") for f in (t.get("fields") or []) + if f.get("type") == "url"), "") + rows = t.get("rows") or {} + handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} + series = ig_master.series_for({h for h in handles.values() if h}) + changes = {} + for rid, row in rows.items(): + s = series.get(handles.get(rid) or "") + for f in mfields: + bag = f["metric"] + val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), + bag.get("agg") or "", today=today) + if s else (None, [])) + want = "" if val is None else str(val) + if str((row or {}).get(f["key"], "")) != want: + changes.setdefault(str(rid), {})[f["key"]] = want + if not changes: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ──────────────── +# +# Owner: *"adding relational database function with links and rollups… exactly like how Airtable +# does it… and this rollup needs to have formula that we can use to calculate things like average +# Views over last N posts."* +# +# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula +# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not +# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced +# write only when something changed — and inherits its flush-ceiling discipline for free. +# +# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching +# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is +# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as +# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED +# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The +# moment one writes a number nothing else can re-derive, it has become a third copy. + +def _ut(): + """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic. + + Pulling this module in at import time initialises the store layer earlier than + `automation_engine` used to, and the last time that happened it moved a store-commit COUNT + from three to four on an unrelated gate. The relational pass + needs three constants and two predicates from the field layer; it does not need to change + when this module is imported. + """ + import core.user_tables as _m + return _m + + +#: The fns by family — how a value is folded, and what a blank means in each. +_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"}) +_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"}) +#: The count/order family — folded by their own arms above the lanes. +_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"}) +#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how +#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED +#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm +#: claims fell past both lanes into the join below — so a "Std deviation" column rendered +#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now +#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question +#: nothing can answer; a comma-joined list is a different question's answer wearing this label). +_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) +#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED. +#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name +#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot +#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the +#: defect above, because the defect was that the two lists already disagreed. +ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS + | _ROLLUP_TEXT_FNS) +_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) +#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing. +_ROLLUP_JOIN = ", " +#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a +#: cell nobody can read is not an aggregate. +ROLLUP_MAX_CHARS = 4000 + + +def _sort_key(value, ftype): + """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell + is blank or does not parse as its declared type. + + ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a + string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides + WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently + averages the wrong twelve posts. + + ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION + SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")` + otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True` + flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent + real one out of the window. A sentinel inside the key cannot mean "last" in both directions, + because the direction is applied to the sentinel too. + ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as + `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at` + only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every + key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed + because its fixture posts all carried dates. + ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it + sorts, and "an unknown date is not a recent one" is finally what the code does. + """ + raw = "" if value is None else str(value).strip() + if raw == "": + return None + if ftype in ("int", "currency", "pct", "rating"): + n = _lane_num(raw) + return (n,) if n is not None else None + if ftype == "date": + d = _parse_iso(raw.replace(" ", "T")) + return (d.timestamp(),) if d is not None else None + return (raw.lower(),) + + +def _sample_stdev(nums): + """SAMPLE standard deviation (n-1) of `nums`, or None under two values. + + ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a + function: `_rollup_fold` RENDERS this number into a "Std deviation" column and + `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy + would let a column and the filter beside it disagree about the same word on the same set — + the exact drift `top_up_views` was extracted to prevent one module over + ([[one-evaluator-per-question]]). + ⛔ None means UNANSWERABLE and no caller may read it as 0. + """ + if len(nums) < 2: + return None + mean = sum(nums) / len(nums) + return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5 + + +def _rollup_ref_threshold(rows, field, sigmas): + """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1). + + ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`, + and NOT yet filtered by the conditions this threshold feeds. That order is the contract + (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders + produce a plausible number: computing it before `limit` answers "2 sigma of everything this + account ever posted" under a column that says "of the last 10", and computing it after the + filter makes the threshold depend on the rows it is choosing — a definition that chases + itself. + + Returns None when the window cannot answer — fewer than two numeric values in `field`. + ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question + with a permissive answer; letting an unanswerable leaf pass everything would silently turn + "the outliers" into "all of them", which is this module's worst failure mode wearing a filter. + """ + nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None] + sd = _sample_stdev(nums) + if sd is None: + return None + return (sum(nums) / len(nums)) + float(sigmas) * sd + + +def _rollup_fold(fn, values, ftype="text"): + """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING. + + Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's + oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a + question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement + that the creator gets no views. + ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records" + over an empty set is genuinely 0, not unknown. + + `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only + `min`/`max` read it today; it defaults to `text` so every existing caller and every test that + folds a bare list keeps its exact previous answer. + """ + if fn == "countall": + return str(len(values)) + if fn == "latest": + # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve + # a blank on the newest row rather than reaching backwards and presenting an older value + # as current. + return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS] + if fn == "counta": + return str(len([v for v in values if str(v or "").strip() != ""])) + if fn == "count": + # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them + # distinct is the whole reason both exist. + return str(len([v for v in values if _lane_num(v) is not None])) + # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN. + # + # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered + # BLANK forever while looking completely configured — the failure this module refuses + # everywhere else, arriving through the one fold that had no type awareness. "Earliest order" + # and "latest invoice due" are the two most ordinary date rollups there are, and + # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates + # and the fold did not. + # + # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a + # string-compare fallback would have worked on well-formed data and silently mis-ordered a + # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is + # already at this call site (`linked_types`), and `_sort_key` is already the one function that + # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather + # than growing a second idea of what a date is. + # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person + # wants in the column is the stored date string exactly as the source row spells it. + # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean + # of two timestamps is a number this product has no column type for, and inventing one here + # would be a value with no author. + if fn in ("min", "max") and ftype == "date": + keyed = [(k, str(v)) for k, v in + ((_sort_key(v, "date"), v) for v in values) if k is not None] + if not keyed: + return "" + return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS] + if fn in _ROLLUP_NUM_FNS: + nums = [n for n in (_lane_num(v) for v in values) if n is not None] + if not nums: + return "" + if fn == "stdev": + # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference + # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to + # be LINKED, which is a sample of an account's posting history and not its entirety. + # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION + # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently + # switched divisors. + # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the + # honest reason is upstream of the arithmetic: one measurement has no spread to + # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the + # single most confident thing this column can say, asserted from a single row. Same + # law as the blank `sum`, and it bites harder here. + out = _sample_stdev(nums) + if out is None: + return "" + else: + out = (sum(nums) if fn == "sum" else min(nums) if fn == "min" + else max(nums) if fn == "max" else sum(nums) / len(nums)) + return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}" + if fn in _ROLLUP_BOOL_FNS: + # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero". + flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values] + if not flags: + return "" + hit = (all(flags) if fn == "and" else any(flags) if fn == "or" + else sum(1 for f in flags if f) % 2 == 1) + return "1" if hit else "" + # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`. + # An fn no arm above recognises returns "" rather than a comma-joined dump of every value, + # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28. + if fn not in _ROLLUP_TEXT_FNS: + return "" + vals = [str(v).strip() for v in values if str(v or "").strip() != ""] + if fn == "arrayunique": + seen, uniq = set(), [] + for v in vals: + if v.lower() not in seen: + seen.add(v.lower()) + uniq.append(v) + vals = uniq + if not vals: + return "" + text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals)) + return text[:ROLLUP_MAX_CHARS] + + +def _link_from_key(fields, bag): + """Which column on THIS table supplies the join value. + + Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That + fallback chain is what makes an Instagram database link up with no configuration at all — + the "automatically" in the owner's instruction — and it is the same chain the grid already + uses to decide a table's identity column, rather than a second opinion about it. + """ + declared = str((bag or {}).get("from") or "").strip() + if declared: + return declared + prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None) + if prof: + return str(prof.get("key") or "") + pin = next((f for f in fields if f.get("pinned") is True), None) + return str(pin.get("key") or "") if pin else "" + + +def _linked_rows_by_join(linked, on_key): + """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE. + + ⚠ Lower-cased because the join values this exists for are Instagram handles, which the + profile flag already normalises to lower case on one side and which a hand-typed cell on the + other side may not. A join that misses on case is a relation that silently reports zero. + """ + idx = {} + for rid, row in ((linked or {}).get("rows") or {}).items(): + k = str((row or {}).get(on_key) or "").strip().lower() + if k: + idx.setdefault(k, []).append((str(rid), row or {})) + return idx + + +def _rollup_condition_matches(row, condition, field_types, ref_value=None): + """Evaluate one Airtable-style linked-record condition against a candidate row. + + `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the + caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed + here because this function sees one row and the statistic is a property of the whole window — + a version that reached for the set from inside would be recomputing the same mean once per + row, and would have to be handed the window anyway. + ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the + threshold helper for why the permissive reading is the dangerous one. + """ + field = str((condition or {}).get("field") or "") + op = str((condition or {}).get("op") or "") + raw = (row or {}).get(field) + text = str(raw or "").strip() + if op == "is_empty": + return text == "" + if op == "is_not_empty": + return text != "" + if (condition or {}).get("ref") is not None: + # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering + # ops, and a row whose cell is blank or unparseable has no position relative to a computed + # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key` + # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]). + left_num = _lane_num(text) + if ref_value is None or left_num is None: + return False + return ((op == "gt" and left_num > ref_value) + or (op == "gte" and left_num >= ref_value) + or (op == "lt" and left_num < ref_value) + or (op == "lte" and left_num <= ref_value)) + wanted = str((condition or {}).get("value") or "").strip() + if op == "contains": + return wanted.casefold() in text.casefold() + if op == "not_contains": + return wanted.casefold() not in text.casefold() + if op in ("eq", "neq"): + left_num, right_num = _lane_num(text), _lane_num(wanted) + equal = (left_num == right_num if left_num is not None and right_num is not None + else text.casefold() == wanted.casefold()) + return equal if op == "eq" else not equal + ftype = (field_types or {}).get(field, "text") + left = _sort_key(text, ftype) + right = _sort_key(wanted, ftype) + if left is None or right is None: + return False + return ((op == "gt" and left > right) or (op == "gte" and left >= right) + or (op == "lt" and left < right) or (op == "lte" and left <= right)) + + +def compute_relation_cells(rt, table_key, tables=None): + """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. + + Zero store reads when the table declares neither kind — the same cheap-by-construction shape + `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. + """ + store = tables if tables is not None else ut_all(rt) + t = (store or {}).get(table_key) + fields = list((t or {}).get("fields") or []) + links = [f for f in fields if _ut().is_derived_link(f)] + rollups = [f for f in fields if isinstance(f.get("rollup"), dict)] + if not links and not rollups: + return 0 + rows = (t or {}).get("rows") or {} + by_key = {str(f.get("key")): f for f in fields} + + # --- resolve every link field ONCE per table, not once per row. + # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]` + resolved, linked_types = {}, {} + for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]: + lk_key = str((f or {}).get("key") or "") + if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict): + continue + bag = f["link"] + linked = (store or {}).get(str(bag.get("table") or "")) or {} + linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text") + for lf in (linked.get("fields") or [])} + if bag.get("inverse"): + # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary + # link cell contains this row id. The source cell remains the one relationship truth. + source_rows = linked.get("rows") or {} + inverse_key = str(bag.get("inverse") or "") + inverse_index = {} + for source_id, source_row in source_rows.items(): + for target_id in [part.strip() for part in + str((source_row or {}).get(inverse_key) or "").split(",")]: + if target_id: + inverse_index.setdefault(target_id, []).append( + (str(source_id), source_row or {})) + resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows} + elif bag.get("on"): + idx = _linked_rows_by_join(linked, str(bag["on"])) + from_key = _link_from_key(fields, bag) + resolved[lk_key] = { + str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), []) + for rid, row in rows.items()} if from_key else {} + else: + # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds + # are dropped rather than carried — a link to a deleted row is not a link. + lrows = (linked.get("rows") or {}) + resolved[lk_key] = { + str(rid): [(i, lrows[i]) for i in + [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] + if i and i in lrows] + for rid, row in rows.items()} + + changes = {} + for rid, row in rows.items(): + rid = str(rid) + row = row or {} + for f in links: + fk = str(f["key"]) + hits = (resolved.get(fk) or {}).get(rid) or [] + if f["link"].get("single"): + hits = hits[:1] + # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell + # is a projection of `resolved`, which is uncapped and is what every rollup below + # consumes — so a profile with 900 posts shows the first 500 ids and still averages + # over all 900. Same argument the `posts` window already makes: shedding is safe + # precisely because the authoritative store still holds everything. + want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + for f in rollups: + fk, bag = str(f["key"]), f["rollup"] + lk_key = str(bag.get("link") or "") + hits = list((resolved.get(lk_key) or {}).get(rid) or []) + # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING + # and therefore to a blank cell — never to a stale number. A column that keeps + # printing yesterday's answer after its input is gone is the worst of the options. + # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE + # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so + # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* + # rather than *the ones over X among the last 10* — two different windows wearing one + # sentence. Harmless while a threshold was a literal; incoherent the moment a + # threshold is a statistic OF the window, because the set being described and the set + # doing the describing would be different sets. + # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s + # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is + # computed over that window, THEN the conditions filter it") and the validator half + # was written against it. + # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH + # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` + # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` + # with no limit), so the blast radius is user-built rollups only. + # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last + # 12 posts, we also want to make it so its last N record, where the record's Status is + # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, + # so they select among the rows the window already kept. `where` selects WHICH rows + # the window is spent on. + # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per + # identity; run it first and a carousel could claim the slot its reel sibling needed, + # so the window would come up short for a reason nothing on screen explains. + # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this + # list, because at this point there is no fixed set for a statistic to be about. + where = list(bag.get("where") or []) + if where: + where_matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=None) + for condition in where] + if str(bag.get("whereConj") or "and") == "or": + hits = [pair for pair in hits if any(where_matches(pair))] + else: + hits = [pair for pair in hits if all(where_matches(pair))] + sort_by = str(bag.get("sortBy") or "") + if sort_by: + ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") + # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not + # rankable, and it must land at the END whichever direction is asked for — which + # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel + # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its + # window on rows that HAVE the value before it falls back to ones that do not. + keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] + rankable = [(p, k) for p, k in keyed if k is not None] + rankable.sort(key=lambda pk: pk[1], + reverse=str(bag.get("sortDir") or "desc") == "desc") + hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] + distinct_by = str(bag.get("distinctBy") or "") + if distinct_by: + seen, unique = set(), [] + for pair in hits: + identity = str(pair[1].get(distinct_by) or "").strip().lower() + # A blank is not an identity. Keep it rather than collapsing every unknown + # record into one synthetic duplicate. + if identity and identity in seen: + continue + if identity: + seen.add(identity) + unique.append(pair) + hits = unique + limit = int(bag.get("limit") or 0) + if limit: + hits = hits[:limit] + # --- the window is now FIXED, so a set-statistic threshold has a set to be about. + conditions = list(bag.get("conditions") or []) + if conditions: + # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the + # window; computing it inside `matches` would recompute the same mean for every + # candidate and — worse — would invite computing it over a set that the filter is + # already shrinking underneath it. + # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond + # the mean") is falsy, and the `or` spelling would silently rewrite it to the same + # number by accident. It reads identically and is right for the wrong reason, + # which is how it survives a review. + refs = [_rollup_ref_threshold( + [r for _i, r in hits], str(c.get("field") or ""), + (c.get("ref") or {}).get("sigmas", 0.0)) + if isinstance(c, dict) and c.get("ref") is not None else None + for c in conditions] + matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=ref) + for condition, ref in zip(conditions, refs)] + if str(bag.get("conditionConj") or "and") == "or": + hits = [pair for pair in hits if any(matches(pair))] + else: + hits = [pair for pair in hits if all(matches(pair))] + src = str(bag.get("field") or "") + # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the + # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at + # all, so the default stands for it. + want = _rollup_fold(str(bag.get("fn") or ""), + [r.get(src) for _i, r in hits] if src else [1] * len(hits), + (linked_types.get(lk_key) or {}).get(src, "text")) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + if not changes: + return 0 + + # A run that just wrote linked rows passes its in-flight user_tables bucket here so the + # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps + # the public helper's old persist-on-change behaviour. + if tables is not None: + tt = tables.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return len(changes) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +def _refresh_relations_inplace(tables, log=print): + """Refresh every relation against one mutable user_tables bucket; perform no store write.""" + touched = 0 + for tk, table in list((tables or {}).items()): + fields = (table or {}).get("fields") or [] + if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): + continue + try: + touched += compute_relation_cells(None, tk, tables=tables) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def refresh_relations(rt, log=print): + """The tick half of the relational pass — the twin of `refresh_metrics`. + + ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale + when table B gains a row, and A has no way to know that happened. Cheap by construction — a + table declaring neither kind costs one dict scan. + """ + snapshot = { + str(key): {**(table or {}), + "rows": {str(rid): dict(row or {}) + for rid, row in ((table or {}).get("rows") or {}).items()}} + for key, table in (ut_all(rt) or {}).items() + } + if not _refresh_relations_inplace(snapshot, log=log): + return 0 + actual = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + actual[0] = _refresh_relations_inplace(cur, log=log) + return cur + + # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). + # + # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and + # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. + # A derived-cell recompute has no business forcing a commit on someone typing. + # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new + # row lives only in the store cache for 2-20s, and this call — on the SAME key + # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the + # result. `POST /rows` answered 201 and the row was gone. + # + # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it + # is what makes the other ~39 sync writers of this key safe. This line is not that fix and + # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both + # landed together on purpose: one is correctness, one is the hot path. + rt.update(UT_STORE_KEY, _up, flush="async") + return actual[0] + + +def refresh_metrics(rt, today=None, log=print): + """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window + metric can never go staler than one tick while a scheduler exists. Cheap by construction — + a table without metric fields costs a dict scan and nothing else.""" + touched = 0 + for tk, t in ut_all(rt).items(): + if metric_fields_of_table(t): + try: + touched += compute_metric_cells(rt, tk, today=today) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def purge_subject(rt, handle): + """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the + tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half + non-optional: a purge that missed the pooled copy would not be erasure). Returns + `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is + now ABSENT, which is the one aggregate whose drill is emptiness.""" + subject = str(handle or "").strip().lstrip("@").lower() + if not subject: + return {} + counts = {} + tables = ut_all(rt) + post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} + codes = {str(r.get("shortcode") or "") for r in post_rows.values() + if str((r or {}).get("influencer_key") or "").strip().lower() == subject} + + keeps = { + "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, + DISCOVER_TABLE: lambda r: str((r or {}).get("handle") + or "").strip().lower() != subject, + } + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for tk, keep in keeps.items(): + t = cur.get(tk) + if t is None: + continue + rows = t.get("rows") or {} + nxt = {rid: r for rid, r in rows.items() if keep(r)} + counts[tk] = len(rows) - len(nxt) + t["rows"] = nxt + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + import ig_master + for bucket, n in (ig_master.purge_handle(subject) or {}).items(): + counts[f"master:{bucket}"] = n + return counts + + +# --------------------------------------------------------------------------------------------- +# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) +# --------------------------------------------------------------------------------------------- +# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | +# webhook | email. The first two are what always existed (Run now; cron via the tick). The four +# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: +# +# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, +# not for being in it. Implemented as per-record ARMED state over successive evaluations +# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms +# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing +# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the +# disarmed set with everything currently matching, so already-matching records do not fire +# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). +# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only +# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers +# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never +# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by +# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as +# a statusNote) catches whatever construction did not foresee. +# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds +# instead of running — a partial run entry names the count and the deliberate way through +# (Run now). C4's discovery guard is this rule's special case. +# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record +# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); +# event_field fires every transition. + +# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── +# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that +# renames the same events is a second vocabulary to learn for no gain. +# +# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". +# Renaming the key would have orphaned every stored trigger in production for a caption; the key +# is the contract with the store, the label is the contract with the reader, and they are allowed +# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the +# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 +# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. +# +# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show +# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a +# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop +# class wearing a friendlier face; here the two lists are separate on purpose and the refusal +# names the state rather than pretending the key is unknown. +# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── +# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the +# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — +# it is the event this flow starts from. Picking it sets the definition's kind to +# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. +# +# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the +# schedule (or on Run now), so it stays out of both lists below and its node switch flips the +# CRON — see `TRIGGER_SCHEDULE_KEYS`. +# +# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and +# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in +# the owner's own words and has not changed; what changed is which tuple it sits in, because +# `clean_trigger` refuses the planned list with a sentence and accepts this one. +TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", + "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", + "tiktok_profile_match") +#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS +#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses +#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper +#: section is not a section of one. +#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an +#: action that drives a browser are different builds; this row must not be read as progress on D-51. +TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") +TRIGGER_LABELS = { + "manual": "Manual", + "schedule": "At a scheduled time", + "event_field": "When a record matches conditions", + "record_updated": "When a record is updated", + "record_created": "When a record is created", + "enters_view": "When a record enters a view", + "webhook": "When a webhook is received", + "email": "When an email arrives", + "form_submitted": "When a form is submitted", + "ig_profile_match": "When an Instagram profile fits a criteria", + "button_clicked": "When a button is clicked", + "comment_added": "When a comment is added", + "web_page_changed": "When a website page changes", + "tiktok_profile_match": "When a TikTok profile fits a criteria", +} + +# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── +# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a +# distinction about where a trigger came FROM rather than about what a person is choosing. The +# question the picker actually asks is: does this fire on TIME, on your own DATA, or because +# something OUTSIDE said so. Three answers, and every trigger has exactly one. +# +# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map +# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the +# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` +# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation +# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is +# the surface. So: +# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule +# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because +# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into +# Database. +# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, +# so the fallback is provably dead code in production and the classification is still +# mandatory. The fallback catches the accident; the gate stops it shipping. +TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, + "database": {"label": "Database", "order": 2}, + "connector": {"label": "Connector", "order": 3}, + "other": {"label": "Other", "order": 99}} +#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to +#: prevent — it is the soft landing, not a category anybody should be adding triggers to. +TRIGGER_GROUP_FALLBACK = "other" +TRIGGER_GROUP_OF = { + "manual": "time", "schedule": "time", + "event_field": "database", "record_updated": "database", "record_created": "database", + "enters_view": "database", "form_submitted": "database", + "button_clicked": "database", "comment_added": "database", + "email": "connector", "webhook": "connector", "ig_profile_match": "connector", + "web_page_changed": "connector", "tiktok_profile_match": "connector", +} +#: The SUB-group inside "Connector" — which connected thing this trigger comes through. +#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two +#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a +#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key +#: against `/connectors/directory` would match `scraper` and `webhooks` and miss `gmail` — so it +#: must group by it and render `label`, never look it up. Said here because the miss would be +#: silent and partial, which is the worst shape. +#: (⚠ that example USED to read "`scraper` and `tiktok`" — R3 retired `tiktok` as a handle, and +#: the sentence is corrected here rather than left to rot into a lie about a key that is gone.) +TRIGGER_CONNECTOR = { + "email": {"key": "gmail", "label": "Gmail"}, + "webhook": {"key": "webhooks", "label": "Webhooks"}, + # ⭐ WAVE 30 · R3 — ONE "SCRAPER" BUCKET, AND IT HOLDS BOTH PLATFORMS. + # The owner, verbatim and for the third wave running: *"I say this multiple times already the + # damn Tiktok and Instagram belongs in the same bucket when creating the automation its under + # Scraper … Only when I click 'Scraper' under each automation trigger and actions would I see + # the option to choose either Instagram OR TikTok. That's it."* + # + # ⛔ THIS SUPERSEDES WAVE 29's RULE, and the old rule was not a typo — it was an argument: + # *"TikTok is its own connector, not the Scraper's … because the sub-group answers WHICH + # PRODUCT and never HOW BUILT."* Coherent, and not what was asked for. Instagram and TikTok are + # two PRODUCTS of one CAPABILITY (a social scraper bought from one vendor); a person opening + # this picker is choosing the capability first and the platform second. `verify_automation` + # asserted the old rule as an assertion AND as prose — a shipped gate forbidding the owner's + # ruling is most of why this complaint survived two waves — so it is INVERTED in this same + # change, comment included. + # + # ⚠ The Scraper sub-group now holds THREE rows: two built (Instagram, TikTok) and one faded + # (the page-change trigger). No display ORDER is emitted here — the client groups on `key` and + # owns its own ordering (contract C1). `tiktok` ceases to exist as a grouping handle. + "ig_profile_match": {"key": "scraper", "label": "Scraper"}, + "web_page_changed": {"key": "scraper", "label": "Scraper"}, + "tiktok_profile_match": {"key": "scraper", "label": "Scraper"}, +} +#: Triggers that watch a database and therefore need one named before they can fire. +TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", + "form_submitted") +#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. +#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, +#: so flipping its node must flip the schedule. +#: +#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read +#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, +#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For +#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a +#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns +#: about eight lines into its own comment. Derived from one named set now, so a trigger added to +#: `TRIGGER_KEYS` cannot silently join the wrong side of it. +#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, +#: and forgetting it is precisely the failure this constant's own note describes: it watches no +#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left +#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while +#: the schedule kept firing — a switch that lies. +TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") +#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. +#: +#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire +#: reason this constant exists rather than the tuple above being sent. That set answers "which +#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual +#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw +#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run +#: now": a control contradicting its own description. +#: +#: D-55's history is why it is DERIVED rather than listed: the client carried +#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact +#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). +#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. +TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} +#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so +#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. +TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") +MAX_WATCH_FIELDS = 12 +#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, +#: and so would a deployment that prefers immediacy over coalescing. +EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) +FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses +FIRE_WINDOW_SECONDS = 300 +FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on +EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation +EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest +CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota + +EMAIL_FIELDS = [ + field_def("email_id", "Email id"), field_def("email_from", "From"), + field_def("email_subject", "Subject"), field_def("email_date", "Date"), + field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), +] + + +def clean_trigger(raw, previous=None): + """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and + means what it always meant: manual + whatever `schedule` says. + + ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry + with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT + rather than refused — the picker writes `{key}` first and the table/field after, the + wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the + wire so the surface says "finish setting this up" instead of snapping back to Manual. It + cannot fire while incomplete (the hooks match on the table it does not name), which is the + fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a + condition on a field the trigger does not watch) are still refused with the sentence — + incomplete is a state, wrong is not. + """ + if raw in (None, "", {}): + return (dict(previous) if isinstance(previous, dict) and previous else None), None + if not isinstance(raw, dict): + return None, "the trigger must be an object" + prev = previous if isinstance(previous, dict) else {} + key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), + 30).strip() + if key in TRIGGER_PLANNED: + # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence + # says WHY rather than "unknown trigger", because the picker legitimately showed it. + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — " + f"it renders so you can see it is coming, and it cannot be saved") + if key not in TRIGGER_KEYS: + return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) + if key == "schedule": + # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` + # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored + # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. + return None, None + if key == "manual": + # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** + # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user + # can just press Run to make the full flow work."* + # + # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys + # together with one argument — *"storing a no-op trigger would be a second copy of that + # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has + # another home; **manual has none.** Nothing anywhere recorded "this automation is + # manual", so storing it is not a duplicate: it is the only record there has ever been. + # + # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so + # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the + # "nobody has decided yet" empty state, and Configuration — including the Database picker + # a plain automation cannot do without — never rendered. The owner reported it twice. The + # previous note reasoned that a manual option *"would bounce straight back to this state + # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion + # was that it should be STORED. + # + # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off + # (Run now always works, which is the whole of what it means) and a switch that governs + # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the + # cron stays reachable — picking Manual says how it fires today, never that it may not be + # scheduled tomorrow. + return {"key": "manual"}, None + out = {"key": key, + "enabled": bool(raw["enabled"]) if "enabled" in raw else + bool(prev.get("enabled", True)), + "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} + if key in TRIGGER_TABLE_KEYS: + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("event triggers watch blank databases (ut_*) this wave — " + f"{table!r} is not one") + out["table"] = table + if key == "event_field": + # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record + # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a + # second, quieter way to express the same narrowing, and the owner asked for one. + # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is + # dropped on this definition's next clean — silently, and exactly once, because nothing + # writes the key back. A refusal here would have 400'd the live automations that carry it. + cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), + where="the trigger") + if cerr: + return None, cerr + out["when"] = cond + if key == "record_updated": + # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty + # is the WIDER reading and it is the default there too, so it stays the default here. + watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") + if watch_raw in (None, ""): + watch = [] + elif not isinstance(watch_raw, list): + return None, "the watched-field list must be a list of field keys" + else: + watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] + if len(watch) > MAX_WATCH_FIELDS: + return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " + f"fields — leave the list empty to watch every field") + out["fields"] = watch + # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A + # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: + # "a record was updated" is an event, and asking it to also be a filter was the overlap + # with `event_field` the owner asked to end. Stated loudly in the contract AND here so + # nobody restores it as a bug fix. + # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` + # naturally returns "no gate" for it — the write itself becomes the event — rather than + # this needing a second removal anywhere. + if key == "enters_view": + out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), + 80).strip() + if key == "form_submitted": + # Blank = any form on that database. Naming one narrows to it, which is what a table + # carrying an intake form AND a correction form needs. + out["formToken"] = _s(raw.get("formToken") if "formToken" in raw + else prev.get("formToken"), 64).strip() + if key == "webhook": + # The token is MINTED here, once, and survives every later patch — rotating it on + # every Save would silently break the external caller the URL was given to. + out["token"] = _s(prev.get("token"), 64) or _secrets_token() + # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. + # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no + # map behaves exactly as it did — it fires the flow and reads nothing — so the live + # webhook automations are untouched. `webhook` deliberately stays OUT of + # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and + # every existing webhook trigger would go unconfigured and stop firing. + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("a webhook writes into a blank database (ut_*) — " + f"{table!r} is not one") + out["table"] = table + fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw + else prev.get("fieldMap")) + if ferr: + return None, ferr + out["fieldMap"] = fmap + if key == "email": + out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), + 200).strip() or "in:inbox is:unread" + out["configured"] = _trigger_configured(out) + return out, None + + +#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody +#: else's schema and is legitimately wider than an action's hand-written value list. +MAX_BODY_FIELDS = 40 +MAX_BODY_DEPTH = 5 + + +def clean_body_map(raw): + """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. + + Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated + rather than half-supported: `items.0.sku` would read as working for the first element and + silently write nothing the day a payload arrives with the list empty, which is the shape of + bug this module keeps paying for. A path that resolves to nothing writes nothing. + """ + if raw in (None, ""): + return {}, None + if not isinstance(raw, dict): + return None, "the webhook field map must be an object of {body path: field key}" + if len(raw) > MAX_BODY_FIELDS: + return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" + out = {} + for path, field in raw.items(): + p = _s(path, 200).strip() + if not p: + return None, "a webhook mapping has an empty body path" + if len(p.split(".")) > MAX_BODY_DEPTH: + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload — " + f"map a shallower value") + fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") + if not fk: + return None, f"the value at {p!r} is not mapped to a field" + out[p] = fk[:60] + return out, None + + +def body_value(body, path): + """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is + an object or a list answers None rather than a stringified `{...}` in a cell, because the + Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" + cur = body + for part in str(path or "").split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur if isinstance(cur, (str, int, float, bool)) else None + + +def webhook_row(rt, defn, body): + """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. + + ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same + `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, + from the only side the caller is on, from a payload whose paths did not resolve. That is the + D-11 class (a table that quietly stops growing), and the caller here is a machine that will + keep posting. The note rides the 200: the flow still fires, and the answer says why no row + was written. + + ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop + prevention law (A2(2)). A sibling automation watching this table does NOT fire on a + webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow + fires, because `hook_fire` fires it explicitly, which is the difference between "this + trigger fired" and "a write happened". + """ + trg = (defn or {}).get("trigger") or {} + table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) + if not table or not fmap or not isinstance(body, dict): + return "", 0, "" # no map configured — nothing to report + t = ut_get(rt, table) + if t is None: + return "", 0, f"{table} no longer exists, so nothing was written" + values = {} + for path, fkey in fmap.items(): + v = body_value(body, path) + if v is not None: + values[fkey] = _s(v, 500) if isinstance(v, str) else v + if not values: + return "", 0, ("none of the mapped paths resolved to a value in this payload — " + "check the paths against what you are sending") + rows = dict((t.get("rows") or {})) + if len(rows) >= row_cap(table): + return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " + f"written (the flow still ran)") + rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) + rows[rid] = values + ut_write_rows(rt, table, rows) + return rid, len(values), "" + + +def _trigger_configured(trg, config=None): + """Is this trigger complete enough to fire? One reader, because "configured" is asserted in + three places (the wire, the graph node, the hooks) and three copies of a boolean is how a + surface says "ready" about a trigger the engine skips. + + ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is + the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) + rather than on the trigger. `clean_trigger` calls this without it and so answers + conservatively (False); `clean_definition` calls it again with the validated config and + refines. Conservative-then-refined is the fail-closed order — the reverse would flash + "ready" on a trigger with nothing to search for. + """ + key = str((trg or {}).get("key") or "") + # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no + # filter is not a search, it is a request for the whole index. Named as a pair rather than + # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a + # boolean expression. + if key in ("ig_profile_match", "tiktok_profile_match"): + return bool((config or {}).get("predicates")) + if key in TRIGGER_TABLE_KEYS and not trg.get("table"): + return False + if key == "event_field": + # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, + # ever" — which is not a trigger, it is a description of the table. + # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone + # and carried no condition becomes `configured: false` on its next clean. It stops + # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it + # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. + return bool(trg.get("when")) + if key == "enters_view": + return bool(trg.get("viewId")) + return True + + +def _secrets_token(): + import secrets as _sec + return _sec.token_urlsafe(24) + + +# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the +# process would keep punishing an automation for a storm that ended with the restart. +_FIRES = {} +_FIRES_LOCK = threading.Lock() + + +def _breaker_trips(tenant, auto_id, now=None): + now = now if now is not None else time.time() + key = (tenant, str(auto_id)) + with _FIRES_LOCK: + log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] + log.append(now) + _FIRES[key] = log + return len(log) > FIRE_LIMIT + + +def _pause_trigger(rt, auto_id, note): + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = _s(note, 200) + return cur + _store_update(rt, _up, flush="sync") + + +def trigger_fire(rt, tenant, auto_id, log=print, rows=None): + """One trigger firing — breaker first, then the ordinary async run. False when it did not + start (breaker, or already running — both are answers, not errors).""" + if _breaker_trips(tenant, auto_id): + note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " + f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's " + f"subject in a loop") + _pause_trigger(rt, auto_id, note) + log(f"[aios-auto] breaker: {auto_id} {note}") + return False + return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) + + +# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE +# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). +_SETTLE = {} +_SETTLE_LOCK = threading.Lock() + + +def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): + """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. + + ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave + dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the + reason is worth stating because it is invisible from this file: an ordinary ut cell typed at + the grid door lands in the editor's PER-USER OVERLAY stratum + (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` + definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for + the common case the value that just changed exists ONLY in this event, and a definition-row + read sees the pre-write value — the trigger would evaluate stale and never fire. + `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree + read the record's other columns) with the written cells on top. + """ + key = (tenant, str(auto_id)) + cell = {str(field): after} if field else {} + if EVENT_SETTLE_SECONDS <= 0: + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + _settle_eval(rt, tenant, auto_id, log=log) + return + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + timer = buf.get("timer") + if timer is not None: + timer.cancel() # the burst continues — push the window out + timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, + args=(rt, tenant, auto_id), kwargs={"log": log}) + timer.daemon = True + buf["timer"] = timer + timer.start() + + +def view_filter(rt, table_key, view_id): + """`(tree, fields, problem)` for one saved view on a user table — the substrate the + `enters_view` trigger tests membership against (C3-v2 / owner R2). + + Personal strata first (`find_view`), then the shared bucket, because a view somebody shared + is exactly the kind an automation gets pointed at. A view that has been deleted answers a + PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one + would turn "when a record enters Overdue" into "on every write", which is the widening this + module refuses everywhere else. + """ + key = str(table_key or "") + vid = str(view_id or "").strip() + if not key or not vid: + return None, [], "the trigger names no view" + try: + import core.table_store as table_store + tops = table_store.make(f"{key}_table_workspace", st=rt) + found = tops.find_view(vid) + view = (found[1] if found else None) or tops.shared_view(vid) + except Exception as e: # noqa: BLE001 + return None, [], f"the view could not be read ({type(e).__name__})" + if not isinstance(view, dict): + return None, [], f"view {vid!r} no longer exists on {key}" + cfg = view.get("config") or {} + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" + + +def _row_gate(rt, defn, trg): + """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. + + None means the trigger has NO match gate — the write itself is the event (wave 22's "the + field changed", and `record_updated` over any field). A problem means the gate cannot be + built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that + passes. + """ + key = trg.get("key") + if key == "enters_view": + tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) + if problem: + return None, problem + import harness.filter_eval as filter_eval + return (lambda row: filter_eval.matches(tree, row, fields)), "" + when = trg.get("when") + if when: + return (lambda row: lane_match(when, row)), "" + return None, "" + + +def _settle_eval(rt, tenant, auto_id, log=print): + """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" + with _SETTLE_LOCK: + buf = _SETTLE.pop((tenant, str(auto_id)), None) + written = dict((buf or {}).get("rows") or {}) + touched = list(written) + if not touched: + return + d = all_definitions(rt).get(str(auto_id)) + trg = (d or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True): + return + gate, problem = _row_gate(rt, d, trg) + if problem: + # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken + # automation, not a quiet no-op — the note is what the surface shows instead of "On". + if (d.get("statusNote") or "") != problem: + _pause_trigger(rt, auto_id, problem) + log(f"[aios-auto] trigger gate: {auto_id} {problem}") + return + fired = [] + if gate is not None: + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) + nxt = set(disarmed) + for rid in touched: + # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 + # tree can read the record's other columns, the just-written cells on top because + # for an ordinary ut column this event is the ONLY place that value exists yet. + if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), + **(written.get(rid) or {})}): + if rid not in disarmed: + fired.append(rid) # false→true THIS evaluation: the edge + nxt.add(rid) + else: + nxt.discard(rid) # left the state — re-armed (Airtable's rule) + if nxt != disarmed: + set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) + if not fired: + return + else: + fired = list(touched) # "the field changed" — the burst is the edge + if len(fired) > FLOOD_LIMIT: + _commit_run(rt, auto_id, "partial", + f"the trigger matched {len(fired)} records in one evaluation — more than " + f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " + f"process them deliberately", {"held": len(fired)}, True) + return + trigger_fire(rt, tenant, auto_id, log=log, rows=fired) + + +def _seed_event_state(rt, defn): + """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so + turning the trigger on fires nothing — the first fire needs a real false→true transition. + Evaluated over the definition rows (the machine-written truth this trigger class watches). + + Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation + reads. Two implementations of "does this row match" is how a seed disagrees with the edge it + is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. + """ + trg = (defn or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or not trg.get("configured"): + return + gate, problem = _row_gate(rt, defn, trg) + if gate is None or problem: + return # no match gate ⇒ nothing to arm; a broken gate seeds nothing + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) + set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) + + +def grid_hook(evt): + """THE listener the human write doors emit into (registered onto + `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may + import both sides). Never raises into a write path; a broken trigger listener must not + break typing into a cell.""" + try: + st = evt.get("st") + if st is None: + return + tenant = str(getattr(st, "key", "") or "royal-imports") + table = str(evt.get("table") or "") + kind = str(evt.get("type") or "") + # ⭐⭐ WAVE 31 · T35 (D-134) — `cached=True`, and this is the ONLY caller that passes it. + # This function runs once per ROW EVENT, so a 20,000-row import used to perform 20,000 + # whole-document deep copies of the automations bucket, under the store lock, to re-read a + # trigger set that had not changed. See `all_definitions` for why the memo is safe here + # and nowhere else. + defs = all_definitions(st, cached=True) + field = str(evt.get("field") or "") + rid = str(evt.get("rowId") or "") + for aid, d in defs.items(): + trg = (d or {}).get("trigger") or {} + key = str(trg.get("key") or "") + if trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True) \ + or key not in TRIGGER_ROW_KEYS \ + or str(trg.get("table") or "") != table: + continue + if kind == "record_created" and key == "record_created": + hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 + ridn = int(rid) if rid.isdigit() else None + if ridn is None or ridn <= hw: + continue # once per record EVER (A2(4)) — undo-proof + set_state(st, aid, {"rcHighwater": ridn}) + # ⛔⛔ W31-T35 — MIRROR THE WRITE INTO THE MEMO'S OWN COPY, IN THE SAME STATEMENT. + # This is the hazard a definitions memo creates and the reason D-134 is not a + # one-line change: this branch READS `rcHighwater` and WRITES it, so within one + # import burst the second row would compare against the highwater the FIRST row + # set — and read the pre-write value out of the memo, fire again, and break + # A2(4)'s *"once per record EVER — undo-proof"*. `set_state` goes through + # `_store_update`, which drops the memo, but `defs` is the object already in hand + # for the rest of THIS event; keeping the two in step is what makes the memo safe + # rather than merely fast [[read-path-cannot-witness-write-path]]. + (d.setdefault("state", {}))["rcHighwater"] = ridn + trigger_fire(st, tenant, aid, rows=[rid]) + elif kind == "event_field" and key == "event_field": + # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every + # write on the watched table settles, and the CONDITION decides whether it fires + # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would + # now always take its first arm anyway; leaving a read of a key the validator no + # longer writes is the drift seat this migration exists to close. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif kind == "event_field" and key == "record_updated" and ( + not (trg.get("fields") or []) or field in (trg.get("fields") or [])): + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif key == "enters_view": + # BOTH kinds feed it: a row can enter a view by being edited into its filter or + # by being CREATED already inside it. Listening only to edits would silently miss + # every new record — the half of the definition a reader assumes is covered. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") + + +def form_fired(rt, table_key, row_id, values=None, form_token=""): + """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). + + One submitted form row → every `form_submitted` automation watching that database fires. + Returns the list of automation ids that started, so the door can log what it set off (and so + the gate can assert it, rather than asserting a side effect nobody can see). + + ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, + so it fires triggers exactly like typing into a cell does. The loop-prevention law is not + weakened by that — the engine's own writers still never reach here (only `routes_forms` calls + it), so an automation cannot create a form row and re-fire itself. + + `values` is accepted and unused today: the row is already written when this is called, and + the flow reads it from the table. It stays in the signature because the caller HAS it and a + later refire policy ("only when field X was submitted") needs it — a parameter added later + would mean changing D's call site in a wave that does not own it. + """ + started, table = [], str(table_key or "") + if not table: + return started + tenant = str(getattr(rt, "key", "") or "royal-imports") + for aid, d in all_definitions(rt).items(): + trg = (d or {}).get("trigger") or {} + if trg.get("key") != "form_submitted" or trg.get("paused") \ + or not trg.get("enabled", True) or not trg.get("configured", True): + continue + if str(trg.get("table") or "") != table: + continue + want = str(trg.get("formToken") or "") + if want and not hmac.compare_digest(want, str(form_token or "")): + continue # this automation watches a DIFFERENT form on that table + if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): + started.append(aid) + return started + + +def hook_fire(rt, tenant, auto_id, token, body=None): + """The webhook trigger's decision, separated from FastAPI so the gate can drive it. + Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, + 409 already running, 200 started. + + ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or + sent something that is not JSON. It is written onto a record BEFORE the flow fires — the + flow's actions walk the record the webhook just created, which is the whole point of mapping + it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. + """ + import hmac as _hmac + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return 404, {"error": "unknown_automation"} + trg = defn.get("trigger") or {} + want = str(trg.get("token") or "") + if trg.get("key") != "webhook" or not want: + return 403, {"error": "no_webhook", "message": + "this automation has no webhook trigger"} + if trg.get("paused") or not trg.get("enabled", True): + return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} + if not _hmac.compare_digest(want, str(token or "")): + return 403, {"error": "bad_token", "message": "that token is not valid"} + # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. + row_id, mapped, note = webhook_row(rt, defn, body) + started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) + out = {"started": bool(started), "at": _iso(), + # Answered even when zero, so a caller wiring a map up can see whether their paths + # resolved. Silence here would make "my JSON is not landing" undebuggable from the + # outside, which is the only side the caller is on. + "rowId": row_id or None, "mapped": mapped} + if note: + out["note"] = note # a 200 that wrote no row SAYS which reason it was + return 200, out + + +def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): + """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a + row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the + gate drives this without a network; production leaves them None. + + Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per + tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most + `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. + """ + trg = defn.get("trigger") or {} + if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): + return None + import oauth_connect + creator = str(defn.get("createdBy") or "").strip() or "admin" + token, err = oauth_connect.google_creds(rt, creator) + if err: + if (defn.get("statusNote") or "") != err: + def _note(cur): + cur = cur if isinstance(cur, dict) else {} + dd = cur.get(str(auto_id)) + if dd is not None: + dd["statusNote"] = _s(err, 200) + return cur + _store_update(rt, _note, flush="sync") + return None + lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) + reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) + ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) + if lerr: + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}", + {}, True) + seen = set((defn.get("state") or {}).get("emailSeen") or []) + fresh = [m for m in ids if m not in seen] + if not fresh: + return None # nothing new is not a run — no history spam + if len(fresh) > FLOOD_LIMIT: + return _commit_run(rt, auto_id, "partial", + f"{len(fresh)} new emails matched in one poll — more than the " + f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " + f"the query, or press Run now after adjusting it", + {"held": len(fresh)}, True) + fresh = fresh[:EMAIL_MAX_PER_POLL] + rows_in, notes = [], [] + for mid in fresh: + row, rerr = reader(mid) + if row: + rows_in.append(row) + elif rerr: + notes.append(rerr) + table_key = (defn.get("config") or {}).get("targetTable") or "" + if not table_key: + return _commit_run(rt, auto_id, "error", + "the email trigger has nowhere to write — the automation names no " + "target database", {}, False) + ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", + EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) + ut_write_rows(rt, table_key, merged) + new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] + set_state(rt, auto_id, {"emailSeen": new_seen}) + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — " + f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") + if notes: + summary += f" — {notes[0][:100]}" + entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, + True, affected=list(merged)[:200]) + trigger_fire(rt, tenant, auto_id, log=log) + return entry + + +def compose_sentence(defn): + """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the + definition so it cannot lie about what runs.""" + cfg = defn.get("config") or {} + trg = defn.get("trigger") or {} + sched = defn.get("schedule") or {} + kind = defn.get("kind") + if trg.get("key") == "event_field": + # Law 4: no watched field any more, so the sentence stops naming one. It said + # "When None changes on ut_x" the moment the key stopped being stored. + head = f"When a record in {trg.get('table')} matches conditions" + elif trg.get("key") == "record_updated": + head = f"When a record in {trg.get('table')} is updated" + elif trg.get("key") == "ig_profile_match": + head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "tiktok_profile_match": + head = "When a TikTok profile fits the criteria" + elif trg.get("key") == "record_created": + head = f"When a record is created in {trg.get('table')}" + elif trg.get("key") == "webhook": + head = "When the webhook is called" + elif trg.get("key") == "email": + head = f"When an email matches {trg.get('query')}" + elif sched.get("enabled"): + head = f"{_cron_label(sched.get('cron'))}" + else: + head = "When you press Run now" + if kind == "scrape_db": + host = urlparse(str(cfg.get("url") or "")).hostname or "the page" + body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" + elif kind == "field_instagram": + # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts + # first" / ", anonymous only" described a choice that no longer exists. + body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" + elif kind in DISCOVERY_KINDS: + # ⭐⭐ WAVE 30 · T05 — the arm covers both kinds, and the network is NAMED from the kind + # rather than hard-coded into the sentence. Instagram-only, a TikTok search fell into the + # `plain` branch below and introduced itself as *"do nothing yet — this automation has no + # actions"*: a flatly false sentence, on the one surface whose stated promise is that it + # cannot lie about what runs. (The `plain` branch's own comment records the mirror-image + # incident — it USED to be this arm, and described every plain automation as an Instagram + # search for 0 profiles. The same two branches have now mis-described each other's + # automations in both directions, which is why neither may be a fallthrough.) + _dplatform, _dtable, _, _ = discovery_facts(kind) + body = (f"search {_dplatform} for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or _dtable}") + else: + # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. + # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every + # plain automation would have introduced itself as "search Instagram for up to 0 profiles + # into ut_ig_candidates" — a sentence composed from a definition that says none of it, + # on the one surface whose whole promise is that it "cannot lie about what runs". + n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) + tbl = cfg.get("targetTable") or trg.get("table") or "" + body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) + if n else "do nothing yet — this automation has no actions") + lanes = cfg.get("lanes") or [] + tail = f", then route each record across {len(lanes)} lanes" if lanes else "" + return f"{head}, {body}{tail}." + + +def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it + directly. Returns the run entry, or None when it was already running (the 409).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None + # ⛔⛔ WAVE 32 · T45 (owner item 10) — AN UNCONFIGURED ACTION BLOCKS THE RUN, HERE, WHERE EVERY + # DOOR PASSES. The route checks too so a person gets a 400 rather than a silent no-op, but the + # tick and the webhook do not go through the route; a client-only block is not a block (D-112). + # ⚠ BEFORE `_claim`, deliberately: claiming and then refusing would leave the automation + # marked running until the release, i.e. a refusal that also produces a phantom 409 for the + # next honest attempt. + _refusal = run_refusal(defn) + if _refusal: + log(f"[aios-auto] refused: {_refusal}") + return None + if not _claim(tenant, auto_id): + return None + # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever + # happened to press Run — and above all not to the scheduler, which is not a person and + # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by + # whether a human or a cron got to the first run first. + owner = str(defn.get("createdBy") or "").strip() + if owner and username in MACHINE_OWNERS: + username = owner + try: + _step(tenant, auto_id, "running") + # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it + # first and do not start another profile scrape while it is outstanding: re-running the + # action would buy duplicate engagement reads and reintroduce the timeout this handoff + # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. + # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one + # rung earlier: a profile snapshot the vendor is still building is paid work, and + # starting a fresh scrape for the same handle would buy the identical row a second time. + # Ahead of the metric collector because the profile IS the thing the run was asked for; + # the engagement batches hang off it. + if _pending_profile_tasks(defn): + state, summary, counts, affected, steps = collect_pending_profile_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + if _pending_metric_tasks(defn): + state, summary, counts, affected, steps = collect_pending_metric_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + runner = RUNNERS.get(defn.get("kind")) + if runner is None: + return _commit_run(rt, auto_id, "error", + f"unknown automation kind {defn.get('kind')!r}", {}, False) + try: + # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. + # `status.step` was already on the wire and D's half renders it; measured against the + # code, it was set exactly ONCE ("running") and never again, so the word would have + # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a + # constant as a progress indicator is worse than rendering nothing: it looks like an + # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. + state, summary, counts, affected, steps = runner( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text), rows=rows) + except Refused as e: + return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") + return _commit_run(rt, auto_id, "error", + f"{type(e).__name__}: {str(e)[:200]}", {}, False) + # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is + # committed, over the records this run actually touched. ONE call site rather than three + # inside the runners: every kind gets actions and endings for free, and a fourth runner + # cannot forget to opt in. + # + # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, + # wired to the wire and covered by twelve gate checks that all called `apply_actions` + # DIRECTLY — so the whole feature was green and unreachable. A person would have built a + # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. + # + # A failing action must not fail the RUN: the machine steps already wrote their rows and + # reporting that as an error would misdescribe what happened. It degrades to `partial` + # with the reason in the summary — the cap_note discipline. + # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, + # which now reads this name — an assignment only on the success path would turn any + # action failure into a NameError inside the handler that exists to prevent exactly that. + # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked + # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the + # run a person stares at wondering why the automation stopped doing anything. + run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) + try: + a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], + username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so + # the runner contract keeps its shape, and they must leave before the merge or they + # would be a "count" everywhere downstream. + run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) + counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} + if a_counts.get("enrichMetricBatchesPending"): + batches = int(a_counts["enrichMetricBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f" — {batches} post-engagement batch" + f"{'' if batches == 1 else 'es'} still building; Views and other " + "metrics will be collected automatically without another paid read") + if a_counts.get("enrichUnbound"): + # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is + # `partial` because it genuinely did part of its job — it walked the records — and + # the sentence names the ONE thing that has to change, in the two places a person + # can change it. The old behaviour was `ok` with an empty table. + state = "partial" if state != "error" else state + summary += (" — the Instagram step did not run: this database has no profile " + "column. Name one on the step, or mark a text column as the " + "Instagram profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichUnbound"): + # ⛔ WAVE 30 · T08 — ITS OWN SENTENCE, not the one above with a word swapped by a + # variable. A flow may carry BOTH steps, and the fix a person has to apply is + # per-column: naming an Instagram profile column does nothing for a TikTok step, + # so a single sentence covering "the enrich step" would send them to the wrong + # place half the time. Both may appear on one run, which is correct. + state = "partial" if state != "error" else state + summary += (" — the TikTok step did not run: this database has no TikTok profile " + "column. Name one on the step, or mark a text column as a TikTok " + "profile" + + _unbound_hint(rt, _flow_table(defn))) + if a_counts.get("ttEnrichBlocked"): + state = "partial" if state != "error" else state + tt_note = next((n for n in run_notes if "(TikTok): " in n), "") + summary += (f" — {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " + "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) + if a_counts.get("enrichProfileBatchesPending"): + # ⭐ The paid profile the vendor is still building. Said out loud so a run that + # looks like a failure is read as the handoff it is — the tick finishes it. + n = int(a_counts["enrichProfileBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f" — {n} profile read{'' if n == 1 else 's'} took longer than the " + "wait allows and will be collected automatically, at no extra cost") + if a_counts.get("enrichBlocked"): + state = "partial" if state != "error" else state + # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile + # read(s) were blocked" is the exact string the owner read three mornings running + # before asking "wtf is going on"; it names a quantity and withholds the one + # thing that would let anybody act. The first note is the vendor's own words. + # ⚠ ...and the Instagram selector SKIPS the tagged TikTok lines for the same + # reason. Two sentences quoting each other's vendor reason is worse than one. + blocked_note = next((n for n in run_notes + if ": " in n and "(TikTok): " not in n), "") + summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + + (f". {_s(blocked_note, 220)}" if blocked_note else "")) + # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the + # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it + # logged the cap and rolled up `ok`, so a flow that had silently stopped writing + # looked exactly like one that had nothing to write. Same rule, same sentence shape. + if a_counts.get("createCapped"): + state = "partial" if state != "error" else state + summary = (f"{summary} — {a_counts['createCapped']} row(s) NOT created: a target " + f"database is at its row cap") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") + state = "partial" if state != "error" else state + summary = f"{summary} — the actions did not finish ({type(e).__name__})" + return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, + steps, notes=run_notes) + finally: + _release(tenant, auto_id) + + +def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Start a run on a background thread. False when one is already in flight (→ 409).""" + if running(tenant, auto_id): + return False + th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows), + daemon=True, name=f"automation-{tenant}-{auto_id}") + th.start() + return True + + +# --------------------------------------------------------------------------------------------- +# THE TICK + the in-process scheduler +# --------------------------------------------------------------------------------------------- + +def pending_collect_ids(rt): + """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. + + ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking + forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and + stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is + true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), + so `due_ids` never returns it, nothing ever collected it, and the results the account had + already been charged for sat ready at the vendor forever. The sentence was not wrong; it was + describing a run nobody had scheduled. + + A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it + regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something + new*, this says *collect what is already running*, and folding the second into the first would + make an unscheduled automation's paid result depend on someone remembering to press a button. + """ + out = [] + for aid, d in all_definitions(rt).items(): + if not isinstance(d, dict): + continue + if (d.get("trigger") or {}).get("paused"): + continue + # ⭐⭐ WAVE 30 · T05 — BOTH discovery kinds, and this one is a MONEY defect rather than a + # cosmetic one. It tested `== "discover_instagram"`, so a TikTok corpus search stored its + # `pendingSnapshot`, told the person *"The next run picks up the results"*, and was then + # never returned by this function — the tick collected nothing, forever. That is EXACTLY + # the live incident quoted in the docstring above (*"why is the SMALL 10 records test + # search taking forever, its not populating"*), reproduced for the second platform by the + # wave that added it: a result the tenant has already been charged for, stranded. + # ⚠ NOT ON THE SCOUT'S LIST OF FIVE. Found by reading this function for a different + # ticket, which is the argument for `DISCOVERY_KINDS` in one line — the sites that test a + # kind string are not enumerable by memory, and this one is three thousand lines from the + # others. Pressing Run again does still collect (the runner's own branch reads the same + # field), so the money was recoverable BY HAND and only ever silently lost on a schedule. + pending_discovery = (d.get("kind") in DISCOVERY_KINDS + and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) + # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile + # deferral would be stored and never collected, which is the same defect it fixes wearing + # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was + # measured on is `trigger: manual`. A capability written down but never walked is what + # this whole change is about, so it must not be reintroduced one function later. + if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): + out.append(aid) + return sorted(out) + + +def due_ids(rt, now=None): + """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" + return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now)) + + +def tick(rt, tenant, now=None, log=print): + """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the + ids started. The email polls are bounded and fail-quiet per automation — one broken + mailbox connection must not stop the tenant's schedules.""" + started = [] + # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the + # tenant has already been charged for; it is collected whether or not this automation is on a + # schedule. `_claim` makes the union safe — an id in both lists starts once. + for aid in dict.fromkeys(list(pending_collect_ids(rt)) + list(due_ids(rt, now))): + if run_async(rt, tenant, aid, username="scheduler", log=log): + started.append(aid) + for aid, d in all_definitions(rt).items(): + if (d.get("trigger") or {}).get("key") == "email": + try: + if email_poll(rt, tenant, aid, d, log=log) is not None: + started.append(aid) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") + # C7's amendment: date-window metrics advance with the tick, so `today` is never staler + # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan. + try: + refresh_metrics(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}") + # ⭐ 2026-08-07 — the relational pass, on the same tick and for the same reason. A rollup on + # table A goes stale when table B gains a row, and A cannot know that happened; the tick is + # the only place that sees both. ⚠ SEPARATE try/except from the metrics above deliberately — + # one pass failing must not silently cancel the other, which a shared block would do. + try: + refresh_relations(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tenant} failed: {type(e).__name__}: {e}") + # ⭐ THE READ-THROUGH ROLLUPS, on the same tick and for a stronger version of the same reason. + # A linked rollup goes stale when the LINKED TABLE gains a row; a source-backed one goes stale + # when ODOO does — and its window moves on its own besides (a `ytd` column is wrong on 1 + # January without anybody writing anything). The tick is the only place that sees either. + # + # ⛔ ITS OWN try/except, like the two passes above, and the reason is the same one stated + # there: this pass REFUSES loudly on a truncated group set, and a shared block would let that + # honest refusal silently cancel the relational pass that had already succeeded. + # + # ⚠ IT WRITES NOTHING WHEN IT REFUSES — see `rollup_sql`'s header. A half-applied rollup mixes + # two vintages of one column and looks completely normal, which is why the refusal is total. + try: + import rollup_sql + for _tk in list((rt.get(UT_STORE_KEY) or {})): + if rollup_sql.source_fields((rt.get(UT_STORE_KEY) or {}).get(_tk) or {}): + # ⚠ `today` is DEFAULTED INSIDE `compute`, not passed: this scope has no + # such local, and `refresh_metrics` above resolves it the same way. + n = rollup_sql.compute(rt, _tk) + if n: + log(f"[aios-auto] source rollups {tenant}/{_tk}: {n}") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] source rollup {tenant} failed: {type(e).__name__}: {e}") + if started: + log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") + return started + + +def tick_all(now=None, log=print): + """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop + the others' schedules.""" + from harness import runtime as _rt + out = {} + for slug in _rt.known_tenants(): + try: + out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}") + return out + + +#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer +#: would be a busy loop against a vocabulary that cannot express it. +TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) +_SCHEDULER = [None] + + +def scheduler_loop(log=print): + """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. + + Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and + `verify_seam` import this module through `main.py`, run against fake stores in seconds and + exit. A loop that ticked on entry would fire inside them. + """ + while True: + time.sleep(TICK_SECONDS) + try: + tick_all(log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") + + +def start_scheduler(log=print): + """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** + + ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave + brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 + seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires + two or three times inside a gate that imports `main.py` — and while a tick over a fake store + finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and + mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread + that can move a gate's subject is a flaky suite waiting to happen. + + So it takes the convention `main.py:353` already established for exactly this hazard — + `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup + event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the + external EventBridge tick (R5) does not depend on it either way, since that POSTs the + endpoint rather than riding this thread. + """ + if os.environ.get("AIOS_AUTOMATIONS") != "1": + return False + if _SCHEDULER[0] is not None: + return False + th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, + daemon=True, name="automation-scheduler") + _SCHEDULER[0] = th + th.start() + return True diff --git a/api/connectors_tt.py b/api/connectors_tt.py index 60d96945a35f7fd8ca79443bf70cdec87e0a4c53..0603f4f23f7075252fdcef06ce19371ac1620523 100644 --- a/api/connectors_tt.py +++ b/api/connectors_tt.py @@ -1,568 +1,568 @@ -"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). - -Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an -automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file -exists at all rather than another thousand lines inside the engine. - -⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 -post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read -live from `GET /datasets/{id}/metadata` for **$0.00** and written down in -`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at -close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a -name below reads through a candidate list it is because the vendor has two names for one fact -(`biography`/`signature`, `region`/`country`), never because the name is uncertain. - -⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — -public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The -vendor key is a key to a SUPPLIER. - -⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE -OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and -`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not -Instagram's — and re-implementing them here would be a second copy of the deferral handling, the -truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. -Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from -`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. -⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because -the sentence above is the kind that quietly stops being true. - -⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** - 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared - envelope keys (`timestamp`, `input`) that no metadata call mentions; - 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` - and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one - TikTok claim that matters most (`play_count`) is exactly a declaration. -""" -from __future__ import annotations - -import automation_engine as engine -# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an -# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were -# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever -# read them off this module (the engine imports them from the transport itself), so they were four -# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. -from connectors_bd import ( - _bd_first_url, - _bd_flag, - _bd_list, - _bd_source_payload, - _first, - _ig_int, - bd_scrape, -) - -# --------------------------------------------------------------------------------------------- -# THE DATASETS -# --------------------------------------------------------------------------------------------- -# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET -# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a -# full field list, and the two DISCOVERY halves answer **404 for our key**: -# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). -# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's -# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. -TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records -TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields -TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields - -#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` -#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. -TT_POST_TYPE_VIDEO = "video" -TT_POST_TYPE_CONTENT = "content" - -#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept -#: beside the dataset ids because it is the same class of vendor fact. -#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical -#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own -#: batch calls, and the gate asserts the two agree rather than trusting that they do. -TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" - - -def tt_profile_url(handle): - """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" - h = str(handle or "").strip().lstrip("@") - return TT_PROFILE_URL.format(handle=h) if h else "" - - -# --------------------------------------------------------------------------------------------- -# THE FIELD MAPS -# --------------------------------------------------------------------------------------------- -# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared -# in `automation_engine`. The rules they all obey, stated once: -# -# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, -# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` -# returns `None` (never 0) when nothing matches, which is what makes that possible. -# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` -# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) -# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the -# vendor's side is preserved rather than silently discarded while our column model catches up. -# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` -# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they -# ride in `source_payload` where they make no claim. - - -def _tt_str(node, *names): - """The first non-empty string among `names`, or None when the vendor sent nothing. - - ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — - an empty string written into a cell claims "we looked and it is empty". - """ - v = _first(node, *names) - if v is None: - return None - s = str(v).strip() - return s or None - - -def _tt_pct(node, *names): - """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. - - ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to - the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — - measured on the Instagram side, and TikTok sends the same shape on all three of its rates. - """ - v = _first(node, *names) - if v is None: - return None - out = engine._pct100(v) - return out or None - - -def _tt_day(node, *names): - """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" - v = _first(node, *names) - if v is None: - return None - return engine._day(v) or None - - -def _drop_blanks(row): - """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" - return {k: v for k, v in row.items() if v is not None and v != ""} - - -def normalize_profile(node, handle=""): - """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. - - ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: - * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` - populated on 85% of rows and they carry the same text; - * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented - two-letter-ISO description, `country` has no description at all. - ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its - `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the - only count of its kind the dataset offers. - """ - node = node if isinstance(node, dict) else {} - account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "handle": account, - "full_name": _tt_str(node, "nickname"), - "tt_id": _tt_str(node, "id"), - "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), - "bio": _tt_str(node, "biography", "signature"), - "external_url": _bd_first_url(_first(node, "bio_link")), - "verified": _bd_flag(node, "is_verified"), - "is_private": _bd_flag(node, "is_private"), - # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* - # on their own description. It is the closest thing TikTok has to Instagram's - # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so - # the approximation only ever fills a cell the vendor actually answered. - "is_business": _bd_flag(node, "is_commerce_user"), - "followers": _ig_int(_first(node, "followers")), - "following": _ig_int(_first(node, "following")), - "posts_count": _ig_int(_first(node, "videos_count")), - # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no - # nulls). It is not a post-level number and it is not our `likes` column, which is why it - # is stored under a different name. - "likes_received": _ig_int(_first(node, "likes")), - "avg_engagement": _tt_pct(node, "awg_engagement_rate"), - "like_engagement": _tt_pct(node, "like_engagement_rate"), - "comment_engagement": _tt_pct(node, "comment_engagement_rate"), - "country_code": _tt_str(node, "region", "country"), - "region": _tt_str(node, "region"), - "predicted_lang": _tt_str(node, "predicted_lang"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT - # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, - # which is why the append law dates a snapshot by when WE read it. - "account_created_at": _tt_day(node, "create_time"), - "source_payload": _bd_source_payload(node), - }) - - -def tt_post_type(node): - """A TikTok post row → one of OUR three type options, or None. - - ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / - `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the - vendor's token would fail `_clean_field`'s option check on the way in and would put an - untranslated API word in front of a user on the way out. - - So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the - row carries more than one `carousel_images` entry, which is what a carousel IS on either - network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the - token cannot express it, so inferring `carousel` from the word would be inventing a fact. - ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and - that is exactly what would make the wrong default invisible). - """ - raw = str((node or {}).get("post_type") or "").strip().lower() - if raw == TT_POST_TYPE_VIDEO: - return "video" - if raw != TT_POST_TYPE_CONTENT: - return None - images = (node or {}).get("carousel_images") - return "carousel" if isinstance(images, list) and len(images) > 1 else "image" - - -def normalize_post(node): - """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. - - ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and - Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same - fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and - `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would - manufacture a second measurement that a rollup could average or double-count, which is a worse - outcome than the missing column it would paper over. - - ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and - the probe measured them as the same shape. That equality is what lets the comments→posts link - join with NO normaliser, which the Instagram side never had. - ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). - ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and - countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares - no equivalent, so `paid_partnership`/`partner` have no column on this family at all. - """ - node = node if isinstance(node, dict) else {} - shortcode = _tt_str(node, "shortcode", "post_id") - if not shortcode: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "shortcode": shortcode, - # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. - # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the - # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so - # one name means one thing across both corpora. Reading the display name silently broke the - # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched - # NOTHING, so a person could not filter posts by creator and a rollup would count zero. - # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a - # different fact, and filling a join key with it is worse than leaving it blank — a blank - # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. - # The URL is the honest second source: it carries the handle by construction. - "influencer_key": (_tt_str(node, "account_id") - or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), - "posted_at": _tt_day(node, "create_time"), - "type": tt_post_type(node), - "caption": _tt_str(node, "description"), - "url": _tt_str(node, "url"), - "hashtags": _bd_list(node, "hashtags"), - "tagged_location": _tt_str(node, "commerce_info"), - "views": _ig_int(_first(node, "play_count")), - "likes": _ig_int(_first(node, "digg_count")), - "comments": _ig_int(_first(node, "comment_count")), - "shares": _ig_int(_first(node, "num_share_count")), - "saves": _ig_int(_first(node, "collect_count")), - "video_duration": _ig_int(_first(node, "video_duration")), - "source_payload": _bd_source_payload(node), - }) - - -def normalize_comment(node): - """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. - - ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR - `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in - `source_payload`. Reading the array's length instead would be a second, disagreeing answer to - a question the vendor already answers — and it would disagree, because a page of replies is not - all of them. - ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is - text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor - that changes its mind cannot change ours. - ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged - PII) stay in `source_payload` and are promoted to no column, which is the same posture the - Instagram comment schema takes for the same D-24 reason. - """ - node = node if isinstance(node, dict) else {} - comment_key = _tt_str(node, "comment_id") - if not comment_key: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "comment_key": comment_key, - "shortcode": _tt_str(node, "post_id"), - # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the - # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); - # primary first, so a row carrying the rich form is not silently served the plain one. - # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. - "text": _tt_str(node, "comment_text", "comment_text_only"), - "commented_at": _tt_day(node, "date_created"), - "likes": _ig_int(_first(node, "num_likes")), - "replies": _ig_int(_first(node, "num_replies")), - "source_payload": _bd_source_payload(node), - }) - - -#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them -#: rather than naming three functions, and so adding a fourth dataset is one entry. -TT_NORMALIZERS = { - "tt_profile": normalize_profile, - "tt_post_metrics": normalize_post, - "tt_comments": normalize_comment, -} - - -# --------------------------------------------------------------------------------------------- -# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) -# --------------------------------------------------------------------------------------------- - -def tt_handle(url): - """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. - - Deliberately permissive about the input and strict about the output, because the two callers - hand it different things: an automation stores whatever a person typed in the profile column - (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean - `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot - resolve to two different handles. - """ - s = str(url or "").strip() - if not s: - return "" - if "tiktok.com" in s.lower(): - # Everything after the first `@`, up to the next path segment or query. - tail = s.split("@", 1)[1] if "@" in s else "" - s = tail.split("/")[0].split("?")[0].split("#")[0] - s = s.strip().lstrip("@").strip() - # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. - return s if s and all(c.isalnum() or c in "._" for c in s) else "" - - -def tt_post_urls(node, limit=0): - """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. - - ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row - we have already bought, so the posts read is a scrape of links we hold, never a search for them. - The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, - so a design that reached for either would not merely be dearer, it would not work. - - ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an - array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe - read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's - `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** - (measured below), so the reader built against the quoted sentence found nothing, forever, in - silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration - of X"* are different claims, and prose cannot be told apart by a reader downstream — which is - why the correction names the method, not just the value. - - ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, - and preferring whichever happened to be longer is how one creator's window silently differs - from another's. - ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, - validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is - never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, - not a number this function may invent. - """ - raw = (node or {}).get("top_videos") - out = [] - for item in raw if isinstance(raw, list) else []: - # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. - # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a - # live handle returns **19 DICTS**, keyed - # `video_url · video_id · playcount · diggcount · commentcount · share_count · - # favorites_count · create_date · cover_image`. - # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and - # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says - # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field - # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's - # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was - # the first. [[reachable-is-not-the-same-as-built]] - # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY - # list on every real profile — post capture could not have worked for anybody, and the - # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture - # written from a schema tests the schema. - # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept - # because it costs nothing and is what a future corpus revision would most likely use. The - # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents - # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as - # before, rather than to a URL built out of a guess. - # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring - # whichever array happened to be longer is how one creator's window silently differs from - # another's, and that reasoning is unchanged by this correction. - if isinstance(item, dict): - u = str(item.get("video_url") or item.get("url") or "").strip() - else: - u = str(item or "").strip() - if u.lower().startswith("http") and u not in out: - out.append(u) - return out[:limit] if limit and limit > 0 else out - - -def pull_posts_tt(post_urls, log=print, deferred=None): - """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. - - ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side - measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still - running at 67 minutes. Nothing here loops per URL. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] posts: {note}") - return [], note - out = [r for r in (normalize_post(n) for n in rows) if r] - return out, "" - - -def pull_comments_tt(post_urls, log=print, deferred=None): - """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. - - ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on - both networks: a comments scrape ingests identifiable third parties who never entered anybody's - list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes - none of them to a column; this function adds no new exposure, it just has to be asked for. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] comments: {note}") - return [], note - out = [r for r in (normalize_comment(n) for n in rows) if r] - return out, "" - - -#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. -#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral -#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's -#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse -#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. -TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) - - -def _media_deferrals(deferred): - """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. - - ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the - handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one - kind here, so the id already carries everything a mapper choice depends on — and importing - Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). - """ - out = [] - for d in deferred or []: - if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: - out.append(dict(d)) - return out - - -def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, - max_posts=0, post_metrics=False, comment_metrics=False): - """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. - - `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, - so the engine's enrich branch treats every network identically and no caller learns a new - shape. - - ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's - do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — - no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` - then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction - rather than by a rule: their input IS a post permalink, so asking for comments with post capture - off is a request with no subject, and it returns none instead of quietly buying posts nobody - asked for. - - ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than - pessimistic. The Instagram contract reads `ok` only when identity AND media both landed - (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower - count and no posts must not paint green over a posts table that did not grow"*). So: posts not - ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → - `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", - never "did the function finish". - - ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the - paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, - with its own note explaining that a multi-provider chain is a promise something walks it and - that nothing walks Instagram's second name today either. So a refusal here is final, and it - says so instead of implying a retry somewhere. - """ - handle = tt_handle(url) - if not handle: - return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", - "note": f"{url!r} is not a TikTok profile URL or handle"} - - # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from - # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not - # happen; a miss falls through to the single-URL call below. - cached = prefetch.get(handle) if isinstance(prefetch, dict) else None - _deferred = [] - if isinstance(cached, dict) and cached: - rows, note = [cached], "" - else: - rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) - - node = rows[0] if rows else {} - profile = normalize_profile(node, handle) if node else {} - # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` - # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and - # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly - # this reason, and answering "0 followers" instead is the failure it exists to prevent. - unreadable = profile.get("followers") is None and profile.get("following") is None - if note or unreadable: - # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still - # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the - # same record. That was live on the Instagram profile path until 2026-08-09 — measured on - # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being - # reintroduced here by omission. - if isinstance(pending_profile, list): - for d in _deferred: - pending_profile.append({**d, "kind": "profile", "influencer": handle}) - why = note or ("the scrape answered, but no follower/following counts were readable in it " - "(the field names may have moved - see tiktok-capture.md)") - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "deferredProfile": [d.get("snapshotId") for d in _deferred], - "note": why} - - # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ - if not post_metrics: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; post capture is off for this step"} - urls = tt_post_urls(node, limit=max_posts) - if not urls: - # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and - # saying so is what stops the next run paying to be told the same thing. - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; this account's row carried no post links"} - posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) - comments, c_note = ([], "") - if comment_metrics and posts: - # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — - # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused - # is not silently asked about again one rung later. - comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], - log=log, deferred=_deferred) - # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is - # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to - # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE - # deferral is impossible — the profile branch above returns `blocked` on any note — so **every - # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose - # collector writes preset profile cells onto somebody's record from post rows. The engine keeps - # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). - # ⇒ So this returns them under their OWN key, filtered by dataset identity - # (`_media_deferrals`), and the engine files them in the metric queue with the handle it - # already holds. Returning rather than appending also keeps the queue's vocabulary out of a - # connector: this module knows which CORPUS deferred, never what the engine calls it. - # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that - # matters most — that is exactly the run where the vendor took too long, so a caller reading - # the ids only from the success path would lose every batch it actually paid for. - deferred_media = _media_deferrals(_deferred) - if not posts: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", "deferredMedia": deferred_media, - "note": p_note or note or "profile read; the post source returned nothing"} - return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, - "via": "brightdata", "deferredMedia": deferred_media, - "note": c_note or note or ""} +"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). + +Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an +automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file +exists at all rather than another thousand lines inside the engine. + +⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 +post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read +live from `GET /datasets/{id}/metadata` for **$0.00** and written down in +`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a +name below reads through a candidate list it is because the vendor has two names for one fact +(`biography`/`signature`, `region`/`country`), never because the name is uncertain. + +⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — +public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The +vendor key is a key to a SUPPLIER. + +⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE +OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and +`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not +Instagram's — and re-implementing them here would be a second copy of the deferral handling, the +truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. +Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from +`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. +⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because +the sentence above is the kind that quietly stops being true. + +⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** + 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared + envelope keys (`timestamp`, `input`) that no metadata call mentions; + 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` + and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one + TikTok claim that matters most (`play_count`) is exactly a declaration. +""" +from __future__ import annotations + +import automation_engine as engine +# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an +# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were +# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever +# read them off this module (the engine imports them from the transport itself), so they were four +# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. +from connectors_bd import ( + _bd_first_url, + _bd_flag, + _bd_list, + _bd_source_payload, + _first, + _ig_int, + bd_scrape, +) + +# --------------------------------------------------------------------------------------------- +# THE DATASETS +# --------------------------------------------------------------------------------------------- +# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET +# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a +# full field list, and the two DISCOVERY halves answer **404 for our key**: +# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). +# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's +# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. +TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records +TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields +TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields + +#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` +#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. +TT_POST_TYPE_VIDEO = "video" +TT_POST_TYPE_CONTENT = "content" + +#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept +#: beside the dataset ids because it is the same class of vendor fact. +#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical +#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own +#: batch calls, and the gate asserts the two agree rather than trusting that they do. +TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" + + +def tt_profile_url(handle): + """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" + h = str(handle or "").strip().lstrip("@") + return TT_PROFILE_URL.format(handle=h) if h else "" + + +# --------------------------------------------------------------------------------------------- +# THE FIELD MAPS +# --------------------------------------------------------------------------------------------- +# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared +# in `automation_engine`. The rules they all obey, stated once: +# +# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, +# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` +# returns `None` (never 0) when nothing matches, which is what makes that possible. +# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` +# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) +# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the +# vendor's side is preserved rather than silently discarded while our column model catches up. +# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` +# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they +# ride in `source_payload` where they make no claim. + + +def _tt_str(node, *names): + """The first non-empty string among `names`, or None when the vendor sent nothing. + + ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — + an empty string written into a cell claims "we looked and it is empty". + """ + v = _first(node, *names) + if v is None: + return None + s = str(v).strip() + return s or None + + +def _tt_pct(node, *names): + """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. + + ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to + the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — + measured on the Instagram side, and TikTok sends the same shape on all three of its rates. + """ + v = _first(node, *names) + if v is None: + return None + out = engine._pct100(v) + return out or None + + +def _tt_day(node, *names): + """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" + v = _first(node, *names) + if v is None: + return None + return engine._day(v) or None + + +def _drop_blanks(row): + """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" + return {k: v for k, v in row.items() if v is not None and v != ""} + + +def normalize_profile(node, handle=""): + """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. + + ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: + * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` + populated on 85% of rows and they carry the same text; + * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented + two-letter-ISO description, `country` has no description at all. + ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its + `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the + only count of its kind the dataset offers. + """ + node = node if isinstance(node, dict) else {} + account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "handle": account, + "full_name": _tt_str(node, "nickname"), + "tt_id": _tt_str(node, "id"), + "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), + "bio": _tt_str(node, "biography", "signature"), + "external_url": _bd_first_url(_first(node, "bio_link")), + "verified": _bd_flag(node, "is_verified"), + "is_private": _bd_flag(node, "is_private"), + # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* + # on their own description. It is the closest thing TikTok has to Instagram's + # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so + # the approximation only ever fills a cell the vendor actually answered. + "is_business": _bd_flag(node, "is_commerce_user"), + "followers": _ig_int(_first(node, "followers")), + "following": _ig_int(_first(node, "following")), + "posts_count": _ig_int(_first(node, "videos_count")), + # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no + # nulls). It is not a post-level number and it is not our `likes` column, which is why it + # is stored under a different name. + "likes_received": _ig_int(_first(node, "likes")), + "avg_engagement": _tt_pct(node, "awg_engagement_rate"), + "like_engagement": _tt_pct(node, "like_engagement_rate"), + "comment_engagement": _tt_pct(node, "comment_engagement_rate"), + "country_code": _tt_str(node, "region", "country"), + "region": _tt_str(node, "region"), + "predicted_lang": _tt_str(node, "predicted_lang"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT + # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, + # which is why the append law dates a snapshot by when WE read it. + "account_created_at": _tt_day(node, "create_time"), + "source_payload": _bd_source_payload(node), + }) + + +def tt_post_type(node): + """A TikTok post row → one of OUR three type options, or None. + + ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / + `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the + vendor's token would fail `_clean_field`'s option check on the way in and would put an + untranslated API word in front of a user on the way out. + + So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the + row carries more than one `carousel_images` entry, which is what a carousel IS on either + network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the + token cannot express it, so inferring `carousel` from the word would be inventing a fact. + ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and + that is exactly what would make the wrong default invisible). + """ + raw = str((node or {}).get("post_type") or "").strip().lower() + if raw == TT_POST_TYPE_VIDEO: + return "video" + if raw != TT_POST_TYPE_CONTENT: + return None + images = (node or {}).get("carousel_images") + return "carousel" if isinstance(images, list) and len(images) > 1 else "image" + + +def normalize_post(node): + """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. + + ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and + Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same + fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and + `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would + manufacture a second measurement that a rollup could average or double-count, which is a worse + outcome than the missing column it would paper over. + + ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and + the probe measured them as the same shape. That equality is what lets the comments→posts link + join with NO normaliser, which the Instagram side never had. + ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). + ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and + countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares + no equivalent, so `paid_partnership`/`partner` have no column on this family at all. + """ + node = node if isinstance(node, dict) else {} + shortcode = _tt_str(node, "shortcode", "post_id") + if not shortcode: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "shortcode": shortcode, + # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. + # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the + # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so + # one name means one thing across both corpora. Reading the display name silently broke the + # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched + # NOTHING, so a person could not filter posts by creator and a rollup would count zero. + # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a + # different fact, and filling a join key with it is worse than leaving it blank — a blank + # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. + # The URL is the honest second source: it carries the handle by construction. + "influencer_key": (_tt_str(node, "account_id") + or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), + "posted_at": _tt_day(node, "create_time"), + "type": tt_post_type(node), + "caption": _tt_str(node, "description"), + "url": _tt_str(node, "url"), + "hashtags": _bd_list(node, "hashtags"), + "tagged_location": _tt_str(node, "commerce_info"), + "views": _ig_int(_first(node, "play_count")), + "likes": _ig_int(_first(node, "digg_count")), + "comments": _ig_int(_first(node, "comment_count")), + "shares": _ig_int(_first(node, "num_share_count")), + "saves": _ig_int(_first(node, "collect_count")), + "video_duration": _ig_int(_first(node, "video_duration")), + "source_payload": _bd_source_payload(node), + }) + + +def normalize_comment(node): + """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. + + ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR + `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in + `source_payload`. Reading the array's length instead would be a second, disagreeing answer to + a question the vendor already answers — and it would disagree, because a page of replies is not + all of them. + ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is + text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor + that changes its mind cannot change ours. + ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged + PII) stay in `source_payload` and are promoted to no column, which is the same posture the + Instagram comment schema takes for the same D-24 reason. + """ + node = node if isinstance(node, dict) else {} + comment_key = _tt_str(node, "comment_id") + if not comment_key: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "comment_key": comment_key, + "shortcode": _tt_str(node, "post_id"), + # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the + # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); + # primary first, so a row carrying the rich form is not silently served the plain one. + # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. + "text": _tt_str(node, "comment_text", "comment_text_only"), + "commented_at": _tt_day(node, "date_created"), + "likes": _ig_int(_first(node, "num_likes")), + "replies": _ig_int(_first(node, "num_replies")), + "source_payload": _bd_source_payload(node), + }) + + +#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them +#: rather than naming three functions, and so adding a fourth dataset is one entry. +TT_NORMALIZERS = { + "tt_profile": normalize_profile, + "tt_post_metrics": normalize_post, + "tt_comments": normalize_comment, +} + + +# --------------------------------------------------------------------------------------------- +# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) +# --------------------------------------------------------------------------------------------- + +def tt_handle(url): + """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. + + Deliberately permissive about the input and strict about the output, because the two callers + hand it different things: an automation stores whatever a person typed in the profile column + (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean + `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot + resolve to two different handles. + """ + s = str(url or "").strip() + if not s: + return "" + if "tiktok.com" in s.lower(): + # Everything after the first `@`, up to the next path segment or query. + tail = s.split("@", 1)[1] if "@" in s else "" + s = tail.split("/")[0].split("?")[0].split("#")[0] + s = s.strip().lstrip("@").strip() + # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. + return s if s and all(c.isalnum() or c in "._" for c in s) else "" + + +def tt_post_urls(node, limit=0): + """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. + + ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row + we have already bought, so the posts read is a scrape of links we hold, never a search for them. + The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, + so a design that reached for either would not merely be dearer, it would not work. + + ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an + array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe + read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's + `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** + (measured below), so the reader built against the quoted sentence found nothing, forever, in + silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration + of X"* are different claims, and prose cannot be told apart by a reader downstream — which is + why the correction names the method, not just the value. + + ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, + and preferring whichever happened to be longer is how one creator's window silently differs + from another's. + ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, + validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is + never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, + not a number this function may invent. + """ + raw = (node or {}).get("top_videos") + out = [] + for item in raw if isinstance(raw, list) else []: + # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. + # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a + # live handle returns **19 DICTS**, keyed + # `video_url · video_id · playcount · diggcount · commentcount · share_count · + # favorites_count · create_date · cover_image`. + # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and + # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says + # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field + # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's + # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was + # the first. [[reachable-is-not-the-same-as-built]] + # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY + # list on every real profile — post capture could not have worked for anybody, and the + # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture + # written from a schema tests the schema. + # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept + # because it costs nothing and is what a future corpus revision would most likely use. The + # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents + # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as + # before, rather than to a URL built out of a guess. + # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring + # whichever array happened to be longer is how one creator's window silently differs from + # another's, and that reasoning is unchanged by this correction. + if isinstance(item, dict): + u = str(item.get("video_url") or item.get("url") or "").strip() + else: + u = str(item or "").strip() + if u.lower().startswith("http") and u not in out: + out.append(u) + return out[:limit] if limit and limit > 0 else out + + +def pull_posts_tt(post_urls, log=print, deferred=None): + """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. + + ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side + measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still + running at 67 minutes. Nothing here loops per URL. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] posts: {note}") + return [], note + out = [r for r in (normalize_post(n) for n in rows) if r] + return out, "" + + +def pull_comments_tt(post_urls, log=print, deferred=None): + """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. + + ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on + both networks: a comments scrape ingests identifiable third parties who never entered anybody's + list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes + none of them to a column; this function adds no new exposure, it just has to be asked for. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] comments: {note}") + return [], note + out = [r for r in (normalize_comment(n) for n in rows) if r] + return out, "" + + +#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. +#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral +#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's +#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse +#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. +TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) + + +def _media_deferrals(deferred): + """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. + + ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the + handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one + kind here, so the id already carries everything a mapper choice depends on — and importing + Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). + """ + out = [] + for d in deferred or []: + if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: + out.append(dict(d)) + return out + + +def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, + max_posts=0, post_metrics=False, comment_metrics=False): + """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. + + `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, + so the engine's enrich branch treats every network identically and no caller learns a new + shape. + + ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's + do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — + no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` + then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction + rather than by a rule: their input IS a post permalink, so asking for comments with post capture + off is a request with no subject, and it returns none instead of quietly buying posts nobody + asked for. + + ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than + pessimistic. The Instagram contract reads `ok` only when identity AND media both landed + (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower + count and no posts must not paint green over a posts table that did not grow"*). So: posts not + ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → + `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", + never "did the function finish". + + ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the + paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, + with its own note explaining that a multi-provider chain is a promise something walks it and + that nothing walks Instagram's second name today either. So a refusal here is final, and it + says so instead of implying a retry somewhere. + """ + handle = tt_handle(url) + if not handle: + return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", + "note": f"{url!r} is not a TikTok profile URL or handle"} + + # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from + # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not + # happen; a miss falls through to the single-URL call below. + cached = prefetch.get(handle) if isinstance(prefetch, dict) else None + _deferred = [] + if isinstance(cached, dict) and cached: + rows, note = [cached], "" + else: + rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) + + node = rows[0] if rows else {} + profile = normalize_profile(node, handle) if node else {} + # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` + # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and + # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly + # this reason, and answering "0 followers" instead is the failure it exists to prevent. + unreadable = profile.get("followers") is None and profile.get("following") is None + if note or unreadable: + # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still + # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the + # same record. That was live on the Instagram profile path until 2026-08-09 — measured on + # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being + # reintroduced here by omission. + if isinstance(pending_profile, list): + for d in _deferred: + pending_profile.append({**d, "kind": "profile", "influencer": handle}) + why = note or ("the scrape answered, but no follower/following counts were readable in it " + "(the field names may have moved - see tiktok-capture.md)") + return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", + "deferredProfile": [d.get("snapshotId") for d in _deferred], + "note": why} + + # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ + if not post_metrics: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; post capture is off for this step"} + urls = tt_post_urls(node, limit=max_posts) + if not urls: + # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and + # saying so is what stops the next run paying to be told the same thing. + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; this account's row carried no post links"} + posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) + comments, c_note = ([], "") + if comment_metrics and posts: + # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — + # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused + # is not silently asked about again one rung later. + comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], + log=log, deferred=_deferred) + # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is + # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to + # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE + # deferral is impossible — the profile branch above returns `blocked` on any note — so **every + # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose + # collector writes preset profile cells onto somebody's record from post rows. The engine keeps + # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). + # ⇒ So this returns them under their OWN key, filtered by dataset identity + # (`_media_deferrals`), and the engine files them in the metric queue with the handle it + # already holds. Returning rather than appending also keeps the queue's vocabulary out of a + # connector: this module knows which CORPUS deferred, never what the engine calls it. + # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that + # matters most — that is exactly the run where the vendor took too long, so a caller reading + # the ids only from the success path would lose every batch it actually paid for. + deferred_media = _media_deferrals(_deferred) + if not posts: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", "deferredMedia": deferred_media, + "note": p_note or note or "profile read; the post source returned nothing"} + return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, + "via": "brightdata", "deferredMedia": deferred_media, + "note": c_note or note or ""} diff --git a/api/deps.py b/api/deps.py index 87d3abd4883ebf297c658145ee0a900cbb9f00bd..3339bd471c143589a0cec3ba60832f76ebb4cf8f 100644 --- a/api/deps.py +++ b/api/deps.py @@ -110,14 +110,7 @@ def _user_for(claims): # (login validated the slug's existence, never the membership) — with # AIOS_ENABLE_QA_TENANT=1 that was a working cross-tenant session. A signature proves # the cookie is ours; THIS proves the account belongs where the cookie says it does. - # ⚠ WAVE 33 (W33-T37) — the default was DELETED, not changed. It read - # `pub.get("tenant", "royal-imports")` and could never fire: `users._public` normalises a - # missing tenant to `royal-imports` before this line ever sees it. A dead default that - # names a real tenant reads like a live fallback, and the cross-tenant audit spent a row - # proving it was not one. Bare `.get` is also the safer shape if that ever stops being - # true: `None != claim_tenant` denies, where the old default would have ADMITTED a - # tenant-less record into tenant #0's session. - if claim_tenant != pub.get("tenant"): + if claim_tenant != pub.get("tenant", "royal-imports"): return None return pub if (uname == "admin" and os.environ.get("APP_PASSWORD", "") diff --git a/api/main.py b/api/main.py index 2d0b50dc62d723fa0a979876531c79568c036460..fce0849c6a534b5c559406db113a7e57924dd6cf 100644 --- a/api/main.py +++ b/api/main.py @@ -1,925 +1,880 @@ -"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. - -It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, -reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the -JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. - -WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: - - * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared - `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same - anonymous principal and no route could scope anything. Now a signed stateless cookie carries - a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the - user RECORD. `/api/health` stays unauthenticated (liveness only). - * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home - for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). - * **The SSL shim is env-gated.** It used to run at import, unconditionally. - -⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the -session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This -is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB -commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. -""" -import os -import sys -from pathlib import Path - -# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- -# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an -# unverified default context; the HF Space and the container are unaffected. This used to run -# unconditionally at import — i.e. the shipped container disabled TLS verification for every -# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a -# man-in-the-middle away from being someone else's data. It is now opt-in, must be set -# deliberately, and is never on by default. -# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. -if os.environ.get("AIOS_INSECURE_SSL") == "1": - import ssl - ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 - -# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container -# (where the layout differs from the local sibling-dir default). --- -_HERE = Path(__file__).resolve() -_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) -for _p in (str(_RI), str(_HERE.parent)): - if _p not in sys.path: - sys.path.insert(0, _p) - -from dotenv import load_dotenv # noqa: E402 -load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed - -from fastapi import Body, Depends, FastAPI, Request # noqa: E402 -from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 -from fastapi.responses import JSONResponse # noqa: E402 -from fastapi.staticfiles import StaticFiles # noqa: E402 -from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 - -import aios_session # noqa: E402 -import routes_admin # noqa: E402 -import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) -import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) -import routes_auth # noqa: E402 -import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) -import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) -import routes_customers # noqa: E402 -import routes_grid # noqa: E402 -import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) -import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) -import routes_nav # noqa: E402 -import routes_pages # noqa: E402 -import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) -import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) -import routes_records # noqa: E402 -import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) -import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) -import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) -import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) -import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) -import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) -import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) -import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) -import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) -import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) -import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) -import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) -import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) -from core import grid_events # noqa: E402 -from deps import Session, module_gate # noqa: E402 - -_WEB_DIST = _HERE.parents[1] / "web" / "dist" - -# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- -# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never -# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the -# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so -# embed == standalone by construction. -# -# TWO REASONS THIS IS HERE and not folded into the routes: -# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR -# layout is broken, and discovering that from a 500 on the first customer request instead of -# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a -# rewrite dropped it and this restores it.) -# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and -# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this -# API have not drifted. The rewrite removed them and the referee went red; retargeting the -# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do -# not delete — but only when the subject genuinely moved. Here it should not have moved). -# -# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from -# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and -# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone -# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the -# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. -import json as _json_contract # noqa: E402 -_FIELDS_PATH = _RI / "aios_grid_fields.json" -if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " - "platform root (it also carries the data layer this API imports)." - ) -_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) -FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract -# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. -# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be -# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. -_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS - if f["source"] == "odoo" and f["type"] in ("text", "status", "select", - "date")} - -app = FastAPI(title="AIOS web API") - -# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with -# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the -# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. -app.add_middleware(GZipMiddleware, minimum_size=1024) - - -@app.exception_handler(StarletteHTTPException) -def _error_shape(request: Request, exc: StarletteHTTPException): - """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. - - `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a - 422 from a malformed path param) is wrapped here so a client never has to branch on two - different error bodies. - """ - detail = exc.detail - if isinstance(detail, dict) and "error" in detail: - body = detail - else: - body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} - return JSONResponse(body, status_code=exc.status_code, - headers=getattr(exc, "headers", None)) - - -@app.exception_handler(grid_events.StoreUnavailable) -def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): - """THE SAFETY NET for "the store is down" → 503, from anywhere. - - ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a - `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own - try/except — surfaced as a 500. A store outage is a normal operational state and every route - here touches the store at least twice (the workspace read, then the write), so "remember to - wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler - means a store outage can only ever be a 503, whichever call raised it. - - The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this - adapter, which has no durable session dict to degrade into. A 200 over a write that - evaporated is the failure the whole rule exists to prevent. - """ - return JSONResponse( - {"error": {"code": "store_unavailable", - "message": "the tenant store is unavailable — no change was saved"}}, - status_code=503) - - -@app.get("/api/health") -def health(): - """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not - reveal anything about the deployment beyond "the process is up". No version, no tenant list, - no config: a health endpoint is the one URL every scanner finds first. - - ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned - release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated - caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind - a session — and the authoritative copy is the `VERSION` file in the Space repo, which - `deploy_web.space_version()` reads without needing the app to be up at all.""" - return {"ok": True} - - -app.include_router(routes_auth.router) -app.include_router(routes_nav.router) -app.include_router(routes_customers.router) -app.include_router(routes_products.router) -app.include_router(routes_assets.router) -app.include_router(routes_tables.router) -app.include_router(routes_grid.router) -app.include_router(routes_records.router) -# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user -# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so -# a new page is a builder module plus one registry line, and nothing here changes. -app.include_router(routes_pages.router) -app.include_router(routes_admin.router) -app.include_router(routes_automation.router) -app.include_router(routes_keychain.router) -app.include_router(routes_statements.router) -# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. -# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path -# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a -# platform-operator route ever shadowing each other. Every path it declares is gated by -# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. -app.include_router(routes_platform_admin.router) -# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are -# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is -# something every user does with their own views/folders/databases. -app.include_router(routes_alerts.router) -app.include_router(routes_shares.router) -app.include_router(routes_uploads.router) -# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this -# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: -# a router included AFTER it answers 404 forever while importing fine, type-checking fine and -# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. -# -# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished -# features that would have shipped dead. The workers each posted a mount ask and said they would -# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The -# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the -# control is that the enumeration must NAME every router in this file.** -app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated -app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory -# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its -# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so -# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token -# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot -# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. -app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door -# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, -# because the wave-23 scar is exactly this: three finished routers shipped with no include_router -# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks -# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. -app.include_router(routes_odoo_tables.router) -# ⭐⭐ WAVE 31 · T49 / C4 — THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY. -# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in -# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by -# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts -# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name -# that lies to every network tab, log line and bug report. -# ⚠ MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar. -app.include_router(routes_connected_tables.router) -# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` -# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep -# copies), which is what lets a filtered view pick up a row created in another tab, by an automation -# or by a connector sync without re-downloading the world. -# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: -# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` -# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 -# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no -# mounted route — which is precisely the control the wave-23 scar above was written to install. -# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this -# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). -app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line -# ⛔ WAVE 31 (R10 / C5) — E's router, taken by the INTEGRATOR under W31-T08's own done-when -# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it. -# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at -# 60/61 for it. Four waves of the same defect — three routers in wave 23, four features in -# wave 29 — is why the assertion exists and why the line is not left for later. -app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's line -# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the -# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. -# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own -# gate before it front-inserted: `include_router` APPENDS, and `app.mount("/", _AppStatic(...), -# html=True)` swallows everything reached after it, so a router added below that mount answers -# **405 on POST and 404 on GET** while every one of its own tests passes. The comment at :213 states -# the rule; E's measurement is what turns it from advice into a number. -# ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in -# app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). -app.include_router(routes_query.router) # R1 / C5 — E's router, A's line -# ⭐⭐ WAVE 33 (C2) — THE SIXTH CONSECUTIVE WAVE IN WHICH THIS BLOCK IS THE THING THE PROTOCOL LOSES. -# Contract C2 is written for exactly that: a lane creating a `routes_*.py` posts an ASK, and the -# INTEGRATOR adds the line in the SAME wave. Both of these arrived that way (C's `ASK C-1`, G's -# `ASK G-1`), and `verify_api::section_w23_mounts` asserts each path in `app.openapi()["paths"]` -# with an NC that comments a mount out and goes RED. -# ⚠ Same placement rule as the line above — ABOVE the `app.mount("/", _AppStatic(...), html=True)` -# at the end of the file, never after it. -# ⛔⛔ THE PUBLISH DOOR IS DELIBERATELY NOT MOUNTED FOR THE WAVE-33 DEPLOY (2026-08-14). -# Owner ruled *"deploy what we can and you can defer any validation later."* Everything else in the -# wave ships; this one router does not, because QA REPRODUCED three HIGH defects on it and **all -# three are armed ONLY by mounting it** — the route 404s on the currently-live build while its -# sibling `/api/v1/forms/{token}` answers 403 from the same request in the same second, so leaving -# it unmounted preserves today's behaviour EXACTLY rather than introducing a new one: -# 1. a published FORM view serves its stored SUBMISSIONS to anyone with the link — `form` is in -# `PUBLISHABLE_MODES` and `PublishedBody` has no `form` branch, so it falls to the ELSE that -# renders rows. Reproduced end to end. -> W33-T68 -# 2. `_visible_keys` falls back to every non-hidden field on an EMPTY intersection, and -# `delete_field` never prunes a view's `visible` — so deleting the columns a published view -# showed silently WIDENS its public page to the table default. -> W33-T69 -# 3. "a wrong passphrase is indistinguishable from a wrong token" is false twice: 1.66 ms vs -# 342.41 ms medians (206x, ZERO overlap across 24 samples) because a bad token returns before -# PBKDF2, plus a zero-timing oracle on the sibling GET. The rate limit keys on the -# caller-supplied `x-forwarded-for`: 100 of 100 allowed against a 30/window ceiling. -> W33-T70 -# ⭐ TO RE-ARM: delete this comment block and uncomment the line below — nothing else. The router, -# its client and its 178-check gate are all shipped and unchanged; only the mount is withheld. -# ⚠ `verify_api::section_w23_mounts` asserts this path in `app.openapi()["paths"]`, so that gate is -# EXPECTED RED on the publish rows until T68-T70 land. That is the alarm doing its job, not a -# regression — do not "fix" it by weakening the assertion. -# app.include_router(routes_publish.router) # R5 / C2 — C's router, A's line (the publish door) -# ⛔ NOT BEHIND `module_gate("product_data")`, WHICH IS THE WHOLE REASON IT IS A SECOND ASSET DOOR. -# `routes_assets.py` gates EVERY one of its routes on that module, so a connector logo served from -# there would 403 for any account without the product-data grant — i.e. the Connectors directory -# would lose its logos for exactly the accounts most likely to be setting a connector up (C6). -app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's line (brand marks) -# ⛔ TWO OF ITS SIX PATHS ARE UNAUTHENTICATED BY DESIGN (`/slack/events`, `/slack/interact`) and -# that is the DOOR, not an omission — Slack posts to them with no session and could not carry one. -# They are built on `routes_forms`' proven public shape: signing-secret verification, a sliding-window -# rate limit, a body cap and ONE non-oracle 403, so a bad signature cannot distinguish "no such -# workspace" from "not yours". ⚠ There is no auth middleware and no exempt-path allow-list in this -# file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why -# `verify_api` asserts the absence rather than an entry in a list that does not exist. -app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) - - -# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) -# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic -# behavior. An alias that kept the old auth would be a bypass of everything above it. -_GATE = module_gate(routes_customers.MODULE) - - -@app.get("/api/customers", deprecated=True) -def _customers_alias(session: Session = Depends(_GATE)): - return routes_customers._payload(session) - - -@app.patch("/api/customers/{pid}", deprecated=True) -def _patch_alias(pid: int, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - return routes_customers.patch_customer(pid, body, session) - - -def _assert_contract_floor(): - """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. - - This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the - canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and - the embedded host are serving two different schemas and the grid's own contract has forked — - the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup - on whatever machine is actually running. - """ - import aios_grid - base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] - if [f["key"] for f in base] != [f["key"] for f in FIELDS]: - raise RuntimeError( - "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " - "disagree on the base field set — embed and standalone would serve different schemas. " - "Run aios-web/verify_fields_contract.py.") - - -def _startup_notes(): - """Say the two things an operator must know, once, at import — never in a response body.""" - if aios_session.EPHEMERAL_SECRET: - print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process " - "key. Sessions will not survive a restart and will not work across workers. " - "Set it in production.") - if os.environ.get("AIOS_INSECURE_SSL") == "1": - print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound " - "requests. Local development only; never in a deployed environment.") - - -_assert_contract_floor() -_startup_notes() - - -# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. -# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login -# page cannot live behind a password prompt, so the shell's own assets must be public — which they -# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. -# -# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also -# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are -# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to -# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` -# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), -# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same -# answer as any other path that is not part of the app. -# -# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); -# this guard is what makes the API safe regardless of what the bundle happens to contain. -_DEV_FIXTURES = {"sample_customers.json"} - - -class _AppStatic(StaticFiles): - async def get_response(self, path, scope): - if Path(path).name in _DEV_FIXTURES: - raise StarletteHTTPException(status_code=404, detail="Not Found") - resp = await super().get_response(path, scope) - # Vite content-hashes everything under assets/ (a change is a NEW url), so those are - # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. - # index.html must stay revalidated or a deploy would strand returning browsers on the old - # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. - # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, - # and `startswith("assets/")` silently skipped every asset (measured on the local probe). - if path.replace("\\", "/").lstrip("/").startswith("assets/"): - resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" - else: - resp.headers["Cache-Control"] = "no-cache" - return resp - - -# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets -if _WEB_DIST.is_dir(): - app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") - - -# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- -# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in -# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's -# default envelope; scoped users still pay their own scope's first build, once. -# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends -# run against fakes) can never fire a live Odoo pull. -def _seed_and_sync_store(): - """Bring the analytical store (harness.datastore) LIVE for this container — the app.py - bootstrap, mirrored (2026-07-31, owner item 1). - - `harness.datastore` powers every measure column/condition, and ONLY app.py used to call - `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store - that never existed and every measure cell served blank. Seeding alone was NOT enough either, - and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless - `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands - EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced - forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the - write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded - passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still - serve. - """ - import time as _t - t0 = _t.time() - try: - from harness import datastore as _ds - if _ds.ensure_seed(): - print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") - # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's - # search_reads must never interleave on the shared client's xmlrpc transport. - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - res = {} - for i in range(12): - res = _ds.sync_all(log=lambda *a, **k: None) - print(f"[aios-api] datastore sync pass {i + 1}: " - + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) - if all(v.get("phase") == "live" for v in res.values()): - break - # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see - # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — - # one reconcile pass at boot removes both classes of phantom row before the first - # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of - # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. - try: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception: - pass - print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " - f"(ready={_ds.ready()})") - # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. - # - # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside - # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a - # freshly booted container could spawn the four locked databases was T+30 MINUTES. The - # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s - # across a ~20-minute window over two boots returned the pre-wave schema on all 20 - # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like - # this — a thing that never ran, mistaken for a thing that ran and failed. - # - # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is - # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, - # then the delete reconcile. Calling it from the other thread would race the seed and hit - # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which - # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild - # refuses — correctly, but for a reason that reads like a data loss scare. - # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. - _pull_meta("boot") - _rebuild_odoo_relational("boot") - # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. - # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and - # this sweep is unrelated to it — putting it last means a slow Odoo sync cannot delay the - # one thing on this path that fixes a grid the owner has asked about twice. - _sweep_automation_schemas("boot") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] datastore seed/sync skipped: {e}") - - -def _pull_meta(why): - """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. - - ⛔ WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the - process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a - developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE - product stays empty — which is the whole difference between "the loader works" and "the - product has the data". Odoo is already arranged this way (`sync_all` runs in the container); - this is the same arrangement for the second connector. - - ⚠ FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no - error: a tenant that has not connected Meta is a normal state, and this runs on every boot. - ⚠ The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot - is not the place for a 90-day backfill — the resync pass widens it. - """ - try: - from harness import meta_store as _meta - if not _meta.token(): - print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this " - f"deployment - the connector is idle, not broken") - return - # ⛔ PASSED, NOT SET IN THE ENVIRONMENT. `meta_store.INSIGHTS_DAYS` binds at - # IMPORT, so an `os.environ.setdefault` here executed after the module was - # already loaded and changed NOTHING: every boot pulled 90 days instead of 7, - # which is the slow path that trips the per-ad-account rate limit and never - # finishes. A knob read at import cannot be turned by a caller at runtime. - rep = _meta.sync("royal-imports", log=lambda *_a: None, insights_days=7) - for p in rep.get("problems") or []: - print(f"[aios-api] meta sync PROBLEM ({why}): {p}") - print(f"[aios-api] meta sync done ({why}): " - + ", ".join(f"{k}={v['in_mirror']}" for k, v in sorted(rep["tables"].items()))) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] meta sync FAILED ({why}): {type(e).__name__}: {e}") - - -def _rebuild_meta_relational(why, rt): - """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. - - ⭐ CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than - tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just - established which tenant's file this process holds open. Spawning Meta here inherits that - binding instead of rebinding it under live readers — which is the documented way to serve one - tenant's rows to another with nothing raised. - - ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables - in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure, - and it must not print an error every 30 minutes for every tenant that does not use Meta. - ⛔ Its own try/except for the reason the two passes above have theirs: a Meta refusal must not - cancel an Odoo rebuild that already succeeded. - """ - try: - import meta_relational as _meta - counts = _meta.refresh(rt, why) - if counts: - print(f"[aios-api] meta relational rebuild done ({why}): " - + ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) - try: - import automation_engine as _engine - _engine.refresh_relations(rt, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] meta relational rebuild FAILED ({why}): {type(e).__name__}: {e}") - - -def _sweep_automation_schemas(why): - """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. - - Owner items 13 and 15 — the TikTok comments lock, and the comment CONTENT column he says he has - asked for *"many times"*. **Both were already correct in the source.** `TT_COMMENT_FIELDS` - carries `field_def("text", "Comment")` and `TT_LOCKED_TABLES` already contains the comments - table, both since wave 31, with his words quoted in the comment beside them. So this ticket is - not a schema change and there is nothing to design: it is DELIVERY. - - ⛔ THE MECHANISM, WHICH IS THE WHOLE OF IT. `ut_ensure` MERGES fields into an existing table and - stamps `recordMode` — but only **when something calls it**, and the only callers are automation - runs. A tenant whose `ut_tt_comments` was spawned before the declaration changed keeps the old - shape until an automation happens to run against it. Nothing sweeps existing tenants. That is - [[a-migration-that-runs-on-the-next-write]], and it is this wave's stated thesis: a declaration - that never reaches a tenant is indistinguishable from one that was never written. - - ⛔⛔ AND IT MUST RUN **IN THE CONTAINER**, WHICH IS WHY THIS IS IN `main.py` AND NOT A SCRIPT. - D-195, measured three times: a developer's CLI write to the tenant store is reverted by the - running Space within a minute (download-modify-upload, last-write-wins) — and **the write - reports success every time**, then a FRESH read confirms it, and it is gone by the next poll. - A connector's tables must be spawned BY THE CONTAINER; a CLI spawn is a dry run that lies. - - ⚠ EVERY TENANT, unlike `_rebuild_odoo_relational` below — and the asymmetry is deliberate - rather than an oversight. That function is scoped to royal because it derives from the DuckDB - mirror, and `harness/datastore` is a ONE-FILE-AT-A-TIME process global (D-29): rebinding it per - tenant in a daemon thread can serve one tenant's rows to another with nothing raised. This - sweep touches only `user_tables` through each tenant's own `rt`, which has no such global — so - the hazard that scopes that one does not exist here, and TikTok automations run in tenants - other than #0. - - ⚠ CHEAP ON A CORRECT TENANT: `ut_ensure` short-circuits when nothing changed, so this is a read - per child table on a tenant that is already right, and the whole delivery on one that is not. - ⚠ ONE TENANT'S FAILURE MUST NOT STOP THE NEXT. Each is wrapped: a tenant whose store is - unreachable at boot is reported and skipped, never allowed to abort the sweep for everyone. - """ - try: - import automation_engine as _eng - from harness import runtime as _runtime - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED — import failed: {e}") - return - try: - schemas = _eng.platform_schemas() - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED — no declarations: {e}") - return - tenants = [] - try: - tenants = _runtime.known_tenants() - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED — tenant list unreadable: {e}") - return - for slug in tenants: - try: - rt = _runtime.get_runtime(slug) - # ⛔⛔ FIXED 2026-08-13 — THIS SWEEP WAS SPAWNING EIGHT DATABASES IN EVERY TENANT. - # Owner: *"Database for Royal Imports, why we have fucking IG and TIktok databases."* - # `ut_ensure`'s first line is *"Create the table if it is missing"*, and this loop fed - # it every child of every platform schema for every tenant — so tenant #0, a floral and - # giftware importer with ZERO automations, woke up on 2026-08-13 at 12:05 UTC owning - # `ut_ig_posts`, `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots` and the - # four TikTok twins, all empty, all in his nav flyout. **The bug is one word wide:** - # this function's own title says *"MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE - # THE TABLES"* and its body called a CREATE-OR-MERGE function to do a MERGE-ONLY job. - # [[reuse-and-delete-are-hypotheses]] — `ut_ensure` was the right function for the - # merge and brought a second behaviour nobody wanted with it. - # - # ⭐ THE PREDICATE IS "DOES THIS TENANT ALREADY HAVE THE TABLE", read ONCE per tenant - # rather than per child — `rt.get` deep-copies the whole tenant document (28.6 MB on - # tenant #0), so asking eight times is eight copies to answer one question. - # ⚠ AND IT MUST NOT WEAKEN THE DELIVERY: a tenant that HAS `ut_tt_comments` still gets - # the `text` column and the `recordMode` stamp, which is the entire point of T07. The - # sweep now delivers to tables that exist and mints none, which is what it always - # claimed to do. - have = set(rt.get("user_tables") or {}) - ensured, skipped = [], [] - for s in schemas: - for key, child in (s.get("children") or {}).items(): - if key not in have: - skipped.append(key) - continue - got = _eng.ut_ensure(rt, child["label"], child["fields"], "automation", - key=key, lock_fields=True, - record_mode=child["record_mode"]) - if got: - ensured.append(got) - if skipped: - # ⭐ SAID OUT LOUD, never silently skipped — this repo's "no silent caps" rule. A - # sweep that quietly does nothing looks identical to a sweep that is not running, - # which is how the previous behaviour survived review in the first place. - print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " - f"present in this tenant, so nothing was created for them " - f"({', '.join(sorted(skipped))}) — they are spawned by an automation that " - f"needs them, never by this sweep") - # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. - # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and - # T07's `done-when` asserts both ("no `ut_tt_*` grid carries an Instagram column"). On a - # tenant that ran TikTok before W30-T08 the 26 machine-authored IG columns are still - # there — the detector was fixed, the damage never was. - # ⚠ `kept` IS THE HONEST HALF: a foreign column that HOLDS DATA is REPORTED, never - # deleted. If it is non-empty, the screenshot shows a column and the report is the - # answer (W30/R6's second sentence). - st = {} - try: - st = _eng.retract_foreign_presets(rt, log=lambda *a, **k: None) or {} - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) {slug}: retraction FAILED: {e}") - print(f"[aios-api] schema sweep ({why}) {slug}: ensured={len(ensured)} " - f"tables={st.get('tables', 0)} columns={st.get('columns', 0)} " - f"cells={st.get('cells', 0)} flags={st.get('flags', 0)} " - f"kept={st.get('kept') or []}") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) {slug}: FAILED: {e}") - # ⭐ A SUCCESS MARKER, for `_rebuild_odoo_relational`'s stated reason: D-107 was chased for a - # day on ABSENT log markers, which cannot tell "it ran and was fine" from "it was never - # reached". Three failure markers and no success marker makes silence ambiguous. - print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)") - - -def _rebuild_odoo_relational(why): - """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or - 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone - could have asked" are different diagnoses. - - ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased - for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran - and was fine" from "the rebuild was never reached". Three failure markers and no success - marker means silence is ambiguous; now it is not. - - ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The - caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and - royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks - straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME - process-global, and rebinding it in a daemon thread under live readers can serve one tenant's - rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. - """ - try: - import odoo_relational as _rel - from harness import runtime as _runtime - if not _rel.is_royal("royal-imports"): - return - _rt = _runtime.get_runtime("royal-imports") - counts = _rel.refresh(_rt, "royal-imports") - # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. - # - # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field - # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to - # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a - # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every - # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the - # tables simply look finished and answer nothing. - # - # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count - # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. - # Calling the inner one here would return a plausible number and change no cell. - # - # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source - # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest - # refusal silently cancel a relational pass that had already succeeded. - try: - import automation_engine as _engine - _engine.refresh_relations(_rt, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") - try: - import rollup_sql as _rollup - for _b, _key, _l, _f in _rel.TABLES: - _rollup.compute(_rt, _key) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") - print(f"[aios-api] odoo relational rebuild done ({why}): " - + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) - _rebuild_meta_relational(why, _rt) - except Exception as e: # noqa: BLE001 - # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only - # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a - # `BinderException` about a missing column is a different action from a timeout or an auth - # failure, and the text alone often does not say which it was. - print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") - - -def _store_resync_loop(): - """Keep the analytical mirror current — the API-process stand-in for the re-sync the - Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a - refreshed pool re-reads the freshly synced store.""" - import time as _t - passes = 0 - while True: - # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. - # - # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 — - # the 30m/1h/4h/daily/manual presets, the server-side clamp, the config door and - # `odoo_relational.sync_seconds()` which turns the stored preset into seconds — and its own - # ticket said the one-line change belonged to A because `main.py` is A's file. That ASK was - # never sent, so the value was stored, displayed, clamped and IGNORED: a person could pick - # "every 4 hours" and the loop would keep resyncing every 30 minutes with nothing anywhere - # reporting the disagreement. Found by `verify_reachability` naming `sync_seconds` as a - # function whose ONLY caller was its own gate [[artifact-with-no-importer]]. - # - # ⚠ `None` MEANS MANUAL AND MUST NOT MEAN ZERO. R11's `manual` preset returns None from - # `sync_seconds`; treating that as a falsy interval would spin this loop with no sleep at - # all. It parks at the default cadence instead and simply does no work — the tenant asked - # not to be synced automatically, not for the server to stop breathing. - # ⚠ RE-READ EVERY PASS, deliberately: a cadence changed in Settings takes effect on the - # next cycle rather than at the next container restart, which is what makes it a setting. - # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is - # enforced in `sync_seconds` as well as at the write door, for the same reason. - _every = 1800 - try: - import odoo_relational as _rel_cad - from harness import runtime as _rt_cad - _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) - _every = 1800 if _secs is None else max(int(_secs), 60) - except Exception: # noqa: BLE001 - pass - _t.sleep(_every) - passes += 1 - try: - from harness import datastore as _ds - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - _ds.sync_all(log=lambda *a, **k: None) - # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot - # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; - # without it deleted Odoo lines inflate every sum on the mirror FOREVER. - if passes % 4 == 0: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] store resync failed: {e}") - # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt - # AFTER the mirror they are derived from, in the same pass and in that order — deriving - # from a mirror this loop is about to advance would publish a worklist one cycle stale - # every single time. - # - # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must - # not swallow the sync's error message, and — worse the other way — a sync failure must - # not skip a rebuild that had nothing wrong with it. Two independent failures, two - # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) - # - # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` - # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so - # "the wiring exists" and "the data is current" are different claims and only the stamp - # can tell a user which one they are looking at. - # - # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which - # is what made the boot path silent for 30 minutes after every restart; it is now the - # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here - # would be a second thing to keep in step, and the two would answer differently on the - # next ruling — the exact shape the Views top-up was just fixed for on the other side of - # this wave. - # ⛔ THE META PULL RIDES THE RESYNC TOO, AND LEAVING IT OUT WAS A REAL GAP — caught by - # reading the deploy's own boot log rather than by any gate. `_pull_meta` was wired into - # `_seed_and_sync_store` ALONE, i.e. it ran exactly once per container, at boot, behind a - # full Odoo sync. So a boot where the Graph call was rate-limited, slow or simply after the - # thread died left the mirror empty with NOTHING to retry it: the relational rebuild below - # would then find no `meta_*` tables every 30 minutes forever and skip, silently and - # correctly. A connector that can only ever be established at boot is one bad boot away - # from being permanently absent. - # ⚠ Cheap when there is nothing to do: no token => one line and return. - _pull_meta("resync") - _rebuild_odoo_relational("resync") - - -def _prewarm(): - import time as _t - t0 = _t.time() - # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can - # take many minutes of XML-RPC to close, and the first live probe of this arrangement - # showed the pool warm (42s) silently queued behind it — the whole app cold for every - # visitor while a background column channel caught up. Measure cells retry via the - # transient rule until the sync lands; nothing else waits on it. - import threading as _th - _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() - try: - from harness import runtime as _runtime - rt = _runtime.get_runtime("royal-imports") - routes_customers.warm_default(rt) - import pages as _pages - _pages.warm_default(rt) - # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent - # from this list, so its memo was always filled by a visitor rather than by boot: call 1 of - # `GET /automations` after every deploy downloaded the whole `user_tables` document (35.8 MB - # ceiling, under the store lock) onto whoever clicked first. Memoising the WARM path — two - # waves of it — could not touch that, because the cold call is the one that fills the memo. - # ⚠ It elects a STRING and keeps no document; see `warm_default`'s own note on why caching - # the bucket would trade a latency for memory this tier does not have. - import routes_automation as _rauto - _rauto.warm_default(rt) - print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") - except Exception as e: # noqa: BLE001 — boot must not die on a warm-up - print(f"[aios-api] prewarm skipped: {e}") - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W31-T46 / D-160 — THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was -# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of -# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`: -# no `royal.duckdb` ⇒ `datastore.ready()` False forever ⇒ `ro_con()` refuses ⇒ every measure -# column blank AND — since wave 30 put the Odoo grids on the mirror — **two user-facing grids -# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the -# symptom was read as a pinned tag and a store problem for days. -# -# ⛔ AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1` -# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) — but that -# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard -# written for the dangerous case that the ordinary case walks straight past. -# -# ⭐ SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its -# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()` -# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file -# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half — -# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop — stays exactly where it was, -# because the env gate's stated reason is still true: importing `api.main` in a gate must never -# fire a live Odoo pull. -# -# ⚠ THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only -# on a genuinely fresh disk — so on every developer box and in every gate run the thread is never -# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids. -#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a -#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause. -MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""} - - -def _seed_mirror_if_absent(): - """Hydrate the analytical mirror when this container has none — INDEPENDENT of `AIOS_PREWARM`. - - Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no - HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator - actions, and a blank grid cannot tell them apart. - """ - from harness import datastore as _ds - MIRROR_SEED["attempted"] = True - try: - if _ds.ensure_seed(): - MIRROR_SEED["seeded"] = True - print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)") - return True - if not _ds.DB_PATH.exists(): - MIRROR_SEED["cause"] = ( - "this container has no analytical mirror and the seed snapshot could not be " - "fetched (no HF_TOKEN, or the dataset was unreachable)") - MIRROR_SEED["recommendation"] = ( - "set HF_TOKEN on the deployment; until then every connected grid and every " - "measure column served from the mirror is empty") - print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}") - except Exception as e: # noqa: BLE001 — boot must not die - MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}" - MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability" - print(f"[aios-api] mirror seed skipped: {e}") - return False - - -try: - from harness import datastore as _ds_boot - if not _ds_boot.DB_PATH.exists(): - import threading as _threading_seed - _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True, - name="mirror-seed").start() -except Exception as e: # noqa: BLE001 - print(f"[aios-api] mirror seed not scheduled: {e}") - -if os.environ.get("AIOS_PREWARM") == "1": - import threading as _threading - _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start() - _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start() +"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. + +It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, +reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the +JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. + +WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: + + * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared + `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same + anonymous principal and no route could scope anything. Now a signed stateless cookie carries + a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the + user RECORD. `/api/health` stays unauthenticated (liveness only). + * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home + for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). + * **The SSL shim is env-gated.** It used to run at import, unconditionally. + +⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the +session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This +is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB +commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. +""" +import os +import sys +from pathlib import Path + +# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- +# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an +# unverified default context; the HF Space and the container are unaffected. This used to run +# unconditionally at import — i.e. the shipped container disabled TLS verification for every +# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a +# man-in-the-middle away from being someone else's data. It is now opt-in, must be set +# deliberately, and is never on by default. +# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. +if os.environ.get("AIOS_INSECURE_SSL") == "1": + import ssl + ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 + +# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container +# (where the layout differs from the local sibling-dir default). --- +_HERE = Path(__file__).resolve() +_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) +for _p in (str(_RI), str(_HERE.parent)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from dotenv import load_dotenv # noqa: E402 +load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed + +from fastapi import Body, Depends, FastAPI, Request # noqa: E402 +from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 +from fastapi.responses import JSONResponse # noqa: E402 +from fastapi.staticfiles import StaticFiles # noqa: E402 +from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 + +import aios_session # noqa: E402 +import routes_admin # noqa: E402 +import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) +import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) +import routes_auth # noqa: E402 +import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) +import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) +import routes_customers # noqa: E402 +import routes_grid # noqa: E402 +import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) +import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) +import routes_nav # noqa: E402 +import routes_pages # noqa: E402 +import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) +import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) +import routes_records # noqa: E402 +import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) +import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) +import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) +import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) +import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) +import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) +import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) +import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) +import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) +import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) +from core import grid_events # noqa: E402 +from deps import Session, module_gate # noqa: E402 + +_WEB_DIST = _HERE.parents[1] / "web" / "dist" + +# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- +# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never +# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the +# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so +# embed == standalone by construction. +# +# TWO REASONS THIS IS HERE and not folded into the routes: +# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR +# layout is broken, and discovering that from a 500 on the first customer request instead of +# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a +# rewrite dropped it and this restores it.) +# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and +# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this +# API have not drifted. The rewrite removed them and the referee went red; retargeting the +# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do +# not delete — but only when the subject genuinely moved. Here it should not have moved). +# +# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from +# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and +# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone +# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the +# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. +import json as _json_contract # noqa: E402 +_FIELDS_PATH = _RI / "aios_grid_fields.json" +if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " + "platform root (it also carries the data layer this API imports)." + ) +_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) +FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract +# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. +# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be +# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. +_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS + if f["source"] == "odoo" and f["type"] in ("text", "status", "select", + "date")} + +app = FastAPI(title="AIOS web API") + +# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with +# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the +# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. +app.add_middleware(GZipMiddleware, minimum_size=1024) + + +@app.exception_handler(StarletteHTTPException) +def _error_shape(request: Request, exc: StarletteHTTPException): + """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. + + `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a + 422 from a malformed path param) is wrapped here so a client never has to branch on two + different error bodies. + """ + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + body = detail + else: + body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} + return JSONResponse(body, status_code=exc.status_code, + headers=getattr(exc, "headers", None)) + + +@app.exception_handler(grid_events.StoreUnavailable) +def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): + """THE SAFETY NET for "the store is down" → 503, from anywhere. + + ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a + `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own + try/except — surfaced as a 500. A store outage is a normal operational state and every route + here touches the store at least twice (the workspace read, then the write), so "remember to + wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler + means a store outage can only ever be a 503, whichever call raised it. + + The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this + adapter, which has no durable session dict to degrade into. A 200 over a write that + evaporated is the failure the whole rule exists to prevent. + """ + return JSONResponse( + {"error": {"code": "store_unavailable", + "message": "the tenant store is unavailable — no change was saved"}}, + status_code=503) + + +@app.get("/api/health") +def health(): + """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not + reveal anything about the deployment beyond "the process is up". No version, no tenant list, + no config: a health endpoint is the one URL every scanner finds first. + + ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned + release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated + caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind + a session — and the authoritative copy is the `VERSION` file in the Space repo, which + `deploy_web.space_version()` reads without needing the app to be up at all.""" + return {"ok": True} + + +app.include_router(routes_auth.router) +app.include_router(routes_nav.router) +app.include_router(routes_customers.router) +app.include_router(routes_products.router) +app.include_router(routes_assets.router) +app.include_router(routes_tables.router) +app.include_router(routes_grid.router) +app.include_router(routes_records.router) +# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user +# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so +# a new page is a builder module plus one registry line, and nothing here changes. +app.include_router(routes_pages.router) +app.include_router(routes_admin.router) +app.include_router(routes_automation.router) +app.include_router(routes_keychain.router) +app.include_router(routes_statements.router) +# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. +# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path +# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a +# platform-operator route ever shadowing each other. Every path it declares is gated by +# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. +app.include_router(routes_platform_admin.router) +# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are +# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is +# something every user does with their own views/folders/databases. +app.include_router(routes_alerts.router) +app.include_router(routes_shares.router) +app.include_router(routes_uploads.router) +# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this +# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: +# a router included AFTER it answers 404 forever while importing fine, type-checking fine and +# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. +# +# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished +# features that would have shipped dead. The workers each posted a mount ask and said they would +# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The +# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the +# control is that the enumeration must NAME every router in this file.** +app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated +app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory +# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its +# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so +# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token +# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot +# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. +app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door +# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, +# because the wave-23 scar is exactly this: three finished routers shipped with no include_router +# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks +# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. +app.include_router(routes_odoo_tables.router) +# ⭐⭐ WAVE 31 · T49 / C4 — THE SOURCE-NEUTRAL DOOR TO THE SAME CAPABILITY. +# `/api/v1/connected-tables/{key}/rows` is an ALIAS: every request lands in +# `routes_odoo_tables.odoo_table_rows`, so "the Odoo path behaves byte-identically" holds by +# CONSTRUCTION rather than by two implementations that agree on the day they were written. R2 puts +# Meta Ads on the same mirror, and a Meta campaign served from a URL with `odoo` in it is a name +# that lies to every network tab, log line and bug report. +# ⚠ MOUNTED IN THE SAME CHANGE AS THE MODULE, per the line above and for the same wave-23 scar. +app.include_router(routes_connected_tables.router) +# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` +# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep +# copies), which is what lets a filtered view pick up a row created in another tab, by an automation +# or by a connector sync without re-downloading the world. +# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: +# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` +# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 +# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no +# mounted route — which is precisely the control the wave-23 scar above was written to install. +# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this +# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). +app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line +# ⛔ WAVE 31 (R10 / C5) — E's router, taken by the INTEGRATOR under W31-T08's own done-when +# ("no router ships unmounted") because `main.py` is D's fence and D's queue did not reach it. +# It was written, gated and 404-dead: `verify_web_agent.py` asserts THIS LINE and was red at +# 60/61 for it. Four waves of the same defect — three routers in wave 23, four features in +# wave 29 — is why the assertion exists and why the line is not left for later. +app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's line +# ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the +# FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses. +# ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own +# gate before it front-inserted: `include_router` APPENDS, and `app.mount("/", _AppStatic(...), +# html=True)` swallows everything reached after it, so a router added below that mount answers +# **405 on POST and 404 on GET** while every one of its own tests passes. The comment at :213 states +# the rule; E's measurement is what turns it from advice into a number. +# ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in +# app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). +app.include_router(routes_query.router) # R1 / C5 — E's router, A's line + + +# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) +# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic +# behavior. An alias that kept the old auth would be a bypass of everything above it. +_GATE = module_gate(routes_customers.MODULE) + + +@app.get("/api/customers", deprecated=True) +def _customers_alias(session: Session = Depends(_GATE)): + return routes_customers._payload(session) + + +@app.patch("/api/customers/{pid}", deprecated=True) +def _patch_alias(pid: int, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + return routes_customers.patch_customer(pid, body, session) + + +def _assert_contract_floor(): + """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. + + This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the + canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and + the embedded host are serving two different schemas and the grid's own contract has forked — + the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup + on whatever machine is actually running. + """ + import aios_grid + base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] + if [f["key"] for f in base] != [f["key"] for f in FIELDS]: + raise RuntimeError( + "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " + "disagree on the base field set — embed and standalone would serve different schemas. " + "Run aios-web/verify_fields_contract.py.") + + +def _startup_notes(): + """Say the two things an operator must know, once, at import — never in a response body.""" + if aios_session.EPHEMERAL_SECRET: + print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process " + "key. Sessions will not survive a restart and will not work across workers. " + "Set it in production.") + if os.environ.get("AIOS_INSECURE_SSL") == "1": + print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound " + "requests. Local development only; never in a deployed environment.") + + +_assert_contract_floor() +_startup_notes() + + +# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. +# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login +# page cannot live behind a password prompt, so the shell's own assets must be public — which they +# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. +# +# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also +# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are +# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to +# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` +# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), +# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same +# answer as any other path that is not part of the app. +# +# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); +# this guard is what makes the API safe regardless of what the bundle happens to contain. +_DEV_FIXTURES = {"sample_customers.json"} + + +class _AppStatic(StaticFiles): + async def get_response(self, path, scope): + if Path(path).name in _DEV_FIXTURES: + raise StarletteHTTPException(status_code=404, detail="Not Found") + resp = await super().get_response(path, scope) + # Vite content-hashes everything under assets/ (a change is a NEW url), so those are + # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. + # index.html must stay revalidated or a deploy would strand returning browsers on the old + # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. + # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, + # and `startswith("assets/")` silently skipped every asset (measured on the local probe). + if path.replace("\\", "/").lstrip("/").startswith("assets/"): + resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" + else: + resp.headers["Cache-Control"] = "no-cache" + return resp + + +# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets +if _WEB_DIST.is_dir(): + app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") + + +# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- +# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in +# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's +# default envelope; scoped users still pay their own scope's first build, once. +# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends +# run against fakes) can never fire a live Odoo pull. +def _seed_and_sync_store(): + """Bring the analytical store (harness.datastore) LIVE for this container — the app.py + bootstrap, mirrored (2026-07-31, owner item 1). + + `harness.datastore` powers every measure column/condition, and ONLY app.py used to call + `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store + that never existed and every measure cell served blank. Seeding alone was NOT enough either, + and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless + `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands + EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced + forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the + write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded + passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still + serve. + """ + import time as _t + t0 = _t.time() + try: + from harness import datastore as _ds + if _ds.ensure_seed(): + print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") + # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's + # search_reads must never interleave on the shared client's xmlrpc transport. + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + res = {} + for i in range(12): + res = _ds.sync_all(log=lambda *a, **k: None) + print(f"[aios-api] datastore sync pass {i + 1}: " + + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) + if all(v.get("phase") == "live" for v in res.values()): + break + # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see + # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — + # one reconcile pass at boot removes both classes of phantom row before the first + # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of + # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. + try: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception: + pass + print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " + f"(ready={_ds.ready()})") + # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. + # + # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside + # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a + # freshly booted container could spawn the four locked databases was T+30 MINUTES. The + # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s + # across a ~20-minute window over two boots returned the pre-wave schema on all 20 + # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like + # this — a thing that never ran, mistaken for a thing that ran and failed. + # + # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is + # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, + # then the delete reconcile. Calling it from the other thread would race the seed and hit + # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which + # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild + # refuses — correctly, but for a reason that reads like a data loss scare. + # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. + _pull_meta("boot") + _rebuild_odoo_relational("boot") + # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. + # ⚠ AFTER the two above and not before: those advance the mirror and can take minutes, and + # this sweep is unrelated to it — putting it last means a slow Odoo sync cannot delay the + # one thing on this path that fixes a grid the owner has asked about twice. + _sweep_automation_schemas("boot") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] datastore seed/sync skipped: {e}") + + +def _pull_meta(why): + """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. + + ⛔ WHY IT HAS TO HAPPEN HERE AND NOT ON A LAPTOP. The mirror is a FILE that lives beside the + process; the Space's copy is seeded from the HF dataset and knows nothing about a DuckDB on a + developer's box. Running `meta_store --sync` locally populates the local mirror and the LIVE + product stays empty — which is the whole difference between "the loader works" and "the + product has the data". Odoo is already arranged this way (`sync_all` runs in the container); + this is the same arrangement for the second connector. + + ⚠ FAIL-QUIET AND SILENT WHEN THERE IS NOTHING TO DO. No token => no Meta => one line, no + error: a tenant that has not connected Meta is a normal state, and this runs on every boot. + ⚠ The window is deliberately SHORT here (`META_INSIGHTS_DAYS`, default 7 at boot) because boot + is not the place for a 90-day backfill — the resync pass widens it. + """ + try: + from harness import meta_store as _meta + if not _meta.token(): + print(f"[aios-api] meta sync skipped ({why}): no META_ADS_ACCESS_TOKEN in this " + f"deployment - the connector is idle, not broken") + return + # ⛔ PASSED, NOT SET IN THE ENVIRONMENT. `meta_store.INSIGHTS_DAYS` binds at + # IMPORT, so an `os.environ.setdefault` here executed after the module was + # already loaded and changed NOTHING: every boot pulled 90 days instead of 7, + # which is the slow path that trips the per-ad-account rate limit and never + # finishes. A knob read at import cannot be turned by a caller at runtime. + rep = _meta.sync("royal-imports", log=lambda *_a: None, insights_days=7) + for p in rep.get("problems") or []: + print(f"[aios-api] meta sync PROBLEM ({why}): {p}") + print(f"[aios-api] meta sync done ({why}): " + + ", ".join(f"{k}={v['in_mirror']}" for k, v in sorted(rep["tables"].items()))) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] meta sync FAILED ({why}): {type(e).__name__}: {e}") + + +def _rebuild_meta_relational(why, rt): + """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used. + + ⭐ CALLED FROM INSIDE `_rebuild_odoo_relational`, ON PURPOSE, and the reason is D-29 rather than + tidiness: `harness/datastore` is a ONE-FILE-AT-A-TIME process global, and that caller has just + established which tenant's file this process holds open. Spawning Meta here inherits that + binding instead of rebinding it under live readers — which is the documented way to serve one + tenant's rows to another with nothing raised. + + ⚠ SILENT WHEN THERE IS NOTHING TO DO. A tenant that never connected Meta has no `meta_*` tables + in its mirror; `refresh` returns `{}` and says so once. That is a normal state, not a failure, + and it must not print an error every 30 minutes for every tenant that does not use Meta. + ⛔ Its own try/except for the reason the two passes above have theirs: a Meta refusal must not + cancel an Odoo rebuild that already succeeded. + """ + try: + import meta_relational as _meta + counts = _meta.refresh(rt, why) + if counts: + print(f"[aios-api] meta relational rebuild done ({why}): " + + ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))) + try: + import automation_engine as _engine + _engine.refresh_relations(rt, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] meta relation cells failed ({why}): {type(e).__name__}: {e}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] meta relational rebuild FAILED ({why}): {type(e).__name__}: {e}") + + +def _sweep_automation_schemas(why): + """⛔⛔ WAVE 32 · `W32-T07` — MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE THE TABLES. + + Owner items 13 and 15 — the TikTok comments lock, and the comment CONTENT column he says he has + asked for *"many times"*. **Both were already correct in the source.** `TT_COMMENT_FIELDS` + carries `field_def("text", "Comment")` and `TT_LOCKED_TABLES` already contains the comments + table, both since wave 31, with his words quoted in the comment beside them. So this ticket is + not a schema change and there is nothing to design: it is DELIVERY. + + ⛔ THE MECHANISM, WHICH IS THE WHOLE OF IT. `ut_ensure` MERGES fields into an existing table and + stamps `recordMode` — but only **when something calls it**, and the only callers are automation + runs. A tenant whose `ut_tt_comments` was spawned before the declaration changed keeps the old + shape until an automation happens to run against it. Nothing sweeps existing tenants. That is + [[a-migration-that-runs-on-the-next-write]], and it is this wave's stated thesis: a declaration + that never reaches a tenant is indistinguishable from one that was never written. + + ⛔⛔ AND IT MUST RUN **IN THE CONTAINER**, WHICH IS WHY THIS IS IN `main.py` AND NOT A SCRIPT. + D-195, measured three times: a developer's CLI write to the tenant store is reverted by the + running Space within a minute (download-modify-upload, last-write-wins) — and **the write + reports success every time**, then a FRESH read confirms it, and it is gone by the next poll. + A connector's tables must be spawned BY THE CONTAINER; a CLI spawn is a dry run that lies. + + ⚠ EVERY TENANT, unlike `_rebuild_odoo_relational` below — and the asymmetry is deliberate + rather than an oversight. That function is scoped to royal because it derives from the DuckDB + mirror, and `harness/datastore` is a ONE-FILE-AT-A-TIME process global (D-29): rebinding it per + tenant in a daemon thread can serve one tenant's rows to another with nothing raised. This + sweep touches only `user_tables` through each tenant's own `rt`, which has no such global — so + the hazard that scopes that one does not exist here, and TikTok automations run in tenants + other than #0. + + ⚠ CHEAP ON A CORRECT TENANT: `ut_ensure` short-circuits when nothing changed, so this is a read + per child table on a tenant that is already right, and the whole delivery on one that is not. + ⚠ ONE TENANT'S FAILURE MUST NOT STOP THE NEXT. Each is wrapped: a tenant whose store is + unreachable at boot is reported and skipped, never allowed to abort the sweep for everyone. + """ + try: + import automation_engine as _eng + from harness import runtime as _runtime + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED — import failed: {e}") + return + try: + schemas = _eng.platform_schemas() + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED — no declarations: {e}") + return + tenants = [] + try: + tenants = _runtime.known_tenants() + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) SKIPPED — tenant list unreadable: {e}") + return + for slug in tenants: + try: + rt = _runtime.get_runtime(slug) + # ⛔⛔ FIXED 2026-08-13 — THIS SWEEP WAS SPAWNING EIGHT DATABASES IN EVERY TENANT. + # Owner: *"Database for Royal Imports, why we have fucking IG and TIktok databases."* + # `ut_ensure`'s first line is *"Create the table if it is missing"*, and this loop fed + # it every child of every platform schema for every tenant — so tenant #0, a floral and + # giftware importer with ZERO automations, woke up on 2026-08-13 at 12:05 UTC owning + # `ut_ig_posts`, `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots` and the + # four TikTok twins, all empty, all in his nav flyout. **The bug is one word wide:** + # this function's own title says *"MAKE D'S DECLARATIONS REACH TENANTS THAT ALREADY HAVE + # THE TABLES"* and its body called a CREATE-OR-MERGE function to do a MERGE-ONLY job. + # [[reuse-and-delete-are-hypotheses]] — `ut_ensure` was the right function for the + # merge and brought a second behaviour nobody wanted with it. + # + # ⭐ THE PREDICATE IS "DOES THIS TENANT ALREADY HAVE THE TABLE", read ONCE per tenant + # rather than per child — `rt.get` deep-copies the whole tenant document (28.6 MB on + # tenant #0), so asking eight times is eight copies to answer one question. + # ⚠ AND IT MUST NOT WEAKEN THE DELIVERY: a tenant that HAS `ut_tt_comments` still gets + # the `text` column and the `recordMode` stamp, which is the entire point of T07. The + # sweep now delivers to tables that exist and mints none, which is what it always + # claimed to do. + have = set(rt.get("user_tables") or {}) + ensured, skipped = [], [] + for s in schemas: + for key, child in (s.get("children") or {}).items(): + if key not in have: + skipped.append(key) + continue + got = _eng.ut_ensure(rt, child["label"], child["fields"], "automation", + key=key, lock_fields=True, + record_mode=child["record_mode"]) + if got: + ensured.append(got) + if skipped: + # ⭐ SAID OUT LOUD, never silently skipped — this repo's "no silent caps" rule. A + # sweep that quietly does nothing looks identical to a sweep that is not running, + # which is how the previous behaviour survived review in the first place. + print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " + f"present in this tenant, so nothing was created for them " + f"({', '.join(sorted(skipped))}) — they are spawned by an automation that " + f"needs them, never by this sweep") + # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. + # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and + # T07's `done-when` asserts both ("no `ut_tt_*` grid carries an Instagram column"). On a + # tenant that ran TikTok before W30-T08 the 26 machine-authored IG columns are still + # there — the detector was fixed, the damage never was. + # ⚠ `kept` IS THE HONEST HALF: a foreign column that HOLDS DATA is REPORTED, never + # deleted. If it is non-empty, the screenshot shows a column and the report is the + # answer (W30/R6's second sentence). + st = {} + try: + st = _eng.retract_foreign_presets(rt, log=lambda *a, **k: None) or {} + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) {slug}: retraction FAILED: {e}") + print(f"[aios-api] schema sweep ({why}) {slug}: ensured={len(ensured)} " + f"tables={st.get('tables', 0)} columns={st.get('columns', 0)} " + f"cells={st.get('cells', 0)} flags={st.get('flags', 0)} " + f"kept={st.get('kept') or []}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] schema sweep ({why}) {slug}: FAILED: {e}") + # ⭐ A SUCCESS MARKER, for `_rebuild_odoo_relational`'s stated reason: D-107 was chased for a + # day on ABSENT log markers, which cannot tell "it ran and was fine" from "it was never + # reached". Three failure markers and no success marker makes silence ambiguous. + print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)") + + +def _rebuild_odoo_relational(why): + """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or + 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone + could have asked" are different diagnoses. + + ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased + for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran + and was fine" from "the rebuild was never reached". Three failure markers and no success + marker means silence is ambiguous; now it is not. + + ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The + caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and + royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks + straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME + process-global, and rebinding it in a daemon thread under live readers can serve one tenant's + rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. + """ + try: + import odoo_relational as _rel + from harness import runtime as _runtime + if not _rel.is_royal("royal-imports"): + return + _rt = _runtime.get_runtime("royal-imports") + counts = _rel.refresh(_rt, "royal-imports") + # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. + # + # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field + # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to + # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a + # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every + # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the + # tables simply look finished and answer nothing. + # + # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count + # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. + # Calling the inner one here would return a plausible number and change no cell. + # + # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source + # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest + # refusal silently cancel a relational pass that had already succeeded. + try: + import automation_engine as _engine + _engine.refresh_relations(_rt, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") + try: + import rollup_sql as _rollup + for _b, _key, _l, _f in _rel.TABLES: + _rollup.compute(_rt, _key) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") + print(f"[aios-api] odoo relational rebuild done ({why}): " + + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) + _rebuild_meta_relational(why, _rt) + except Exception as e: # noqa: BLE001 + # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only + # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a + # `BinderException` about a missing column is a different action from a timeout or an auth + # failure, and the text alone often does not say which it was. + print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") + + +def _store_resync_loop(): + """Keep the analytical mirror current — the API-process stand-in for the re-sync the + Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a + refreshed pool re-reads the freshly synced store.""" + import time as _t + passes = 0 + while True: + # ⭐⭐ WAVE 32 · OWNER ITEM 11 / R11 — THE TENANT'S OWN CADENCE, NOT A HARDCODED 1800. + # + # ⛔ THIS LINE IS WHY THE SETTING WAS NOT A SETTING. SESSION B built the whole of item 11 — + # the 30m/1h/4h/daily/manual presets, the server-side clamp, the config door and + # `odoo_relational.sync_seconds()` which turns the stored preset into seconds — and its own + # ticket said the one-line change belonged to A because `main.py` is A's file. That ASK was + # never sent, so the value was stored, displayed, clamped and IGNORED: a person could pick + # "every 4 hours" and the loop would keep resyncing every 30 minutes with nothing anywhere + # reporting the disagreement. Found by `verify_reachability` naming `sync_seconds` as a + # function whose ONLY caller was its own gate [[artifact-with-no-importer]]. + # + # ⚠ `None` MEANS MANUAL AND MUST NOT MEAN ZERO. R11's `manual` preset returns None from + # `sync_seconds`; treating that as a falsy interval would spin this loop with no sleep at + # all. It parks at the default cadence instead and simply does no work — the tenant asked + # not to be synced automatically, not for the server to stop breathing. + # ⚠ RE-READ EVERY PASS, deliberately: a cadence changed in Settings takes effect on the + # next cycle rather than at the next container restart, which is what makes it a setting. + # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is + # enforced in `sync_seconds` as well as at the write door, for the same reason. + _every = 1800 + try: + import odoo_relational as _rel_cad + from harness import runtime as _rt_cad + _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) + _every = 1800 if _secs is None else max(int(_secs), 60) + except Exception: # noqa: BLE001 + pass + _t.sleep(_every) + passes += 1 + try: + from harness import datastore as _ds + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + _ds.sync_all(log=lambda *a, **k: None) + # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot + # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; + # without it deleted Odoo lines inflate every sum on the mirror FOREVER. + if passes % 4 == 0: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] store resync failed: {e}") + # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt + # AFTER the mirror they are derived from, in the same pass and in that order — deriving + # from a mirror this loop is about to advance would publish a worklist one cycle stale + # every single time. + # + # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must + # not swallow the sync's error message, and — worse the other way — a sync failure must + # not skip a rebuild that had nothing wrong with it. Two independent failures, two + # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) + # + # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` + # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so + # "the wiring exists" and "the data is current" are different claims and only the stamp + # can tell a user which one they are looking at. + # + # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which + # is what made the boot path silent for 30 minutes after every restart; it is now the + # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here + # would be a second thing to keep in step, and the two would answer differently on the + # next ruling — the exact shape the Views top-up was just fixed for on the other side of + # this wave. + # ⛔ THE META PULL RIDES THE RESYNC TOO, AND LEAVING IT OUT WAS A REAL GAP — caught by + # reading the deploy's own boot log rather than by any gate. `_pull_meta` was wired into + # `_seed_and_sync_store` ALONE, i.e. it ran exactly once per container, at boot, behind a + # full Odoo sync. So a boot where the Graph call was rate-limited, slow or simply after the + # thread died left the mirror empty with NOTHING to retry it: the relational rebuild below + # would then find no `meta_*` tables every 30 minutes forever and skip, silently and + # correctly. A connector that can only ever be established at boot is one bad boot away + # from being permanently absent. + # ⚠ Cheap when there is nothing to do: no token => one line and return. + _pull_meta("resync") + _rebuild_odoo_relational("resync") + + +def _prewarm(): + import time as _t + t0 = _t.time() + # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can + # take many minutes of XML-RPC to close, and the first live probe of this arrangement + # showed the pool warm (42s) silently queued behind it — the whole app cold for every + # visitor while a background column channel caught up. Measure cells retry via the + # transient rule until the sync lands; nothing else waits on it. + import threading as _th + _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() + try: + from harness import runtime as _runtime + rt = _runtime.get_runtime("royal-imports") + routes_customers.warm_default(rt) + import pages as _pages + _pages.warm_default(rt) + # ⭐⭐ WAVE 30 · T12, THE COLD PATH (owner items 4/5). Automation was the ONE module absent + # from this list, so its memo was always filled by a visitor rather than by boot: call 1 of + # `GET /automations` after every deploy downloaded the whole `user_tables` document (35.8 MB + # ceiling, under the store lock) onto whoever clicked first. Memoising the WARM path — two + # waves of it — could not touch that, because the cold call is the one that fills the memo. + # ⚠ It elects a STRING and keeps no document; see `warm_default`'s own note on why caching + # the bucket would trade a latency for memory this tier does not have. + import routes_automation as _rauto + _rauto.warm_default(rt) + print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") + except Exception as e: # noqa: BLE001 — boot must not die on a warm-up + print(f"[aios-api] prewarm skipped: {e}") + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W31-T46 / D-160 — THE MIRROR IS SEEDED WHETHER OR NOT `AIOS_PREWARM` IS SET. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# THE DEFECT, and it has cost a release once already (owner item 12, wave 20). `_prewarm()` was +# the ONLY caller of `_seed_and_sync_store()`, which is the ONLY caller of +# `datastore.ensure_seed()`. So on a fresh Space disk with `AIOS_PREWARM` anything but `1`: +# no `royal.duckdb` ⇒ `datastore.ready()` False forever ⇒ `ro_con()` refuses ⇒ every measure +# column blank AND — since wave 30 put the Odoo grids on the mirror — **two user-facing grids +# serve nothing**, with the app RUNNING, the deploy green and the tag correct. Last time the +# symptom was read as a pinned tag and a store problem for days. +# +# ⛔ AND THE FLAG IS EASIER TO LOSE THAN IT LOOKS: `deploy_web.py` pushes `AIOS_PREWARM=1` +# EXPLICITLY (to overwrite a stale `0`, because a Space secret survives a redeploy) — but that +# push sits under `if TARGET:`, so an ordinary bare `python deploy_web.py` skips it. A guard +# written for the dangerous case that the ordinary case walks straight past. +# +# ⭐ SO THE SEED IS SPLIT OFF AND MADE UNCONDITIONAL, which is T46's first branch rather than its +# fallback. It is the right half to move because it is the CHEAP, SAFE one: `ensure_seed()` +# touches no Odoo (it is one `hf_hub_download` of a snapshot), returns immediately when the file +# is already there, and returns immediately without `HF_TOKEN`. The EXPENSIVE, live half — +# `sync_all()`'s XML-RPC passes, the pool warm, the resync loop — stays exactly where it was, +# because the env gate's stated reason is still true: importing `api.main` in a gate must never +# fire a live Odoo pull. +# +# ⚠ THE `DB_PATH.exists()` PRE-CHECK IS WHAT KEEPS THIS FREE. It is a stat, and it is False only +# on a genuinely fresh disk — so on every developer box and in every gate run the thread is never +# started at all, and on a fresh Space it does exactly the thing whose absence blanks the grids. +#: What the boot seed did, so a surface can REPORT it instead of an operator inferring it from a +#: blank grid. R6's second sentence: a limit that cannot be removed is reported with its cause. +MIRROR_SEED = {"attempted": False, "seeded": False, "cause": "", "recommendation": ""} + + +def _seed_mirror_if_absent(): + """Hydrate the analytical mirror when this container has none — INDEPENDENT of `AIOS_PREWARM`. + + Fail-quiet by design, and it records WHY rather than only whether: "there is no mirror and no + HF_TOKEN to fetch one" and "there is no mirror and the fetch failed" are different operator + actions, and a blank grid cannot tell them apart. + """ + from harness import datastore as _ds + MIRROR_SEED["attempted"] = True + try: + if _ds.ensure_seed(): + MIRROR_SEED["seeded"] = True + print("[aios-api] analytical mirror seeded at boot (independent of AIOS_PREWARM)") + return True + if not _ds.DB_PATH.exists(): + MIRROR_SEED["cause"] = ( + "this container has no analytical mirror and the seed snapshot could not be " + "fetched (no HF_TOKEN, or the dataset was unreachable)") + MIRROR_SEED["recommendation"] = ( + "set HF_TOKEN on the deployment; until then every connected grid and every " + "measure column served from the mirror is empty") + print(f"[aios-api] NO ANALYTICAL MIRROR: {MIRROR_SEED['cause']}") + except Exception as e: # noqa: BLE001 — boot must not die + MIRROR_SEED["cause"] = f"the boot seed raised {type(e).__name__}: {e}" + MIRROR_SEED["recommendation"] = "check HF_TOKEN and the seed dataset's availability" + print(f"[aios-api] mirror seed skipped: {e}") + return False + + +try: + from harness import datastore as _ds_boot + if not _ds_boot.DB_PATH.exists(): + import threading as _threading_seed + _threading_seed.Thread(target=_seed_mirror_if_absent, daemon=True, + name="mirror-seed").start() +except Exception as e: # noqa: BLE001 + print(f"[aios-api] mirror seed not scheduled: {e}") + +if os.environ.get("AIOS_PREWARM") == "1": + import threading as _threading + _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start() + _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start() diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 3659f76145d0bfdeec4ec26fc75c2d8d6878faa0..f28a4fc365e17ecc722663b748df5af48148fb5e 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -1,1824 +1,1606 @@ -"""odoo_relational.py — Odoo entities as LOCKED relational databases. - -Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset -Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually -replace. - -⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the -owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports -tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the -partners who owed money — so most Odoo ids were simply absent, and the missing rows were the -reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: - - ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a - posted customer document - ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code - ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund - ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order - -⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's -own `plan()` refused above it — but the cap was never a property of the store (see the measured -banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). -The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until -wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory -budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). -The four tables together are -17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split -is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); -they are answered by the read-through rollup, which never copies a row. - -⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts -GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables -dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and -carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner -holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. -Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you -filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps -excluding, because `modules/ar` is its oracle and an oracle answers ONE question. - -⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over -ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate -**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside -`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the -`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the -wider link would count 31,418 documents and call them open invoices, so those two rollups carry -the oracle's predicate as an explicit `payment_state` condition pair. -""" -import datetime as _dt - -#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: -#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. -RI_SLUGS = ("", "royal-imports") - -INVOICES_KEY = "ut_odoo_invoices" -CUSTOMERS_KEY = "ut_odoo_customers" -ORDERS_KEY = "ut_odoo_orders" -PRODUCTS_KEY = "ut_odoo_products" -#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, -#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to -#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). -AGENTS_KEY = "ut_odoo_agents" -ACCOUNTS_KEY = "ut_odoo_accounts" -BILLS_KEY = "ut_odoo_bills" -VENDORS_KEY = "ut_odoo_vendors" - -#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. -#: -#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it -#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced -#: rather than left standing beside its own contradiction. The obstacle was never the number of -#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED -#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) -#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 -#: settles what that means: *"there is no cap in how many data from the API source … can be pulled -#: into the app"*, so the answer is a different residency, never a bigger ceiling. -#: -#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the -#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this -#: database stores no rows HERE", which is a different statement from `None` ("connected and -#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator -#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work -#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with -#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working -#: grid; that is the whole shape of the conversion. -ORDER_LINES_KEY = "ut_odoo_order_lines" -GL_LINES_KEY = "ut_odoo_gl_lines" - -#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). -JOIN_KEY = "partner_id" -#: The product-grain equivalent. -PRODUCT_JOIN_KEY = "product_id" -#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but -#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the -#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. -AGENT_JOIN_KEY = "agent_id" -#: A vendor is also a `res.partner`; same reasoning, its own column. -VENDOR_JOIN_KEY = "vendor_id" -ACCOUNT_JOIN_KEY = "account_code" - -#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two -#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. -_AR_OPEN = "payment_state IN ('not_paid','partial')" -_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" -_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" -_CONFIRMED = "state IN ('sale','done')" - -#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. -#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that -#: left the population, which is right — a reversed invoice must not keep inflating a total. But -#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded -#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible -#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. -MAX_SHRINK = 0.5 - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _registry(): - """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the - route layer before `platform/` is necessarily on the path.""" - import core.registry as registry - return registry - - -def _iso_today(): - return _dt.date.today().strftime("%Y-%m-%d") - - -def _preset(field, flow="odoo_relational"): - """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the - grid renders it grey and the preset walls refuse a rename or a delete.""" - field = dict(field) - field["automation"] = {"flowId": flow, "preset": True} - return field - - -#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. -#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing -#: one here would be a second condition vocabulary beside `_clean_rollup`'s. -_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, - {"field": "payment_state", "op": "eq", "value": "partial"}], - "conditionConj": "or"} - - -# --------------------------------------------------------------------------------------------- -# FIELD CONTRACTS -# --------------------------------------------------------------------------------------------- -# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None -# for an unknown one — which DELETES the column silently on the next read rather than erroring. -def _scope_field(): - return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", - "source": "overlay", "default": False, - "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " - "metric in this product excludes. The row is kept so no Odoo id is " - "missing; filter on this column to reconcile against the AR page."} - - -def _refreshed_field(): - return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", - "default": False, "description": "When this row was last reconciled against Odoo."} - - -def agent_fields(): - """One row per SALES AGENT, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it - is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 - carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without - the flag and 3 are flagged with no commission yet**. Either source alone silently drops real - agents. Same shape as `read_customers`' two document universes, for the same reason. - """ - return [_preset(f) for f in ( - {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " - "their commission lines instead - both are real, which is why this " - "table is the union of the two."}, - {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", - "source": "overlay", "default": True}, - # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 - # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). - # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` - # for the ruling. ⚠ This one costs more than the other two and the difference is worth - # recording: those were a click-through to a record whose id stays on the row, while this - # was an agent's BOOK — the list of customers assigned to them. The id side survives - # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in - # the data and only the rendered list is gone; the same question is answerable on the - # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on - # `sales_lines` / `sales_orders`. - _refreshed_field(), - )] - - -def account_fields(): - """One row per `account.account` — the GL chart, the owner's "GL codes". - - ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest - of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on - expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest - binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a - link into a table that does not exist. Declaring one here would render a permanently blank - column, which is the exact trap D-87 warns about from the value site. - """ - return [_preset(f) for f in ( - {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.account` id. Also this row's id."}, - # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its - # options omit is wave-26 item 24: the filter panel answers with a list that cannot match - # what is stored. - {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", - "default": True, - "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", - "income_other", "asset_cash", "asset_current", "asset_receivable", - "asset_fixed", "asset_non_current", "asset_prepayments", - "liability_current", "liability_payable", "liability_credit_card", - "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, - {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Ticked for the expense family - the same predicate the semantic layer's " - "gl_lines topic uses, so this column and that topic cannot disagree."}, - _refreshed_field(), - )] - - -def vendor_fields(): - """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. - - ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 - vendors, of which only 9 also appear in the customer population. Pointing bills at - `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly - empty column, not an error. - """ - return [_preset(f) for f in ( - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. - # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) - # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by - # default would push the useful ones off the first screen, and DESIGN.md's "never - # over-explain" applies to columns as much as to prose. Every one is one click away in the - # field picker, and a locked database still allows fields (item 4's vocabulary). - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": True}, - {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, - {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, - {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", - "default": False}, - {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, - "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, - # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already - # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a - # shared spelling across two linked grids is how a rollup ends up summing the wrong column. - {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, - {"key": "website", "label": "Website", "type": "url", "source": "overlay", - "default": False}, - {"key": "street", "label": "Street", "type": "text", "source": "overlay", - "default": False}, - {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", - "default": False}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, - {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, - {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, - "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def bill_fields(): - """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. - - ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 - expense GL lines. What a person calls "expenses" is both, and they are different tables - the - bill is what you pay, the line is what it was coded to. This is the payable; the line ledger - is the read-through mirror grid (R2). - """ - return [_preset(f) for f in ( - {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, - {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's - # direction, so a refund reduces a total without anybody re-deriving a sign here. - {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["in_invoice", "in_refund"]}, - {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, - _refreshed_field(), - )] - - -def invoice_fields(): - """One row per POSTED customer invoice or refund — the full history, not just what is open.""" - return [_preset(f) for f in ( - {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", - "default": True}, - {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", - "default": True}, - {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " - "why AR rollups need no filter."}, - {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the - # table held open AR only; the full posted history also carries paid / in_payment / - # reversed. A select holding a value its options do not declare is the wave-26 item-24 - # defect — the filter panel answers with a list that cannot match what is stored. - {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", - "default": True, - "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, - {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", - "default": False, "options": ["out_invoice", "out_refund"]}, - _scope_field(), - # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like - # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. - # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather - # than left to be inferred from a green gate. - # - # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). - # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data - # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: - # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, - # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. - # - # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: - # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` - # prefix test relaxed — which makes EVERY GATE GREEN while - # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` - # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids - # touching that line this wave for exactly that reason. - # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was - # raised from, and the mirror did not sync it until this wave — so order->invoice was a - # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. - # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold - # anything; a merged one can hold several origins space-separated). The link resolves - # against `order_no` and finds nothing when the text is not an order name — the honest - # outcome, and the reason this is a join HINT rather than a foreign key. - {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", - "default": False, - "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, - {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, - _refreshed_field(), - )] - - -def order_fields(): - """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order` id. Also this row's id."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", - "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", - "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, - _scope_field(), - # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same - # column in `invoice_fields` above for the ruling and the refused alternative. This row - # keeps `customer` and `partner_id`, so only the click-through is gone. - # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS - # order, matched on the order's own name. - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, - _refreshed_field(), - )] - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE -# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two -# lists must name the SAME columns, and both failure directions are silent: -# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); -# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. -# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just -# add a column". -# -# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings -# rather than omissions: -# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw -# document, not the mirror), so a link at a read-through grain resolves against nothing and -# renders a permanently blank column — the trap `account_fields` names from the other end. -# The join ids ride as ordinary filterable columns instead, so the relationships are all -# still reachable by a person and by SQL. -# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by -# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column -# would be blank for every row forever. `section_line_grids` tolerates the key; the honest -# thing is not to declare it. - - -def order_line_fields(): - """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` - scope every wholesale metric in this product uses. - - MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded - sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — - but `done` stays in the option list because it is in the SCOPE, and a filter offering only - what happens to be stored today goes stale the first time an order is marked done. - - ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` - carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — - which is also what makes `order_no` a readable primary cell instead of a line id. - - ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a - FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the - type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both - render through the client's `numberText`, which rounds nothing without a `format.decimals` - bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so - it takes the type that does not paint one. - """ - return [_preset(f) for f in ( - {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The sale order this line belongs to. Zero orders have a blank name, " - "which is why it is the primary cell rather than the line id."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `sale.order.line` id. Also this row's id."}, - {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", - "default": False, - "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 101 section and note lines, which carry no product."}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", - "default": False}, - {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, - "agg": "sum", - "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " - "(returns), so nothing here is rounded."}, - {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", - "default": False, "agg": "sum", - "description": "Odoo's own line margin. Populated on every line."}, - {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", - "default": False, - "description": "The cost Odoo priced this line's margin against, per unit."}, - {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", - "default": True}, - {"key": "state", "label": "State", "type": "select", "source": "overlay", - "default": False, "options": ["sale", "done"]}, - _scope_field(), - )] - - -def gl_line_fields(): - """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the - grain a person can actually browse. - - MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 - draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are - invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the - reader chooses. That is the same decision the binding states from the SQL side. - - ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: - **61,911 lines carry no partner** (journal entries that are not about a customer), and - **1,727 carry no account** at all, which is also why `account_code` — the key - `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies - it is a LEFT one. - - ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: - five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · - `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. - A select whose options go stale answers a filter with a list that cannot match a stored value - (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were - measured COMPLETE against the whole table. - """ - return [_preset(f) for f in ( - {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", - "default": True, "pinned": True, - "description": "The journal entry this line belongs to. Never blank."}, - {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `account.move.line` id. Also this row's id."}, - {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", - "default": False}, - {"key": "account", "label": "Account", "type": "text", "source": "overlay", - "default": True}, - {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", - "default": True, - "description": "The GL code, from the joined chart of accounts — the key " - "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " - "account."}, - {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", - "default": True, - "description": "Blank on the 61,911 lines that are not about a partner."}, - {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", - "default": False}, - {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, - {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", - "default": True, "agg": "sum"}, - {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", - "default": True, "agg": "sum", - "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, - {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", - "default": False, - "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, - {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", - "default": False}, - {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", - "default": True, "options": ["draft", "posted", "cancel"]}, - _scope_field(), - )] - - -def product_fields(): - """One row per `product.product`, keyed on its id — EVERY product, archived ones included. - - ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes - map to more than one product id (re-SKU / merge history). The code is what a human reads and - the id is what `sales_lines.product` groups by, so both are columns and only the id is the - identity. - - ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products - carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a - blank name — measured. Pinning `code` would give those rows a blank primary cell, which is - exactly D-80: a first column nothing populates quietly becoming the row's identity - ([[fallback-that-became-the-rule]]). - """ - return [_preset(f) for f in ( - {"key": "product", "label": "Product", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `product.product` id. Also this row's id, and what " - "every product rollup groups by."}, - {"key": "code", "label": "SKU", "type": "text", "source": "overlay", - "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " - "products that are not stocked SKUs."}, - {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", - "default": True, - "description": "Unticked = archived. Archived products are kept because they still " - "carry sales history — two of them sold this year."}, - # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a - # select whose options go stale answers a filter with a list that cannot match a stored - # value (wave-26 item 24). Text filters honestly and never goes out of date. - {"key": "category", "label": "Category", "type": "text", "source": "overlay", - "default": True}, - {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", - "default": False, "options": ["product", "consu", "service"]}, - {"key": "standard_price", "label": "Standard cost", "type": "currency", - "source": "overlay", "default": True}, - # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in - # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; - # `sales_lines` holds 256,810 rows that are never copied into this table. - {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "units", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -def customer_fields(): - """One row per partner Odoo has transacted with, keyed on the `res.partner` id. - - ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a - relation Odoo already decided. The rollups come in two kinds on purpose: - * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer - anything about a document the table holds, including a date rank; - * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query - over the whole mirror — they can answer a DATE-WINDOWED money question, which a link - rollup cannot, because a condition can only compare against a literal and a literal year - start is right until 1 January. - """ - return [_preset(f) for f in ( - {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", - "default": True, "pinned": True}, - {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", - "default": False, "description": "The `res.partner` id. Also this row's id."}, - {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, - {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, - {"key": "country", "label": "Country", "type": "text", "source": "overlay", - "default": False}, - {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", - "default": True, - "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " - "Customers-module convention)."}, - # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this - # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are - # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to - # itself and look plausible doing it. - {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", - "default": False}, - _scope_field(), - - # --- the relations ------------------------------------------------------------------- - {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", - "default": True, - "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", - "default": True, - "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, - # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the - # agents table — zero dangling, which is why this ships as a link rather than a lookup. - {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, - - # --- link rollups over the invoice history -------------------------------------------- - # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual - # is exactly 0, so summing the full history gives the open balance to the cent. - # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. - # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while - # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / - # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two - # numbers with one name, 4x apart, in one product is how a correct figure gets reported - # as a bug; the scope belongs in the label, not only in a column somebody has to filter. - {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "description": "Open balance across every posted document, INCLUDING the Amazon " - "channel. Filter `In wholesale scope` to reconcile with the AR page.", - "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, - {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", - "source": "overlay", "default": False, "agg": "sum", - "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, - # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every - # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right - # this module refuses everywhere else. - {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", - "source": "overlay", "default": True, - "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, - # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a - # date column finds no numbers and returns BLANK — a column that renders empty forever - # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. - {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", - "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, - - # --- link rollups over the order history ---------------------------------------------- - {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "fn": "countall"}}, - {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", - "default": True, - "rollup": {"link": "orders", "field": "order_date", "fn": "latest", - "sortBy": "order_date", "sortDir": "desc"}}, - - # --- source-backed (read-through) rollups --------------------------------------------- - # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted - # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced - # flag. Different grain, different question — the metric KEY carries the distinction, - # which is the whole reason a rollup may not carry SQL of its own. - {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", - "source": "overlay", "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", - "default": True, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, - {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", - "default": False, "agg": "sum", - "rollup": {"source": {"topic": "sales_lines", "measure": "margin", - "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, - {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", - "default": False, - "rollup": {"source": {"topic": "sales_orders", "measure": "orders", - "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, - _refreshed_field(), - )] - - -# --------------------------------------------------------------------------------------------- -# READING THE MIRROR -# --------------------------------------------------------------------------------------------- -def excluded_names(): - """The partner names out of wholesale scope, from the ONE place that defines them. - - Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and - the day somebody adds a channel this module would keep answering the old question. - """ - try: - import core.odoo as odoo - names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() - return {str(n).strip().lower() for n in names if str(n).strip()} - except Exception: # noqa: BLE001 - return set() - - -def excluded_ids(cur, names=None): - """The out-of-scope partner IDS, resolved against the MIRROR. - - ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. - `account_move.partner_name` is a copy taken when the document was written, and this module's - own `customers_from` says so out loud — *"a partner's name can differ across documents - (renames land on new invoices only)"*. So a rename would put some of one partner's documents - in scope and the rest out, silently, and the totals would stop reconciling with nothing to - point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. - - ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a - LIVE `search_read`, so importing it here would make spawning four locked databases fail - whenever Odoo is unreachable — including on this developer machine, where the handshake dies - on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. - - ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which - is the correct direction. 44 transacting partners carry no name at all; treating an - unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. - """ - names = names if names is not None else excluded_names() - if not names: - return set() - rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() - return {int(pid) for pid, name in rows if str(name).strip().lower() in names} - - -def columns(cur, table): - """The column names a mirror table actually has, lowercased. `set()` if the table is absent. - - ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY - phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns - added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, - `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` - keys, which `ready()` does not read. So there is a real window, right after a boot, where the - store reports READY and a column this module names does not exist yet — and DuckDB answers a - missing identifier with a Binder error, which reached the operator as a bare `500`. - ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the - whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers - degrade the COLUMN to blank and still write every id. - """ - try: - rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") - return {str(d[0]).lower() for d in rows.description} - except Exception: # noqa: BLE001 - return set() - - -def _col(have, name, default="NULL"): - """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" - return name if str(name).split(".")[-1].lower() in have else default - - -def _as_date(value): - """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the - STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" - if not value: - return "" - return str(value)[:10] - - -def _in_scope(pid, excluded): - """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" - return "" if int(pid) in excluded else "1" - - -def read_invoices(cur, excluded=None, open_only=False): - """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. - - ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the - out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and - TAGS the channel instead. Two queries would be two populations, and they drift the moment - either is edited. - - Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - where = _AR_WHERE if open_only else _POSTED_DOCS - # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in - # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does - # not have the column yet — and DuckDB answers a missing identifier with a Binder error that - # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the - # link degrades to blank for one sync cycle instead of refusing the whole spawn. - have = columns(cur, "account_move") - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " - f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " - f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r - scope = _in_scope(pid, excluded) - if open_only and not scope: - continue - out.append({ - "_id": str(mid), - "invoice_no": str(name or ""), - "odoo_id": int(mid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "invoice_date": _as_date(inv_date), - "due_date": _as_date(due), - "residual": float(residual or 0.0), - "amount_untaxed": float(untaxed or 0.0), - "payment_state": str(pay_state or ""), - "move_type": str(mtype or ""), - "origin_order": str(origin or "").strip(), - "wholesale_scope": scope, - }) - return out - - -def read_open_ar(cur, excluded=None): - """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. - - Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an - oracle answers exactly one question. It is a projection of `read_invoices`, never a second - query. - """ - return read_invoices(cur, excluded=excluded, open_only=True) - - -def read_orders(cur, excluded=None): - """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "sale_order") - sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " - f" state, amount_untaxed, {_col(have, 'invoice_status')} " - f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r - out.append({ - "_id": str(oid), - "order_no": str(name or ""), - "odoo_id": int(oid), - "customer": str(pname or ""), - JOIN_KEY: int(pid), - "order_date": _as_date(when), - "amount_untaxed": float(untaxed or 0.0), - "team": str(team or ""), - "state": str(state or ""), - "invoice_status": str(inv_status or ""), - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_products(cur): - """[(row dict)] — EVERY `product.product`, keyed on its id. - - ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they - were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products - that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three - uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product - grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to - land — silently. 119 extra rows is the whole cost of the claim being literally true. - """ - have = columns(cur, "product_product") - sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " - f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " - "FROM product_product") - out = [] - for r in cur.execute(sql).fetchall(): - (prid, code, name, categ, ptype, cost, active) = r - out.append({ - "_id": str(prid), - "product": str(name or ""), - PRODUCT_JOIN_KEY: int(prid), - "code": str(code or ""), - "active": "1" if active else "", - "category": str(categ or ""), - "product_type": str(ptype or ""), - "standard_price": float(cost or 0.0), - }) - return out - - -def read_customers(cur, excluded=None): - """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. - - The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale - order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. - - ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — - the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old - docstring said partners with no document are *"left out on purpose — a row that can never - appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same - join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** - (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer - a salesperson has not sold to yet is exactly the row a prospecting view needs. - - ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the - document legs for the rank leg would drop **16 partners that hold posted documents** (7 - archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry - `partner_id` LINKS straight back here — so those links would dangle with nothing reporting - it. When other tables point AT a population, a widening must be a SUPERSET. - - ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same - `columns()`/`_col` discipline every other optional column here uses — but note the difference - honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the - POPULATION back to the document union. It degrades to today's behaviour rather than to an - empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the - column is missing so the narrowing can never pass for done. - - ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above - states the rule for this module and it applies with more force to a POPULATION than to a name - list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its - mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this - time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the - weather. One source, always present at spawn time: the mirror. - - ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR - partners; sourcing a customer registry from its own receivables is what kept most Odoo ids - out of the store in the first place. - """ - excluded = excluded if excluded is not None else excluded_ids(cur) - have = columns(cur, "res_partner") - # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join - # itself names the column, so `_col` on the SELECT list alone would still fail to bind. - agent = ("ag.name" if "agent_id" in have else "NULL") - agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") - join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") - # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an - # archived prospect out, and a rank test without it would re-admit the 47 archived partners - # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. - rank_leg = (" OR (p.customer_rank > 0 AND p.active) " - if {"customer_rank", "active"} <= have else "") - sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " - f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " - "FROM res_partner p " - f"{join}" - "WHERE p.id IN (" - f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " - " UNION " - " SELECT partner_id FROM account_move " - f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" - f"{rank_leg}") - out = [] - for r in cur.execute(sql).fetchall(): - (pid, name, city, state, country, agent, agent_id) = r - out.append({ - "_id": str(pid), - "customer": str(name or ""), - JOIN_KEY: int(pid), - "city": str(city or ""), - "state": str(state or ""), - "country": str(country or ""), - "agent": str(agent or ""), - AGENT_JOIN_KEY: int(agent_id) if agent_id else "", - "wholesale_scope": _in_scope(pid, excluded), - }) - return out - - -def read_agents(cur): - """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. - - ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: - Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list - silently becomes NULL and every row would read "not an agent" indistinguishably from - "unknown". Read it as a truth value, never as a presence test. - """ - have = columns(cur, "res_partner") - if "id" not in have: - return [] - flagged = "p.agent" if "agent" in have else "FALSE" - # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that - # distinction is the whole point of this block. `columns()` was written for a missing COLUMN - # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that - # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent - # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table - # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is - # precisely D-107's shape, and it would have arrived on the first deploy of this feature. - # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the - # commission table the population falls back to the FLAGGED partners alone and `commissioned` - # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. - has_comm = bool(columns(cur, "account_invoice_line_agent")) - commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " - " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") - union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " - " UNION " if has_comm else "") - sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " - "FROM res_partner p WHERE p.id IN (" - f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") - out = [] - for (aid, name, flag, comm) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(aid), - "agent": str(name or ""), - "odoo_id": int(aid), - AGENT_JOIN_KEY: int(aid), - "flagged": "1" if flag else "", - "commissioned": "1" if comm else "", - }) - return out - - -def read_accounts(cur): - """[(row dict)] — the whole GL chart, keyed on the `account.account` id. - - ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: - `harness/semantic.py`'s `gl_lines` topic scopes expenses as - `account_type in ('expense','expense_depreciation')`. A second definition here is how a - column and a topic start disagreeing about the same word. - ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so - there is nothing to filter and every account is a row. - """ - have = columns(cur, "account_account") - if not have: - return [] - sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " - f" {_col(have, 'name', chr(39) + chr(39))}, " - f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") - out = [] - for (aid, code, name, atype) in cur.execute(sql).fetchall(): - t = str(atype or "") - out.append({ - "_id": str(aid), - ACCOUNT_JOIN_KEY: str(code or ""), - "account_name": str(name or ""), - "odoo_id": int(aid), - "account_type": t, - "is_expense": "1" if t in ("expense", "expense_depreciation") else "", - }) - return out - - -_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" - - -def read_bills(cur): - """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" - have = columns(cur, "account_move") - if not have: - return [] - sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " - f" {_col(have, 'amount_untaxed_signed', '0')}, " - f" {_col(have, 'amount_residual_signed', '0')}, " - f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " - f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") - out = [] - for r in cur.execute(sql).fetchall(): - (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r - out.append({ - "_id": str(mid), - "bill_no": str(name or ""), - "odoo_id": int(mid), - "vendor": str(pname or ""), - VENDOR_JOIN_KEY: int(pid), - "invoice_date": _as_date(when), - "due_date": _as_date(due), - "amount_untaxed": float(untaxed or 0.0), - "residual": float(residual or 0.0), - "payment_state": str(pay or ""), - "move_type": str(mtype or ""), - }) - return out - - -def read_vendors(cur): - """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. - - ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its - invoices. The asymmetry is intentional and the reason is what that function's own comment - says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. - There is no second document universe for vendors — a partner with no bill has no payable - history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills - carry a null partner; all 393 vendors resolve in `res_partner`). - """ - have = columns(cur, "res_partner") - if not have or not columns(cur, "account_move"): - return [] - # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured - # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a - # vendor record for — how to reach them, who they are for tax, and where they are. - # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the - # column. That is not defensive habit: this projection has to keep working against a mirror - # that has not been re-synced since the widening, and the alternative is a reader that raises - # on the exact box the widening was meant to help. The columns arrive when the backfill runs; - # until then these read blank rather than failing. - blank = chr(39) + chr(39) - sql = ("SELECT p.id, p.name, " - f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " - f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " - f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " - f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " - f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " - f"{_col(have, 'p.zip', blank)} " - "FROM res_partner p WHERE p.id IN " - f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " - " AND partner_id IS NOT NULL)") - out = [] - for (pid, name, country, email, phone, mobile, website, vat, ref, - street, street2, city, zipc) in cur.execute(sql).fetchall(): - out.append({ - "_id": str(pid), - "vendor": str(name or ""), - "odoo_id": int(pid), - VENDOR_JOIN_KEY: int(pid), - "country": str(country or ""), - "email": str(email or ""), - "phone": str(phone or ""), - "mobile": str(mobile or ""), - "website": str(website or ""), - "vat": str(vat or ""), - "vendor_ref": str(ref or ""), - "street": str(street or ""), - "street2": str(street2 or ""), - "city": str(city or ""), - "zip": str(zipc or ""), - }) - return out - - -def customers_from(invoice_rows): - """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. - - ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a - pure function over rows and the gate uses it to prove the FOLD against a fixture without a - mirror; deleting it would cost a test its independence from the SQL. - """ - out = {} - for row in invoice_rows: - pid = row[JOIN_KEY] - entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], - JOIN_KEY: pid}) - # A partner's name can differ across documents (renames land on new invoices only); - # the newest non-empty one wins so the locked table shows what Odoo shows today. - if row["customer"]: - entry["customer"] = row["customer"] - return list(out.values()) - - -# --------------------------------------------------------------------------------------------- -# THE SPAWN -# --------------------------------------------------------------------------------------------- -class Refused(Exception): - """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have - been written wrong.""" - - -#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole -#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. -#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. -TABLES = ( - # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / - # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules - # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current - # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it - # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries - # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). - # Dropping the row here is what stops them being planned, built or re-created; - # `RETIRED_KEYS` below is what removes the rows a tenant already has. - ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), - ("orders", ORDERS_KEY, "Odoo orders", order_fields), - # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders - # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" - # is four more spec rows and four readers rather than a new substrate. - ("agents", AGENTS_KEY, "Odoo agents", agent_fields), - ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), - ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), - ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), - # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that - # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) - # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of - # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` - # does not name, so the grids would be bound to the mirror and unreachable, which is this - # wave's own [[reachable-is-not-the-same-as-built]] shape. - ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), - ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), -) - -#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` -#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. -#: -#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: -#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so -#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that -#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the -#: key, and this is the shape that says so. -#: -#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all -#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the -#: database, never over which Odoo model was queried. Same for `account_move`, which is the -#: customer-invoice book under one WHERE and the vendor-bill book under another. -TABLE_SUBJECTS = { - # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects - # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` - # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one - # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future - # spec row tried to bring it back. - INVOICES_KEY: "odoo:account.move.customer", - ORDERS_KEY: "odoo:sale.order", - AGENTS_KEY: "odoo:res.partner.agent", - ACCOUNTS_KEY: "odoo:account.account", - VENDORS_KEY: "odoo:res.partner.vendor", - BILLS_KEY: "odoo:account.move.vendor", - ORDER_LINES_KEY: "odoo:sale.order.line", - GL_LINES_KEY: "odoo:account.move.line", -} - -#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. -#: -#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the -#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the -#: owner actually sees in the nav. This set is what removes them, and it is applied inside -#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every -#: resync. -#: -#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a -#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms -#: it — and the running Space reverts it within a minute, because the container holds the document -#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a -#: script is a dry run that reports success. -#: -#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job -#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This -#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, -#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes -#: and re-adds the table every 30 minutes. -RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) - -#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. -#: -#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness -#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was -#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote -#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new -#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned -#: the gate RED until this constant went with them. It did, and that is the ratchet working. -#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. - -#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no -#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a -#: policy `plan()` could forget. The two line grains are absent for that reason and no other. -_READERS = { - "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), - "products": lambda cur, excluded: read_products(cur), - "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), - "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), - "agents": lambda cur, excluded: read_agents(cur), - "accounts": lambda cur, excluded: read_accounts(cur), - "vendors": lambda cur, excluded: read_vendors(cur), - "bills": lambda cur, excluded: read_bills(cur), -} - -#: The table keys this module can never materialise — DERIVED from the absence of a reader, never -#: typed out, so it cannot drift from the fact it describes. -#: -#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE -#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a -#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a -#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps -#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early -#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is -#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the -#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. -#: The conversion writes the stamp; a table that needs no conversion still needs the statement. -READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) - - -class _LentDoc: - """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. - - ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` - resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE - 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per - table per loop, so passing the live handle would have added ~16 full document copies to a - function that already reads it exactly once — and with `st=None` (the gate's fixture posture, - and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset - fetch per table, on a path that has no business touching the network at all. - `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` - takes `st` rather than `defn`, so the lending happens one level up, here. - - ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim - that quietly proxied other keys would be a second store with a partial view, which is worse - than one that says what it knows. - """ - - def __init__(self, doc, key): - self._doc, self._key = doc if isinstance(doc, dict) else {}, key - - def get(self, name): - return self._doc if name == self._key else None - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner opened the Odoo connector and found nothing to configure: no key, no server database, -# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all -# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey -# it — a config the route knows and the sync path does not is a switch that flips nothing. -# -#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` -CONFIG_KEY = "odoo_connector_config" - -#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The -#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a -#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. -#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None -#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather -#: than a silent once-a-century schedule. -SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} -SYNC_FLOOR_SECONDS = 1800 -DEFAULT_SYNC = "30m" - - -def read_config(rt): - """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not - make a connector look disconnected.""" - try: - cur = rt.get(CONFIG_KEY) if rt is not None else None - except Exception: # noqa: BLE001 - cur = None - cur = cur if isinstance(cur, dict) else {} - grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} - every = cur.get("syncEvery") - return {"grids": {str(k): bool(v) for k, v in grids.items()}, - "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, - "frozen": bool(cur.get("frozen")), - "frozenAt": str(cur.get("frozenAt") or "")} - - -def grid_choices(rt): - """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from - `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). - - ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is - what they have today; only an explicit untick turns one off. The alternative — an empty - config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. - """ - chosen = read_config(rt)["grids"] - return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} - for _bucket, key, label, _fields in TABLES] - - -def enabled_buckets(rt): - """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" - chosen = read_config(rt)["grids"] - return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} - - -def sync_seconds(rt): - """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). - - ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that - predates the preset list, or one written by any path that is not the route, must still not be - able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that - holds until somebody finds the second way in [[limit-with-no-enforcer]]. - """ - secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) - if secs is None: - return None - return max(int(secs), SYNC_FLOOR_SECONDS) - - -def frozen(rt): - """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and - stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" - return bool(read_config(rt)["frozen"]) - - -def plan(cur, rt=None): - """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. - - Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and - so **every cap is checked before anything is committed**. - - Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty - list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — - `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's - second sentence carried as data rather than left for a reader to infer from an empty list. - - ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn - bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would - create some and refuse the rest — leaving a locked invoices database with no rollup host, - while the route answered as though nothing had happened. A partial spawn is worse than a - refused one, so the count is checked against the tables that ALREADY exist, before the first - write. `rt` also carries the stored row counts the shrink guard compares against. - """ - ut = _ut() - # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from - # three places in falling authority: the registry, a stored `connected: True`, then the - # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a - # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one - # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently - # not apply on exactly the run that creates the databases. This module DECLARES these keys, so - # it is the honest place to say what they are. Idempotent (a set add), and it fills the - # evaluator's input rather than becoming a second evaluator. - ut.register_connected(*[key for _b, key, _l, _f in TABLES]) - excluded = excluded_ids(cur) - # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is - # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = - # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this - # function as the caller that reads it, which is the seam working as designed — one evaluator, - # so the spawn, the write doors and the wire cannot disagree about whether a table is capped - # ([[one-evaluator-per-question]]). - # - # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: - # * no reader at all — structural, permanent, and the case that must not depend on a store - # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); - # * a reader exists but the table has already been converted to read-through — the - # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them - # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` - # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table - # D-87's conversion had already emptied. - # - # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used - # to be read after the build loop; it moved up because `row_limit` needs it and reading it per - # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face - # fetch per table on a path that must never touch the network). - existing = {} - if rt is not None: - try: - existing = dict(rt.get(ut.STORE_KEY) or {}) - except Exception: # noqa: BLE001 - existing = {} - # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and - # without it R6's report is silently absent on the run that matters most — the FIRST spawn. - # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so - # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and - # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids - # were skipped with no stated reason. The structural `reader is None` guard still did its job; - # what went missing was the half of R6 that has to SAY WHY. - # - # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` - # would work in one line, and `core.user_tables` explicitly reserves that registrar — - # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for - # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold - # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could - # be eligible FOR), but the law is written without an exception, so this lends the evaluator - # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so - # what is lent is the document as it stands the moment this plan applies. - lent = _LentDoc({**existing, - **{k: {**(existing.get(k) or {}), "readThrough": True} - for k in READ_THROUGH_KEYS}}, - ut.STORE_KEY) - - built, limits = {}, {} - caps = {} - for bucket, key, _label, _fields in TABLES: - reader = _READERS.get(bucket) - caps[key] = cap = ut.row_limit(key, st=lent) - if reader is None or cap == 0: - built[bucket] = [] - report = ut.limit_report(key, st=lent) - if report: - limits[key] = report - continue - built[bucket] = reader(cur, excluded) - problems = [] - - if rt is not None: - needed = [k for _b, k, _l, _f in TABLES if k not in existing] - if needed and len(existing) + len(needed) > ut.MAX_TABLES: - problems.append( - f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " - f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " - f"part of a linked set") - - for bucket, key, _label, _fields in TABLES: - rows = built[bucket] - cap = caps[key] # asked ONCE per table, above — never re-read per loop - # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED - # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in - # how many data from the API source (as long as its from a connected source like Odoo) that - # can be pulled into the app… Now if there is lag or it can't be done, you need to - # explicitly tell me why and recommend a fix."* Both halves are here: a connected table - # answers `None` and is never refused for its size, and anything that IS still bounded is - # reported with its cause and its recommendation instead of a hand-typed line. - # - # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: - # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and - # the loop above already handed it an empty list. Neither may reach the refusal. The - # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped - # table understates every total it feeds while looking exactly like a complete one. - if cap and len(rows) > cap: - report = ut.limit_report(key, st=lent) or {} - limits[key] = report - problems.append( - f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " - f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " - f"{report.get('recommendation', '')}".strip()) - # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every - # spawn after the first conversion: zero rows against a stored population is the INTENDED - # end state there, not the partial mirror read this guard exists to catch. - if cap == 0: - continue - stored = len(((existing.get(key) or {}).get("rows")) or {}) - if stored and len(rows) < stored * MAX_SHRINK: - problems.append( - f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " - f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " - f"shrinking; refusing rather than deleting rows that still exist") - built["problems"] = problems - # R6's second sentence as DATA rather than prose: every table whose rows this plan did not - # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. - # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that - # is not a bucket name is inert there, exactly as `problems` has always been. - built["limits"] = limits - return built - - -def apply_plan(rt, built, username="automation", today=None, report=None): - """Create-or-merge every table in `built` and its rows. Idempotent by construction. - - Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the - same record — which is also what makes "every Odoo unique id is in the database" a checkable - statement rather than a hopeful one. - - ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. - """ - if built.get("problems"): - raise Refused("; ".join(built["problems"])) - stamp = today or _iso_today() - written = {} - # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and - # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. - retired = [] - plans = [(key, label, fields(), built[bucket]) - for bucket, key, label, fields in TABLES if bucket in built] - - # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. - # - # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full - # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB - # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits - # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). - # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. - # - # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at - # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had - # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to - # create. Inside one updater, a raise aborts before anything is persisted. - def _apply_all(cur): - cur = cur if isinstance(cur, dict) else {} - # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE - # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every - # resync), which is the only place a change to the tenant document survives (D-195: the - # same edit from a CLI reports success and is reverted within a minute). - # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot - # contain them, and there is no door here that could re-add one. `removed` is reported so - # a caller can SAY what happened rather than infer it from a table going missing. - # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them - # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its - # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die - # with `TypeError: list indices must be integers`. One dict, two value shapes, is a - # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through - # its OWN channel and the return type stays exactly what it was. - for key in RETIRED_KEYS: - if cur.pop(key, None) is not None: - retired.append(key) - for key, label, fields, rows in plans: - written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _apply_all, flush="sync") - if report is not None and retired: - report["retired"] = sorted(retired) - return written - - -def _ensure_table(rt, key, label, fields, rows, username, stamp): - """One table, written on its own. Kept because the gate drives a single table directly, and - because a caller with one table to reconcile should not have to compose an updater.""" - written = {} - - def _one(cur): - cur = cur if isinstance(cur, dict) else {} - written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) - return cur - - rt.update(_ut().STORE_KEY, _one, flush="sync") - return written["counts"] - - -def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): - """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. - - ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — - but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` - was built FROM open invoices, removal meant a customer who paid their bill vanished from the - registry; now a partner leaves only when their last document does. The shrink guard in - `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. - """ - ut = _ut() - wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} - for row in wanted.values(): - row["refreshed"] = stamp - counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} - subject = TABLE_SUBJECTS.get(key) - table = cur.get(key) - if table is None: - if len(cur) >= ut.MAX_TABLES: - # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a - # successful refresh over a table that does not exist. - raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") - # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. - # - # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This - # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key - # on every pass. A claim test outside this `if` refuses the table it created last boot, - # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would - # take the product down to prevent a duplicate that already exists. Refusing the SECOND - # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to - # remove, and it is removed by dropping its `TABLES` row, not by a guard here. - # - # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. - # Never a module-level cache: one Space process serves every tenant, and two tenants both - # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated - # here). A stored definition with no `subject` claims nothing. - held = {t["subject"]: k for k, t in cur.items() - if isinstance(t, dict) and t.get("subject")} - # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered - # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. - other = _registry().subject_conflict(subject, key, claimed=held) - if other: - raise Refused( - f"{key}: refusing to create a second database for {subject!r} — {other!r} " - f"already presents it. One subject, one database (item 12a); if this table is " - f"meant to replace {other!r}, retire {other!r} first rather than shipping both") - table = cur[key] = { - "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, - "createdBy": username, "created": stamp, "fields": [], "rows": {}, - # recordMode = a LOCKED database (item-3 nomenclature): no human may add or - # delete records, while fields stay addable. Odoo owns this population. - "recordMode": ut.AUTOMATION_RECORD_MODE, - } - table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) - # W30-T35 — the durable "my rows are not in this document" statement, on the tables no - # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not - # `setdefault`: it is derived from the code's own structure, so the code is what it must agree - # with, and a definition that somehow lost the flag should regain it rather than keep serving - # an empty grid. - if key in READ_THROUGH_KEYS: - table["readThrough"] = True - # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same - # reason `readThrough` is: it is derived from this module's own structure, so the code is what - # it has to agree with, and a definition that lost the stamp should regain it rather than go - # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees - # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold - # process reading a fresh document has nothing else to read the claim off. - if subject: - table["subject"] = subject - have = {str(f.get("key")): f for f in (table.get("fields") or [])} - for field in fields: - # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. - # - # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND - # patch, so every field that goes through the normal door has one. This function does NOT - # go through that door — it writes definitions straight into the document — and - # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that - # ever omitted it would 500 the entire rows route rather than degrade one column. Measured - # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is - # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that - # prompted it came from a hand-written test fixture, not from production). - # - # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a - # missing `source` means the definition does not say which stratum owns the column, and - # rendering it as blank would bury that. The default matches what the validator would - # have stamped, so the bypass stops being a hole without inventing a second rule. - if isinstance(field, dict) and not field.get("source"): - field = {**field, "source": "overlay"} - fkey = str(field.get("key")) - if fkey not in have: - table.setdefault("fields", []).append(dict(field)) - continue - # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The - # widening moved `payment_state`'s option list and every rollup's conditions; a - # create-only merge would have left the LIVE table declaring the old contract - # forever, so the column would render but its filter could not match what is stored. - # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a - # column a user added to a locked database is never rewritten. - stored = have[fkey] - # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, - # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the - # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on - # an Odoo database would silently revert at the next boot rebuild. - if _ut().user_edited(stored): - continue - if (stored.get("automation") or {}).get("preset"): - for prop in ("label", "type", "options", "link", "rollup", "description", - "agg", "pinned", "default"): - if prop in field: - stored[prop] = field[prop] - else: - stored.pop(prop, None) - stored_rows = table.setdefault("rows", {}) - for rid, values in wanted.items(): - current = stored_rows.get(rid) - if current is None: - stored_rows[rid] = dict(values) - counts["added"] += 1 - elif any(str(current.get(k, "")) != str(v) for k, v in values.items() - if k != "refreshed"): - current.update(values) - counts["updated"] += 1 - else: - current["refreshed"] = values["refreshed"] - for rid in [r for r in stored_rows if r not in wanted]: - stored_rows.pop(rid, None) - counts["removed"] += 1 - return counts - - -def is_royal(tenant): - return str(tenant or "").strip().lower() in RI_SLUGS - - -def refresh(rt, tenant, username="automation", cur=None, today=None): - """THE entry point — the store-resync path and the route both call this. - - ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row - still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than - silently authoritative. - """ - if not is_royal(tenant): - raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " - f"Imports only); refusing to spawn empty locked databases") - # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, - # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is - # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process - # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a - # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked - # databases from another customer's rows — a spawn, not a read, so the wrong numbers become - # durable. Entitlement is not residency. - # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot - # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. - if rt is not None: - rt.assert_datastore_matches() - # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. - # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing - # the credential alone is not a freeze — this function is also reached by the boot rebuild and - # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a - # disconnected workspace would silently re-materialise from `.env` on the next tick and the - # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: - # the resync loop calling this every cycle must not be handed an exception as a status. - if rt is not None and frozen(rt): - return {"tables": {}, "frozen": True, - "note": "this workspace has disconnected Odoo; its databases are frozen as " - "static data and are not being refreshed"} - if cur is None: - from harness import datastore - cur = datastore.ro_con() - built = plan(cur, rt=rt) - # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the - # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a - # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so - # every cap, refusal and limit report is still computed over the full set — a config must not - # be able to hide a problem by hiding the table that has it. - # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from - # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not - # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. - if rt is not None: - keep = enabled_buckets(rt) - skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) - for bucket, _key, _l, _f in TABLES: - if bucket not in keep: - built.pop(bucket, None) - else: - skipped = [] - # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. - # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no - # longer built) SAYS SO, with the keys. - plan_report = {} - written = apply_plan(rt, built, username=username, today=today, report=plan_report) - return {"tables": written, - **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), - # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. - **({"skipped": skipped} if skipped else {}), - **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} +"""odoo_relational.py — Odoo entities as LOCKED relational databases. + +Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset +Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually +replace. + +⭐⭐ 2026-08-09 — THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the +owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports +tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices — the +partners who owed money — so most Odoo ids were simply absent, and the missing rows were the +reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself: + + ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a + posted customer document + ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code + ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund + ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order + +⛔ WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's +own `plan()` refused above it — but the cap was never a property of the store (see the measured +banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time). +The cap is now 60,000, DERIVED from what a row actually weighs (⚠ this line said 100,000 until +wave 28 — that was the FIRST candidate and its own derivation REJECTED it for clearing the memory +budget by 0.6%; the prose was written before the number lost, and two sibling files said it too). +The four tables together are +17.81 MB in one `user_tables` document — real, bounded, and booked: the per-table row-key split +is D-87's next increment. ⛔ ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy); +they are answered by the read-through rollup, which never copies a row. + +⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts +GIFTWARE DEALS (partner 6369 — the Amazon channel) outside WHOLESALE scope, and the old tables +dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and +carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner +holds **$1,755,779.95 of the $2,347,608.49** raw open balance — 75% of it — across 25 invoices. +Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you +filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps +excluding, because `modules/ar` is its oracle and an oracle answers ONE question. + +⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over +ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate +**to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside +`payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the +`ar_outstanding` rollup needs no condition. ⛔ THE COUNT AND THE DATE DO — `countall` over the +wider link would count 31,418 documents and call them open invoices, so those two rollups carry +the oracle's predicate as an explicit `payment_state` condition pair. +""" +import datetime as _dt + +#: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn: +#: nurilab has no Odoo mirror behind these tables and would get empty locked databases. +RI_SLUGS = ("", "royal-imports") + +INVOICES_KEY = "ut_odoo_invoices" +CUSTOMERS_KEY = "ut_odoo_customers" +ORDERS_KEY = "ut_odoo_orders" +PRODUCTS_KEY = "ut_odoo_products" +#: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents, +#: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to +#: fit far inside `MAX_ROWS` (19 / 192 / 6,538 / 393 against 60,000). +AGENTS_KEY = "ut_odoo_agents" +ACCOUNTS_KEY = "ut_odoo_accounts" +BILLS_KEY = "ut_odoo_bills" +VENDORS_KEY = "ut_odoo_vendors" + +#: ⭐⭐ WAVE 30 / R7 / W30-T35 — THE TWO LINE GRAINS, AND THEY ARRIVE THE ONLY WAY THEY EVER COULD. +#: +#: ⚠ THE PARAGRAPH THAT STOOD HERE SAID THESE WERE "DELIBERATELY NOT HERE … at any cap", and it +#: was RIGHT ABOUT THE CAP AND WRONG ABOUT THE CONCLUSION — which is exactly why it is replaced +#: rather than left standing beside its own contradiction. The obstacle was never the number of +#: rows; it was that every row had to be COPIED into the shared `user_tables` document. MEASURED +#: on this box's mirror 2026-08-12: 254,189 order lines in the confirmed scope (256,810 unscoped) +#: and 963,783 GL lines — 4.2x and 16x `MAX_ROWS`, 63.9 MB and ~240 MB as JSON. Owner ruling R6 +#: settles what that means: *"there is no cap in how many data from the API source … can be pulled +#: into the app"*, so the answer is a different residency, never a bigger ceiling. +#: +#: ⛔ THESE TWO TABLES STORE NO ROWS HERE AND NEVER WILL. `routes_odoo_tables` binds them to the +#: DuckDB mirror (`GRID_SOURCES`) and `core.user_tables.row_limit` answers **0** for them — "this +#: database stores no rows HERE", which is a different statement from `None` ("connected and +#: uncapped") and from `MAX_ROWS` ("the editable substrate"). `plan()` below reads that evaluator +#: and builds no python row for either grain: 963,783 dicts in one process is the dangerous work +#: the answer exists to prevent. What DOES get written is the DEFINITION — a locked database with +#: fields, a label, grants and a nav entry, and zero rows. A definition with no rows is a working +#: grid; that is the whole shape of the conversion. +ORDER_LINES_KEY = "ut_odoo_order_lines" +GL_LINES_KEY = "ut_odoo_gl_lines" + +#: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`). +JOIN_KEY = "partner_id" +#: The product-grain equivalent. +PRODUCT_JOIN_KEY = "product_id" +#: ⚠ AN AGENT IS A `res.partner`, so its id shares the partner namespace with a customer's — but +#: it is a DIFFERENT COLUMN on the customer row (`agent_id`, the customer's assigned agent) and the +#: two must never be joined through `JOIN_KEY`, which would link every customer to itself. +AGENT_JOIN_KEY = "agent_id" +#: A vendor is also a `res.partner`; same reasoning, its own column. +VENDOR_JOIN_KEY = "vendor_id" +ACCOUNT_JOIN_KEY = "account_code" + +#: The oracle's own predicate — `modules/ar._open_docs`, copied rather than re-derived so the two +#: cannot drift. It now selects a SUBSET of the invoices table rather than defining it. +_AR_OPEN = "payment_state IN ('not_paid','partial')" +_POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')" +_AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}" +_CONFIRMED = "state IN ('sale','done')" + +#: A refresh that would delete more than this share of a table's stored rows REFUSES instead. +#: ⛔ THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that +#: left the population, which is right — a reversed invoice must not keep inflating a total. But +#: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded +#: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible +#: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read. +MAX_SHRINK = 0.5 + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _iso_today(): + return _dt.date.today().strftime("%Y-%m-%d") + + +def _preset(field, flow="odoo_relational"): + """Stamp a field as machine-owned + preset — the `ut_ensure` lock_fields convention, so the + grid renders it grey and the preset walls refuse a rename or a delete.""" + field = dict(field) + field["automation"] = {"flowId": flow, "preset": True} + return field + + +#: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup. +#: ⚠ Two `eq` legs joined by OR, not one `in` — `ROLLUP_CONDITION_OPS` has no `in`, and inventing +#: one here would be a second condition vocabulary beside `_clean_rollup`'s. +_OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"}, + {"field": "payment_state", "op": "eq", "value": "partial"}], + "conditionConj": "or"} + + +# --------------------------------------------------------------------------------------------- +# FIELD CONTRACTS +# --------------------------------------------------------------------------------------------- +# ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None +# for an unknown one — which DELETES the column silently on the next read rather than erroring. +def _scope_field(): + return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox", + "source": "overlay", "default": False, + "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale " + "metric in this product excludes. The row is kept so no Odoo id is " + "missing; filter on this column to reconcile against the AR page."} + + +def _refreshed_field(): + return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay", + "default": False, "description": "When this row was last reconciled against Odoo."} + + +def agent_fields(): + """One row per SALES AGENT, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF TWO DISAGREEING SOURCES, and the disagreement is the reason it + is a union rather than a pick. MEASURED 2026-08-09: 16 partners carry commission lines, 17 + carry `res_partner.agent = TRUE`, and the union is 19 — so **2 agents earn commission without + the flag and 3 are flagged with no commission yet**. Either source alone silently drops real + agents. Same shape as `read_customers`' two document universes, for the same reason. + """ + return [_preset(f) for f in ( + {"key": "agent", "label": "Agent", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + {"key": "flagged", "label": "Flagged in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked = `res.partner.agent` is set. Unticked agents were found by " + "their commission lines instead - both are real, which is why this " + "table is the union of the two."}, + {"key": "commissioned", "label": "Has commission lines", "type": "checkbox", + "source": "overlay", "default": True}, + # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 + # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). + {"key": "customers", "label": "Customers", "type": "link", "source": "overlay", + "default": True, + "link": {"table": CUSTOMERS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, + _refreshed_field(), + )] + + +def account_fields(): + """One row per `account.account` — the GL chart, the owner's "GL codes". + + ⚠ NO LINK COLUMN, and that is a finding rather than an omission. A GL account meets the rest + of this schema only at LINE grain (963,783 `account_move_line` rows, 154,917 of them on + expense-type accounts), and a `ut_*` link folds rows that live in the store. The honest + binding is a read-through rollup naming a governed topic, or the mirror grid (R2) — never a + link into a table that does not exist. Declaring one here would render a permanently blank + column, which is the exact trap D-87 warns about from the value site. + """ + return [_preset(f) for f in ( + {"key": "account_code", "label": "Code", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "account_name", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.account` id. Also this row's id."}, + # ⚠ 15 DISTINCT VALUES MEASURED IN THE MIRROR, all declared. A `select` storing a value its + # options omit is wave-26 item 24: the filter panel answers with a list that cannot match + # what is stored. + {"key": "account_type", "label": "Type", "type": "select", "source": "overlay", + "default": True, + "options": ["expense", "expense_direct_cost", "expense_depreciation", "income", + "income_other", "asset_cash", "asset_current", "asset_receivable", + "asset_fixed", "asset_non_current", "asset_prepayments", + "liability_current", "liability_payable", "liability_credit_card", + "liability_non_current", "equity", "equity_unaffected", "off_balance"]}, + {"key": "is_expense", "label": "Expense account", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Ticked for the expense family - the same predicate the semantic layer's " + "gl_lines topic uses, so this column and that topic cannot disagree."}, + _refreshed_field(), + )] + + +def vendor_fields(): + """One row per partner we have POSTED a vendor bill to, keyed on the `res.partner` id. + + ⚠ A VENDOR IS NOT A CUSTOMER TABLE ROW, even though both are `res.partner`. MEASURED: 393 + vendors, of which only 9 also appear in the customer population. Pointing bills at + `ut_odoo_customers` would have dangled 384 of 393 links — the failure would have been a mostly + empty column, not an error. + """ + return [_preset(f) for f in ( + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": True}, + {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, + "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def bill_fields(): + """One row per POSTED vendor bill or refund — the owner's "expenses", at DOCUMENT grain. + + ⚠ DOCUMENT GRAIN IS A CHOICE AND IT IS THE ONLY ONE THAT FITS: 6,538 bills against 154,917 + expense GL lines. What a person calls "expenses" is both, and they are different tables - the + bill is what you pay, the line is what it was coded to. This is the payable; the line ledger + is the read-through mirror grid (R2). + """ + return [_preset(f) for f in ( + {"key": "bill_no", "label": "Bill", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "vendor", "label": "Vendor", "type": "text", "source": "overlay", "default": True}, + {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Bill date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + # ⚠ SIGNED, like the customer side: Odoo's `_signed` fields already carry the refund's + # direction, so a refund reduces a total without anybody re-deriving a sign here. + {"key": "amount_untaxed", "label": "Billed $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["in_invoice", "in_refund"]}, + {"key": "vendor_link", "label": "Vendor record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": VENDORS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, + _refreshed_field(), + )] + + +def invoice_fields(): + """One row per POSTED customer invoice or refund — the full history, not just what is open.""" + return [_preset(f) for f in ( + {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "invoice_date", "label": "Invoice date", "type": "date", "source": "overlay", + "default": True}, + {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay", + "default": True}, + {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Odoo's signed residual. Exactly 0 on every settled document, which is " + "why AR rollups need no filter."}, + {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + # ⛔ THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the + # table held open AR only; the full posted history also carries paid / in_payment / + # reversed. A select holding a value its options do not declare is the wave-26 item-24 + # defect — the filter panel answers with a list that cannot match what is stored. + {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay", + "default": True, + "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]}, + {"key": "move_type", "label": "Document", "type": "select", "source": "overlay", + "default": False, "options": ["out_invoice", "out_refund"]}, + _scope_field(), + # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like + # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. + {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was + # raised from, and the mirror did not sync it until this wave — so order->invoice was a + # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. + # ⚠ IT IS A NAME, NOT AN ID, and Odoo writes free text there (a manual invoice can hold + # anything; a merged one can hold several origins space-separated). The link resolves + # against `order_no` and finds nothing when the text is not an order name — the honest + # outcome, and the reason this is a join HINT rather than a foreign key. + {"key": "origin_order", "label": "Source order", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo's `invoice_origin` - usually the order name, sometimes blank."}, + {"key": "order_link", "label": "Order record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": ORDERS_KEY, "on": "order_no", "from": "origin_order"}}, + _refreshed_field(), + )] + + +def order_fields(): + """One row per CONFIRMED sale order — `state in (sale, done)`, the fixed wholesale scope.""" + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order` id. Also this row's id."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "team", "label": "Business unit", "type": "select", "source": "overlay", + "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", + "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, + _scope_field(), + {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS + # order, matched on the order's own name. + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": "origin_order", "from": "order_no"}}, + _refreshed_field(), + )] + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# THE TWO READ-THROUGH GRAINS (W30-T35). Their rows are SERVED FROM THE MIRROR, never stored. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⛔⛔ THE KEY SET IS HALF OF A CONTRACT AND `routes_odoo_tables.GRID_SOURCES[…]["cols"]` IS THE +# OTHER HALF. It binds a key to SQL; the label, type and order are declared HERE, once. The two +# lists must name the SAME columns, and both failure directions are silent: +# * a bound key with no declaration -> a silent NO-CELL (the row projection is strict); +# * a declared key with no binding -> an INACTIVE filter leaf, which WIDENS the result set. +# `verify_scopes.section_line_grids` compares the two sets, which is why neither side may "just +# add a column". +# +# ⛔ NO LINK COLUMN AND NO `refreshed` STAMP ON EITHER TABLE, and both absences are findings +# rather than omissions: +# * a LINK folds the rows the TARGET table STORES (`compute_relation_cells` reads the raw +# document, not the mirror), so a link at a read-through grain resolves against nothing and +# renders a permanently blank column — the trap `account_fields` names from the other end. +# The join ids ride as ordinary filterable columns instead, so the relationships are all +# still reachable by a person and by SQL. +# * `refreshed` means "when this row was last reconciled against Odoo", and it is stamped by +# `_ensure_table_inplace` onto rows it WRITES. Nothing here is ever written, so the column +# would be blank for every row forever. `section_line_grids` tolerates the key; the honest +# thing is not to declare it. + + +def order_line_fields(): + """One row per `sale.order.line` on a CONFIRMED order — the same `state in (sale, done)` + scope every wholesale metric in this product uses. + + MEASURED on the mirror 2026-08-12: **254,189 lines in scope** of 256,810 (the 2,621 excluded + sit on draft/sent/cancelled orders). Every line in scope is on a `sale` order — zero `done` — + but `done` stays in the option list because it is in the SCOPE, and a filter offering only + what happens to be stored today goes stale the first time an order is marked done. + + ⛔ THE SCOPE, THE ORDER DATE AND THE ORDER NAME ALL LIVE ACROSS A JOIN. `sale_order_line` + carries no `state` at all (12 columns, measured), so the binding is a join to `sale_order` — + which is also what makes `order_no` a readable primary cell instead of a line id. + + ⚠ `qty` IS `int`, NOT `currency`, AND THAT IS A MEASUREMENT. 3,888 of 256,810 lines carry a + FRACTIONAL quantity (0.2, 0.4, 0.5, 1.66 …) and the minimum is -1.0, so the question "does the + type truncate?" had to be answered rather than assumed: it does not. `int` and `currency` both + render through the client's `numberText`, which rounds nothing without a `format.decimals` + bag — the only difference is the `$` a `currency` column prepends. A quantity is not money, so + it takes the type that does not paint one. + """ + return [_preset(f) for f in ( + {"key": "order_no", "label": "Order", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The sale order this line belongs to. Zero orders have a blank name, " + "which is why it is the primary cell rather than the line id."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `sale.order.line` id. Also this row's id."}, + {"key": "order_id", "label": "Odoo order id", "type": "int", "source": "overlay", + "default": False, + "description": "The `sale.order` id — the key `ut_odoo_orders` is keyed on."}, + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 101 section and note lines, which carry no product."}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo product id", "type": "int", "source": "overlay", + "default": False}, + {"key": "qty", "label": "Qty", "type": "int", "source": "overlay", "default": True, + "agg": "sum", + "description": "Ordered quantity. 3,888 lines carry a fraction and some are negative " + "(returns), so nothing here is rounded."}, + {"key": "price_subtotal", "label": "Line $", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "margin", "label": "Margin $", "type": "currency", "source": "overlay", + "default": False, "agg": "sum", + "description": "Odoo's own line margin. Populated on every line."}, + {"key": "purchase_price", "label": "Unit cost", "type": "currency", "source": "overlay", + "default": False, + "description": "The cost Odoo priced this line's margin against, per unit."}, + {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay", + "default": True}, + {"key": "state", "label": "State", "type": "select", "source": "overlay", + "default": False, "options": ["sale", "done"]}, + _scope_field(), + )] + + +def gl_line_fields(): + """One row per `account.move.line` — the general ledger, and the owner's "expenses" at the + grain a person can actually browse. + + MEASURED 2026-08-12: **963,783 lines**, of which 944,846 posted, 18,885 cancelled and 52 + draft. ⛔ UNSCOPED ON PURPOSE — a general ledger whose draft and cancelled entries are + invisible is a ledger that cannot be reconciled, so `parent_state` rides as a COLUMN and the + reader chooses. That is the same decision the binding states from the SQL side. + + ⚠ TWO COLUMNS ARE LEGITIMATELY BLANK ON REAL ROWS, named here so neither reads as a defect: + **61,911 lines carry no partner** (journal entries that are not about a customer), and + **1,727 carry no account** at all, which is also why `account_code` — the key + `ut_odoo_accounts` is keyed on — is blank on exactly those 1,727 and the join that supplies + it is a LEFT one. + + ⚠ `move_type` IS `text`, NOT `select`, and it is the `product_fields.category` argument: + five values exist today (`out_invoice` 515,634 · `entry` 414,992 · `in_invoice` 18,901 · + `out_refund` 14,061 · `in_refund` 195) and Odoo's enum is longer than what we happen to hold. + A select whose options go stale answers a filter with a list that cannot match a stored value + (wave-26 item 24). `line_type` and `parent_state` ARE selects because their option lists were + measured COMPLETE against the whole table. + """ + return [_preset(f) for f in ( + {"key": "entry", "label": "Entry", "type": "text", "source": "overlay", + "default": True, "pinned": True, + "description": "The journal entry this line belongs to. Never blank."}, + {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `account.move.line` id. Also this row's id."}, + {"key": "move_id", "label": "Odoo entry id", "type": "int", "source": "overlay", + "default": False}, + {"key": "account", "label": "Account", "type": "text", "source": "overlay", + "default": True}, + {"key": ACCOUNT_JOIN_KEY, "label": "Account code", "type": "text", "source": "overlay", + "default": True, + "description": "The GL code, from the joined chart of accounts — the key " + "`ut_odoo_accounts` is keyed on. Blank on the 1,727 lines with no " + "account."}, + {"key": "customer", "label": "Partner", "type": "text", "source": "overlay", + "default": True, + "description": "Blank on the 61,911 lines that are not about a partner."}, + {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay", + "default": False}, + {"key": "date", "label": "Date", "type": "date", "source": "overlay", "default": True}, + {"key": "debit", "label": "Debit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "credit", "label": "Credit", "type": "currency", "source": "overlay", + "default": True, "agg": "sum"}, + {"key": "balance", "label": "Balance", "type": "currency", "source": "overlay", + "default": True, "agg": "sum", + "description": "Debit minus credit, as Odoo stores it. Sums to zero over a whole entry."}, + {"key": "line_type", "label": "Line type", "type": "select", "source": "overlay", + "default": False, + "options": ["product", "cogs", "payment_term", "line_note", "line_section"]}, + {"key": "move_type", "label": "Document type", "type": "text", "source": "overlay", + "default": False}, + {"key": "parent_state", "label": "Entry state", "type": "select", "source": "overlay", + "default": True, "options": ["draft", "posted", "cancel"]}, + _scope_field(), + )] + + +def product_fields(): + """One row per `product.product`, keyed on its id — EVERY product, archived ones included. + + ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes + map to more than one product id (re-SKU / merge history). The code is what a human reads and + the id is what `sales_lines.product` groups by, so both are columns and only the id is the + identity. + + ⛔ THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products + carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a + blank name — measured. Pinning `code` would give those rows a blank primary cell, which is + exactly D-80: a first column nothing populates quietly becoming the row's identity + ([[fallback-that-became-the-rule]]). + """ + return [_preset(f) for f in ( + {"key": "product", "label": "Product", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `product.product` id. Also this row's id, and what " + "every product rollup groups by."}, + {"key": "code", "label": "SKU", "type": "text", "source": "overlay", + "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service " + "products that are not stocked SKUs."}, + {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay", + "default": True, + "description": "Unticked = archived. Archived products are kept because they still " + "carry sales history — two of them sold this year."}, + # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a + # select whose options go stale answers a filter with a list that cannot match a stored + # value (wave-26 item 24). Text filters honestly and never goes out of date. + {"key": "category", "label": "Category", "type": "text", "source": "overlay", + "default": True}, + {"key": "product_type", "label": "Type", "type": "select", "source": "overlay", + "default": False, "options": ["product", "consu", "service"]}, + {"key": "standard_price", "label": "Standard cost", "type": "currency", + "source": "overlay", "default": True}, + # ⭐ SOURCE-BACKED (read-through) — the product-grain half of "compute all of the data in + # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU; + # `sales_lines` holds 256,810 rows that are never copied into this table. + {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "units", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +def customer_fields(): + """One row per partner Odoo has transacted with, keyed on the `res.partner` id. + + ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a + relation Odoo already decided. The rollups come in two kinds on purpose: + * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` — they can answer + anything about a document the table holds, including a date rank; + * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query + over the whole mirror — they can answer a DATE-WINDOWED money question, which a link + rollup cannot, because a condition can only compare against a literal and a literal year + start is right until 1 January. + """ + return [_preset(f) for f in ( + {"key": "customer", "label": "Customer", "type": "text", "source": "overlay", + "default": True, "pinned": True}, + {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", + "default": False, "description": "The `res.partner` id. Also this row's id."}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, + {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, + {"key": "country", "label": "Country", "type": "text", "source": "overlay", + "default": False}, + {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", + "default": True, + "description": "The customer's assigned agent (res.partner.agent_ids[0] — the " + "Customers-module convention)."}, + # ⭐ WAVE 28 — the agent's ID beside its NAME, because a link joins on an id and this + # table carried only the display string. ⚠ It is `agent_id`, NEVER `partner_id`: both are + # `res.partner` ids, and joining agents through `JOIN_KEY` would link every customer to + # itself and look plausible doing it. + {"key": AGENT_JOIN_KEY, "label": "Odoo agent id", "type": "int", "source": "overlay", + "default": False}, + _scope_field(), + + # --- the relations ------------------------------------------------------------------- + {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", + "default": True, + "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + {"key": "orders", "label": "Orders", "type": "link", "source": "overlay", + "default": True, + "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # MEASURED: 2,093 customers carry an `agent_id` and all 2,093 resolve to a row in the + # agents table — zero dangling, which is why this ships as a link rather than a lookup. + {"key": "agent_link", "label": "Agent record", "type": "link", "source": "overlay", + "default": False, + "link": {"table": AGENTS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, + + # --- link rollups over the invoice history -------------------------------------------- + # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual + # is exactly 0, so summing the full history gives the open balance to the cent. + # ⚠ THE LABEL SAYS "ALL CHANNELS" BECAUSE THE COLUMN TOTAL DOES NOT MATCH THE AR PAGE. + # Per customer this is exactly right. Summed down the column it is $2,347,608.49 while + # `Settings → AR` shows $591,828.54 — a 4x gap that is entirely the GIFTWARE DEALS / + # Amazon partner, which wholesale scope excludes and this table deliberately keeps. Two + # numbers with one name, 4x apart, in one product is how a correct figure gets reported + # as a bug; the scope belongs in the label, not only in a column somebody has to filter. + {"key": "ar_outstanding", "label": "AR outstanding $ - all channels", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "description": "Open balance across every posted document, INCLUDING the Amazon " + "channel. Filter `In wholesale scope` to reconcile with the AR page.", + "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}}, + {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup", + "source": "overlay", "default": False, "agg": "sum", + "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}}, + # ⛔ THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every + # document ever posted and labels it "open invoices" — the wrong-number-that-looks-right + # this module refuses everywhere else. + {"key": "open_invoices", "label": "Open invoices #", "type": "rollup", + "source": "overlay", "default": True, + "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}}, + # ⛔ NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a + # date column finds no numbers and returns BLANK — a column that renders empty forever + # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for. + {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "invoices", "field": "due_date", "fn": "latest", + "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}}, + + # --- link rollups over the order history ---------------------------------------------- + {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "fn": "countall"}}, + {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay", + "default": True, + "rollup": {"link": "orders", "field": "order_date", "fn": "latest", + "sortBy": "order_date", "sortDir": "desc"}}, + + # --- source-backed (read-through) rollups --------------------------------------------- + # ⛔ THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted + # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced + # flag. Different grain, different question — the metric KEY carries the distinction, + # which is the whole reason a rollup may not carry SQL of its own. + {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup", + "source": "overlay", "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay", + "default": True, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "revenue", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}}, + {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay", + "default": False, "agg": "sum", + "rollup": {"source": {"topic": "sales_lines", "measure": "margin", + "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}}, + {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay", + "default": False, + "rollup": {"source": {"topic": "sales_orders", "measure": "orders", + "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}}, + _refreshed_field(), + )] + + +# --------------------------------------------------------------------------------------------- +# READING THE MIRROR +# --------------------------------------------------------------------------------------------- +def excluded_names(): + """The partner names out of wholesale scope, from the ONE place that defines them. + + Read through `core.odoo` rather than re-listed here: a second literal is a second scope, and + the day somebody adds a channel this module would keep answering the old question. + """ + try: + import core.odoo as odoo + names = getattr(odoo, "EXCLUDE_PARTNER_NAMES", None) or set() + return {str(n).strip().lower() for n in names if str(n).strip()} + except Exception: # noqa: BLE001 + return set() + + +def excluded_ids(cur, names=None): + """The out-of-scope partner IDS, resolved against the MIRROR. + + ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite. + `account_move.partner_name` is a copy taken when the document was written, and this module's + own `customers_from` says so out loud — *"a partner's name can differ across documents + (renames land on new invoices only)"*. So a rename would put some of one partner's documents + in scope and the rest out, silently, and the totals would stop reconciling with nothing to + point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID. + + ⛔ RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a + LIVE `search_read`, so importing it here would make spawning four locked databases fail + whenever Odoo is unreachable — including on this developer machine, where the handshake dies + on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network. + + ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match — which + is the correct direction. 44 transacting partners carry no name at all; treating an + unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope. + """ + names = names if names is not None else excluded_names() + if not names: + return set() + rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall() + return {int(pid) for pid, name in rows if str(name).strip().lower() in names} + + +def columns(cur, table): + """The column names a mirror table actually has, lowercased. `set()` if the table is absent. + + ⛔ WHY THIS EXISTS, AND IT COST A LIVE 500. `harness.datastore.ready()` gates on ENTITY + phases, and a Space hydrates its mirror from `store_seed/royal.duckdb` — a SNAPSHOT. Columns + added to `ENTITIES` after that snapshot was taken (`res_partner.agent_id`, + `account_move_line.product_id`, …) are backfilled by `sync_all()` under their OWN `_sync_state` + keys, which `ready()` does not read. So there is a real window, right after a boot, where the + store reports READY and a column this module names does not exist yet — and DuckDB answers a + missing identifier with a Binder error, which reached the operator as a bare `500`. + ⚠ The absent columns are all DISPLAY ones (an agent name, a category, a team). Refusing the + whole spawn over a cosmetic column would be worse than the gap it is reporting, so the readers + degrade the COLUMN to blank and still write every id. + """ + try: + rows = cur.execute(f"SELECT * FROM {table} LIMIT 0") + return {str(d[0]).lower() for d in rows.description} + except Exception: # noqa: BLE001 + return set() + + +def _col(have, name, default="NULL"): + """`name` when the mirror has it, else a literal that keeps the SELECT's arity intact.""" + return name if str(name).split(".")[-1].lower() in have else default + + +def _as_date(value): + """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the + STORED value stays ISO — a formatted string in the cell is a value the filters cannot sort.""" + if not value: + return "" + return str(value)[:10] + + +def _in_scope(pid, excluded): + """`'1'` | `''` — the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '').""" + return "" if int(pid) in excluded else "1" + + +def read_invoices(cur, excluded=None, open_only=False): + """[(row dict)] — posted customer invoices and refunds, keyed on the `account.move` id. + + ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the + out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and + TAGS the channel instead. Two queries would be two populations, and they drift the moment + either is edited. + + Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + where = _AR_WHERE if open_only else _POSTED_DOCS + # ⚠ `invoice_origin` (D-88) is read through `_col` DELIBERATELY. It was added to `ENTITIES` in + # this same wave, so a Space whose mirror is still hydrating from a pre-wave seed snapshot does + # not have the column yet — and DuckDB answers a missing identifier with a Binder error that + # reaches the operator as a bare 500. This is the exact class `columns()` was written for: the + # link degrades to blank for one sync cycle instead of refusing the whole spawn. + have = columns(cur, "account_move") + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + " amount_untaxed_signed, amount_residual_signed, payment_state, move_type, " + f" {_col(have, 'invoice_origin', chr(39) + chr(39))} " + f"FROM account_move WHERE {where} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype, origin) = r + scope = _in_scope(pid, excluded) + if open_only and not scope: + continue + out.append({ + "_id": str(mid), + "invoice_no": str(name or ""), + "odoo_id": int(mid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "invoice_date": _as_date(inv_date), + "due_date": _as_date(due), + "residual": float(residual or 0.0), + "amount_untaxed": float(untaxed or 0.0), + "payment_state": str(pay_state or ""), + "move_type": str(mtype or ""), + "origin_order": str(origin or "").strip(), + "wholesale_scope": scope, + }) + return out + + +def read_open_ar(cur, excluded=None): + """The OPEN, wholesale-scoped subset — `modules/ar._open_docs`' own population. + + Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an + oracle answers exactly one question. It is a projection of `read_invoices`, never a second + query. + """ + return read_invoices(cur, excluded=excluded, open_only=True) + + +def read_orders(cur, excluded=None): + """[(row dict)] — confirmed sale orders, keyed on the `sale.order` id.""" + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "sale_order") + sql = (f"SELECT id, name, date_order, partner_id, partner_name, {_col(have, 'team_name')}, " + f" state, amount_untaxed, {_col(have, 'invoice_status')} " + f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r + out.append({ + "_id": str(oid), + "order_no": str(name or ""), + "odoo_id": int(oid), + "customer": str(pname or ""), + JOIN_KEY: int(pid), + "order_date": _as_date(when), + "amount_untaxed": float(untaxed or 0.0), + "team": str(team or ""), + "state": str(state or ""), + "invoice_status": str(inv_status or ""), + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_products(cur): + """[(row dict)] — EVERY `product.product`, keyed on its id. + + ⛔ NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they + were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products + that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three + uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product + grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to + land — silently. 119 extra rows is the whole cost of the claim being literally true. + """ + have = columns(cur, "product_product") + sql = (f"SELECT id, default_code, name, {_col(have, 'categ_name')}, type, " + f" {_col(have, 'standard_price', '0')}, {_col(have, 'active', 'TRUE')} " + "FROM product_product") + out = [] + for r in cur.execute(sql).fetchall(): + (prid, code, name, categ, ptype, cost, active) = r + out.append({ + "_id": str(prid), + "product": str(name or ""), + PRODUCT_JOIN_KEY: int(prid), + "code": str(code or ""), + "active": "1" if active else "", + "category": str(categ or ""), + "product_type": str(ptype or ""), + "standard_price": float(cost or 0.0), + }) + return out + + +def read_customers(cur, excluded=None): + """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. + + The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale + order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. + + ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — + the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old + docstring said partners with no document are *"left out on purpose — a row that can never + appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same + join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** + (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer + a salesperson has not sold to yet is exactly the row a prospecting view needs. + + ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the + document legs for the rank leg would drop **16 partners that hold posted documents** (7 + archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry + `partner_id` LINKS straight back here — so those links would dangle with nothing reporting + it. When other tables point AT a population, a widening must be a SUPERSET. + + ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same + `columns()`/`_col` discipline every other optional column here uses — but note the difference + honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the + POPULATION back to the document union. It degrades to today's behaviour rather than to an + empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the + column is missing so the narrowing can never pass for done. + + ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above + states the rule for this module and it applies with more force to a POPULATION than to a name + list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its + mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this + time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the + weather. One source, always present at spawn time: the mirror. + + ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR + partners; sourcing a customer registry from its own receivables is what kept most Odoo ids + out of the store in the first place. + """ + excluded = excluded if excluded is not None else excluded_ids(cur) + have = columns(cur, "res_partner") + # ⚠ THE AGENT JOIN IS DROPPED WHOLE when `agent_id` is absent, not merely NULL-ed: the join + # itself names the column, so `_col` on the SELECT list alone would still fail to bind. + agent = ("ag.name" if "agent_id" in have else "NULL") + agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") + join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") + # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an + # archived prospect out, and a rank test without it would re-admit the 47 archived partners + # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. + rank_leg = (" OR (p.customer_rank > 0 AND p.active) " + if {"customer_rank", "active"} <= have else "") + sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " + f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " + "FROM res_partner p " + f"{join}" + "WHERE p.id IN (" + f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " + " UNION " + " SELECT partner_id FROM account_move " + f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" + f"{rank_leg}") + out = [] + for r in cur.execute(sql).fetchall(): + (pid, name, city, state, country, agent, agent_id) = r + out.append({ + "_id": str(pid), + "customer": str(name or ""), + JOIN_KEY: int(pid), + "city": str(city or ""), + "state": str(state or ""), + "country": str(country or ""), + "agent": str(agent or ""), + AGENT_JOIN_KEY: int(agent_id) if agent_id else "", + "wholesale_scope": _in_scope(pid, excluded), + }) + return out + + +def read_agents(cur): + """[(row dict)] — the UNION of both agent sources, keyed on the `res.partner` id. + + ⛔ `res_partner.agent` is a BOOLEAN and `datastore.BOOL_FIELDS` lists it for a measured reason: + Odoo returns False both for "empty" and for "boolean false", so a bool missing from that list + silently becomes NULL and every row would read "not an agent" indistinguishably from + "unknown". Read it as a truth value, never as a presence test. + """ + have = columns(cur, "res_partner") + if "id" not in have: + return [] + flagged = "p.agent" if "agent" in have else "FALSE" + # ⛔⛔ THE COMMISSION TABLE IS GUARDED AS A **TABLE**, not just as a column, and that + # distinction is the whole point of this block. `columns()` was written for a missing COLUMN + # (a backfill that has not run yet); `account_invoice_line_agent` is an OCA module entity that + # a mirror hydrated from an older seed snapshot may not have AT ALL. A SELECT naming an absent + # table is a DuckDB Binder error, and this reader runs inside `plan()` — so one missing table + # would fail the WHOLE eight-table spawn and reach the operator as a bare 500. That is + # precisely D-107's shape, and it would have arrived on the first deploy of this feature. + # ⚠ DEGRADE, NEVER REFUSE, which is the posture `columns()`'s own docstring sets: without the + # commission table the population falls back to the FLAGGED partners alone and `commissioned` + # reads blank for every row — fewer agents and an honestly empty column, rather than no spawn. + has_comm = bool(columns(cur, "account_invoice_line_agent")) + commissioned = ("(p.id IN (SELECT agent_id FROM account_invoice_line_agent " + " WHERE agent_id IS NOT NULL))" if has_comm else "FALSE") + union_leg = (" SELECT agent_id FROM account_invoice_line_agent WHERE agent_id IS NOT NULL " + " UNION " if has_comm else "") + sql = (f"SELECT p.id, p.name, {flagged}, {commissioned} AS commissioned " + "FROM res_partner p WHERE p.id IN (" + f"{union_leg}SELECT id FROM res_partner WHERE {flagged})") + out = [] + for (aid, name, flag, comm) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(aid), + "agent": str(name or ""), + "odoo_id": int(aid), + AGENT_JOIN_KEY: int(aid), + "flagged": "1" if flag else "", + "commissioned": "1" if comm else "", + }) + return out + + +def read_accounts(cur): + """[(row dict)] — the whole GL chart, keyed on the `account.account` id. + + ⚠ THE EXPENSE PREDICATE IS THE SEMANTIC LAYER'S, copied rather than invented: + `harness/semantic.py`'s `gl_lines` topic scopes expenses as + `account_type in ('expense','expense_depreciation')`. A second definition here is how a + column and a topic start disagreeing about the same word. + ⛔ `account.account` has NO `active` column in this Odoo version (a domain naming it 500s), so + there is nothing to filter and every account is a row. + """ + have = columns(cur, "account_account") + if not have: + return [] + sql = (f"SELECT id, {_col(have, 'code', chr(39) + chr(39))}, " + f" {_col(have, 'name', chr(39) + chr(39))}, " + f" {_col(have, 'account_type', chr(39) + chr(39))} FROM account_account") + out = [] + for (aid, code, name, atype) in cur.execute(sql).fetchall(): + t = str(atype or "") + out.append({ + "_id": str(aid), + ACCOUNT_JOIN_KEY: str(code or ""), + "account_name": str(name or ""), + "odoo_id": int(aid), + "account_type": t, + "is_expense": "1" if t in ("expense", "expense_depreciation") else "", + }) + return out + + +_VENDOR_DOCS = "state = 'posted' AND move_type IN ('in_invoice','in_refund')" + + +def read_bills(cur): + """[(row dict)] — posted vendor bills and refunds, keyed on the `account.move` id.""" + have = columns(cur, "account_move") + if not have: + return [] + sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, " + f" {_col(have, 'amount_untaxed_signed', '0')}, " + f" {_col(have, 'amount_residual_signed', '0')}, " + f" {_col(have, 'payment_state', chr(39) + chr(39))}, move_type " + f"FROM account_move WHERE {_VENDOR_DOCS} AND partner_id IS NOT NULL") + out = [] + for r in cur.execute(sql).fetchall(): + (mid, name, pid, pname, when, due, untaxed, residual, pay, mtype) = r + out.append({ + "_id": str(mid), + "bill_no": str(name or ""), + "odoo_id": int(mid), + "vendor": str(pname or ""), + VENDOR_JOIN_KEY: int(pid), + "invoice_date": _as_date(when), + "due_date": _as_date(due), + "amount_untaxed": float(untaxed or 0.0), + "residual": float(residual or 0.0), + "payment_state": str(pay or ""), + "move_type": str(mtype or ""), + }) + return out + + +def read_vendors(cur): + """[(row dict)] — every partner carrying a posted vendor bill, keyed on the `res.partner` id. + + ⚠ DERIVED FROM THE BILLS, unlike `read_customers` which is deliberately NOT derived from its + invoices. The asymmetry is intentional and the reason is what that function's own comment + says: a customer registry sourced from receivables is what kept most Odoo ids out of the store. + There is no second document universe for vendors — a partner with no bill has no payable + history to show — so the bill IS the population, and MEASURED it dangles nothing (0 bills + carry a null partner; all 393 vendors resolve in `res_partner`). + """ + have = columns(cur, "res_partner") + if not have or not columns(cur, "account_move"): + return [] + sql = (f"SELECT p.id, p.name, {_col(have, 'p.country_name', chr(39) + chr(39))} " + "FROM res_partner p WHERE p.id IN " + f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " + " AND partner_id IS NOT NULL)") + out = [] + for (pid, name, country) in cur.execute(sql).fetchall(): + out.append({ + "_id": str(pid), + "vendor": str(name or ""), + "odoo_id": int(pid), + VENDOR_JOIN_KEY: int(pid), + "country": str(country or ""), + }) + return out + + +def customers_from(invoice_rows): + """The partners carrying the given invoice rows — the pre-2026-08-09 population builder. + + ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a + pure function over rows and the gate uses it to prove the FOLD against a fixture without a + mirror; deleting it would cost a test its independence from the SQL. + """ + out = {} + for row in invoice_rows: + pid = row[JOIN_KEY] + entry = out.setdefault(str(pid), {"_id": str(pid), "customer": row["customer"], + JOIN_KEY: pid}) + # A partner's name can differ across documents (renames land on new invoices only); + # the newest non-empty one wins so the locked table shows what Odoo shows today. + if row["customer"]: + entry["customer"] = row["customer"] + return list(out.values()) + + +# --------------------------------------------------------------------------------------------- +# THE SPAWN +# --------------------------------------------------------------------------------------------- +class Refused(Exception): + """A refusal a caller should SHOW, not swallow. Every raise names what would otherwise have + been written wrong.""" + + +#: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole +#: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. +#: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. +TABLES = ( + ("customers", CUSTOMERS_KEY, "Odoo customers", customer_fields), + ("products", PRODUCTS_KEY, "Odoo products", product_fields), + ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), + ("orders", ORDERS_KEY, "Odoo orders", order_fields), + # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders + # of magnitude inside `MAX_ROWS`, which is why the answer to "every unique id is a database" + # is four more spec rows and four readers rather than a new substrate. + ("agents", AGENTS_KEY, "Odoo agents", agent_fields), + ("accounts", ACCOUNTS_KEY, "Odoo GL accounts", account_fields), + ("vendors", VENDORS_KEY, "Odoo vendors", vendor_fields), + ("bills", BILLS_KEY, "Odoo vendor bills", bill_fields), + # ⭐⭐ W30-T35 / R7 — the two READ-THROUGH grains. They are spec rows like any other, and that + # is the point: `apply_plan` creates their DEFINITION (label, fields, lock, nav entry, grants) + # exactly as it does for the eight above, and `plan` hands them ZERO rows. Leaving them out of + # this tuple was the alternative and it is the wrong one — the route 404s on a key `TABLES` + # does not name, so the grids would be bound to the mirror and unreachable, which is this + # wave's own [[reachable-is-not-the-same-as-built]] shape. + ("order_lines", ORDER_LINES_KEY, "Odoo order lines", order_line_fields), + ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), +) + +#: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no +#: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a +#: policy `plan()` could forget. The two line grains are absent for that reason and no other. +_READERS = { + "customers": lambda cur, excluded: read_customers(cur, excluded=excluded), + "products": lambda cur, excluded: read_products(cur), + "invoices": lambda cur, excluded: read_invoices(cur, excluded=excluded), + "orders": lambda cur, excluded: read_orders(cur, excluded=excluded), + "agents": lambda cur, excluded: read_agents(cur), + "accounts": lambda cur, excluded: read_accounts(cur), + "vendors": lambda cur, excluded: read_vendors(cur), + "bills": lambda cur, excluded: read_bills(cur), +} + +#: The table keys this module can never materialise — DERIVED from the absence of a reader, never +#: typed out, so it cannot drift from the fact it describes. +#: +#: ⛔⛔ IT IS STAMPED ONTO THE DEFINITION AT SPAWN, AND THAT IS NOT BELT-AND-BRACES — IT IS THE +#: ONLY WAY THESE TWO TABLES EVER GET THE DURABLE FLAG. `core.user_tables.materialises` reads a +#: process-global registry first and falls back to a stored `readThrough` stamp, "which is what a +#: cold process reads" — but the only writer of that stamp is `strip_materialised`, and it stamps +#: exclusively tables it found rows on (`if isinstance(t, dict) and t.get('rows')`, after an early +#: return when nothing is fat). A table that was BORN read-through has no rows to strip, so it is +#: never stamped, so a process that cannot reach the mirror reads `rows: {}` and calls that the +#: answer — an EMPTY GRID with nothing going red, which is the exact failure that docstring names. +#: The conversion writes the stamp; a table that needs no conversion still needs the statement. +READ_THROUGH_KEYS = frozenset(key for bucket, key, _l, _f in TABLES if bucket not in _READERS) + + +class _LentDoc: + """A store handle that serves the ONE `user_tables` document `plan()` has ALREADY read. + + ⛔⛔ THIS IS NOT A MICRO-OPTIMISATION AND IT IS NOT OPTIONAL. `core.user_tables.row_limit` + resolves through `materialises` → `get` → `all_tables(st)`, and every one of those is a WHOLE + 20 MB document read, deep-copied under `Store._lock`. `plan()` asks the evaluator once per + table per loop, so passing the live handle would have added ~16 full document copies to a + function that already reads it exactly once — and with `st=None` (the gate's fixture posture, + and any dry run) those reads resolve to the MODULE-GLOBAL store, i.e. a Hugging Face dataset + fetch per table, on a path that has no business touching the network at all. + `materialises`' own docstring asks callers to lend the definition they are holding; `row_limit` + takes `st` rather than `defn`, so the lending happens one level up, here. + + ⚠ It answers ONLY the user-tables document and `None` for anything else, deliberately: a shim + that quietly proxied other keys would be a second store with a partial view, which is worse + than one that says what it knows. + """ + + def __init__(self, doc, key): + self._doc, self._key = doc if isinstance(doc, dict) else {}, key + + def get(self, name): + return self._doc if name == self._key else None + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE CONNECTOR'S OWN CONFIGURATION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner opened the Odoo connector and found nothing to configure: no key, no server database, +# no choice of which grids to materialise, no sync cadence, no way off. R9/R10/R11 answer all +# four, and the state lives HERE rather than in the route because `refresh()` is what has to obey +# it — a config the route knows and the sync path does not is a switch that flips nothing. +# +#: `{"grids": {table_key: bool}, "syncEvery": "", "frozen": bool, "frozenAt": ""}` +CONFIG_KEY = "odoo_connector_config" + +#: ⭐ R11's presets, and the FLOOR IS THE POINT. Owner: *"30m / 1h / 4h / daily / manual. The +#: floor is 30 minutes, server enforced; no free-text interval."* An interval box would let a +#: tenant ask for 60 s against an ERP over XML-RPC and a Hugging Face free-tier container. +#: ⚠ `manual` is not "very slow" — it is NO scheduled sync at all, which is why it maps to None +#: rather than to a large number. A caller that treats None as a duration gets a TypeError rather +#: than a silent once-a-century schedule. +SYNC_PRESETS = {"30m": 1800, "1h": 3600, "4h": 14400, "daily": 86400, "manual": None} +SYNC_FLOOR_SECONDS = 1800 +DEFAULT_SYNC = "30m" + + +def read_config(rt): + """This tenant's stored connector config, defaulted. Never raises — a store hiccup must not + make a connector look disconnected.""" + try: + cur = rt.get(CONFIG_KEY) if rt is not None else None + except Exception: # noqa: BLE001 + cur = None + cur = cur if isinstance(cur, dict) else {} + grids = cur.get("grids") if isinstance(cur.get("grids"), dict) else {} + every = cur.get("syncEvery") + return {"grids": {str(k): bool(v) for k, v in grids.items()}, + "syncEvery": every if every in SYNC_PRESETS else DEFAULT_SYNC, + "frozen": bool(cur.get("frozen")), + "frozenAt": str(cur.get("frozenAt") or "")} + + +def grid_choices(rt): + """`[{key, label, enabled}]` for every grid this connector can materialise — DERIVED from + `TABLES`, never a second hand-typed list (contract C2's parity leg asserts exactly that). + + ⚠ ABSENT MEANS ENABLED. A tenant that has never opened the panel has every grid, which is + what they have today; only an explicit untick turns one off. The alternative — an empty + config meaning "nothing enabled" — would silently unspawn ten live databases on deploy. + """ + chosen = read_config(rt)["grids"] + return [{"key": key, "label": label, "enabled": bool(chosen.get(key, True))} + for _bucket, key, label, _fields in TABLES] + + +def enabled_buckets(rt): + """The BUCKET names `plan()` speaks, for the grids this tenant has left ticked.""" + chosen = read_config(rt)["grids"] + return {bucket for bucket, key, _l, _f in TABLES if chosen.get(key, True)} + + +def sync_seconds(rt): + """How often this tenant's Odoo mirror should resync, or None for `manual` (R11). + + ⛔ THE FLOOR IS ENFORCED HERE AS WELL AS AT THE WRITE DOOR, deliberately. A stored value that + predates the preset list, or one written by any path that is not the route, must still not be + able to ask this loop for a 60-second cycle — a limit with only one enforcer is a limit that + holds until somebody finds the second way in [[limit-with-no-enforcer]]. + """ + secs = SYNC_PRESETS.get(read_config(rt)["syncEvery"], SYNC_PRESETS[DEFAULT_SYNC]) + if secs is None: + return None + return max(int(secs), SYNC_FLOOR_SECONDS) + + +def frozen(rt): + """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and + stop being refreshed — distinct from PAUSED, which is temporary and keeps the credential.""" + return bool(read_config(rt)["frozen"]) + + +def plan(cur, rt=None): + """The rows that WOULD be written, plus the refusals that apply — no store WRITE at all. + + Separated from `apply_plan` so a route, a gate and a dry run all measure the same thing, and + so **every cap is checked before anything is committed**. + + Returns one key per bucket plus two that are not buckets: `problems` (refusals — a non-empty + list makes `apply_plan` raise before it writes anything) and, since W30-T35, **`limits`** — + `{table_key: limit_report}` for every table this plan did not fully materialise, which is R6's + second sentence carried as data rather than left for a reader to infer from an empty list. + + ⛔ `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn + bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would + create some and refuse the rest — leaving a locked invoices database with no rollup host, + while the route answered as though nothing had happened. A partial spawn is worse than a + refused one, so the count is checked against the tables that ALREADY exist, before the first + write. `rt` also carries the stored row counts the shrink guard compares against. + """ + ut = _ut() + # ⭐ R6, AND WITHOUT THIS LINE THE RULE IS ONLY ACCIDENTALLY TRUE. `is_connected` answers from + # three places in falling authority: the registry, a stored `connected: True`, then the + # `ut_odoo_` naming convention — and that last leg needs the table to ALREADY EXIST. So on a + # FIRST spawn, in a process that has not yet built `routes_odoo_tables.GRID_SOURCES`, every one + # of these tables reads as unconnected and earns `MAX_ROWS`: R6's cap removal would silently + # not apply on exactly the run that creates the databases. This module DECLARES these keys, so + # it is the honest place to say what they are. Idempotent (a set add), and it fills the + # evaluator's input rather than becoming a second evaluator. + ut.register_connected(*[key for _b, key, _l, _f in TABLES]) + excluded = excluded_ids(cur) + # ⭐⭐ W30-T35 / R6 / R7 — WHICH BUCKETS ARE BUILT AT ALL IS NOW ASKED, NOT ASSUMED, and it is + # `core.user_tables.row_limit` that answers: 0 = "stores no rows HERE" (read-through), None = + # "connected and uncapped", MAX_ROWS = "the editable substrate". Its own docstring names this + # function as the caller that reads it, which is the seam working as designed — one evaluator, + # so the spawn, the write doors and the wire cannot disagree about whether a table is capped + # ([[one-evaluator-per-question]]). + # + # ⛔ TWO DIFFERENT REASONS NOT TO BUILD, AND THEY ARE KEPT SEPARATE ON PURPOSE: + # * no reader at all — structural, permanent, and the case that must not depend on a store + # read succeeding (a cold process with no mirror still must not try to build 963,783 rows); + # * a reader exists but the table has already been converted to read-through — the + # `ut_odoo_accounts` case. Building 192 rows and letting `strip_materialised` delete them + # again on the next pass "works", and it is exactly the wasted, dangerous work `row_limit` + # was built to prevent. It also stops a refresh from silently RE-MATERIALISING a table + # D-87's conversion had already emptied. + # + # ⚠ THE DOCUMENT IS READ **ONCE**, HERE, AND LENT TO THE EVALUATOR — see `_LentDoc`. It used + # to be read after the build loop; it moved up because `row_limit` needs it and reading it per + # table per loop is ~16 more whole-document deep copies (or, with `st=None`, a Hugging Face + # fetch per table on a path that must never touch the network). + existing = {} + if rt is not None: + try: + existing = dict(rt.get(ut.STORE_KEY) or {}) + except Exception: # noqa: BLE001 + existing = {} + # ⛔ THE LENT DOCUMENT CARRIES THE `readThrough` STAMP THIS MODULE IS RESPONSIBLE FOR, and + # without it R6's report is silently absent on the run that matters most — the FIRST spawn. + # MEASURED: with an empty store, `row_limit` finds no registry entry and no stored stamp, so + # it answers `None` ("connected and uncapped") for a grain that stores nothing at all, and + # `limit_report` answers None with it — so `plan()["limits"]` came back EMPTY and the grids + # were skipped with no stated reason. The structural `reader is None` guard still did its job; + # what went missing was the half of R6 that has to SAY WHY. + # + # ⚠ THE OBVIOUS FIX IS THE ONE I DID NOT TAKE: `ut.register_read_through(*READ_THROUGH_KEYS)` + # would work in one line, and `core.user_tables` explicitly reserves that registrar — + # *"IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`"* — for + # `routes_odoo_tables.sync_read_through`, because ELIGIBILITY needs the mirror and the fold + # matrix. That reasoning does not apply to a grain with no reader (there is nothing it could + # be eligible FOR), but the law is written without an exception, so this lends the evaluator + # the definition instead of taking one. `_ensure_table_inplace` writes exactly this stamp, so + # what is lent is the document as it stands the moment this plan applies. + lent = _LentDoc({**existing, + **{k: {**(existing.get(k) or {}), "readThrough": True} + for k in READ_THROUGH_KEYS}}, + ut.STORE_KEY) + + built, limits = {}, {} + caps = {} + for bucket, key, _label, _fields in TABLES: + reader = _READERS.get(bucket) + caps[key] = cap = ut.row_limit(key, st=lent) + if reader is None or cap == 0: + built[bucket] = [] + report = ut.limit_report(key, st=lent) + if report: + limits[key] = report + continue + built[bucket] = reader(cur, excluded) + problems = [] + + if rt is not None: + needed = [k for _b, k, _l, _f in TABLES if k not in existing] + if needed and len(existing) + len(needed) > ut.MAX_TABLES: + problems.append( + f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and " + f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating " + f"part of a linked set") + + for bucket, key, _label, _fields in TABLES: + rows = built[bucket] + cap = caps[key] # asked ONCE per table, above — never re-read per loop + # ⭐⭐ R6: THE `MAX_ROWS` REFUSAL IS GONE FOR A CONNECTED SOURCE, AND THE SENTENCE IT USED + # TO PRINT IS NOW `limit_report`'s STRUCTURED ANSWER. Owner, verbatim: *"there is no cap in + # how many data from the API source (as long as its from a connected source like Odoo) that + # can be pulled into the app… Now if there is lag or it can't be done, you need to + # explicitly tell me why and recommend a fix."* Both halves are here: a connected table + # answers `None` and is never refused for its size, and anything that IS still bounded is + # reported with its cause and its recommendation instead of a hand-typed line. + # + # ⛔ THE `cap and` GUARD IS THE WHOLE CHANGE AND ITS TWO FALSY CASES MEAN OPPOSITE THINGS: + # `None` = connected, uncapped, build every row Odoo has; `0` = stores no rows here, and + # the loop above already handed it an empty list. Neither may reach the refusal. The + # editable substrate still gets `MAX_ROWS` and is still REFUSED, never truncated — a capped + # table understates every total it feeds while looking exactly like a complete one. + if cap and len(rows) > cap: + report = ut.limit_report(key, st=lent) or {} + limits[key] = report + problems.append( + f"{key}: {len(rows):,} rows exceeds the {cap:,}-row ceiling; refusing " + f"({report.get('cause', 'a truncated table understates every rollup it feeds')}). " + f"{report.get('recommendation', '')}".strip()) + # ⚠ THE SHRINK GUARD SKIPS A READ-THROUGH GRAIN, and without this it would refuse every + # spawn after the first conversion: zero rows against a stored population is the INTENDED + # end state there, not the partial mirror read this guard exists to catch. + if cap == 0: + continue + stored = len(((existing.get(key) or {}).get("rows")) or {}) + if stored and len(rows) < stored * MAX_SHRINK: + problems.append( + f"{key}: the mirror answered {len(rows)} rows against {stored} stored — a drop of " + f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history " + f"shrinking; refusing rather than deleting rows that still exist") + built["problems"] = problems + # R6's second sentence as DATA rather than prose: every table whose rows this plan did not + # (or may not) materialise, with the cause and the recommendation `core.user_tables` derives. + # ⚠ NOT a bucket — `apply_plan` iterates `TABLES` and asks `if bucket in built`, so a key that + # is not a bucket name is inert there, exactly as `problems` has always been. + built["limits"] = limits + return built + + +def apply_plan(rt, built, username="automation", today=None): + """Create-or-merge every table in `built` and its rows. Idempotent by construction. + + Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the + same record — which is also what makes "every Odoo unique id is in the database" a checkable + statement rather than a hopeful one. + + ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset. + """ + if built.get("problems"): + raise Refused("; ".join(built["problems"])) + stamp = today or _iso_today() + written = {} + plans = [(key, label, fields(), built[bucket]) + for bucket, key, label, fields in TABLES if bucket in built] + + # ⭐⭐ ONE SYNC WRITE FOR ALL FOUR TABLES, not one per table — measured, not tidied. + # + # ⛔ A `flush="sync"` update of `user_tables` is a FULL DOWNLOAD of the document plus a full + # UPLOAD of it (`Store.update` -> `_read_strict` -> `put`). Four of them against the 20.6 MB + # document these tables produce is ~165 MB of Hugging Face traffic and four dataset commits + # EVERY resync — and `main.py` runs this at boot and after every `sync_all()` (~30 min). + # Composed into one pass it is ~41 MB and one commit: the same rows, a quarter of the bill. + # + # ⭐ AND IT IS ATOMIC, WHICH IS THE BIGGER WIN. `_ensure_table_inplace` raises `Refused` at + # `MAX_TABLES`; with four separate writes that refusal landed AFTER earlier tables had + # already been committed, leaving exactly the half-spawn `plan()` opens by refusing to + # create. Inside one updater, a raise aborts before anything is persisted. + def _apply_all(cur): + cur = cur if isinstance(cur, dict) else {} + for key, label, fields, rows in plans: + written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _apply_all, flush="sync") + return written + + +def _ensure_table(rt, key, label, fields, rows, username, stamp): + """One table, written on its own. Kept because the gate drives a single table directly, and + because a caller with one table to reconcile should not have to compose an updater.""" + written = {} + + def _one(cur): + cur = cur if isinstance(cur, dict) else {} + written["counts"] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) + return cur + + rt.update(_ut().STORE_KEY, _one, flush="sync") + return written["counts"] + + +def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): + """One table INSIDE a caller's updater: definition merged, rows reconciled, dict mutated. + + ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening — + but only because the populations widened to "everything Odoo has". While `ut_odoo_customers` + was built FROM open invoices, removal meant a customer who paid their bill vanished from the + registry; now a partner leaves only when their last document does. The shrink guard in + `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read. + """ + ut = _ut() + wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows} + for row in wanted.values(): + row["refreshed"] = stamp + counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} + table = cur.get(key) + if table is None: + if len(cur) >= ut.MAX_TABLES: + # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a + # successful refresh over a table that does not exist. + raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") + table = cur[key] = { + "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, + "createdBy": username, "created": stamp, "fields": [], "rows": {}, + # recordMode = a LOCKED database (item-3 nomenclature): no human may add or + # delete records, while fields stay addable. Odoo owns this population. + "recordMode": ut.AUTOMATION_RECORD_MODE, + } + table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE) + # W30-T35 — the durable "my rows are not in this document" statement, on the tables no + # conversion will ever stamp (see `READ_THROUGH_KEYS`). Written on every pass, not + # `setdefault`: it is derived from the code's own structure, so the code is what it must agree + # with, and a definition that somehow lost the flag should regain it rather than keep serving + # an empty grid. + if key in READ_THROUGH_KEYS: + table["readThrough"] = True + have = {str(f.get("key")): f for f in (table.get("fields") or [])} + for field in fields: + fkey = str(field.get("key")) + if fkey not in have: + table.setdefault("fields", []).append(dict(field)) + continue + # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The + # widening moved `payment_state`'s option list and every rollup's conditions; a + # create-only merge would have left the LIVE table declaring the old contract + # forever, so the column would render but its filter could not match what is stored. + # ⚠ Only machine-owned keys are touched — `automation.preset` is the wall — so a + # column a user added to a locked database is never rewritten. + stored = have[fkey] + # ⭐⭐ 2026-08-09 — a column a human has taken over keeps its own definition. Same stamp, + # same reader (`user_tables.user_edited`) and the same reason as the IG reconciler: the + # loop below overwrites `rollup` from the shipped contract, so an edited preset rollup on + # an Odoo database would silently revert at the next boot rebuild. + if _ut().user_edited(stored): + continue + if (stored.get("automation") or {}).get("preset"): + for prop in ("label", "type", "options", "link", "rollup", "description", + "agg", "pinned", "default"): + if prop in field: + stored[prop] = field[prop] + else: + stored.pop(prop, None) + stored_rows = table.setdefault("rows", {}) + for rid, values in wanted.items(): + current = stored_rows.get(rid) + if current is None: + stored_rows[rid] = dict(values) + counts["added"] += 1 + elif any(str(current.get(k, "")) != str(v) for k, v in values.items() + if k != "refreshed"): + current.update(values) + counts["updated"] += 1 + else: + current["refreshed"] = values["refreshed"] + for rid in [r for r in stored_rows if r not in wanted]: + stored_rows.pop(rid, None) + counts["removed"] += 1 + return counts + + +def is_royal(tenant): + return str(tenant or "").strip().lower() in RI_SLUGS + + +def refresh(rt, tenant, username="automation", cur=None, today=None): + """THE entry point — the store-resync path and the route both call this. + + ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row + still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than + silently authoritative. + """ + if not is_royal(tenant): + raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal " + f"Imports only); refusing to spawn empty locked databases") + # ⭐⭐ W31-T45 / D-169 — THE SLUG GATE ABOVE AND THE FILE GATE HERE ANSWER DIFFERENT QUESTIONS, + # and this is the one place in the codebase where that is easy to miss. `is_royal` asks "is + # this tenant ENTITLED to Odoo databases"; it says nothing about WHICH DuckDB file this process + # has open. A worker pinned to another tenant's store (AIOS_DUCKDB_PATH, or a `use_path` in a + # provisioning script) passes `is_royal("royal-imports")` and then WRITES tenant #0's locked + # databases from another customer's rows — a spawn, not a read, so the wrong numbers become + # durable. Entitlement is not residency. + # ⚠ It runs when `cur` is LENT too, not only when we open one: the resync loop and the boot + # rebuild both hand a cursor in, and a lent cursor is exactly the case where nobody re-checks. + if rt is not None: + rt.assert_datastore_matches() + # ⛔⛔ W32-T16 / R10 — A DISCONNECTED CONNECTOR DOES NOT REFRESH, AND THAT IS THE WHOLE FREEZE. + # R10: *"Disconnect removes the credential and FREEZES the grids as static data."* Removing + # the credential alone is not a freeze — this function is also reached by the boot rebuild and + # the resync loop, and for tenant #0 the ENVIRONMENT still holds Odoo credentials, so a + # disconnected workspace would silently re-materialise from `.env` on the next tick and the + # "disconnect" would last until the container restarted. The refusal is a REPORT, not a raise: + # the resync loop calling this every cycle must not be handed an exception as a status. + if rt is not None and frozen(rt): + return {"tables": {}, "frozen": True, + "note": "this workspace has disconnected Odoo; its databases are frozen as " + "static data and are not being refreshed"} + if cur is None: + from harness import datastore + cur = datastore.ro_con() + built = plan(cur, rt=rt) + # ⭐ W32-T15 / R9 — THE GRID PICKER, ENFORCED WHERE IT COUNTS. `apply_plan` writes only the + # buckets present in `built`, so dropping an unticked one here is the whole of "unticking a + # grid stops it materialising on the next sync". Done AFTER `plan` rather than inside it so + # every cap, refusal and limit report is still computed over the full set — a config must not + # be able to hide a problem by hiding the table that has it. + # ⚠ It does NOT delete a grid that was already spawned. Unticking stops the next refresh from + # rewriting it; dropping the rows a tenant already has is `disconnect`'s job, and it does not + # do that either (R10 keeps them). Silent data deletion behind a checkbox is not on offer. + if rt is not None: + keep = enabled_buckets(rt) + skipped = sorted(key for bucket, key, _l, _f in TABLES if bucket not in keep) + for bucket, _key, _l, _f in TABLES: + if bucket not in keep: + built.pop(bucket, None) + else: + skipped = [] + written = apply_plan(rt, built, username=username, today=today) + return {"tables": written, + # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. + **({"skipped": skipped} if skipped else {}), + **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} diff --git a/api/routes_admin.py b/api/routes_admin.py index f397997c73a726cbec52dd3841e8ba1bf6c3941b..b588db85769adb835b18d15c99734329b084a1f4 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -1,1012 +1,839 @@ -"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). - -These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` -modals, over the SAME `core/users` account store — so both front-ends administer one set of -accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. - -ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit -dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record -is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. - -⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` -has no caller anywhere in the shipped app — module grants are only settable by editing the store -JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both -asserted in `verify_api.py` section A: - - * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin - clearing every checkbox to lock an account down would grant it everything. - * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to - `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose - whole point is strict isolation. - -Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in -`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the -WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both -read semantics are untouched — the seam is write-strict / read-unchanged. - -⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` -one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no -accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". -""" -import os -import re - -from fastapi import APIRouter, Body, Depends, Response - -import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag -import core.registry as registry -import core.store as store - -from deps import Session, err, perms, require_session, users -from routes_auth import _public_user - -router = APIRouter(prefix="/api/v1") - -#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under -#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no -#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. -_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") -#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog -#: enforces nothing; this does, and the two are allowed to differ because only one of them is -#: reachable from the internet. -_MIN_PW = 8 -_ROLES = ("user", "admin") - - -def admin_gate(session: Session = Depends(require_session)) -> Session: - """401 without a session, 403 without the admin role. Role, not a module grant — mirroring - `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - return session - - -def _require_store(): - """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" - try: - ok = store.available() - except Exception: - ok = False - if not ok: - raise err(503, "store_unavailable", - "the tenant store is unavailable — accounts cannot be read or changed") - - -def _registry(): - _require_store() - try: - reg = users.registry() or {} - except Exception: - raise err(503, "store_unavailable", "the tenant store is unavailable") - return reg - - -def _view(uname, rec): - """What an administrator may see about an account. NEVER `salt` or `hash` — this function is - the only projection these routes use, so there is one place that can leak and it does not.""" - return {"username": uname, - "name": rec.get("name") or uname, - "role": rec.get("role", "user"), - "bus": rec.get("bus", "all"), - "bu_labels": perms.allowed_bu_labels(rec), - "modules": rec.get("modules", "all"), - "agent": rec.get("agent") or None, - "email": rec.get("email") or None, - "tenant": _tenant_of(rec), - "active": bool(rec.get("active", True)), - # The session-revocation handle. An admin needs it: it is the only visible evidence - # that "sign them out everywhere" actually happened. - "epoch": int(rec.get("epoch") or 0), - # S3 ask 4 — one sentence for the roster's Access column, so the list does not need - # a /perms round trip per row to fill one cell. Computed from the same record the - # editor will open, so the two cannot disagree. - "accessSummary": _access_summary(rec), - "perms_v": int(rec.get("perms_v") or 0)} - - -def _access_summary(rec): - """One sentence describing what this account may reach. Deliberately says LESS than the - editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters - would be wrong the moment a filter got interesting.""" - import core.perm_scope as perm_scope - - if perms.is_admin(rec): - # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if - # they applied is the one misreading of that clause that could actually hurt. - return "Everything (admin)" - if not perm_scope.is_migrated(rec): - mods = perms.allowed_modules(rec) - return "All modules (legacy rules)" if mods is None else \ - f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" - entries = (rec.get("perms") or {}) - opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] - restricted = sum(1 for k in opened - if (entries[k].get("filter") or entries[k].get("hiddenFields"))) - if not opened: - return "No modules" - base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" - return f"{base}, {restricted} restricted" if restricted else base - - -# ── request validation: every rule below is fail-closed ────────────────────────────────────────── -def _clean_username(v): - uname = str(v or "").strip().lower() - if not _UNAME_RE.match(uname): - raise err(400, "bad_username", - "a username is 2-32 characters: lowercase letters, digits, dot, dash or " - "underscore, starting with a letter or digit") - return uname - - -def _clean_password(v): - pw = str(v or "") - if len(pw) < _MIN_PW: - raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") - return pw - - -def _clean_role(v): - role = str(v or "").strip().lower() - if role not in _ROLES: - raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") - return role - - -def _clean_bus(v): - """'all', or a non-empty list of KNOWN business-unit ids. - - Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no - recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_bus", - "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " - "would read as unrestricted)") - out = [] - for b in v: - try: - b = int(b) - except (TypeError, ValueError): - raise err(400, "bad_bus", "a business-unit id must be a number") - if b not in users.BU_LABELS: - raise err(400, "bad_bus", - f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " - "unrecognised id would widen access to every BU") - out.append(b) - return sorted(set(out)) - - -def _clean_modules(v): - """'all', or a non-empty list of KNOWN module keys. - - `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is - refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would - silently lock the account out of the page they meant to grant. - """ - if isinstance(v, str): - if v.strip().lower() == "all": - return "all" - raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") - if not isinstance(v, (list, tuple)) or not v: - raise err(400, "bad_modules", - "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " - "reads as UNRESTRICTED, which is the opposite of locking an account down") - known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) - out, bad = [], [] - for k in v: - k = str(k or "").strip() - (out if k in known else bad).append(k) - if bad: - raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") - return sorted(set(out)) - - -def _tenant_of(rec): - """The account's company, with the pre-wave default. ONE spelling of the default, used by - every admin route — two spellings is how a tenant wall grows a gap.""" - return str((rec or {}).get("tenant") or "royal-imports").strip().lower() - - -# ── the routes ─────────────────────────────────────────────────────────────────────────────────── -@router.get("/admin/users") -def list_users(session: Session = Depends(admin_gate)): - """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global - control-plane bucket, so without this filter a Nurilab admin would read Royal's whole - roster — names, emails, agents — from one URL.""" - reg = _registry() - mine = _tenant_of(session.user) - return {"users": [_view(u, reg[u]) for u in sorted(reg) - if _tenant_of(reg[u]) == mine]} - - -@router.post("/admin/users", status_code=201) -def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): - """Create an account. **409 if the username already exists — `PATCH` is the update path.** - - ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh - record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and - RESURRECT every cookie minted before its last password change. `core/users.py` now - preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / - update a user" calls the same function), but this route still refuses: an upsert that silently - replaces an account's password, role and BU access because someone reused a username is not an - update anybody asked for. - """ - body = body or {} - _require_store() - uname = _clean_username(body.get("username")) - pw = _clean_password(body.get("password")) - role = _clean_role(body.get("role") or "user") - bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") - modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") - name = str(body.get("name") or "").strip() or uname - agent = str(body.get("agent") or "").strip() or None - email = str(body.get("email") or "").strip() or None - - if uname in _registry(): - raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") - # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — - # the tenant is stamped from the session, not taken from the body. - users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, - agent=agent, email=email, tenant=_tenant_of(session.user)) - reg = _registry() - if uname not in reg: - # The store accepted the write and does not have it. Reporting 201 here would be the - # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. - raise err(503, "store_unavailable", "the account was not saved — try again") - return {"user": _view(uname, reg[uname])} - - -@router.patch("/admin/users/{username}") -def update_user(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Change access on an existing account. Only the keys PRESENT in the body change. - - ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every - request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very - next call — bumping the epoch would only add a forced re-login on top. `active: false` is the - exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its - session rather than keep working until the cookie expires. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if not body: - raise err(400, "empty_patch", "no fields to update") - - unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) - if unknown: - # A silently-ignored field is how a UI ends up believing it saved something it did not. - raise err(400, "unknown_fields", f"cannot update: {unknown}") - - is_self = uname == session.uname - role = _clean_role(body["role"]) if "role" in body else None - active = bool(body["active"]) if "active" in body else None - # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is - # not the thing that keeps the product reachable — it just stops an administrator removing - # their own access with a toggle and having to go find the master password. - if is_self and role is not None and role != "admin": - raise err(400, "self_demote", - "you cannot remove your own administrator role — ask another admin") - if is_self and active is False: - raise err(400, "self_deactivate", "you cannot deactivate your own account") - - kw = {} - if role is not None: - kw["role"] = role - if "name" in body: - kw["name"] = str(body["name"] or "").strip() or uname - if "bus" in body: - kw["bus"] = _clean_bus(body["bus"]) - if "modules" in body: - kw["modules"] = _clean_modules(body["modules"]) - if "agent" in body: - kw["agent"] = str(body["agent"] or "").strip() # '' clears the link - if "email" in body: - kw["email"] = str(body["email"] or "").strip() - if kw: - users.set_access(uname, **kw) - if active is not None and active != bool(reg[uname].get("active", True)): - users.set_active(uname, active) # bumps the epoch: kills live sessions - - fresh = _registry() - return {"user": _view(uname, fresh[uname])} - - -@router.post("/admin/users/{username}/password", status_code=204) -def set_password(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump - happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two - can never disagree. - - 204 with no body: there is nothing to say that the caller did not already know, and echoing the - account back would invite a client to diff a response for a password change. - """ - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - pw = _clean_password((body or {}).get("password")) - before = int(reg[uname].get("epoch") or 0) - users.set_password(uname, pw) - after = int((_registry().get(uname) or {}).get("epoch") or 0) - if after <= before: - # The whole value of this route is the revocation. If the epoch did not move, the sessions - # were not revoked, and answering 204 would say they were. - raise err(503, "store_unavailable", - "the password change did not persist — outstanding sessions were NOT revoked") - return Response(status_code=204) - - -# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── -#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema -#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list -#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` -#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), -#: so a migrated account would be locked out of Product with no control anywhere to grant it: -#: fail-closed, but unadministrable. The editor grows its section with zero client edits -#: (S3 built it off this route's `modules` + `fields_by_module`). -#: -#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be -#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple -#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid -#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` -#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, -#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` -#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s -#: `section_tenant_derived` NC does exactly that and reds. -#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from -#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, -#: non-group_only registry row that declares a `topic`. What stays hand-written is -#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` -#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this -#: editor at all. The intersection is the honest answer to "what can this module build pickers for". -#: -#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads -#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the -#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise -#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red -#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives -#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. -_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) - - -def _governable_catalogue(): - """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. - - ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is - tenant-blind by design. Two filters in two files answering one question is how the perms route - and `routes_nav` came apart in the first place. - """ - import core.registry as registry - return [{"key": v["key"], "label": v["label"]} - for v in registry.governable_modules().values() - if v.get("key") in _FIELD_PROVIDER_KEYS] - - -_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) - - -def _perm_modules(session): - """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. - - Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned - module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and - the tenant's own `ut_*` databases read through the DEFINITIONS projection. - - ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE - resolver for "which databases may this account see", it is `may_open`-filtered, and it reads - the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. - The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver - instead of assuming that is what keeps this from becoming the SECOND idea of who may see a - table (`user_tables.may_open`'s own docstring is the record of the first time that happened). - """ - import core.perms as _perms - import core.user_tables as user_tables - try: - ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, - st=session.runtime) - except Exception: - # A store that cannot list the tenant's databases must not take the whole editor down — - # the topic half is still administrable, and an empty `ut_*` half is visibly empty. - ut = () - return _perms.tenant_governable_modules(session.runtime, _governable_catalogue(), - ut_entries=ut) - - -def _enforced_keys(session): - """The module keys a permission block may actually be STORED against for this tenant.""" - import core.perms as _perms - return _perms.enforced_module_keys(_perm_modules(session)) - - -def _module_fields(key): - """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, - so an admin filters on exactly what the user sees. - - ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing - what to hide, so hiding the choices would make a field unhideable the moment it was hidden - once for somebody. The route is admin-gated; the projection a MEMBER receives is the one - `perm_scope.visible_fields` narrows. - - ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the - editor the customer schema for it would let an admin build a wall out of columns that - module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, - a filter naming a field the product rows lack would DENY every row via `permits()`. - The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader - is served fewer columns, but the ADMIN is choosing what may ever be hidden. - - ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read - `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that - was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being - a two-element literal, a `ut_*` database or a third topic would have been handed customer - columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the - stored wall would name columns that module does not have — which `permits()` resolves by - DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: - a topic without a schema arm is a missing arm, never a customer grid in disguise. - """ - import aios_grid - providers = {"customer_data": lambda: aios_grid.FIELDS, - "product_data": aios_grid.product_fields} - provider = providers.get(str(key or "")) - if provider is None: - # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative - # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is truthy. - raise err(400, "ungoverned_module", - f"{key!r} has no permission-editor field schema — the editor governs " - f"{sorted(providers)}") - src = provider() - known = _server_side_vocabularies() - out = [] - for f in src: - if not isinstance(f, dict) or not f.get("key"): - continue - opts = f.get("options") - if not isinstance(opts, list) or not opts: - # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See - # `_server_side_vocabularies`. - opts = known.get(f["key"]) - out.append({"key": f["key"], "label": f.get("label") or f["key"], - "type": f.get("type") or "text", - **({"options": list(opts)} if isinstance(opts, list) and opts else {}), - **({"pinned": True} if f.get("pinned") else {})}) - return out - - -def _server_side_vocabularies(): - """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, - but which the grid contract does not declare. - - ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid - resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the - field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so - every column that leans on the second branch renders `Select…` with nothing in it — the - identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix - belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied - list as authoritative *including when it is empty* ("this column has no values" is an answer), - which is correct and must not be weakened. So the caller stops handing it nothing. - - ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, - the same constant `_bucket()` mints from — a copy here would be a second vocabulary that - drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy - for the same reason `aios_grid`'s is: this route should not pull the analytics stack at - module load. - - ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than - let them look handled: - · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no - closed list to declare, and discovering it would mean a pool read on an admin route. - · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E - answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which - is the right home (the grid picks a declared list up for free) and is why this function - never needed an arm for it. Recorded because the ask, not a guess, is what settled it. - """ - try: - import modules.inventory as inventory - labels = list(inventory.COVERAGE_LABELS or ()) - except Exception: - # A missing analytics import must not take the permission editor down; the dropdown - # degrades to today's empty one rather than the page to a 500. - return {} - return {"stock_bucket": labels} if labels else {} - - -def _clean_perms(v, enforced_keys=None, listed_keys=None): - """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. - - ⭐ WAVE 33 (W33-T33) — `enforced_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It - used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 - flagged: an admin of a tenant provisioned for neither grid module could still store a wall - for both. It now comes from `_enforced_keys(session)`, i.e. from the tenant's own catalogue. - The default is the catalogue itself so a test or a future caller cannot silently widen it. - - ⛔ AND "LISTED" IS NOT "ENFORCED". `listed_keys` is every database the editor SHOWS, which - includes this tenant's `ut_*` ones. `perm_scope` is never consulted on those (visibility - there is `user_tables.may_open` — creator, admin, or a `core.shares` grant), so a wall stored - against one would be INERT: the editor would say DENY and the table routes would keep - serving. It is refused with a message that says WHICH of the two it is, because - "unknown module key" for a database the admin can see on screen is the wrong sentence. - - ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately - drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION - filter is the opposite situation — an admin types "agent is Tara", a leaf is silently - dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole - book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: - the filter is re-validated with `clean_filter_tree` and the result must come back with the - SAME leaf count it went in with. - - Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` - validates the NODE LIST alone and never sees the root conjunction, so the two are checked - separately and a bare node list is refused rather than silently read as `and`). - """ - import aios_grid - - if v is None: - return None - if not isinstance(v, dict): - raise err(400, "bad_perms", "perms must be an object keyed by module") - governed = set(enforced_keys) if enforced_keys is not None else set(_PERM_MODULES) - shown = set(listed_keys) if listed_keys is not None else governed - inert = sorted(set(v) & (shown - governed)) - if inert: - raise err(400, "unenforced_module", - f"{inert}: this tenant can SEE these databases, but a permission wall stored " - f"against them would never be applied — `routes_tables` gates a `ut_*` " - f"database on `user_tables.may_open` (creator, admin, or a share), not on " - f"`perm_scope`. Saved silently, it would read as a restriction that is not " - f"there. Share the database instead, or arm `perm_scope` over `ut_*` first.") - unknown = sorted(set(v) - shown) - if unknown: - raise err(400, "bad_perms", - f"unknown module keys: {unknown} — this tenant's permission editor governs " - f"{sorted(governed)}") - out = {} - for key, raw in v.items(): - if not isinstance(raw, dict): - raise err(400, "bad_perms", f"{key}: each entry must be an object") - valid_keys = {f["key"] for f in _module_fields(key)} - hidden = raw.get("hiddenFields") or [] - if not isinstance(hidden, list): - raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") - bad = sorted({str(h) for h in hidden} - valid_keys) - if bad: - # A hiddenFields entry naming nothing hides nothing — and reads as a restriction - # that is not there. - raise err(400, "bad_perms", - f"{key}: hiddenFields names unknown fields {bad}") - tree = raw.get("filter") - clean_tree = None - if tree not in (None, {}, []): - if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): - raise err(400, "bad_filter", - f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " - f"the root conjunction, and an 'or' wall read as 'and' restricts " - f"nothing it was meant to") - conj = tree.get("conj", "and") - if conj not in ("and", "or"): - raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") - nodes = tree["nodes"] - cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) - if _leaf_count(cleaned) != _leaf_count(nodes): - raise err(400, "bad_filter", - f"{key}: the filter contains conditions this module cannot evaluate " - f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " - f"are per-user and cannot be a permanent rule). Refused rather than " - f"saved with the bad conditions silently removed, which would store a " - f"weaker wall than the one on screen.") - clean_tree = {"conj": conj, "nodes": cleaned} - out[key] = {"access": bool(raw.get("access", True)), - "filter": clean_tree, - "hiddenFields": sorted({str(h) for h in hidden})} - _refuse_unshapeable_bu(out) - return out - - -def _refuse_unshapeable_bu(perms_out): - """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. - - Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes - that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which - is what `perm_migrate` emits and what the condition builder produces for "is any of". An - admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a - `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through - `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries - Fisch+Royal numbers. The row list looks right. The revenue is another BU's. - - That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just - shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with - nothing anomalous about them. - - So it is refused at the WRITE, where a person is present to fix it. The discriminator is - narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like - `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and - consolidated values are the correct answer for it, so it is not caught here (`derive` also - refuses it) and must not be. - """ - import core.perm_scope as perm_scope - - for key, e in perms_out.items(): - tree = e.get("filter") - if not tree or not _mentions_dba(tree.get("nodes")): - continue - probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, - "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} - team_id, _ = perm_scope.derive_pool_scope(probe, key) - if team_id is None and _confines_to_one_bu(tree): - raise err(400, "bu_condition_shape", - f"{key}: this filter restricts the business unit in a way the data layer " - f"cannot push into the query, so the rows would be correct while the " - f"revenue figures on them stayed consolidated across both units. Express " - f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " - f"or an 'any of' over the brands — and keep any other rules alongside it.") - - -def _guard_bu_widening(rec, cleaned, confirmed): - """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. - - `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the - first save the filter IS the whole wall and the admin owns it. But the first save also - CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted - rather than the one the server returned will, on its next save, PUT a payload with no BU - condition — and a whole-record replace deletes it. Measured, not theorised: the same client - payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both - business units. - - So a save that REMOVES the brand confinement is refused unless it says it means to. Widening - BU access is a legitimate thing for an administrator to do — it is just never a thing to do - by accident, and the difference between the two is one explicit flag. - - ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU - restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the - admin edits it). The right shape is "you may widen, say so". - """ - import core.perm_scope as perm_scope - - if confirmed: - return - stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} - for key, new_entry in cleaned.items(): - if not new_entry.get("access", True): - # Revoking the module entirely is the OPPOSITE of widening, and the filter on a - # denied entry governs nothing. Guarding it would make "lock this account out" - # require a confirm_widen flag, which reads as nonsense to whoever hits it. - continue - old = stored.get(key) - if not isinstance(old, dict): - continue # nothing stored yet: the fold already handled it - old_tree, new_tree = old.get("filter"), new_entry.get("filter") - if not old_tree or not _confines_to_one_bu(old_tree): - continue # was not confined -> this save cannot widen it - if new_tree and _confines_to_one_bu(new_tree): - continue # still confined -> not a widening - raise err(400, "bu_widening", - f"{key}: this would REMOVE the business-unit restriction and give the account " - f"both units. If that is intended, resend with confirm_widen: true. If it is " - f"not, reload the account's permissions first — saving a filter loaded before " - f"the last change drops the conditions it did not know about.") - # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees - # the connection to derive_pool_scope, which is what the removed condition was feeding. - _ = perm_scope - - -def _mentions_dba(nodes): - for n in nodes or (): - if not isinstance(n, dict): - continue - if isinstance(n.get("children"), list): - if _mentions_dba(n["children"]): - return True - elif n.get("colId") == "dba": - return True - return False - - -def _confines_to_one_bu(tree): - """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? - - ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard - refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a - high-balance Royal customer satisfies it — so consolidated values are the correct answer and - refusing it would be a false positive. My first attempt evaluated synthetic rows carrying - only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact - wall into a "confinement" it is not. - - So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the - reader satisfies everything else; can they still see both brands?" If yes, the wall does not - confine and the pool is correctly consolidated. If no, the brand structure is doing the - confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. - """ - def admits(node, brand): - if isinstance(node.get("children"), list): - kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] - if not kids: - return True - return any(kids) if node.get("conj") == "or" else all(kids) - if node.get("colId") != "dba": - return True # every other condition: assume satisfied - want = str(node.get("value") or "").strip().lower() - op = node.get("op") - if op == "eq": - return brand.lower() == want - if op == "neq": - return brand.lower() != want - if op == "contains": - return want in brand.lower() - if op == "doesNotContain": - return want not in brand.lower() - if op == "isEmpty": - return False # a probe brand is never blank - if op == "isNotEmpty": - return True - return True # an op that cannot judge a brand does not confine - - nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") - root = {"conj": conj, "children": nodes} - admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} - if not admitted: - return False # admits nothing at all — a different problem - # 'Both' belongs to either unit, so it cannot by itself widen the answer. - return not ({"Fisch", "Royal"} <= admitted) - - -def _leaf_count(nodes): - n = 0 - for node in nodes or (): - if isinstance(node, dict) and isinstance(node.get("children"), list): - n += _leaf_count(node["children"]) - else: - n += 1 - return n - - -def _fold_legacy_scope(rec, cleaned): - """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. - - Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` - fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write - whose filter carries no BU condition silently promotes a Royal-only account to BOTH business - units: the wall that used to come from `bus` is gone and nothing replaced it. - - Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check - passed, because the model was doing exactly what it was told. The account simply received - the other BU's customers on the wire. - - So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as - `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a - perms block. From then on the admin sees those leaves in the editor (GET returns the folded - filter) and can change them like any other condition — which is R1's whole point, and why - this is a one-time fold rather than a permanent floor AND-ed on every write. - - Idempotent by construction: it runs only while the record is un-migrated, and the write that - calls it is the write that migrates it. - - ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE - NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` - has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent - filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into - the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for - a `bus:[5]` account the moment an admin pressed Save. - - Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against - `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a - cross-topic leaf could come through, which is why it is the only place that needs the rule. - - ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the - BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU - is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). - The pushdown falls back to the record's own `bus` for exactly the modules whose field - vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves - ship together or the second one is a leak. - """ - import core.perm_migrate as perm_migrate - import core.perm_scope as perm_scope - - if perm_scope.is_migrated(rec): - return cleaned - legacy = perm_migrate.filter_for(rec) - if not legacy: - return cleaned # nothing to preserve: the account was unrestricted - out = {} - for key, e in cleaned.items(): - legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) - tree = e.get("filter") - if not legacy_here: - # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than - # storing a wall it will read as "deny every row". - out[key] = e - continue - if not tree or not tree.get("nodes"): - folded = legacy_here - else: - # AND the two roots together. The submitted tree keeps its own conjunction by being - # nested as a GROUP — flattening an `or` tree into an `and` root would turn the - # admin's "A or B" into "A and B", which is a different and much narrower wall. - folded = {"conj": "and", - "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), - "children": list(tree["nodes"])}]} - out[key] = dict(e, filter=folded) - return out - - -def _prune_to_module(tree, valid_keys): - """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. - - Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies - every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the - reason its own docstring gives (a silently-weakened wall reads as "Saved"). - - ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than - the admin's intent and half of an `and` group is a wider one; neither is the thing that was - written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps - the surviving conditions ones somebody actually declared. - """ - if not isinstance(tree, dict): - return None - - def keep(node): - if not isinstance(node, dict): - return None - kids = node.get("children") - if isinstance(kids, list): - surviving = [keep(k) for k in kids] - if not kids or any(s is None for s in surviving): - return None - return dict(node, children=surviving) - return node if node.get("colId") in valid_keys else None - - nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] - return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None - - -@router.get("/admin/users/{username}/perms") -def get_perms(username: str, session: Session = Depends(admin_gate)): - """This account's permission block plus the field schema the editor needs to render it.""" - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - import core.perm_scope as perm_scope - - rec = reg[uname] - stored = rec.get("perms") or {} - # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. - # The PUT is a whole-record replace over this same module list, so a key declared here and - # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The - # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the - # access its legacy grant currently gives rather than a guess in either direction. - # - # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything - # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a - # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product - # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` - # has applied since wave 18) and merges the tenant's own `ut_*` databases. - modules = _perm_modules(session) - enforced = [m["key"] for m in modules if m.get("enforced")] - perms_out = {} - for k in enforced: - e = stored.get(k) - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_access(rec, k)), "filter": None, "hiddenFields": []} - return {"username": uname, - "perms": perms_out, - # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a - # migrated record and LEGACY on an un-migrated one: opposite meanings for identical - # JSON, so the client cannot render honestly without knowing which world it is in. - "perms_v": int(rec.get("perms_v") or 0), - # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can - # say "Everything (admin)" instead of rendering stored rules that do not apply. - "is_admin": perms.is_admin(rec), - # ⭐ `enforced` rides every row. A `ut_*` database is LISTED because the owner asked to - # see every database, and declared UNENFORCED because `perm_scope` is not consulted on - # that path — so the editor can say which it is instead of painting a dead panel - # (wave 26 item 24: an empty answer that reads as an answer). The LABEL comes from the - # registry / the table's own definition; `_MODULE_LABELS` was a hardcoded copy of two - # registry labels and is deleted with this change. - "modules": modules, - # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav - # schema envelope; the editor needs the field list and nothing else, and one shape - # beats two readings of a sentence. - # ENFORCED MODULES ONLY, and that is the honest shape: a picker for a database whose - # wall is never applied is a control that lies. An unenforced row carries no entry - # here at all, so a client cannot mistake `[]` for "this database has no fields". - "fields_by_module": {k: _module_fields(k) for k in enforced}} - - -@router.put("/admin/users/{username}/perms") -def put_perms(username: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this account's permission block wholesale. - - WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot - express a deletion without a second vocabulary for it. - - ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: - `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the - target's LIVE session on its very next call. Bumping would only add a forced re-login on top - of a change that has already taken effect. - """ - body = body if isinstance(body, dict) else {} - reg = _registry() - uname = str(username or "").strip().lower() - if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): - # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — - # 404, never 403, because "that account exists in another company" is itself a leak. - raise err(404, "no_such_user", f"no account named {uname!r}") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - - # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses - # perm_scope entirely, so this cannot brick them today — but it stops an administrator - # writing a wall for themselves that would bite the moment their role changed. - if uname == session.uname: - raise err(400, "self_perms", - "you cannot set permissions on your own account — ask another admin") - - _mods = _perm_modules(session) - cleaned = _clean_perms(body.get("perms"), - enforced_keys={m["key"] for m in _mods if m.get("enforced")}, - listed_keys={m["key"] for m in _mods}) or {} - cleaned = _fold_legacy_scope(reg[uname], cleaned) - _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) - users.set_access(uname, perms=cleaned) - fresh = _registry() - rec = fresh.get(uname) or {} - if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: - # The store took the write and did not record it. A 200 here would tell an administrator - # the wall is up when it is not — the one direction this must never fail in. - raise err(503, "store_unavailable", - "the permissions were not saved — the account is UNCHANGED") - return {"username": uname, "perms": rec.get("perms") or {}, - "perms_v": int(rec.get("perms_v") or 0)} - - -@router.get("/settings") -def settings(session: Session = Depends(require_session)): - """The session's OWN settings — any authenticated user, not admin-only. - - `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a - client may know about itself (never a hash, never the epoch). `scope` restates it as the two - things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others - would answer a question about our customer list. - """ - modules = perms.allowed_modules(session.user) - return { - # The build this Space is running. Behind the session on purpose — see `main.py::health`, - # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a - # Space secret at every deploy; absent means "somebody started this container by hand". - "version": os.environ.get("AIOS_VERSION") or "unknown", - "user": _public_user(session.user), - "scope": { - "bus": perms.allowed_bu_labels(session.user), - "team_id": perms.scope_team_id(session.user), - "agent": perms.scope_agent(session.user), - "modules": sorted(modules) if modules is not None else "all", - # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey - # pickers rather than to enforce — hidden fields are stripped from every data wire - # regardless, so a client that ignores this is narrowed anyway, never widened. - "perms": session.user.get("perms") or {}, - }, - "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", - session.tenant)}, - # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client - # does not paint a "Manage users" button that 403s. - "admin": perms.is_admin(session.user), - # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail - # paints its entry iff this is true, and `routes_platform_admin` refuses every request - # that is not, so a client that ignored this would gain exactly nothing. Derived from the - # ONE predicate rather than restated: a second copy of a two-condition wall is a second - # place for one of the conditions to go missing. - "platformAdmin": platform_admin.is_platform_admin(session.user), - } +"""routes_admin.py — Y4: user administration + the session's own settings (W2-3). + +These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog` +modals, over the SAME `core/users` account store — so both front-ends administer one set of +accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model. + +ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit +dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record +is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route. + +⛔ THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)` +has no caller anywhere in the shipped app — module grants are only settable by editing the store +JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both +asserted in `verify_api.py` section A: + + * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin + clearing every checkbox to lock an account down would grant it everything. + * `bus: []` falls through `allowed_bus_labels`'s "no recognisable label" branch to + `['All','Fisch','Royal']` — so one typo'd BU id is FULL cross-BU access, in the model whose + whole point is strict isolation. + +Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in +`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the +WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both +read semantics are untouched — the seam is write-strict / read-unchanged. + +⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}` +one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no +accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong". +""" +import os +import re + +from fastapi import APIRouter, Body, Depends, Response + +import core.platform_admin as platform_admin # wave 19 R3 — the /settings chrome flag +import core.registry as registry +import core.store as store + +from deps import Session, err, perms, require_session, users +from routes_auth import _public_user + +router = APIRouter(prefix="/api/v1") + +#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under +#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no +#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. +_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") +#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog +#: enforces nothing; this does, and the two are allowed to differ because only one of them is +#: reachable from the internet. +_MIN_PW = 8 +_ROLES = ("user", "admin") + + +def admin_gate(session: Session = Depends(require_session)) -> Session: + """401 without a session, 403 without the admin role. Role, not a module grant — mirroring + `app.py`'s `users_dialog`, whose only wall is `is_admin()`.""" + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + return session + + +def _require_store(): + """A 503 the moment the store cannot serve, so no route below can report emptiness as truth.""" + try: + ok = store.available() + except Exception: + ok = False + if not ok: + raise err(503, "store_unavailable", + "the tenant store is unavailable — accounts cannot be read or changed") + + +def _registry(): + _require_store() + try: + reg = users.registry() or {} + except Exception: + raise err(503, "store_unavailable", "the tenant store is unavailable") + return reg + + +def _view(uname, rec): + """What an administrator may see about an account. NEVER `salt` or `hash` — this function is + the only projection these routes use, so there is one place that can leak and it does not.""" + return {"username": uname, + "name": rec.get("name") or uname, + "role": rec.get("role", "user"), + "bus": rec.get("bus", "all"), + "bu_labels": perms.allowed_bu_labels(rec), + "modules": rec.get("modules", "all"), + "agent": rec.get("agent") or None, + "email": rec.get("email") or None, + "tenant": _tenant_of(rec), + "active": bool(rec.get("active", True)), + # The session-revocation handle. An admin needs it: it is the only visible evidence + # that "sign them out everywhere" actually happened. + "epoch": int(rec.get("epoch") or 0), + # S3 ask 4 — one sentence for the roster's Access column, so the list does not need + # a /perms round trip per row to fill one cell. Computed from the same record the + # editor will open, so the two cannot disagree. + "accessSummary": _access_summary(rec), + "perms_v": int(rec.get("perms_v") or 0)} + + +def _access_summary(rec): + """One sentence describing what this account may reach. Deliberately says LESS than the + editor: it is a signpost, not a rule listing, and a summary that tried to spell out filters + would be wrong the moment a filter got interesting.""" + import core.perm_scope as perm_scope + + if perms.is_admin(rec): + # amendment 4: an admin bypasses `perms` entirely, so rendering their stored rules as if + # they applied is the one misreading of that clause that could actually hurt. + return "Everything (admin)" + if not perm_scope.is_migrated(rec): + mods = perms.allowed_modules(rec) + return "All modules (legacy rules)" if mods is None else \ + f"{len(mods)} module{'' if len(mods) == 1 else 's'} (legacy rules)" + entries = (rec.get("perms") or {}) + opened = [k for k, e in entries.items() if isinstance(e, dict) and e.get("access", True)] + restricted = sum(1 for k in opened + if (entries[k].get("filter") or entries[k].get("hiddenFields"))) + if not opened: + return "No modules" + base = f"{len(opened)} module{'' if len(opened) == 1 else 's'}" + return f"{base}, {restricted} restricted" if restricted else base + + +# ── request validation: every rule below is fail-closed ────────────────────────────────────────── +def _clean_username(v): + uname = str(v or "").strip().lower() + if not _UNAME_RE.match(uname): + raise err(400, "bad_username", + "a username is 2-32 characters: lowercase letters, digits, dot, dash or " + "underscore, starting with a letter or digit") + return uname + + +def _clean_password(v): + pw = str(v or "") + if len(pw) < _MIN_PW: + raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters") + return pw + + +def _clean_role(v): + role = str(v or "").strip().lower() + if role not in _ROLES: + raise err(400, "bad_role", f"role must be one of {list(_ROLES)}") + return role + + +def _clean_bus(v): + """'all', or a non-empty list of KNOWN business-unit ids. + + Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no + recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_bus", + "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list " + "would read as unrestricted)") + out = [] + for b in v: + try: + b = int(b) + except (TypeError, ValueError): + raise err(400, "bad_bus", "a business-unit id must be a number") + if b not in users.BU_LABELS: + raise err(400, "bad_bus", + f"{b} is not a known business unit {sorted(users.BU_LABELS)} — an " + "unrecognised id would widen access to every BU") + out.append(b) + return sorted(set(out)) + + +def _clean_modules(v): + """'all', or a non-empty list of KNOWN module keys. + + `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is + refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would + silently lock the account out of the page they meant to grant. + """ + if isinstance(v, str): + if v.strip().lower() == "all": + return "all" + raise err(400, "bad_modules", "modules must be 'all' or a list of module keys") + if not isinstance(v, (list, tuple)) or not v: + raise err(400, "bad_modules", + "modules must be 'all' or a NON-EMPTY list of module keys — an empty list " + "reads as UNRESTRICTED, which is the opposite of locking an account down") + known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS) + out, bad = [], [] + for k in v: + k = str(k or "").strip() + (out if k in known else bad).append(k) + if bad: + raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}") + return sorted(set(out)) + + +def _tenant_of(rec): + """The account's company, with the pre-wave default. ONE spelling of the default, used by + every admin route — two spellings is how a tenant wall grows a gap.""" + return str((rec or {}).get("tenant") or "royal-imports").strip().lower() + + +# ── the routes ─────────────────────────────────────────────────────────────────────────────────── +@router.get("/admin/users") +def list_users(session: Session = Depends(admin_gate)): + """Wave 18 (C1-TENANT): scoped to the CALLER'S tenant. The user registry is a global + control-plane bucket, so without this filter a Nurilab admin would read Royal's whole + roster — names, emails, agents — from one URL.""" + reg = _registry() + mine = _tenant_of(session.user) + return {"users": [_view(u, reg[u]) for u in sorted(reg) + if _tenant_of(reg[u]) == mine]} + + +@router.post("/admin/users", status_code=201) +def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)): + """Create an account. **409 if the username already exists — `PATCH` is the update path.** + + ⛔ WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh + record has NO `epoch` — so overwriting an existing account used to drop its epoch back to 0 and + RESURRECT every cookie minted before its last password change. `core/users.py` now + preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add / + update a user" calls the same function), but this route still refuses: an upsert that silently + replaces an account's password, role and BU access because someone reused a username is not an + update anybody asked for. + """ + body = body or {} + _require_store() + uname = _clean_username(body.get("username")) + pw = _clean_password(body.get("password")) + role = _clean_role(body.get("role") or "user") + bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all") + modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all") + name = str(body.get("name") or "").strip() or uname + agent = str(body.get("agent") or "").strip() or None + email = str(body.get("email") or "").strip() or None + + if uname in _registry(): + raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — + # the tenant is stamped from the session, not taken from the body. + users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, + agent=agent, email=email, tenant=_tenant_of(session.user)) + reg = _registry() + if uname not in reg: + # The store accepted the write and does not have it. Reporting 201 here would be the + # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent. + raise err(503, "store_unavailable", "the account was not saved — try again") + return {"user": _view(uname, reg[uname])} + + +@router.patch("/admin/users/{username}") +def update_user(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Change access on an existing account. Only the keys PRESENT in the body change. + + ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every + request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very + next call — bumping the epoch would only add a forced re-login on top. `active: false` is the + exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its + session rather than keep working until the cookie expires. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if not body: + raise err(400, "empty_patch", "no fields to update") + + unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"}) + if unknown: + # A silently-ignored field is how a UI ends up believing it saved something it did not. + raise err(400, "unknown_fields", f"cannot update: {unknown}") + + is_self = uname == session.uname + role = _clean_role(body["role"]) if "role" in body else None + active = bool(body["active"]) if "active" in body else None + # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is + # not the thing that keeps the product reachable — it just stops an administrator removing + # their own access with a toggle and having to go find the master password. + if is_self and role is not None and role != "admin": + raise err(400, "self_demote", + "you cannot remove your own administrator role — ask another admin") + if is_self and active is False: + raise err(400, "self_deactivate", "you cannot deactivate your own account") + + kw = {} + if role is not None: + kw["role"] = role + if "name" in body: + kw["name"] = str(body["name"] or "").strip() or uname + if "bus" in body: + kw["bus"] = _clean_bus(body["bus"]) + if "modules" in body: + kw["modules"] = _clean_modules(body["modules"]) + if "agent" in body: + kw["agent"] = str(body["agent"] or "").strip() # '' clears the link + if "email" in body: + kw["email"] = str(body["email"] or "").strip() + if kw: + users.set_access(uname, **kw) + if active is not None and active != bool(reg[uname].get("active", True)): + users.set_active(uname, active) # bumps the epoch: kills live sessions + + fresh = _registry() + return {"user": _view(uname, fresh[uname])} + + +@router.post("/admin/users/{username}/password", status_code=204) +def set_password(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Rotate a password. **Revokes every outstanding session for that account** — the epoch bump + happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two + can never disagree. + + 204 with no body: there is nothing to say that the caller did not already know, and echoing the + account back would invite a client to diff a response for a password change. + """ + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + pw = _clean_password((body or {}).get("password")) + before = int(reg[uname].get("epoch") or 0) + users.set_password(uname, pw) + after = int((_registry().get(uname) or {}).get("epoch") or 0) + if after <= before: + # The whole value of this route is the revocation. If the epoch did not move, the sessions + # were not revoked, and answering 204 would say they were. + raise err(503, "store_unavailable", + "the password change did not persist — outstanding sessions were NOT revoked") + return Response(status_code=204) + + +# ── C-PERM (wave 15): per-module access + permanent filters + hidden fields ────────────────── +#: The modules the permission editor governs — the GRID surfaces, the ones with a field schema +#: to filter and hide. Amendment 6 shipped the wall on `customer_data` alone and said this list +#: is the one place that changes when C-TOPIC lands. WAVE 16: it landed, so `product_data` +#: joins — and it is not cosmetic. `perms_v: 1` means an UNDECLARED module DENIES (amendment 4), +#: so a migrated account would be locked out of Product with no control anywhere to grant it: +#: fail-closed, but unadministrable. The editor grows its section with zero client edits +#: (S3 built it off this route's `modules` + `fields_by_module`). +_PERM_MODULES = ("customer_data", "product_data") + + +def _module_fields(key): + """The field schema the editor builds its pickers from — the SAME vocabulary the grid uses, + so an admin filters on exactly what the user sees. + + ⛔ Returned WITHOUT any per-user hiding applied, deliberately: this is the admin choosing + what to hide, so hiding the choices would make a field unhideable the moment it was hidden + once for somebody. The route is admin-gated; the projection a MEMBER receives is the one + `perm_scope.visible_fields` narrows. + + ⚠ PER TOPIC (wave 16). `product_data` has its OWN canonical contract, and handing the + editor the customer schema for it would let an admin build a wall out of columns that + module does not have — `_clean_perms` would then 400 on the admin's own choices, or worse, + a filter naming a field the product rows lack would DENY every row via `permits()`. + The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader + is served fewer columns, but the ADMIN is choosing what may ever be hidden. + """ + import aios_grid + src = aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS + out = [] + for f in src: + if not isinstance(f, dict) or not f.get("key"): + continue + out.append({"key": f["key"], "label": f.get("label") or f["key"], + "type": f.get("type") or "text", + **({"options": f["options"]} if isinstance(f.get("options"), list) else {}), + **({"pinned": True} if f.get("pinned") else {})}) + return out + + +def _clean_perms(v): + """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. + + ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately + drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION + filter is the opposite situation — an admin types "agent is Tara", a leaf is silently + dropped, and the stored wall is EMPTY. The admin sees "Saved", the account sees the whole + book, and nothing anywhere says so. So anything that would be dropped is a 400 instead: + the filter is re-validated with `clean_filter_tree` and the result must come back with the + SAME leaf count it went in with. + + Shape (C-PERM amendment 2 — a `FilterTree` is a PAIR, `{conj?, nodes}`; `clean_filter_tree` + validates the NODE LIST alone and never sees the root conjunction, so the two are checked + separately and a bare node list is refused rather than silently read as `and`). + """ + import aios_grid + + if v is None: + return None + if not isinstance(v, dict): + raise err(400, "bad_perms", "perms must be an object keyed by module") + unknown = sorted(set(v) - set(_PERM_MODULES)) + if unknown: + raise err(400, "bad_perms", + f"unknown module keys: {unknown} — the permission editor governs " + f"{list(_PERM_MODULES)}") + out = {} + for key, raw in v.items(): + if not isinstance(raw, dict): + raise err(400, "bad_perms", f"{key}: each entry must be an object") + valid_keys = {f["key"] for f in _module_fields(key)} + hidden = raw.get("hiddenFields") or [] + if not isinstance(hidden, list): + raise err(400, "bad_perms", f"{key}: hiddenFields must be a list") + bad = sorted({str(h) for h in hidden} - valid_keys) + if bad: + # A hiddenFields entry naming nothing hides nothing — and reads as a restriction + # that is not there. + raise err(400, "bad_perms", + f"{key}: hiddenFields names unknown fields {bad}") + tree = raw.get("filter") + clean_tree = None + if tree not in (None, {}, []): + if not isinstance(tree, dict) or not isinstance(tree.get("nodes"), list): + raise err(400, "bad_filter", + f"{key}: filter must be {{conj?, nodes:[…]}} — a bare list would lose " + f"the root conjunction, and an 'or' wall read as 'and' restricts " + f"nothing it was meant to") + conj = tree.get("conj", "and") + if conj not in ("and", "or"): + raise err(400, "bad_filter", f"{key}: conj must be 'and' or 'or'") + nodes = tree["nodes"] + cleaned = aios_grid.clean_filter_tree(nodes, valid_keys, cohort_ids=None) + if _leaf_count(cleaned) != _leaf_count(nodes): + raise err(400, "bad_filter", + f"{key}: the filter contains conditions this module cannot evaluate " + f"(an unknown field, an unknown operator, or a cohort leaf — cohorts " + f"are per-user and cannot be a permanent rule). Refused rather than " + f"saved with the bad conditions silently removed, which would store a " + f"weaker wall than the one on screen.") + clean_tree = {"conj": conj, "nodes": cleaned} + out[key] = {"access": bool(raw.get("access", True)), + "filter": clean_tree, + "hiddenFields": sorted({str(h) for h in hidden})} + _refuse_unshapeable_bu(out) + return out + + +def _refuse_unshapeable_bu(perms_out): + """⛔ A BU CONDITION THE PUSHDOWN CANNOT READ IS A VALUE LEAK WEARING A CORRECT ROW LIST. + + Amendment 3 in one more place. `perm_scope.derive_pool_scope` recognises exactly the shapes + that PIN a business unit: a top-level `dba eq` leaf, or a top-level OR-group of them — which + is what `perm_migrate` emits and what the condition builder produces for "is any of". An + admin composing the same intent a slightly different way (a NESTED group, `dba neq Royal`, a + `dba` leaf one level down) produces a filter that still narrows the ROWS correctly through + `permits()` — and leaves the pool built CONSOLIDATED, so every surviving row carries + Fisch+Royal numbers. The row list looks right. The revenue is another BU's. + + That is precisely the defect amendment 3 exists for, re-entering through the editor R9 just + shipped, and it cannot be caught downstream: by then the numbers are simply wrong, with + nothing anomalous about them. + + So it is refused at the WRITE, where a person is present to fix it. The discriminator is + narrow on purpose — the filter must MENTION `dba` and must fail to pin a team. A wall like + `dba is Fisch OR revenue > 1000` genuinely does not confine anyone to one BU, and + consolidated values are the correct answer for it, so it is not caught here (`derive` also + refuses it) and must not be. + """ + import core.perm_scope as perm_scope + + for key, e in perms_out.items(): + tree = e.get("filter") + if not tree or not _mentions_dba(tree.get("nodes")): + continue + probe = {"role": "user", "perms_v": perm_scope.PERMS_VERSION, + "perms": {key: {"access": True, "filter": tree, "hiddenFields": []}}} + team_id, _ = perm_scope.derive_pool_scope(probe, key) + if team_id is None and _confines_to_one_bu(tree): + raise err(400, "bu_condition_shape", + f"{key}: this filter restricts the business unit in a way the data layer " + f"cannot push into the query, so the rows would be correct while the " + f"revenue figures on them stayed consolidated across both units. Express " + f"the business-unit condition as a TOP-LEVEL condition — 'DBA is Fisch', " + f"or an 'any of' over the brands — and keep any other rules alongside it.") + + +def _guard_bu_widening(rec, cleaned, confirmed): + """⛔ THE TWO-CLICK LEAK: a SECOND save that silently drops the business-unit condition. + + `_fold_legacy_scope` runs only while a record is un-migrated, which is correct — after the + first save the filter IS the whole wall and the admin owns it. But the first save also + CHANGES the tree (it folds the BU condition in), so a client holding the tree it submitted + rather than the one the server returned will, on its next save, PUT a payload with no BU + condition — and a whole-record replace deletes it. Measured, not theorised: the same client + payload saved twice takes the pool scope from `(6, 'Ann')` to `(None, 'Ann')`, which is both + business units. + + So a save that REMOVES the brand confinement is refused unless it says it means to. Widening + BU access is a legitimate thing for an administrator to do — it is just never a thing to do + by accident, and the difference between the two is one explicit flag. + + ⚠ Deliberately NOT solved by always folding the legacy scope: that would make a BU + restriction permanent and unremovable, which contradicts R1 (the filter is the wall, and the + admin edits it). The right shape is "you may widen, say so". + """ + import core.perm_scope as perm_scope + + if confirmed: + return + stored = (rec.get("perms") or {}) if isinstance(rec.get("perms"), dict) else {} + for key, new_entry in cleaned.items(): + if not new_entry.get("access", True): + # Revoking the module entirely is the OPPOSITE of widening, and the filter on a + # denied entry governs nothing. Guarding it would make "lock this account out" + # require a confirm_widen flag, which reads as nonsense to whoever hits it. + continue + old = stored.get(key) + if not isinstance(old, dict): + continue # nothing stored yet: the fold already handled it + old_tree, new_tree = old.get("filter"), new_entry.get("filter") + if not old_tree or not _confines_to_one_bu(old_tree): + continue # was not confined -> this save cannot widen it + if new_tree and _confines_to_one_bu(new_tree): + continue # still confined -> not a widening + raise err(400, "bu_widening", + f"{key}: this would REMOVE the business-unit restriction and give the account " + f"both units. If that is intended, resend with confirm_widen: true. If it is " + f"not, reload the account's permissions first — saving a filter loaded before " + f"the last change drops the conditions it did not know about.") + # A guard is only as good as its trigger: `perm_scope` is imported so a future reader sees + # the connection to derive_pool_scope, which is what the removed condition was feeding. + _ = perm_scope + + +def _mentions_dba(nodes): + for n in nodes or (): + if not isinstance(n, dict): + continue + if isinstance(n.get("children"), list): + if _mentions_dba(n["children"]): + return True + elif n.get("colId") == "dba": + return True + return False + + +def _confines_to_one_bu(tree): + """Does the BRAND STRUCTURE ALONE confine the reader to one business unit? + + ⚠ "Alone" is the whole precision of this function, and getting it wrong makes the guard + refuse legitimate walls. `dba is Fisch OR ar_open > 1000` does NOT confine anybody — a + high-balance Royal customer satisfies it — so consolidated values are the correct answer and + refusing it would be a false positive. My first attempt evaluated synthetic rows carrying + only `dba`, which made `ar_open > 1000` read false on every probe row and turned that exact + wall into a "confinement" it is not. + + So every NON-`dba` leaf is treated as TRUE — the most generous reading, i.e. "suppose the + reader satisfies everything else; can they still see both brands?" If yes, the wall does not + confine and the pool is correctly consolidated. If no, the brand structure is doing the + confining and `derive_pool_scope` must be able to see it, or the numbers are wrong. + """ + def admits(node, brand): + if isinstance(node.get("children"), list): + kids = [admits(k, brand) for k in node["children"] if isinstance(k, dict)] + if not kids: + return True + return any(kids) if node.get("conj") == "or" else all(kids) + if node.get("colId") != "dba": + return True # every other condition: assume satisfied + want = str(node.get("value") or "").strip().lower() + op = node.get("op") + if op == "eq": + return brand.lower() == want + if op == "neq": + return brand.lower() != want + if op == "contains": + return want in brand.lower() + if op == "doesNotContain": + return want not in brand.lower() + if op == "isEmpty": + return False # a probe brand is never blank + if op == "isNotEmpty": + return True + return True # an op that cannot judge a brand does not confine + + nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and") + root = {"conj": conj, "children": nodes} + admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)} + if not admitted: + return False # admits nothing at all — a different problem + # 'Both' belongs to either unit, so it cannot by itself widen the answer. + return not ({"Fisch", "Royal"} <= admitted) + + +def _leaf_count(nodes): + n = 0 + for node in nodes or (): + if isinstance(node, dict) and isinstance(node.get("children"), list): + n += _leaf_count(node["children"]) + else: + n += 1 + return n + + +def _fold_legacy_scope(rec, cleaned): + """⛔ MIGRATING A RECORD MUST NEVER WIDEN IT. The invariant this function exists to hold. + + Writing perms stamps `perms_v: 1`, and that marker SWITCHES OFF the legacy `bus`/`agent` + fallback in `perm_scope` (amendment 4 — absence must start meaning DENY). So a first write + whose filter carries no BU condition silently promotes a Royal-only account to BOTH business + units: the wall that used to come from `bus` is gone and nothing replaced it. + + Caught by the end-to-end leg in `verify_api` and by nothing else — every model-level check + passed, because the model was doing exactly what it was told. The account simply received + the other BU's customers on the wire. + + So on the FIRST perms write we fold the legacy scope in, as ordinary conditions, exactly as + `perm_migrate` would: this IS R1's migration, performed at the moment the record acquires a + perms block. From then on the admin sees those leaves in the editor (GET returns the folded + filter) and can change them like any other condition — which is R1's whole point, and why + this is a one-time fold rather than a permanent floor AND-ed on every write. + + Idempotent by construction: it runs only while the record is un-migrated, and the write that + calls it is the write that migrates it. + + ⛔ THE FOLD IS PER MODULE, BECAUSE THE LEGACY FILTER IS CUSTOMER-SHAPED AND THE MODULES ARE + NOT. `perm_migrate.filter_for` speaks `dba` and `agent` — both CUSTOMER columns. `product_data` + has neither (19 fields, no brand column at all), and `apply_row_scope` evaluates the permanent + filter with `permits()`, which DENIES anything unanswerable. So folding the customer wall into + the product entry does not narrow the product grid, it EMPTIES it: measured at 0 rows of 2 for + a `bus:[5]` account the moment an admin pressed Save. + + Nothing was wrong upstream — `_clean_perms` already refuses a `dba` condition typed against + `product_data` (its leaf keys are validated per topic). This fold was the ONLY door a + cross-topic leaf could come through, which is why it is the only place that needs the rule. + + ⚠ WHAT IS LOST HERE IS RECOVERED IN `perm_scope.derive_pool_scope`, NOT DISCARDED. Dropping the + BU condition from an AND root WIDENS that module's row scope, and for the product grid the BU + is a VALUES question anyway (`team_id` shapes `rev_ytd`/`qty_ytd`/`inv_value` — amendment 3). + The pushdown falls back to the record's own `bus` for exactly the modules whose field + vocabulary cannot express a BU, so the product pool is still BUILT as Fisch. The two halves + ship together or the second one is a leak. + """ + import core.perm_migrate as perm_migrate + import core.perm_scope as perm_scope + + if perm_scope.is_migrated(rec): + return cleaned + legacy = perm_migrate.filter_for(rec) + if not legacy: + return cleaned # nothing to preserve: the account was unrestricted + out = {} + for key, e in cleaned.items(): + legacy_here = _prune_to_module(legacy, {f["key"] for f in _module_fields(key)}) + tree = e.get("filter") + if not legacy_here: + # This module cannot evaluate ANY of the legacy conditions. Fold nothing rather than + # storing a wall it will read as "deny every row". + out[key] = e + continue + if not tree or not tree.get("nodes"): + folded = legacy_here + else: + # AND the two roots together. The submitted tree keeps its own conjunction by being + # nested as a GROUP — flattening an `or` tree into an `and` root would turn the + # admin's "A or B" into "A and B", which is a different and much narrower wall. + folded = {"conj": "and", + "nodes": list(legacy_here["nodes"]) + [{"conj": tree.get("conj", "and"), + "children": list(tree["nodes"])}]} + out[key] = dict(e, filter=folded) + return out + + +def _prune_to_module(tree, valid_keys): + """`tree` with every leaf this module cannot evaluate removed, or None if nothing survives. + + Used ONLY on the legacy fold above, where the alternative to pruning is a filter that denies + every row. It is not a general sanitiser: `_clean_perms` REFUSES rather than drops, for the + reason its own docstring gives (a silently-weakened wall reads as "Saved"). + + ⚠ A GROUP THAT LOSES ANY CHILD IS DROPPED WHOLE. Half of an `or` group is a narrower rule than + the admin's intent and half of an `and` group is a wider one; neither is the thing that was + written, and a fold has no standing to invent a third meaning. All-or-nothing per group keeps + the surviving conditions ones somebody actually declared. + """ + if not isinstance(tree, dict): + return None + + def keep(node): + if not isinstance(node, dict): + return None + kids = node.get("children") + if isinstance(kids, list): + surviving = [keep(k) for k in kids] + if not kids or any(s is None for s in surviving): + return None + return dict(node, children=surviving) + return node if node.get("colId") in valid_keys else None + + nodes = [n for n in (keep(n) for n in tree.get("nodes") or ()) if n is not None] + return {"conj": tree.get("conj", "and"), "nodes": nodes} if nodes else None + + +@router.get("/admin/users/{username}/perms") +def get_perms(username: str, session: Session = Depends(admin_gate)): + """This account's permission block plus the field schema the editor needs to render it.""" + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + import core.perm_scope as perm_scope + + rec = reg[uname] + stored = rec.get("perms") or {} + # S3 ask 3 — AN ENTRY FOR EVERY DECLARED MODULE, never a gap the client has to default. + # The PUT is a whole-record replace over this same module list, so a key declared here and + # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The + # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the + # access its legacy grant currently gives rather than a guess in either direction. + perms_out = {} + for k in _PERM_MODULES: + e = stored.get(k) + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(perm_scope.may_access(rec, k)), "filter": None, "hiddenFields": []} + return {"username": uname, + "perms": perms_out, + # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a + # migrated record and LEGACY on an un-migrated one: opposite meanings for identical + # JSON, so the client cannot render honestly without knowing which world it is in. + "perms_v": int(rec.get("perms_v") or 0), + # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can + # say "Everything (admin)" instead of rendering stored rules that do not apply. + "is_admin": perms.is_admin(rec), + "modules": [{"key": k, "label": _MODULE_LABELS.get(k, k)} for k in _PERM_MODULES], + # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav + # schema envelope; the editor needs the field list and nothing else, and one shape + # beats two readings of a sentence. + "fields_by_module": {k: _module_fields(k) for k in _PERM_MODULES}} + + +@router.put("/admin/users/{username}/perms") +def put_perms(username: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this account's permission block wholesale. + + WHOLESALE, not a merge: "remove this restriction" has to be expressible, and a merge cannot + express a deletion without a second vocabulary for it. + + ⚠ NO EPOCH BUMP, for the same reason `PATCH /admin/users/{u}` does not bump one: + `deps._user_for` re-reads the record on every request, so a narrowed wall applies to the + target's LIVE session on its very next call. Bumping would only add a forced re-login on top + of a change that has already taken effect. + """ + body = body if isinstance(body, dict) else {} + reg = _registry() + uname = str(username or "").strip().lower() + if uname not in reg or _tenant_of(reg[uname]) != _tenant_of(session.user): + # Wave 18 (C1-TENANT): a cross-tenant username answers exactly like a missing one — + # 404, never 403, because "that account exists in another company" is itself a leak. + raise err(404, "no_such_user", f"no account named {uname!r}") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + + # THE SELF-LOCKOUT GUARD, the twin of `self_demote` on the PATCH route. An admin bypasses + # perm_scope entirely, so this cannot brick them today — but it stops an administrator + # writing a wall for themselves that would bite the moment their role changed. + if uname == session.uname: + raise err(400, "self_perms", + "you cannot set permissions on your own account — ask another admin") + + cleaned = _clean_perms(body.get("perms")) or {} + cleaned = _fold_legacy_scope(reg[uname], cleaned) + _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) + users.set_access(uname, perms=cleaned) + fresh = _registry() + rec = fresh.get(uname) or {} + if int(rec.get("perms_v") or 0) < users.PERMS_VERSION: + # The store took the write and did not record it. A 200 here would tell an administrator + # the wall is up when it is not — the one direction this must never fail in. + raise err(503, "store_unavailable", + "the permissions were not saved — the account is UNCHANGED") + return {"username": uname, "perms": rec.get("perms") or {}, + "perms_v": int(rec.get("perms_v") or 0)} + + +_MODULE_LABELS = {"customer_data": "Customer", "product_data": "Product"} + + +@router.get("/settings") +def settings(session: Session = Depends(require_session)): + """The session's OWN settings — any authenticated user, not admin-only. + + `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a + client may know about itself (never a hash, never the epoch). `scope` restates it as the two + things the Settings UI shows, and `tenant` names only THIS tenant — enumerating the others + would answer a question about our customer list. + """ + modules = perms.allowed_modules(session.user) + return { + # The build this Space is running. Behind the session on purpose — see `main.py::health`, + # which refuses it for being the first URL a scanner finds. `deploy_web.py` stamps it as a + # Space secret at every deploy; absent means "somebody started this container by hand". + "version": os.environ.get("AIOS_VERSION") or "unknown", + "user": _public_user(session.user), + "scope": { + "bus": perms.allowed_bu_labels(session.user), + "team_id": perms.scope_team_id(session.user), + "agent": perms.scope_agent(session.user), + "modules": sorted(modules) if modules is not None else "all", + # C-PERM: the caller's OWN effective wall, read-only. The grid uses it to grey + # pickers rather than to enforce — hidden fields are stripped from every data wire + # regardless, so a client that ignores this is narrowed anyway, never widened. + "perms": session.user.get("perms") or {}, + }, + "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", + session.tenant)}, + # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client + # does not paint a "Manage users" button that 403s. + "admin": perms.is_admin(session.user), + # Wave 19 (R3, contract C2): the same courtesy for the LOOPABLE plane — the Settings rail + # paints its entry iff this is true, and `routes_platform_admin` refuses every request + # that is not, so a client that ignored this would gain exactly nothing. Derived from the + # ONE predicate rather than restated: a second copy of a two-condition wall is a second + # place for one of the conditions to go missing. + "platformAdmin": platform_admin.is_platform_admin(session.user), + } diff --git a/api/routes_alerts.py b/api/routes_alerts.py index 3d2de1f509c606851031800122329a2fa9a7c9fc..5db8ccd3fe295c43e401817adff02344918291af 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -1,664 +1,537 @@ -"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). - - GET /api/v1/alerts -> {alerts:[...]} - POST /api/v1/alerts <- {viewId, topic, label?} - DELETE /api/v1/alerts/{alert_id} - POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) - GET /api/v1/notifications -> {unread, items:[...]} - POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} - -The semantics — an alert is a view plus a remembered matched set, a notification is a NEW -ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. -This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. - -⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool -for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a -full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole -book, and the notification would name customers that user may not see — a permission leak wearing -a notification's clothes. The owner's own scope is the only correct basis for their alert. - -⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same -`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an -alert can see is by construction a row its owner could open. Re-implementing the filter here -would be a second definition of "matches", and those two would drift. -""" -import re - -from fastapi import APIRouter, Body, Depends - -import core.alerts as alerts -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. -_TOPICS = ("customer", "product") - -# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── -# -# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). -# -# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three -# keys onto the record at write time would give them to notifications minted AFTER the deploy and -# to nothing else — every notification already sitting in every tenant's inbox would open nothing, -# and the feature would be correct in the source and absent from the product -# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a -# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is -# another lane's file this wave — but that is the convenience, not the reason. -# -# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them -# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets -# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding -# here means the client branches on ONE field instead of re-deriving the same split. -# -# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` -# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by -# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) -# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, -# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review -# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real -# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored -# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary -# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed -# at was deleted two waves ago. - -#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them -#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). -NOTIF_KIND_ALERT = "alert" -NOTIF_KIND_AUTOMATION = "automation" -NOTIF_KIND_SHARE = "share" - -#: C3's `target.module` vocabulary, and the automation sub-selection. -TARGET_MODULE_DATABASE = "database" -TARGET_MODULE_AUTOMATION = "automation" -TARGET_TAB_RUNS = "runs" - -#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with -#: no producer is a string that reads as a feature — the reason this constant is named here and -#: cited from `routes_shares` rather than typed twice). -SHARE_TOPIC = "share" - -#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. -AUTOMATION_TOPIC = "automation" - -_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") - - -def route_for_topic(topic): - """A grid SCOPE key -> the registry route that renders it, or None. - - ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED - (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only - pair that differ — the registry names the surface (`customer_data`) while the grid names the - scope (`customer`) — so a topic passed through as a route sends every click to a page that - does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT - rather than plausible, because an absent target renders as a row that does not pretend to be - clickable, and a wrong one renders as a click that silently goes nowhere. - """ - t = str(topic or "").strip() - if t == "customer": - return "customer_data" - if t == "product": - return "product_data" - if _UT_TOPIC.match(t): - return t - return None - - -def _refusal_code(exc): - """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. - - ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` - (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own - cause. Anything that reduces all four to one word is throwing away the only information the - reader could have acted on. Returns `""` for a plain exception, so a caller can tell - "refused, and here is why" apart from "broke, and we do not know why". - """ - detail = getattr(exc, "detail", None) - if isinstance(detail, dict): - inner = detail.get("error") - if isinstance(inner, dict): - return str(inner.get("code") or "") - return "" - - -def notification_view(item): - """One STORED notification -> the shape the Inbox renders. PURE, and total. - - Never raises and never drops a row: an item it cannot classify comes back as an `alert` with - no `target`, which the client renders as an unclickable row rather than hiding. An inbox that - silently omits what it does not understand is the one failure a reader cannot detect. - """ - if not isinstance(item, dict): - return item - topic = str(item.get("topic") or "").strip() - alert_id = str(item.get("alertId") or "").strip() - - # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic - # says `automation` but whose producer key never arrived (a truncated payload, a server - # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work - # and silently does not, which is this repo's most-repeated failure shape. Failing the test - # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. - if topic == AUTOMATION_TOPIC and alert_id: - kind = NOTIF_KIND_AUTOMATION - target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} - elif topic == SHARE_TOPIC and alert_id: - # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, - # `row_id=`. - # - # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it - # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the - # target read `{module: "database", id: "view_42"}` — an instruction to open a database - # called `view_42`. It looked right in the payload and would have opened nothing. The - # producer resolves the object to its topic and hands over the route; this branch only - # shapes what it is given. - kind = NOTIF_KIND_SHARE - row_id = str(item.get("rowId") or "").strip() - target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, - **({"tab": row_id} if row_id else {})} - else: - kind = NOTIF_KIND_ALERT - route = route_for_topic(topic) - view_id = str(item.get("viewId") or "").strip() - target = None if route is None else ( - {"module": TARGET_MODULE_DATABASE, "id": route, - **({"tab": view_id} if view_id else {})}) - - # The email split: `subject` is the HEADER (what this is about — the alert, the automation, - # the database), `label` stays the BODY (what happened — the record that entered, the run - # summary). They were one field, which is why a notification read as a sentence with no - # sender and the pane could not be laid out like mail. - subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() - # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it - # is D-101's dead one; leaving it through would give the client two vocabularies for one - # question, which is the defect this wave's item 6 is about in a different file. - # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. - # - # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was - # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — - # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in - # the model or in the markup. Mail has a from. This is it. - # - # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. - # An alert firing and an automation landing rows have no person behind them; their honest - # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a - # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and - # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with - # you") — a sender recovered by regexing a sentence breaks the first time the sentence is - # reworded, and it would break silently, in the header. - # - # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an - # empty from column reads as a broken inbox rather than as an old notification. - actor = str(item.get("actor") or "").strip() - if kind == NOTIF_KIND_SHARE: - sender = actor or "A teammate" - elif kind == NOTIF_KIND_AUTOMATION: - sender = actor or "Automation" - else: - sender = actor or "Alerts" - out = {**item, "read": bool(item.get("read")), "kind": kind, - "subject": subject or "Notification", "sender": sender} - if target is not None: - out["target"] = target - return out - - -def inbox_view(box): - """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. - - ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a - recount here would make the badge a function of whatever this page happened to include, which - is the exact defect `alertsModel.parseInbox`'s own header records from the other side. - """ - if not isinstance(box, dict): - return box - items = box.get("items") - if not isinstance(items, list): - return box - # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client - # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. - # - # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and - # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH - # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" - # against a browser clock would re-introduce the drift the offset exists to remove — a reader a - # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands - # now come from the same machine. - # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — - # the note two lines up records what happened last time only one of them was enriched. - return {**box, "now": alerts._now_iso(), - "items": [notification_view(n) for n in items]} - - -def _view_by_id(g, view_id): - """One saved view out of an assembly, by id. `None` when there is no such view. - - ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS - FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND - `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes - it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on - it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the - `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** - - ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In - `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare - FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our - side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above - the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical - line is swallowed by `/notifications`' `except Exception: continue`, so **every stored - view-alert was silently dropped from the Inbox** and nobody had anything to report at all. - One expression, one loud symptom and one silent one. - - ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site - be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. - - ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was - keyed `{id: view}` — which is precisely why the gate was green while production raised on - every call. The fixture is moving to the production shape in this same change, and tolerating - both here means a caller that legitimately holds one cannot resurrect the bug. - """ - want = str(view_id or "") - if not want: - return None - views = (g or {}).get("views") - if isinstance(views, dict): - found = views.get(want) - return found if isinstance(found, dict) else None - if not isinstance(views, list): - return None - for v in views: - if isinstance(v, dict) and str(v.get("id") or "") == want: - return v - return None - - -def _topic_or_400(raw): - topic = str(raw or "").strip().lower() - if topic.startswith("ut_") or topic in _TOPICS: - return topic - raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") - - -def _owner_session(session: Session, owner: str): - """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). - - ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an - owner session is built by swapping the `user` RECORD and letting both derive themselves. An - earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the - first write hook of the wave; the properties are the single definition of who a session is, - and going around them is how a session with an admin flag and a non-admin record exists. - - Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather - than evaluating as somebody else, which is the fail-closed direction. - """ - import core.users as users - - if str(owner) == str(session.uname): - return session - rec = (users.registry() or {}).get(str(owner)) - if not isinstance(rec, dict) or not rec.get("active", True): - return None - # `_public` is THE definition of what a session may know about its own account (never a hash - # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a - # second definition, and the one that leaks is always the copy. - return Session(tenant=session.tenant, user=users._public(str(owner), rec), - claims=session.claims, runtime=session.runtime) - - -def _evaluate(session: Session, rec: dict, assemblies=None): - """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. - - ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole - of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each - one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two - alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, - because each `_evaluate` was a closed call. - ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module - note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak - wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and - must never share an entry. Getting that key wrong is the one way this optimisation could leak. - ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: - they evaluate ONE alert and a memo for a single call is pure overhead. - """ - import aios_grid - from harness import filter_eval - - owner_sess = _owner_session(session, rec.get("owner")) - if owner_sess is None: - return {"skipped": "owner_unavailable"} - topic = str(rec.get("topic") or "") - memo_key = (topic, str(owner_sess.uname)) - g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None - if g is None: - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. - # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the - # one-shot field-name correction acks for every `ut_` topic that has an alert — - # taking them from the `/workspace` refresh that exists to show them to the person - # who made the edit. The customer branch below has always passed False; this one - # inherited a default nobody re-read. An inbox poll must never consume a one-shot. - g = ut_assembly(owner_sess, topic, - storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", - consume_corrections=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) - except Exception as e: # noqa: BLE001 - # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not - # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it - # keeps a blanket catch — but it now reports the refusal's OWN code where there is - # one. `type(e).__name__` said `HTTPException` for four different causes, and - # `lastError` is the only place a user ever learns why an alert stopped firing. - # - # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an - # evaluation genuinely needs the rows to run the filter over. So an alert on a - # read-through grid is created (T22) and then skips at evaluation with - # `window_required` naming why — which is D-184's remaining half, and it is a - # SENTENCE now rather than silence. - return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} - if isinstance(assemblies, dict): - assemblies[memo_key] = g - - view = _view_by_id(g, rec.get("viewId")) - if not isinstance(view, dict): - # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than - # deleting the alert: an alert that silently vanishes is indistinguishable from one that - # never fires, and the user cannot debug what is not there. - return {"skipped": "view_missing"} - - # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived - # and overlay values on a row. Evaluating a filter against raw pool dicts would silently - # never match any condition on a user-created or measure column. - # - # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND - # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. - # - # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows - # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner - # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: - # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell - # evaluated as BLANK. Measured on one assembly, same view, same rows: - # rows_src state='unpaid' / 'paid' - # _evaluate saw state='' / '' ⇐ every base cell blank - # the GRID saw state='unpaid' / 'paid' - # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` - # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it - # inverted.** End to end: a row whose value arrived by import, automation, paste or the create - # door never fired; only a value typed as a hand EDIT did. - # - # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; - # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); - # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns - # `skipped: window_required`). - # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it - # would let a stale definition value shadow an edit the user has just made — the same defect - # `table_rows`' own note records, arriving from the other side. - _ov = (g.get("ws") or {}).get("overlays") or {} - _merged = {} - for _r in g["rows_src"]: - _pid = str(_r.get("pid")) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _o = _ov.get(_pid) - if isinstance(_o, dict): - _cells.update(_o) - _merged[_pid] = _cells - rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, - derived=g.get("derived")) - config = view.get("config") or view - ctx = filter_eval.EvalCtx( - cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} - for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, - measure_sets=g.get("measure_sets") or {}, - today=g.get("today")) - pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, - member_pids=config.get("memberPids")) - labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} - return alerts.evaluate(rec.get("id"), [str(p) for p in pids], - labels=labels, partial=False, st=session.runtime) - - -@router.get("/alerts") -def list_alerts(session: Session = Depends(require_session)): - return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime)} - - -@router.post("/alerts") -def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): - body = body or {} - view_id = str(body.get("viewId") or "").strip() - if not view_id: - raise err(400, "bad_view", "an alert needs the id of the view it watches") - topic = _topic_or_400(body.get("topic")) - _require_filtered_view(session, topic, view_id) - import uuid - aid = f"al_{uuid.uuid4().hex[:12]}" - rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, - label=body.get("label") or "", st=session.runtime) - # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". - # Deferring this to the first write hook would mean the next edit announces the whole view. - outcome = _evaluate(session, rec) - return {"alert": {**rec, "seeded": True}, "first": outcome} - - -def _require_filtered_view(session: Session, topic: str, view_id: str): - """400 unless `view_id` exists on `topic` AND actually narrows something. - - ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one - that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" - (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an - entrant again — there is nothing left to enter. The owner's words are *"when a Record gets - into that Filter's criteria"*: no criteria, no alert, and said at creation rather than - discovered by never being notified. - - `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a - half-typed rule is not a filter, and this must agree with what actually narrows or it would - accept a view whose one rule the engine then ignores. - - ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and - the second one hid the first. - - (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is - `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a - read-through grid built the whole pool — and `scoped_pool` refuses that with - `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` - (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, - runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, - closed** — an alert on a read-through grid can now be made at all. - (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` - is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and - `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by - *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED - A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design - (a 5xx body is the server's internals), substituting *"Something went wrong on our - side."* — which is the owner's screenshot, word for word. A knowable cause returned as a - 5xx is invisible by construction, so re-wording the 503 could never have fixed this. - ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is - honest. What it may no longer do is catch a refusal that already knows its own name. - """ - from fastapi import HTTPException - - from harness import filter_eval - - try: - if topic.startswith("ut_"): - from routes_tables import ut_assembly - # ⚠ `consume_corrections=False` — the customer branch has always passed it and this - # one inherited a default nobody re-read. Creating an alert must not eat the one-shot - # field-name correction acks belonging to the `/workspace` refresh that exists to show - # them to the person who made the edit. Same defect `_evaluate`'s header records. - g = ut_assembly(session, topic, - storage_key=f"{session.tenant}:{topic}:{session.uname}", - consume_corrections=False, with_rows=False) - else: - from routes_customers import grid_assembly - g = grid_assembly(session, scope=topic, consume_corrections=False) - except HTTPException: - raise # it already names its own cause - except Exception as e: # noqa: BLE001 - # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, - # the one path that reaches this branch is also the one path with nothing to debug from. - raise err(503, "unavailable", - f"the table could not be read ({type(e).__name__}) — try again in a moment") - view = _view_by_id(g, view_id) - if not isinstance(view, dict): - raise err(404, "no_view", "that view does not exist on this table") - nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) - - # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT - # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS - # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have - # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines - # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the - # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. - # - # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the - # fix was right, and the three reasons it hid are the most transferable thing in this file. - # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each - # naming themselves as the origin of the same screenshot are mutually exclusive, and the next - # reader believes whichever they meet first — which is why this correction sits above rather - # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead - # of the working tree. [[grep-output-is-not-source]] - # - # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, - # that this was owner item 17 — see the correction directly above). - # - # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other - # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF - # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. - # - # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the - # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. - # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever - # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that - # worked was the refusal path: "Alert me about new records" had never once created an alert - # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the - # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went - # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. - # - # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check - # arity until the line RUNS, and this line runs only on the success path of a feature whose - # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, - # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal - # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, - # thorough and honest about everything except the one path the feature exists for. - # - # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a - # column up in; building a second dict here would be a second answer to one question, which - # is this wave's other headline defect in a different file. Its leading underscore is a real - # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. - columns = filter_eval._columns_map(g.get("fields") or []) - - def _any_active(ns): - for n in ns or (): - if isinstance(n, dict) and isinstance(n.get("children"), list): - if _any_active(n["children"]): - return True - elif filter_eval.is_rule_active(n, columns): - return True - return False - - if not _any_active(nodes): - raise err(400, "no_filter", - "this view has no active filter, so no record can ever ENTER it — add a " - "condition to the view first, then create the alert") - - -@router.delete("/alerts/{alert_id}") -def delete_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - if str(rec.get("owner")) != str(session.uname) and not session.admin: - raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") - alerts.delete(alert_id, st=session.runtime) - return {"ok": True} - - -@router.post("/alerts/{alert_id}/run") -def run_alert(alert_id: str, session: Session = Depends(require_session)): - rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, - st=session.runtime) - if str(r.get("id")) == str(alert_id)), None) - if rec is None: - raise err(404, "no_alert", "that alert does not exist") - return _evaluate(session, rec) - - -@router.get("/notifications") -def notifications(session: Session = Depends(require_session)): - """The inbox — RE-EVALUATED on read, which is a deliberate design choice. - - ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The - plan was a push hook: the automation engine calls `after_write` when it lands rows. But - `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be - evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside - a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity - construction that leaks scope. - - Pulling on read has none of that: the caller IS a session, the assemblies are already - scope-cached, and the user cannot observe the difference — an inbox is only ever read by - someone opening it. The cost is that a notification is minted when you LOOK rather than when - the row landed, so the `at` stamp is detection time, not arrival time. - - `after_write` stays exported for the day the engine can hand over a real identity. - - ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. - ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is - **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** - (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the - re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the - body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole - grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which - is the difference between "fine" and "three seconds per alert" the day somebody uses the - feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what - this does: every alert is still evaluated, against the same rows, in the same order. - """ - assemblies = {} - for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): - try: - _evaluate(session, rec, assemblies=assemblies) - except Exception: # noqa: BLE001 - continue # one bad alert must not empty the pane - # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before - # this wave carries a `target` too. See `notification_view`'s header for why it is derived. - return inbox_view(alerts.inbox(session.uname, st=session.runtime)) - - -@router.post("/notifications/read") -def read_notifications(body: dict = Body(default=None), - session: Session = Depends(require_session)): - body = body or {} - ids = body.get("ids") - if ids is not None and not isinstance(ids, list): - raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") - # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox - # module re-renders from it — an un-enriched answer here would strip `target` off every row - # the moment somebody marked one read, i.e. the feature would work until first use. - return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), - st=session.runtime)) - - -def after_write(session: Session, topic_key: str): - """THE WRITE HOOK — call after a write that could change what a view matches. - - Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation - upserts reach it the same way. It never raises: an alert evaluation failing must not fail the - edit that triggered it. - - ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes - one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. - ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the - push hook was resolved the other way). Booked rather than wired: minting a session inside the - engine's worker thread is the ad-hoc identity construction this file exists to avoid. - """ - try: - assemblies = {} - return alerts.after_write(topic_key, st=session.runtime, - runner=lambda rec: _evaluate(session, rec, - assemblies=assemblies)) - except Exception: # noqa: BLE001 - return {"evaluated": 0} +"""routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT). + + GET /api/v1/alerts -> {alerts:[...]} + POST /api/v1/alerts <- {viewId, topic, label?} + DELETE /api/v1/alerts/{alert_id} + POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh) + GET /api/v1/notifications -> {unread, items:[...]} + POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool} + +The semantics — an alert is a view plus a remembered matched set, a notification is a NEW +ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning. +This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated. + +⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool +for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a +full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole +book, and the notification would name customers that user may not see — a permission leak wearing +a notification's clothes. The owner's own scope is the only correct basis for their alert. + +⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same +`routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an +alert can see is by construction a row its owner could open. Re-implementing the filter here +would be a second definition of "matches", and those two would drift. +""" +import re + +from fastapi import APIRouter, Body, Depends + +import core.alerts as alerts +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else. +_TOPICS = ("customer", "product") + +# ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ──────────────────────── +# +# `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there). +# +# ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three +# keys onto the record at write time would give them to notifications minted AFTER the deploy and +# to nothing else — every notification already sitting in every tenant's inbox would open nothing, +# and the feature would be correct in the source and absent from the product +# ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a +# notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is +# another lane's file this wave — but that is the convenience, not the reason. +# +# ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them +# apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets +# `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding +# here means the client branches on ONE field instead of re-deriving the same split. +# +# ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'` +# + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by +# W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`) +# records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch, +# its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review +# branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real +# caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored +# review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary +# `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed +# at was deleted two waves ago. + +#: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them +#: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row). +NOTIF_KIND_ALERT = "alert" +NOTIF_KIND_AUTOMATION = "automation" +NOTIF_KIND_SHARE = "share" + +#: C3's `target.module` vocabulary, and the automation sub-selection. +TARGET_MODULE_DATABASE = "database" +TARGET_MODULE_AUTOMATION = "automation" +TARGET_TAB_RUNS = "runs" + +#: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with +#: no producer is a string that reads as a feature — the reason this constant is named here and +#: cited from `routes_shares` rather than typed twice). +SHARE_TOPIC = "share" + +#: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`. +AUTOMATION_TOPIC = "automation" + +_UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") + + +def route_for_topic(topic): + """A grid SCOPE key -> the registry route that renders it, or None. + + ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED + (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only + pair that differ — the registry names the surface (`customer_data`) while the grid names the + scope (`customer`) — so a topic passed through as a route sends every click to a page that + does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT + rather than plausible, because an absent target renders as a row that does not pretend to be + clickable, and a wrong one renders as a click that silently goes nowhere. + """ + t = str(topic or "").strip() + if t == "customer": + return "customer_data" + if t == "product": + return "product_data" + if _UT_TOPIC.match(t): + return t + return None + + +def _refusal_code(exc): + """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`. + + ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden` + (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own + cause. Anything that reduces all four to one word is throwing away the only information the + reader could have acted on. Returns `""` for a plain exception, so a caller can tell + "refused, and here is why" apart from "broke, and we do not know why". + """ + detail = getattr(exc, "detail", None) + if isinstance(detail, dict): + inner = detail.get("error") + if isinstance(inner, dict): + return str(inner.get("code") or "") + return "" + + +def notification_view(item): + """One STORED notification -> the shape the Inbox renders. PURE, and total. + + Never raises and never drops a row: an item it cannot classify comes back as an `alert` with + no `target`, which the client renders as an unclickable row rather than hiding. An inbox that + silently omits what it does not understand is the one failure a reader cannot detect. + """ + if not isinstance(item, dict): + return item + topic = str(item.get("topic") or "").strip() + alert_id = str(item.get("alertId") or "").strip() + + # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic + # says `automation` but whose producer key never arrived (a truncated payload, a server + # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work + # and silently does not, which is this repo's most-repeated failure shape. Failing the test + # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None. + if topic == AUTOMATION_TOPIC and alert_id: + kind = NOTIF_KIND_AUTOMATION + target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS} + elif topic == SHARE_TOPIC and alert_id: + # ⭐ W32-T28: the sharer writes `key=` and, for a shared VIEW, + # `row_id=`. + # + # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it + # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the + # target read `{module: "database", id: "view_42"}` — an instruction to open a database + # called `view_42`. It looked right in the payload and would have opened nothing. The + # producer resolves the object to its topic and hands over the route; this branch only + # shapes what it is given. + kind = NOTIF_KIND_SHARE + row_id = str(item.get("rowId") or "").strip() + target = {"module": TARGET_MODULE_DATABASE, "id": alert_id, + **({"tab": row_id} if row_id else {})} + else: + kind = NOTIF_KIND_ALERT + route = route_for_topic(topic) + view_id = str(item.get("viewId") or "").strip() + target = None if route is None else ( + {"module": TARGET_MODULE_DATABASE, "id": route, + **({"tab": view_id} if view_id else {})}) + + # The email split: `subject` is the HEADER (what this is about — the alert, the automation, + # the database), `label` stays the BODY (what happened — the record that entered, the run + # summary). They were one field, which is why a notification read as a sentence with no + # sender and the pane could not be laid out like mail. + subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip() + # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it + # is D-101's dead one; leaving it through would give the client two vocabularies for one + # question, which is the defect this wave's item 6 is about in a different file. + out = {**item, "read": bool(item.get("read")), "kind": kind, + "subject": subject or "Notification"} + if target is not None: + out["target"] = target + return out + + +def inbox_view(box): + """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}. + + ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a + recount here would make the badge a function of whatever this page happened to include, which + is the exact defect `alertsModel.parseInbox`'s own header records from the other side. + """ + if not isinstance(box, dict): + return box + items = box.get("items") + if not isinstance(items, list): + return box + return {**box, "items": [notification_view(n) for n in items]} + + +def _topic_or_400(raw): + topic = str(raw or "").strip().lower() + if topic.startswith("ut_") or topic in _TOPICS: + return topic + raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table") + + +def _owner_session(session: Session, owner: str): + """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller). + + ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an + owner session is built by swapping the `user` RECORD and letting both derive themselves. An + earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the + first write hook of the wave; the properties are the single definition of who a session is, + and going around them is how a session with an admin flag and a non-admin record exists. + + Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather + than evaluating as somebody else, which is the fail-closed direction. + """ + import core.users as users + + if str(owner) == str(session.uname): + return session + rec = (users.registry() or {}).get(str(owner)) + if not isinstance(rec, dict) or not rec.get("active", True): + return None + # `_public` is THE definition of what a session may know about its own account (never a hash + # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a + # second definition, and the one that leaks is always the copy. + return Session(tenant=session.tenant, user=users._public(str(owner), rec), + claims=session.claims, runtime=session.runtime) + + +def _evaluate(session: Session, rec: dict, assemblies=None): + """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in. + + ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole + of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each + one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two + alerts on one view built that table twice; ten built it ten times. Nothing dedupes them, + because each `_evaluate` was a closed call. + ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module + note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak + wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and + must never share an entry. Getting that key wrong is the one way this optimisation could leak. + ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want: + they evaluate ONE alert and a memo for a single call is pure overhead. + """ + import aios_grid + from harness import filter_eval + + owner_sess = _owner_session(session, rec.get("owner")) + if owner_sess is None: + return {"skipped": "owner_unavailable"} + topic = str(rec.get("topic") or "") + memo_key = (topic, str(owner_sess.uname)) + g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None + if g is None: + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up. + # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the + # one-shot field-name correction acks for every `ut_` topic that has an alert — + # taking them from the `/workspace` refresh that exists to show them to the person + # who made the edit. The customer branch below has always passed False; this one + # inherited a default nobody re-read. An inbox poll must never consume a one-shot. + g = ut_assembly(owner_sess, topic, + storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}", + consume_corrections=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(owner_sess, scope=topic, consume_corrections=False) + except Exception as e: # noqa: BLE001 + # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not + # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it + # keeps a blanket catch — but it now reports the refusal's OWN code where there is + # one. `type(e).__name__` said `HTTPException` for four different causes, and + # `lastError` is the only place a user ever learns why an alert stopped firing. + # + # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an + # evaluation genuinely needs the rows to run the filter over. So an alert on a + # read-through grid is created (T22) and then skips at evaluation with + # `window_required` naming why — which is D-184's remaining half, and it is a + # SENTENCE now rather than silence. + return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__} + if isinstance(assemblies, dict): + assemblies[memo_key] = g + + view = (g.get("views") or {}).get(str(rec.get("viewId"))) + if not isinstance(view, dict): + # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than + # deleting the alert: an alert that silently vanishes is indistinguishable from one that + # never fires, and the user cannot debug what is not there. + return {"skipped": "view_missing"} + + # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived + # and overlay values on a row. Evaluating a filter against raw pool dicts would silently + # never match any condition on a user-created or measure column. + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"), + derived=g.get("derived")) + config = view.get("config") or view + ctx = filter_eval.EvalCtx( + cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())} + for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)}, + measure_sets=g.get("measure_sets") or {}, + today=g.get("today")) + pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, + member_pids=config.get("memberPids")) + labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} + return alerts.evaluate(rec.get("id"), [str(p) for p in pids], + labels=labels, partial=False, st=session.runtime) + + +@router.get("/alerts") +def list_alerts(session: Session = Depends(require_session)): + return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime)} + + +@router.post("/alerts") +def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)): + body = body or {} + view_id = str(body.get("viewId") or "").strip() + if not view_id: + raise err(400, "bad_view", "an alert needs the id of the view it watches") + topic = _topic_or_400(body.get("topic")) + _require_filtered_view(session, topic, view_id) + import uuid + aid = f"al_{uuid.uuid4().hex[:12]}" + rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname, + label=body.get("label") or "", st=session.runtime) + # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news". + # Deferring this to the first write hook would mean the next edit announces the whole view. + outcome = _evaluate(session, rec) + return {"alert": {**rec, "seeded": True}, "first": outcome} + + +def _require_filtered_view(session: Session, topic: str, view_id: str): + """400 unless `view_id` exists on `topic` AND actually narrows something. + + ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one + that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows" + (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an + entrant again — there is nothing left to enter. The owner's words are *"when a Record gets + into that Filter's criteria"*: no criteria, no alert, and said at creation rather than + discovered by never being notified. + + `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a + half-typed rule is not a filter, and this must agree with what actually narrows or it would + accept a view whose one rule the engine then ignores. + + ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and + the second one hid the first. + + (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is + `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a + read-through grid built the whole pool — and `scoped_pool` refuses that with + `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False` + (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`, + runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half, + closed** — an alert on a read-through grid can now be made at all. + (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException` + is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and + `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by + *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED + A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design + (a 5xx body is the server's internals), substituting *"Something went wrong on our + side."* — which is the owner's screenshot, word for word. A knowable cause returned as a + 5xx is invisible by construction, so re-wording the 503 could never have fixed this. + ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is + honest. What it may no longer do is catch a refusal that already knows its own name. + """ + from fastapi import HTTPException + + from harness import filter_eval + + try: + if topic.startswith("ut_"): + from routes_tables import ut_assembly + # ⚠ `consume_corrections=False` — the customer branch has always passed it and this + # one inherited a default nobody re-read. Creating an alert must not eat the one-shot + # field-name correction acks belonging to the `/workspace` refresh that exists to show + # them to the person who made the edit. Same defect `_evaluate`'s header records. + g = ut_assembly(session, topic, + storage_key=f"{session.tenant}:{topic}:{session.uname}", + consume_corrections=False, with_rows=False) + else: + from routes_customers import grid_assembly + g = grid_assembly(session, scope=topic, consume_corrections=False) + except HTTPException: + raise # it already names its own cause + except Exception as e: # noqa: BLE001 + # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it, + # the one path that reaches this branch is also the one path with nothing to debug from. + raise err(503, "unavailable", + f"the table could not be read ({type(e).__name__}) — try again in a moment") + view = (g.get("views") or {}).get(str(view_id)) + if not isinstance(view, dict): + raise err(404, "no_view", "that view does not exist on this table") + nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) + + # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT, AND THAT IS OWNER ITEM 17.** + # + # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other + # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF + # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`. + # + # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the + # view HAS a condition — and a view with a condition is the only kind an alert is allowed on. + # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever + # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that + # worked was the refusal path: "Alert me about new records" had never once created an alert + # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the + # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went + # wrong on our side. Try again in a moment."*, the owner's screenshot word for word. + # + # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check + # arity until the line RUNS, and this line runs only on the success path of a feature whose + # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct, + # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal + # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green, + # thorough and honest about everything except the one path the feature exists for. + # + # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a + # column up in; building a second dict here would be a second answer to one question, which + # is this wave's other headline defect in a different file. Its leading underscore is a real + # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around. + columns = filter_eval._columns_map(g.get("fields") or []) + + def _any_active(ns): + for n in ns or (): + if isinstance(n, dict) and isinstance(n.get("children"), list): + if _any_active(n["children"]): + return True + elif filter_eval.is_rule_active(n, columns): + return True + return False + + if not _any_active(nodes): + raise err(400, "no_filter", + "this view has no active filter, so no record can ever ENTER it — add a " + "condition to the view first, then create the alert") + + +@router.delete("/alerts/{alert_id}") +def delete_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + if str(rec.get("owner")) != str(session.uname) and not session.admin: + raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it") + alerts.delete(alert_id, st=session.runtime) + return {"ok": True} + + +@router.post("/alerts/{alert_id}/run") +def run_alert(alert_id: str, session: Session = Depends(require_session)): + rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin, + st=session.runtime) + if str(r.get("id")) == str(alert_id)), None) + if rec is None: + raise err(404, "no_alert", "that alert does not exist") + return _evaluate(session, rec) + + +@router.get("/notifications") +def notifications(session: Session = Depends(require_session)): + """The inbox — RE-EVALUATED on read, which is a deliberate design choice. + + ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The + plan was a push hook: the automation engine calls `after_write` when it lands rows. But + `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be + evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside + a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity + construction that leaks scope. + + Pulling on read has none of that: the caller IS a session, the assemblies are already + scope-cached, and the user cannot observe the difference — an inbox is only ever read by + someone opening it. The cost is that a notification is minted when you LOOK rather than when + the row landed, so the `at` stamp is detection time, not arrival time. + + `after_write` stays exported for the day the engine can hand over a real identity. + + ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT. + ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is + **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts** + (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the + re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the + body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole + grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which + is the difference between "fine" and "three seconds per alert" the day somebody uses the + feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what + this does: every alert is still evaluated, against the same rows, in the same order. + """ + assemblies = {} + for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime): + try: + _evaluate(session, rec, assemblies=assemblies) + except Exception: # noqa: BLE001 + continue # one bad alert must not empty the pane + # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before + # this wave carries a `target` too. See `notification_view`'s header for why it is derived. + return inbox_view(alerts.inbox(session.uname, st=session.runtime)) + + +@router.post("/notifications/read") +def read_notifications(body: dict = Body(default=None), + session: Session = Depends(require_session)): + body = body or {} + ids = body.get("ids") + if ids is not None and not isinstance(ids, list): + raise err(400, "bad_ids", "ids must be a list, or null to mark every notification") + # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox + # module re-renders from it — an un-enriched answer here would strip `target` off every row + # the moment somebody marked one read, i.e. the feature would work until first use. + return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), + st=session.runtime)) + + +def after_write(session: Session, topic_key: str): + """THE WRITE HOOK — call after a write that could change what a view matches. + + Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation + upserts reach it the same way. It never raises: an alert evaluation failing must not fail the + edit that triggered it. + + ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes + one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table. + ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the + push hook was resolved the other way). Booked rather than wired: minting a session inside the + engine's worker thread is the ad-hoc identity construction this file exists to avoid. + """ + try: + assemblies = {} + return alerts.after_write(topic_key, st=session.runtime, + runner=lambda rec: _evaluate(session, rec, + assemblies=assemblies)) + except Exception: # noqa: BLE001 + return {"evaluated": 0} diff --git a/api/routes_automation.py b/api/routes_automation.py index 6de4ef3aa25c8bce0e14abbb702d192815b380d7..214128a74849f05cb9b698bbea3522ea568948e2 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -1,1155 +1,998 @@ -"""routes_automation.py — wave-18 item 5 (contract C4-AUTO): the automation surface's API. - -The router is thin on purpose: everything that can be wrong about an automation — a malformed -cron, a URL the SSRF rail refuses, a key field that is not one of the mapped columns — is decided -in `automation_engine`, which is a pure-ish module a gate can drive without a server. This file -does auth, shape and status codes. - -⛔ THE TICK ENDPOINT IS THE ONE UNAUTHENTICATED ROUTE, AND IT IS FAIL-CLOSED TWICE OVER. It takes -no session (an external cron has no cookie), so it is gated on a shared secret in -`X-AIOS-TICK-TOKEN`; and when `AIOS_AUTOMATION_TICK_TOKEN` is UNSET the route refuses everything -rather than admitting everyone. A "no token configured means no check" default is how an internal -trigger becomes a public one — the same class of mistake as an empty-200 permission answer. -""" -import os -import time - -from fastapi import APIRouter, Body, Depends, Header, Request - -import ai_review -import automation_engine as engine -import oauth_connect -import routes_oauth -import scope_cache -from deps import Session, err, module_gate, require_session - -router = APIRouter(prefix="/api/v1") - -# C5 (wave 22): the OAuth connector surface rides INSIDE this router — main.py belongs to no -# session this wave, and this router is already mounted there. `/api/v1` + `/oauth/...`. -router.include_router(routes_oauth.router) -# ⛔⛔ D-203 — THE NESTED INCLUDE OF `routes_connectors` IS REMOVED, and the reason it existed is -# worth keeping because it was a GOOD reason that expired. -# -# Wave 23 (C11) mounted the connectors directory here rather than in `main.py`, because this router -# was already mounted and `main.py` belonged to another session — a sensible way to avoid a -# cross-fence ask. `main.py:227` includes `routes_connectors` DIRECTLY now, so this line stopped -# being the only door and became a SECOND one. -# ⚠ AND A ROUTER MOUNTED TWICE DOES NOT SERVE THE SAME PATHS TWICE — it serves the prefix twice. -# `routes_connectors` carries its own `/api/v1`, so nesting it inside this router's `/api/v1` -# produced **`/api/v1/api/v1/connectors/directory`**: a live, session-gated, entirely dead path -# that nothing links to and every route audit has to explain. MEASURED before removal: 123 served -# paths, exactly 1 of them doubled. -# ⚠ `routes_oauth` above is NOT the same case and stays: `main.py` does not mount it, so this -# router is genuinely its only door. Deleting it because its neighbour was wrong is how a real -# route dies for a tidy-up. -import routes_connectors # noqa: E402,F401 - -#: The registry key this surface carries (C-AUTONAV — A adds the row; the gate is live now, so -#: the day the row lands the wall is already the one that was tested). -MODULE = "automation" - -_GATE = module_gate(MODULE) - - -def _wire(defn, tenant): - """One automation, as the client reads it. `running` is PROCESS state, never store state — - see the engine header on why a persisted 'running' is a permanent lock.""" - live = engine.running(tenant, defn.get("id")) - sched = defn.get("schedule") or {} - nxt = engine.next_fire(sched.get("cron")) if sched.get("enabled") else None - status = dict(defn.get("status") or {}) - if live: - status = {**status, "state": "running", "startedAt": live.get("startedAt"), - "step": live.get("step")} - return { - "id": defn.get("id"), "name": defn.get("name"), "kind": defn.get("kind"), - "config": defn.get("config") or {}, "schedule": sched, "status": status, - "runs": list(defn.get("runs") or [])[:engine.MAX_RUNS], - "created": defn.get("created"), "createdBy": defn.get("createdBy"), - "nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "", - "running": bool(live), - # ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the - # vendor". THE MONEY GUARD, and it was INERT for a whole wave because this line was - # missing: the client declared `awaitingResults` REQUIRED and read it in `runBlock()`, - # nothing ever sent it, so `undefined` was falsy and the Run button stayed armed. - # - # ⛔ WHY IT IS NOT `running`. `running` above is PROCESS state (`engine.running`) and is - # FALSE for the entire 20–30 MINUTE vendor wait, which IS the hazard window: the ticks - # are 3 minutes apart, so between them the automation is idle and the button re-arms. - # MEASURED the expensive way on 2026-08-06 (~$0.025): with no snapshot outstanding a - # Run-now press returned 200 and started a BRAND-NEW billable corpus search. - # - # ⛔ AND NOT A PERSISTED "running" FLAG EITHER — the engine header forbids one (it - # outlives the process and locks the automation forever). `state.pendingSnapshot` is the - # field that already exists, is already persisted, is already carried untouched through - # `clean_definition`, and is the SAME expression `pending_collect_ids()` uses to decide - # what to collect. One truth, two readers. - "awaitingResults": bool(str((defn.get("state") or {}).get("pendingSnapshot") - or "").strip()), - # C3 (wave 22): the trigger config rides whole — the webhook token included, because - # the person configuring the external caller has to be shown the URL somewhere, and - # this payload is session-gated behind the same wall as everything else here. - "trigger": defn.get("trigger") or None, - "statusNote": defn.get("statusNote") or "", - # The one-sentence summary (airtable-brief rec 7), composed from the definition so it - # cannot describe steps the engine does not run. - "sentence": engine.compose_sentence(defn), - # THE CANVAS TOPOLOGY (R5) rides with the automation rather than being rebuilt in the - # client, for the same reason `cronPresets` does: the engine that RUNS the steps is the - # only thing entitled to say what the steps are. - "graph": engine.graph(defn), - # ⭐ WAVE 23 (C4/C5) — THE BUILDER'S OWN STATE, and its absence here was a silent-drop - # bug caught in review rather than by a gate: `flow` was stored, patchable and validated, - # but never sent back. B would have saved a flow through PATCH, got a 200, and watched - # every action vanish on reload — the classic "it didn't save" with nothing red anywhere. - "flow": defn.get("flow") or {"actions": []}, - # ⭐⭐ WAVE 32 · T45 (owner item 10) — CONFIGURED / UNCONFIGURED, per action, on the wire. - # - # ⛔ THE SERVER SAYS IT, not the client, and that is the whole reason it is here: the same - # `engine.action_needs` answers this label, `engine.run_refusal`'s 400 and `run_now`'s own - # refusal, so a card cannot read "Configured" over an action the run will refuse. Three - # readers, one predicate — the alternative is a client-side rule that agrees with the - # server until somebody adds a key to one of them (`awaitingResults` above is this file's - # own record of what the other shape costs). - # ⚠ IT IS A LIST, KEYED BY ACTION ID, and it names the NESTED actions too — an unconfigured - # step inside an If / then branch is exactly the one a person cannot see. - # ⚠ COSTS NO STORE READ. `action_needs` is pure over `(kind, config)`; this route is the - # one W30-T12 took the `user_tables` deep copy off, and a label is not worth putting it - # back. What needs the target database's schema (an enrich binding resolved by the profile - # FLAG) stays a run-time refusal — see `ACTION_REQUIRED`'s note. - "unconfigured": engine.unconfigured_actions(defn), - } - - -def _triggers_vocab(session): - """C3's server-owned trigger list: `[{key, label, ready, needs, planned}]`, keys EXACTLY - `engine.TRIGGER_KEYS` + `engine.TRIGGER_PLANNED`. `ready:false` + `needs` renders as a - not-configured state — never a dead control, never a client-side union. - - ⭐ WAVE 23 (R2): the list is the WHOLE Airtable-parity vocabulary, and the two triggers we - have not built ride it with `planned: true`. That is the honest version of "show all, wire - eight": the picker paints them faded with a reason instead of a shorter list that quietly - implies the missing ones do not exist. `clean_trigger` refuses them, so the faded state is - enforced at the door and not merely in the client's `disabled` attribute. - """ - tick_on = _tick_state()["enabled"] - g = oauth_connect.status(session.runtime, session.uname).get("google") or {} - email_ready = bool(g.get("configured")) and bool(g.get("connected")) \ - and not g.get("reconnect") - email_needs = "" if email_ready else ( - "connect_gmail" if g.get("configured") else "configure_google") - per = { - "manual": (True, ""), - "schedule": (tick_on, "" if tick_on else "arm_tick"), - "event_field": (True, ""), - "record_updated": (True, ""), - "record_created": (True, ""), - "enters_view": (True, ""), - "webhook": (True, ""), - "email": (email_ready, email_needs), - "form_submitted": (True, ""), - # ⭐ WAVE 24 (C-TRIG) — Instagram discovery, now a trigger. Readiness is the vendor key, - # the same bit `paidReady` carries: with no key the search door is closed and the picker - # must say so rather than offering a control that silently finds nothing. `clean_trigger` - # still ACCEPTS it either way — readiness is a deployment fact, not a validity one, which - # is the same split `email` already makes. - "ig_profile_match": (engine.bd_ready(), "" if engine.bd_ready() else "configure_brightdata"), - # ⭐ WAVE 29 (D-9 / R1) — TikTok, live. ⛔ IT NEEDS ITS OWN ROW EVEN THOUGH THE ANSWER IS - # IDENTICAL, and the reason is the `.get(k, (True, ""))` default below: a trigger this dict - # forgets is reported READY, so a deployment with no vendor key would offer TikTok search - # as configured and the search would find nothing. Same key, same readiness bit, stated. - "tiktok_profile_match": (engine.bd_ready(), - "" if engine.bd_ready() else "configure_brightdata"), - } - #: ⭐ WAVE 24 — the server's own one-line description per trigger. C-TYPES: the picker renders - #: THIS under the option, because a CLIENT paraphrase of a server vocabulary is a second copy - #: of it, free to drift. Absent = the client shows nothing, never something invented. - detail = { - "manual": "It runs only when you press Run now", - "schedule": "It runs on a repeating schedule", - "event_field": "A record in the database starts matching a condition you set", - "record_updated": "Any of the columns you watch is changed", - "record_created": "A new record is added to the database", - "enters_view": "A record starts appearing in a saved view", - "webhook": "Something outside calls this automation's URL", - "email": "A message arrives in the connected mailbox", - "form_submitted": "Somebody submits one of this database's forms", - "ig_profile_match": "Search Instagram for profiles matching your filters, on a schedule", - "button_clicked": "Somebody presses a button on a record", - "comment_added": "Somebody comments on a record", - "web_page_changed": "A page you are watching is different from last time", - "tiktok_profile_match": "Search TikTok for profiles matching your filters, on a schedule", - } - - def _taxonomy(k): - """⭐ WAVE 25 · C2 — the four taxonomy keys, composed from the ENGINE's maps. - - ⛔ `.get(k) or FALLBACK`, NEVER `TRIGGER_GROUP_OF[k]`. The first draft of this indexed the - map on the reasoning that a default is how a Connector trigger quietly appears under - Database — and the gate rejected it, correctly, against the incident `per.get(k, ...)` - eight lines below records: a key added to `TRIGGER_KEYS` without remembering a dict beside - it raised KeyError and took `GET /automations` down, i.e. the whole surface, which polls - this every 2.5 s. A mis-grouped row is cosmetic; a 500 is not, and the ranking is not - close. - ⚠ THE CLASSIFICATION IS STILL MANDATORY — it is enforced at the GATE (no shipped trigger - may land in `other`) rather than at the request. Soft here, hard there. - """ - g = engine.TRIGGER_GROUP_OF.get(k) or engine.TRIGGER_GROUP_FALLBACK - return {"group": g, - "groupLabel": engine.TRIGGER_GROUPS[g]["label"], - "groupOrder": engine.TRIGGER_GROUPS[g]["order"], - # The SUB-group inside "Connector"; None everywhere else. ⚠ Group by this key, - # render its label — it is NOT a connector-directory slug (see the engine's note). - "connector": engine.TRIGGER_CONNECTOR.get(k), - # D-55: "the cron drives this one". Derived from the engine's schedule set MINUS - # `manual`, so the client's `CRON_DRIVEN_TRIGGERS` copy can be deleted. - "schedules": k in engine.TRIGGER_CRON_KEYS} - - out = [] - for k in engine.TRIGGER_KEYS: - # ⚠ `.get` WITH A DEFAULT, NOT `per[k]`. This loop walks the ENGINE's vocabulary and - # indexed a hand-maintained dict beside it: adding a key to `TRIGGER_KEYS` without - # remembering this dict raised KeyError and 500'd `GET /automations` — the payload the - # whole automation surface polls every 2.5 s — with every gate and `tsc` still green. - # Defaulting to "ready, needs nothing" is the honest fallback: a trigger the engine - # offers and this route has no readiness opinion about is simply available. - ready, needs = per.get(k, (True, "")) - row = {"key": k, "label": engine.TRIGGER_LABELS[k], "ready": ready, "needs": needs, - "planned": False, "detail": detail.get(k, ""), - # A3(3): the connect affordance is SERVER-COMPOSED — the client never maps a - # `needs` token to a route, so B's CONNECT_PROVIDERS shim deletes itself. - "connect": None, **_taxonomy(k)} - if not ready and needs == "connect_gmail": - row["connect"] = {"provider": "google", - "startUrl": "/api/v1/oauth/google/start"} - out.append(row) - for k in engine.TRIGGER_PLANNED: - out.append({"key": k, "label": engine.TRIGGER_LABELS[k], "ready": False, - "needs": "coming_soon", "planned": True, "connect": None, - "detail": detail.get(k, ""), **_taxonomy(k)}) - return out - - -def _tick_state(): - """⭐ WAVE 21 (C6 amendment A1) — can a SCHEDULE fire on this deployment? - - TWO independent paths can: the in-process scheduler (`AIOS_AUTOMATIONS=1`, - `automation_engine.py` module bottom) and an external cron POSTing `/automations/tick`, - gated on `AIOS_AUTOMATION_TICK_TOKEN` (AWS EventBridge in production). The Step-1 Trigger - card must be honest in both directions: "schedules won't fire" on a deployment where - EventBridge demonstrably fires them daily is the exact lie R9 forbids. `external` means - "the door is OPEN", never "the caller is alive" — the client's copy says so.""" - inproc = os.environ.get("AIOS_AUTOMATIONS") == "1" - ext = bool(os.environ.get("AIOS_AUTOMATION_TICK_TOKEN")) - return {"enabled": bool(inproc or ext), - "source": "in-process" if inproc else ("external" if ext else "")} - - -#: W29-T01 — the Board retirement ran, per tenant, this process. Same shape and same reasoning as -#: `_C8_MIGRATED` below: a one-way cleanup of data no living code path creates any more, whose -#: cost is a full walk of every row-cell in the tenant and whose result on pass 2..N is always -#: "nothing to do". ⚠ A module-global keyed by tenant is deliberately NOT reset by a table create -#: or delete — a new database cannot contain the legacy stage cells this retires. -_BOARD_RETIRED = set() - -#: ⭐⭐ WAVE 30 · T12 — THE PICKER DEFAULT, MEMOISED. `{tenant: (stamp, table_key)}`, the shape -#: `scope_cache` stores. -#: -#: ⛔ WHY THE DERIVED STRING AND NOT THE DOCUMENT. The obvious cache here is the `user_tables` -#: bucket itself, and it is the wrong one: that bucket is up to 35.8 MB per tenant and this box is -#: the HF free tier, so caching it would trade a latency problem for a memory one. What the warm -#: path actually needs is `discover_default_table`'s ANSWER — one short string. -#: -#: ⛔ AND WHY NOT A PLAIN `_BOARD_RETIRED`-STYLE ONCE-PER-PROCESS SET, which would have been less -#: code: the election reads which profile databases exist and which hold rows, and BOTH change -#: while the process lives (somebody creates a database, an automation writes the first row). A -#: once-per-process memo would pin the picker's default to whatever was true at boot and never -#: correct itself — a default that disagrees with the save door, which is exactly the wave-25 R2 -#: defect the `"table"` line's own comment records. A TTL bounds the staleness instead. -#: -#: ⚠ `scope_cache` rather than a hand-rolled dict, because it is the house pattern for precisely -#: this (`routes_customers`, `routes_products`, `pages` all use it) and it is stale-while-refresh: -#: once a copy exists NO request blocks on a rebuild. Automation was the one module importing it -#: nowhere, which the wave-30 scout named as the reason every other surface feels fast. -_DISCOVER_DEFAULT = {} -#: 5 minutes — the same order as `apiBridge.ts:CUSTOMERS_FRESH_MS` on the client. Overridable so a -#: gate can pin it rather than sleep. -_DISCOVER_DEFAULT_TTL = float(os.environ.get("AIOS_AUTOMATION_DEFAULT_TTL") or 300) - - -#: The one store key `_LentTables` intercepts. DERIVED from the engine's own constant rather than -#: written out here: `engine.ut_all` reads the bucket through it, so if that key ever moves, the -#: lend moves with it instead of silently becoming a pass-through that still looks correct. -_UT_STORE_KEY = engine.UT_STORE_KEY - - -def _LentTables(runtime, tables): - """⭐⭐ WAVE 30 · T13, NOW W31-C1 — a read-only `st` that serves ONE already-read `user_tables` - document and passes every other key straight through to the real runtime. - - ⭐⭐ WAVE 31 · C1 — THE BODY IS NOW `core.user_tables.lend`, AND THE CLASS THAT USED TO BE HERE - IS GONE. W30's version carried its own note that the general fix was unavailable at the time: - *"adding a `tables=` parameter to `may_open` would mean editing platform/core/user_tables.py, - which belongs to another lane this wave."* It is session B's lane THIS wave, they built the - generalisation (`user_tables.lend` + the `_LENDABLE` allow-list) for W31-T10, and this is the - second caller adopting it rather than becoming a third copy of the shape. - - ⭐ AND THE SWAP FIXES SOMETHING THIS FUNCTION NEVER COVERED. The old class intercepted exactly - one key, so `may_open`'s LAST branch — `shares.may_see` → `role_for` → `st.get('object_shares')` - — still took a fresh read PER TABLE. `_LENDABLE` covers both buckets, so a shared database no - longer costs a second whole-document read per row of the picker. The name is kept because - `verify_automation`'s NC66 pins this symbol as the thing it replaces to restore the `1 + N` - state; keeping it a function means that control still has exactly one seam to swap. - - ⛔ THE OBVIOUS FIX IS STILL THE FORBIDDEN ONE. Inlining the creator-or-admin test here would - remove the N reads and re-create the exact defect wave 20 fixed: this route USED to carry its - own wider rule (`createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a - database in the picker and was refused the moment they opened it. `may_open` stays THE one - resolver, unmodified and still called per table; it is simply no longer charged for a document - the caller is already holding. - """ - # ⚠ IMPORTED HERE, not at module scope — `core` is deliberately kept out of this module's - # import-time graph (the same reason `automation_tables` does it locally 300 lines down). - import core.user_tables as _ut - return _ut.lend(runtime, **{_UT_STORE_KEY: tables}) - - -def _discover_default(session, tables): - """The discovery picker's default table for this tenant — WITHOUT a bucket read on a warm call. - - ⚠ `tables` is the document the caller ALREADY holds on a cold call (the retirement pass reads - one). Passing it through means the cold path elects from the copy it has rather than taking a - second one, so this is never an extra read — only ever a saved one. - """ - if not session.runtime.available(): - return "" - if tables is not None: - # Cold call: the document is in hand. Elect from it and prime the memo in the same pass. - value = engine.discover_default_table(session.runtime, tables=tables) - _DISCOVER_DEFAULT[session.tenant] = (time.time(), value) - return value - return scope_cache.get(_DISCOVER_DEFAULT, session.tenant, _DISCOVER_DEFAULT_TTL, - lambda: engine.discover_default_table(session.runtime)) - - -def warm_default(rt): - """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT. - - ⛔ THE HALF THE MEMO CANNOT FIX, and it is why this exists rather than being a nicety. The memo - above makes calls 2..N free; **call 1 is still a full `user_tables` download**, and it lands on - whoever clicks Automation first after a deploy — the one visitor with no cache anywhere, - waiting on a document whose documented ceiling is 35.8 MB / ~1.4 s, taken under the store's - single lock. That is the owner's *"only automation has a loading screen"* on a cold Space, and - two waves of memoising the warm path could never touch it. Automation was the only module - importing `scope_cache` nowhere AND the only one absent from `_prewarm`. - - ⭐ IT ELECTS, IT DOES NOT CACHE THE DOCUMENT — deliberately, and this is the whole design. - `discover_default_table` reads the bucket once here, in the prewarm daemon thread where nobody - is waiting, and what survives is a DERIVED STRING. Holding the 35.8 MB document resident would - trade a latency the tenant notices for memory the HF free tier does not have, which is the - argument `_DISCOVER_DEFAULT`'s own note makes against caching it. - - ⚠ SAME TTL AS THE REQUEST PATH, not a permanent set: the election reads which databases exist - and which hold rows, and both change while the process lives. Priming by assignment is exactly - what the cold request path already does one function up, so there is one way this memo is - filled, not two. - - Returns the elected key (`""` when the store is unavailable), so a caller can log it rather - than guess whether the warm-up did anything. Called from `main.py:_prewarm` only. - """ - tenant = str(getattr(rt, "key", "") or "") - if not tenant or not rt.available(): - return "" - value = engine.discover_default_table(rt) - _DISCOVER_DEFAULT[tenant] = (time.time(), value) - return value - - -@router.get("/automations") -def list_automations(session: Session = Depends(_GATE)): - """`{automations: [...], kinds: [...], cronPresets: [...]}` — the rail's whole payload. - - The vocabularies ride WITH the list rather than sitting in a client constant: the cron - presets and the kind list are the server's, and a client copy of either is a thing that goes - stale silently (the editor would offer a preset the parser rejects).""" - # ⭐⭐ WAVE 29 (W29-T01, owner item 5: "Automation takes a while to appear"). THE COMPLAINT WAS - # THIS ROUTE, and the cost was never the automations: it read the whole `user_tables` bucket - # THREE TIMES per call — once for the stage-field retirement scan, twice more inside - # `discover_default_table`. That bucket's documented ceiling is 35.8 MB / ~1.4 s to serialize - # (`automation_engine.py` header) and `core/store.py` re-serializes on EVERY `.get()`, hit or - # miss, UNDER THE STORE LOCK — so the reads also serialize behind each other. ~4 s of deep - # copying to render a rail that shows a name and a toggle. - # - # Now: ONE read, lent to both callers. And the retirement SCAN — O(all row-cells in the - # tenant), which hits its own `if not stage_keys: return` only AFTER walking every row of - # every table — runs once per tenant per process, mirroring the `_C8_MIGRATED` guard below. - # - # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent": - # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s - # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can - # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either — - # wave 26's R6 deleted the board that made them. - # ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves). - # ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The - # line below used to be unconditional — every single request deep-copied the tenant's entire - # `user_tables` document (documented ceiling 35.8 MB / ~1.4 s, and `core/store.py:Store.get` - # re-serializes on EVERY `.get()`, hit or miss, UNDER THE STORE LOCK so the copies also queue - # behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two - # consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING. - # A whole-tenant document, per request, to render a rail showing a name and a toggle. - tables = None - if session.runtime.available() and session.tenant not in _BOARD_RETIRED: - tables = engine.ut_all(session.runtime) - _BOARD_RETIRED.add(session.tenant) - # Idempotent Board retirement removes only engine-marked stage fields and legacy Board - # state. User-created Status/Stage columns remain intact. - engine.retire_automation_board_state(session.runtime, tables=tables) - defs = engine.all_definitions(session.runtime) - items = [_wire(d, session.tenant) for _, d in - sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] - return {"automations": items, - # ⭐ WAVE 24 — DERIVED from the engine's own `KINDS`, not a hand-written trio. It was - # three literals that happened to match, which is a second copy of a server - # vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a - # hand list would now be wrong in three ways at once. `creatable` carries R6 onto the - # wire, so the ruling is a fact the client can read rather than one it must remember. - "kinds": [{"key": k, "label": engine.KIND_LABELS.get(k, k), - "creatable": k not in engine.RETIRED_KINDS} - for k in engine.KINDS], - "cronPresets": engine.CRON_PRESETS, - # ⚠ A BOOLEAN, NEVER THE KEY. The surface needs to say "the paid rung is not - # configured" honestly instead of offering a tier that will silently refuse — and - # that needs exactly one bit. Shipping the key itself to a browser would put a - # billable secret in every user's devtools. - "paidReady": engine.bd_ready(), - # THE SOURCE REGISTRY (D-9's seam), on the wire for the same reason `cronPresets` is: - # the module that RUNS a source is the only thing entitled to say what it can do, and - # a client copy of "Instagram can discover, TikTok cannot" goes stale in silence. - "sources": engine.source_status(), - # The discovery vocabulary, likewise server-owned: every name here is MEASURED- - # accepted by the vendor's own validator, so a client that invented one would build a - # query the API rejects. `lead` is the subset seen carrying VALUES on real rows. - # ⭐⭐ WAVE 32 · T46 (D-167) — THE VOCABULARY HAS A PLATFORM, AND `byKind` IS ADDITIVE - # ON PURPOSE. `fields`/`lead` keep INSTAGRAM's 21 and 3, so no stored automation and no - # client that has not adopted this changes behaviour today; `byKind` carries the per- - # corpus answer, and the SERVER already refuses a TikTok predicate naming one of the 16 - # fields TikTok's dataset does not have (`clean_predicates(..., kind)`). The door is - # closed either way — this is what lets the Find panel stop OFFERING them. - # ⚠ Derived through `engine.filter_fields`, the same accessor the validator uses, so - # the published vocabulary and the enforced one cannot drift — which is precisely what - # D-167 was: a route serving 21 names and a validator checking the same 21, both wrong - # about TikTok together, with nothing able to notice. - "discoverByKind": {k: {"fields": list(engine.filter_fields(k)[0]), - "lead": list(engine.filter_fields(k)[1])} - for k in engine.DISCOVERY_KINDS}, - "discover": {"fields": list(engine.BD_FILTER_FIELDS), - "lead": list(engine.BD_FILTER_LEAD), - "operators": list(engine.BD_FILTER_OPS), - "nullaryOperators": list(engine.BD_NULLARY_OPS), - "maxRecords": engine.BD_MAX_RECORDS, - # Wave 22 C4 — ADDITIVE: per-field flags for the toggle rows (the 3 - # PII fields stay structurally absent, R3) + the too-big guard's - # numbers, so the surface can say WHY a filter is refused before the - # server has to. - "filterMeta": engine.filter_meta(), - "guard": {"minNarrowing": engine.BD_MIN_NARROWING, - "maxRecords": engine.BD_MAX_RECORDS}, - # ⛔ D-59's `categoryOptions` IS DELIBERATELY NOT HERE — it is served by - # `GET /automations/discover/categories`. It sat on this payload for one - # commit and that was a real defect: THIS ROUTE IS POLLED EVERY 2.5 s by - # the whole automation surface, and deriving the options reads two user - # tables plus the PLATFORM MASTER, which is a different HF repo — i.e. a - # network round-trip per poll, per open tab. Caught by the gate's own - # output, which started carrying `store:get:master_snapshots` errors from - # a suite whose stated contract is that it touches no network. - # C6/R5's vocabulary, so the seed picker cannot offer a source the - # validator refuses. - "seedSources": list(engine.SEED_SOURCES), - "seedMaxRows": engine.SEED_MAX_ROWS, - # W29-T01: `tables` is the snapshot read once at the top of this handler. - # ⚠ IT WAS TAKEN BEFORE THE RETIREMENT WROTE, and that is deliberate and - # harmless: retirement only removes `stage_auto_*` FIELDS and pops those - # same keys off rows, and the election reads the preset-profile vocabulary - # (`handle` + N preset keys) and whether a table has ANY rows. A stage key - # is in neither set, and no row is ever deleted — so the pre-write - # snapshot and the post-write bucket cannot elect different tables. - # 2026-08-10 — the OFFER must name the table the SAVE will actually use. - # This was the bare `DISCOVER_TABLE` constant while `create`/`patch` now - # resolve a targetless discovery flow to the profile database the tenant - # already has, so the picker would have shown `ut_ig_candidates` and the - # save would have written somewhere else — a default that disagrees with - # itself across two panels, which is the shape wave 25's R2 fixed for - # `targetTable` vs the action's `table`. - # ⭐ WAVE 30 · T12 — through the per-tenant memo. The ELECTION rule and - # everything the comment above says about it are unchanged; what changed - # is that a warm request no longer re-reads a 35.8 MB document to - # recompute a string that did not move. - "table": (_discover_default(session, tables) - or engine.DISCOVER_TABLE)}, - "storeAvailable": bool(session.runtime.available()), - # Wave 22 C3 — the trigger vocabulary, session-scoped because email readiness is a - # per-USER fact (the poll runs through the creator's own Gmail connection). - "triggers": _triggers_vocab(session), - # ⭐ WAVE 23 C4 (R3) — the ACTION MENU, including what we have not built. Each row - # carries `ready`, so B paints "Send email" and "Run script" faded with the server's - # own reason instead of omitting them — the owner asked for Airtable's full menu, and - # a shorter list would imply those actions do not exist. `clean_actions` REFUSES an - # unready kind, so the faded state is a wall rather than a styling choice. - "actionsCatalog": engine.action_catalog(), - # The builder's own vocabulary: how deep a condition tree may nest, how deep groups - # may nest, and the ceilings. B reads these instead of hard-coding the same numbers - # into its "+ Add condition" affordance. - "flow": {"condOps": list(engine.LANE_OPS), - "nullaryCondOps": list(engine.LANE_NULLARY_OPS), - "maxCondDepth": engine.MAX_COND_DEPTH, - "maxCondChildren": engine.MAX_COND_CHILDREN, - "maxGroupDepth": engine.MAX_GROUP_DEPTH, - "maxActions": engine.MAX_ACTIONS, - # ⭐ W29-T09 (owner item 11) — THE POST-GROUP VOCABULARY, on the wire for the - # same reason `cronPresets` is: `clean_post_groups` REFUSES a type it does - # not know, so a client that invented one would build a config the save door - # rejects with a sentence about a word the person never typed. Both halves - # ride — the stored key and the label a person reads — because a client-side - # translation of `video` into "Reels" is a second copy of this list. - "postTypes": [{"key": t, "label": engine.POST_TYPE_LABELS.get(t, t)} - for t in engine.POST_TYPES], - # The ceiling a group's limit is judged against. `clean_post_groups` bounds a - # group by the action's OWN `maxPosts`, not by a constant, so the control can - # only warn honestly if it reads the same number the validator uses. - "maxPostsPerPull": engine.MAX_POSTS_PER_PULL, - # ⭐⭐ W33-T52 (owner item 16) — WHICH CONFIG KEYS A KIND REQUIRES, on the - # wire, for exactly the reason `postTypes` above is: the panel paints a red - # `*` beside a required control, and a client-side table of which keys those - # are would be a SECOND copy of `ACTION_REQUIRED` living in another file. The - # two would agree on the day they were written and diverge the first time a - # kind gained a key — the panel would then mark a control optional that the - # runner blocks on, and the person would read "this is fine" from the one - # surface that is meant to tell them it is not. - # ⛔ BOTH HALVES RIDE, phrase AND key, because the phrase is the ONLY human - # wording of that requirement anywhere ("a column to write into"), and the - # runner's own refusal sentence is built from it. A client that re-worded it - # would give the same requirement two names. - # ⚠ It is `ACTION_REQUIRED`, not `WEB_REQUIRED`: the table is the one the - # run refusal and `unconfigured_actions` already read, so a kind added to it - # later paints its `*` with no client change at all. - "actionRequired": {kind: [{"phrase": phrase, "key": key} - for phrase, key in reqs] - for kind, reqs in engine.ACTION_REQUIRED.items()}}, - # Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see - # `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie. - "tick": _tick_state()} - - -@router.post("/automations/discover/estimate") -def discover_estimate(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """What would this search cost? Shown BEFORE the run, never after. - - ⚠ THE ANSWER IS AN ESTIMATE AND SAYS SO IN ITS OWN PAYLOAD (`basis: "SPEC"`). Bright Data - never returns a price before a run — the funds gate fires first and `price: 0` means "not - priced", not "free" — and this account's token cannot read a balance (`/customer/balance` - answers 403). A number presented as billed would be the invented measurement this whole - module refuses to make. - """ - return engine.discover_estimate((body or {}).get("recordsLimit")) - - -#: C8's migration ran, per tenant, this process. Once is the point: the sweep only writes when -#: an unbound bag exists, so after the first pass this is a read that finds nothing. -_C8_MIGRATED = set() - - -def _table_views(session, key): - """⭐ WAVE 24 (item 8) — the saved views on ONE user table as `[{id, label}]`, or **None** - when the workspace bucket could not be read at all. - - The client said "This server did not offer a view list" because the payload genuinely had - no `views` key. C-TYPES: **absent stays ABSENT.** `[]` is a MEASUREMENT ("this database has - no saved views") and `None` is a STATE ("nobody looked") — collapsing them is precisely how - an empty picker comes to read as a claim about the database. - - ⛔ SOURCED FROM `core.table_store`, WHICH IS THE READER `view_filter` ALREADY USES — not a - second one, the same one, listed instead of looked up. The brief said to source it the way - the grid does; the grid's projection (`aios_grid.views_from_defs`) is the wrong list HERE and - the difference matters: it INJECTS the system view ("All records") and PROJECTS cohorts as - views, and neither of those lives in the stored bucket — so `view_filter` answers "view no - longer exists" for every one of them. A picker built from that projection would offer options - the `enters_view` trigger cannot resolve, which is a worse bug than the missing key it fixes. - (An `enters_view` trigger on "All records" would also mean "fire on every record", so its - absence is correct rather than a gap.) - - ⚠ `consume_corrections=False`: this is a READ for a picker, and the default MUTATES — it - takes and clears the pending field-correction acknowledgement, so listing views would eat a - protocol message meant for the grid. - """ - try: - import core.table_store as table_store - # ⛔ THE REACHABILITY PROBE IS NOT REDUNDANT, and leaving it out was a real defect this - # gate's own NC caught: EVERY `TableStore` accessor wraps its store read in `try/except` - # and returns `{}` on failure. So a bucket that could not be read is indistinguishable - # from one with no views — which collapses exactly the ABSENT/EMPTY distinction this - # function exists to preserve, and the caller would ship `views: []` as a measurement - # nobody took. The one read that is allowed to raise has to be ours. - session.runtime.get(f"{key}_table_workspace") - tops = table_store.make(f"{key}_table_workspace", st=session.runtime) - own = (tops.workspace(session.uname, consume_corrections=False) or {}).get("views") or {} - shared = tops.shared_views(session.uname, session.admin) or {} - except Exception: # noqa: BLE001 - return None - merged = {**shared, **own} # a view lives in ONE home; the merge is belt-and-braces - return sorted( - ({"id": str(vid), "label": str((v or {}).get("name") or vid)} - for vid, v in merged.items() if isinstance(v, dict)), - key=lambda r: r["label"].lower()) - - -@router.get("/automations/tables") -def automation_tables(session: Session = Depends(_GATE)): - """The blank databases an automation can target, with their fields. - - ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the - two can never collide. It exists because the automation editor needs the table+field list to - build a config at all, and D must not block on A's route landing. When C3-UT is live this - keeps working (same bucket) — it is a duplicate reader, never a second writer. - - ⛔ WAVE 20, item 3 — IT NO LONGER MIRRORS THE WALL, IT CALLS IT. This route re-implemented - `may_open` and got it WIDER: it also admitted `createdBy in ('automation', 'scheduler')`, so - a non-admin saw automation-created databases here and was refused the moment they opened, - edited or deleted one. A duplicate READER is fine; a duplicate WALL is not, because the two - only disagree in front of a user. One resolver now, and the engine stamps a human owner so - the merge takes nothing legitimate away (`ut_ensure`, `MACHINE_OWNERS`). - """ - import core.user_tables as user_tables - - # C8 (wave 22): bind pre-law automation bags to the definitions that write them — once - # per tenant per process, and a no-op read after the first real pass. A write-on-read, - # stated out loud: this is the surface whose stale bags mislead (the column picker), so - # it is where the truth gets repaired. - if session.tenant not in _C8_MIGRATED and session.runtime.available(): - _C8_MIGRATED.add(session.tenant) - try: - engine.bind_unbound_fields(session.runtime) - except Exception: # noqa: BLE001 - pass - - out = [] - # ⭐⭐ WAVE 30 · T13 — ONE read of the tenant document, LENT to the wall for every table. - # It was `1 + N` full deep copies (this `ut_all`, then `may_open` → `get` → `all_tables` per - # table), each of a document with a 35.8 MB / ~370 ms ceiling, all of them queued behind - # `Store._lock`. `may_open` is unchanged and still asked about every table — see `_LentTables` - # on why re-implementing the wall here is the one fix that is NOT available. - _tables_doc = engine.ut_all(session.runtime) - _lent = _LentTables(session.runtime, _tables_doc) - for key, t in sorted(_tables_doc.items(), - key=lambda kv: (kv[1].get("label") or "").lower()): - if not user_tables.may_open(key, session.uname, session.admin, st=_lent): - continue - row = {"key": key, "label": t.get("label") or key, - "source": t.get("source") or "Blank", - "rowCount": len(t.get("rows") or {}), - "fields": [{"key": f.get("key"), "label": f.get("label"), - "type": f.get("type") or "text", - "automation": f.get("automation") or None} - for f in (t.get("fields") or [])]} - views = _table_views(session, key) - if views is not None: - row["views"] = views # absent when the bucket did not answer — never [] - out.append(row) - return {"tables": out} - - -@router.get("/automations/presets") -def automation_presets(table: str = "", session: Session = Depends(_GATE)): - """⭐ WAVE 25 · C1 / R2b — the Instagram preset columns, diffed against ONE database. - - `{fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` — the - "already in this database" / "will be created" split the owner asked for, so the Create record - action's configuration can SHOW what pointing it here does before anything is spent. - - ⛔ DECLARED ABOVE `/automations/{auto_id}`, AND THAT IS LOAD-BEARING RATHER THAN TIDY. - FastAPI matches routes in DECLARATION order, so a literal path registered after a sibling - path-parameter route is never reached — this handler would simply never run and - `get_automation` would answer "no automation with that id" for the id `presets`. A 404 with a - plausible sentence is the worst possible failure here, because it reads as "the endpoint is - fine, the data is missing". `/automations/tables` sits above the same route for the same - reason; this follows it rather than inventing a second convention. - - ⚠ WALLED BY `may_open`, LIKE EVERY OTHER TABLE READER. Without it, asking about a table you - cannot open would answer which of its columns exist — a small disclosure, and exactly the - duplicate-wall mistake `automation_tables` records at wave 20. - """ - import core.user_tables as user_tables - - key = str(table or "").strip() - if key and not user_tables.may_open(key, session.uname, session.admin, - st=session.runtime): - raise err(404, "unknown_table", "no such database") - return engine.preset_plan(session.runtime, key) - - -@router.get("/automations/discover/categories") -def automation_categories(session: Session = Depends(_GATE)): - """⭐ DEBT D-59 — the Category combobox's options: the values this deployment has ACTUALLY - SEEN, each with its observed count. `{options: [{value, count}]}`. - - ⛔ ITS OWN ROUTE, ON PURPOSE. This belongs to the Find surface and is asked for when that - panel opens — it must never ride `GET /automations`, which the whole automation surface polls - every 2.5 s: deriving these reads two user tables AND the platform master (a different HF - repo), so on the polled payload it is a network round-trip per tab per poll. - - ⚠ THE CONTROL STAYS A COMBOBOX. These are the values we have seen, not the values that exist. - D-59 is explicit that transcribing Instagram's published taxonomy would be worse than no - dropdown — a filter on a value the corpus does not use returns zero rows and looks exactly - like an honest "no such accounts exist", which misleads precisely when it looks authoritative. - """ - return {"options": engine.observed_categories(session.runtime)} - - -@router.post("/automations/seed/derive") -def automation_seed_derive(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """⭐ WAVE 25 · C6 / R5 — "find me more accounts like the ones in this view". - - Body `{source: "view"|"cohort", table: "", id: ""}` → - `{derived: [...], basis: {rows, fields:[{name,label,value,coverage}], related, note}}` - - ⛔ IT DERIVES AND RETURNS; IT SAVES NOTHING. R5: the derived conditions are "visible and - editable, never hidden" — so the client drops them into the ordinary condition rows, where the - user edits them like anything else, and the ordinary PATCH stores them. A door that both - derived and saved would make the suggestion feel like a decision. - - ⚠ `basis` IS NOT DECORATION AND MUST BE RENDERED. It carries how many rows were read and the - MEASURED coverage of each characteristic, which is the difference between "12 of 12 of these - bios say florist" and "7 of 12 do" — presented identically, the weak one reads as authority. - - ⚠ Declared ABOVE `/automations/{auto_id}` for the reason `/automations/presets` is (FastAPI - matches in declaration order); that ordering is gated. - """ - import core.user_tables as user_tables - - body = body or {} - table = str(body.get("table") or "").strip() - if not table: - raise err(400, "no_table", "name the database to read the seed records from") - if not user_tables.may_open(table, session.uname, session.admin, st=session.runtime): - raise err(404, "unknown_table", "no such database") - source = str(body.get("source") or "view").strip().lower() - if source not in engine.SEED_SOURCES: - raise err(400, "bad_seed_source", - f"a seed comes from one of: {', '.join(engine.SEED_SOURCES)}") - rows, problem = engine.seed_rows(session.runtime, table, str(body.get("id") or "")) - if problem: - raise err(400, "bad_seed", problem) - derived, basis = engine.seed_predicates(rows) - return {"source": source, "table": table, "id": str(body.get("id") or ""), - "derived": derived, "basis": basis} - - -#: The injected chat transport, for gates only — `None` in every shipped path, so the route takes -#: the real ladder. Set by `verify_automation.py` to prove this door end to end with NO API key and -#: NO spend (`routes_query._call_model`'s `chat` argument is the same idea and the same reason). -_DRAFT_CHAT = [None] - - -@router.post("/automations/draft") -def draft_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A DESCRIPTION BECOMES A DRAFT FLOW. - - `{prompt}` → `{draft: {name, trigger, table, actions:[{id,kind,config,why}]}, provider, - dropped: [...], saved: false}` — or a 400 carrying ONE plain sentence. - - ⛔⛔ NOTHING IS SAVED HERE, AND THE `saved: false` ON THE WIRE IS NOT DECORATION. T54's - `done-when` is *"gets back a DRAFT flow they can see BEFORE anything is saved"*; a door that - wrote first and showed second would satisfy every check about the flow's contents and none about - the promise. Accepting a draft is the ORDINARY `POST /automations` — R3's *"indistinguishable - from a hand-built one"* is achieved by there being no second write path, not by making the - second one look similar. - - ⛔⛔ CONTRACT C7, AND THIS ROUTE IS WHERE IT IS ENFORCED: *"the output must pass `clean_actions` - unchanged."* So it is RUN, here, before the person ever sees the draft — and the result is - DIFFED against what the model wrote. `clean_actions` has no disclosure channel (D-75): it drops - a config key it does not recognise and answers 200. Without this diff a person would accept a - flow, open it, and find a step configured differently from the one they were shown, with nothing - anywhere having said so. `dropped` is that channel, built here because this is the first caller - that needed one. - ⚠ A draft whose actions `clean_actions` REFUSES outright is a 400, never a partial flow: half a - flow presented as a whole one is the one outcome worse than a refusal. - - ⚠ DECLARED ABOVE `/automations/{auto_id}`, and that is load-bearing rather than tidy — FastAPI - matches in DECLARATION order, so a literal path registered after a sibling path-parameter route - is never reached, and `get_automation` would answer *"no automation with that id"* for the id - `draft`. The same reason `/automations/tables` and `/automations/presets` sit up here. - """ - prompt = str((body or {}).get("prompt") or "").strip() - if not prompt: - raise err(400, "no_prompt", "type what you want the automation to do") - # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open` - # per table, so the model is shown exactly the databases this caller may already see and cannot - # name one they were not granted — the permission wall re-used, never a second one built beside - # it. (`QueryPage`'s `granted` prop carries the same clause on the client side.) - tables = (automation_tables(session) or {}).get("tables") or [] - draft, refusal, provider = ai_review.draft_flow( - prompt=prompt, - catalog=engine.action_catalog(), - required=engine.ACTION_REQUIRED, - triggers=_triggers_vocab(session), - tables=tables, - chat=_DRAFT_CHAT[0]) - if refusal or not draft: - raise err(400, "draft_refused", refusal or "no automation could be drafted from that") - - # ── C7: run the real save-door validator and DIFF it ────────────────────────────────── - wrote = draft.get("actions") or [] - cleaned, why = engine.clean_actions([{"kind": a.get("kind"), "config": a.get("config") or {}} - for a in wrote]) - if why or cleaned is None: - raise err(400, "draft_invalid", - f"the assistant produced a flow this deployment will not accept: {why}") - dropped = [] - for before, after in zip(wrote, cleaned): - b_cfg = before.get("config") or {} - a_cfg = after.get("config") or {} - gone = sorted(k for k in b_cfg - if k not in a_cfg or str(a_cfg.get(k)) != str(b_cfg.get(k))) - if gone: - dropped.append({"kind": str(after.get("kind") or ""), "keys": gone}) - # ⛔⛔ WHAT IS STILL MISSING FROM EACH STEP, PER STEP — and this is the fix for the sharpest - # thing the verifier found. The system prompt TELLS the model *"leave one blank rather than - # inventing a web address, a CSS selector or a column name"* (which is right), and the panel - # skips empty values (which is also right, on its own). Together they meant a `web_read` with - # no selector and no target column painted as a FINISHED step — url, attr, timeout, nothing - # else — and the ordinary create door then accepted it. A person approved a step that reads - # nothing into nowhere, having been shown no blank at all. - # ⚠ `engine.action_needs` IS THE PREDICATE, not a second list: the same function the builder's - # Configured/Unconfigured label and the run refusal read. Three readers, one answer. - _tables_by_key = {str(t.get("key")): t for t in tables} - _target_cols = {str(f.get("key")) for f - in (_tables_by_key.get(str(draft.get("table") or "")) or {}).get("fields") or []} - shown = [] - for i, after in enumerate(cleaned): - row = dict(after, why=str((wrote[i] or {}).get("why") or ""), - needs=engine.action_needs(after)) - # ⛔ AND A COLUMN NAME THE DATABASE DOES NOT HAVE. `field` is a free string that nobody - # validated at draft, at accept or at store — so a model writing the column's LABEL - # ("Price") instead of its key ("price"), which is exactly what a person would say and an - # obedient model would echo, stored a write into a column that does not exist. No schema - # was violated; the flow was well-formed and did something other than what was asked. - _f = str((after.get("config") or {}).get("field") or "") - if _f and _target_cols and _f not in _target_cols: - row["unknownField"] = _f - shown.append(row) - # ⚠ A TRIGGER THE CREATE DOOR WOULD REFUSE IS CORRECTED HERE, NOT SHOWN AND THEN 400'd. The - # enum makes this need a model that ignores its own schema, but the failure mode is ugly: the - # draft paints, the person clicks Accept, and the save refuses with a sentence about a word - # they never chose. Falling back to `manual` and SAYING SO keeps the draft usable. - _trig = str(draft.get("trigger") or "").strip() - _notes = [] - if _trig and _trig not in tuple(engine.TRIGGER_KEYS): - _notes.append(f"the assistant asked for a trigger this deployment does not have " - f"({_trig}) — set to manual instead") - _trig = "" - _asked = int(draft.get("asked") or len(wrote)) - if _asked > len(shown): - _notes.append(f"the assistant wrote {_asked} steps and a draft carries at most " - f"{ai_review.MAX_DRAFT_ACTIONS}; the last {_asked - len(shown)} were not " - f"kept. Describe the job in two automations, or shorten it.") - return {"draft": {"name": draft.get("name") or "New automation", - "trigger": _trig or "manual", - "table": draft.get("table") or "", - "actions": shown}, - "provider": provider, "dropped": dropped, "notes": _notes, "saved": False} - - -@router.get("/automations/{auto_id}") -def get_automation(auto_id: str, session: Session = Depends(_GATE)): - defn = engine.all_definitions(session.runtime).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - return {"automation": _wire(defn, session.tenant)} - - -@router.post("/automations") -def create_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """Create an automation. ⭐ WAVE 23 (C2): the body may carry - `target: {mode: "existing"|"new"|"automated", table?, label?}` — the wizard's FIRST question, - answered before the kind. `new` mints the blank database in the same call, so the automation - is never saved pointing at a table that does not exist yet.""" - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - # ⭐⭐ D-75 — THE DISCLOSURE CHANNEL, CONSUMED. `clean_actions` silently drops a - # `create_record` condition (D-65's law: dropping is recoverable, refusing locks the door) and - # any config key an arm's allowlist does not keep. Both are deliberate; what was missing was - # anybody being TOLD, so a person saved a flow and got a different one with nothing said. - # ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the - # person is looking — a note in a log they never open is the same silence with extra steps. - _notes = [] - defn, error = engine.create(session.runtime, body or {}, username=session.uname, - notes=_notes) - if error: - raise err(400, "invalid_automation", error) - return {"automation": _wire(defn, session.tenant), "notes": _notes} - - -@router.patch("/automations/{auto_id}") -def patch_automation(auto_id: str, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - # D-75, same channel as `create` above — an EDIT is where this matters most, because the - # person has just typed the thing that gets dropped. - _notes = [] - defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname, - notes=_notes) - if error: - raise err(400 if error != "no such automation" else 404, - "invalid_automation" if error != "no such automation" else "unknown_automation", - error) - return {"automation": _wire(defn, session.tenant), "notes": _notes} - - -@router.delete("/automations/{auto_id}") -def delete_automation(auto_id: str, session: Session = Depends(_GATE)): - """Delete the DEFINITION. ⚠ The database it filled is NOT touched — an automation is the - thing that writes rows, not the thing that owns them, and deleting a job must never be a way - to lose data (the same rule the orphan count encodes).""" - if engine.running(session.tenant, auto_id): - raise err(409, "automation_running", "it is running — wait for it to finish") - engine.remove(session.runtime, auto_id) - return {"deleted": str(auto_id)} - - -@router.post("/automations/preview") -def preview_source(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """What does this URL actually offer? The field-map step — never writes anything.""" - body = body or {} - url = str(body.get("url") or "").strip() - if not url: - raise err(400, "no_url", "give a page URL to read") - try: - return engine.preview(url, str(body.get("extract") or "table"), - int(body.get("tableIndex") or 0)) - except engine.Refused as e: - raise err(400, "refused_url", str(e)) - except Exception as e: # noqa: BLE001 - raise err(502, "fetch_failed", f"could not read that page — {type(e).__name__}: " - f"{str(e)[:160]}") - - -@router.post("/automations/{auto_id}/run") -def run_automation(auto_id: str, session: Session = Depends(_GATE)): - """Start a run on a background thread. 409 when one is already in flight.""" - defn = engine.all_definitions(session.runtime).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - # ⛔ WAVE 32 · T45 (owner item 10) — REFUSED, NAMING THE ACTION. `run_now` refuses too, for the - # tick and the webhook; this one exists so the person who pressed the button reads the reason - # instead of watching a run start and end with nothing done. The two ask the SAME function, so - # they cannot come to disagree about what "configured" means. - refusal = engine.run_refusal(defn) - if refusal: - raise err(400, "action_unconfigured", refusal) - if not engine.run_async(session.runtime, session.tenant, auto_id, username=session.uname): - raise err(409, "automation_running", "that automation is already running") - return {"started": str(auto_id), "startedAt": time.strftime("%Y-%m-%dT%H:%M:%S")} - - -@router.post("/automations/{auto_id}/nodes/{node_id}/toggle") -def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depends(_GATE)): - """Flip one step on the canvas. The SERVER decides what a node's switch means (see - `engine.NODE_TOGGLES`) — the client only reports which node was clicked. - - A node with no switch answers 400 with the sentence saying why, rather than silently doing - nothing: a control that appears to work and does not is worse than one that refuses. - """ - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - if engine.running(session.tenant, auto_id): - raise err(409, "automation_running", "it is running — wait for it to finish") - defn, error = engine.toggle_node(session.runtime, auto_id, node_id) - if error: - raise err(404 if error == "no such automation" else 400, - "unknown_automation" if error == "no such automation" else "node_not_toggleable", - error) - return {"automation": _wire(defn, session.tenant)} - - -@router.post("/automations/{auto_id}/hook/{token}") -async def automation_hook(auto_id: str, token: str, request: Request): - """C3's webhook trigger — the tick-endpoint pattern one level down: unauthenticated BY - DESIGN (the external caller has no cookie), gated on a per-automation token minted when the - trigger was configured, constant-time compared. The tenant is FOUND by the (id, token) - pair — a wrong token answers 403 for every tenant, so the route confirms nothing about - which slugs exist. - - ⭐ WAVE 24 (D-41) — THE BODY IS READ DEFENSIVELY AND IS NEVER A REASON TO REFUSE. - ⛔ It is deliberately NOT declared as `body: dict = Body(...)`, which is the obvious way to - write this and would be a live regression: FastAPI would then VALIDATE the payload, so an - existing caller posting text, form-encoding, an empty body or slightly malformed JSON would - start getting a 422 from a door that has accepted anything since wave 22. A webhook sender is - somebody else's system; we do not get to change what it must send in order to fire a flow. - An unreadable body simply maps nothing — the flow still fires, exactly as it did before. - - `run_in_threadpool` keeps the store I/O off the event loop: `hook_fire` walks every tenant - and may commit a row, and this handler had to become `async` only to read the request body. - """ - from starlette.concurrency import run_in_threadpool - from harness import runtime as _rt - - try: - payload_in = await request.json() - except Exception: # noqa: BLE001 - payload_in = None - - def _fire(): - last = (404, {"error": "unknown_automation", - "message": "no automation with that id and token"}) - for slug in _rt.known_tenants(): - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - status, payload = engine.hook_fire(rt, slug, auto_id, token, body=payload_in) - if status == 200: - return 200, payload - if status != 404: - last = (status, payload) - return last - - status, payload = await run_in_threadpool(_fire) - if status == 200: - return payload - raise err(status, str(payload.get("error") or "refused"), - str(payload.get("message") or "refused")) - - -@router.post("/automations/tick") -def tick(request: Request, x_aios_tick_token: str = Header(default="")): - """Fire every due schedule, for every tenant. The durable-cron entry point (R5). - - Unauthenticated BY DESIGN and gated on a shared secret instead — an EventBridge rule has no - cookie. Refuses when the secret is not configured (see the module header).""" - want = os.environ.get("AIOS_AUTOMATION_TICK_TOKEN") or "" - if not want: - raise err(403, "tick_disabled", - "AIOS_AUTOMATION_TICK_TOKEN is not configured — the tick endpoint is closed") - got = x_aios_tick_token or request.headers.get("X-AIOS-TICK-TOKEN") or "" - if got != want: - raise err(403, "bad_tick_token", "that token is not valid for this deployment") - started = engine.tick_all() - return {"started": started, "at": time.strftime("%Y-%m-%dT%H:%M:%S")} - - -@router.post("/automations/ig/purge") -def purge_ig_subject(body: dict = Body(default=None), session: Session = Depends(_GATE)): - """D-24 (wave 22): the right-to-erasure door for ONE Instagram subject — walks the - tenant's four `ut_ig_*` tables AND the platform master (R2 pooled a copy there, so a purge - that skipped it would not be erasure). Admin-only: erasure is a compliance act, not a - grid gesture. Answers the per-table removal counts — the one aggregate whose drill is the - rows' ABSENCE.""" - if not session.admin: - raise err(403, "admin_only", "erasing a subject is an admin action") - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was " - "purged") - handle = str((body or {}).get("handle") or "").strip() - if not handle: - raise err(400, "no_handle", "name the Instagram handle to erase") - counts = engine.purge_subject(session.runtime, handle) - return {"handle": handle.lstrip("@").lower(), "removed": counts, - "total": sum(counts.values())} - - -@router.get("/automations/metrics/{table_key}/{field_key}/{row_id}/rows") -def metric_drill(table_key: str, field_key: str, row_id: str, - session: Session = Depends(_GATE)): - """C7's drill: the EXACT master snapshot rows behind one metric cell - ([[no-unverifiable-aggregates]]) — recomputed on ask with the same function that filled - the cell, so the drill can never disagree with the number by construction.""" - import core.user_tables as user_tables - - if not user_tables.may_open(table_key, session.uname, session.admin, st=session.runtime): - raise err(404, "unknown_table", "no such database") - t = engine.ut_get(session.runtime, table_key) or {} - fdef = next((f for f in (t.get("fields") or []) if f.get("key") == field_key), None) - if not fdef or not isinstance(fdef.get("metric"), dict): - raise err(404, "not_a_metric", "that column is not a metric field") - row = (t.get("rows") or {}).get(str(row_id)) - if row is None: - raise err(404, "unknown_row", "that record is not in the database") - import ig_master - url_field = next((f.get("key") for f in (t.get("fields") or []) - if f.get("type") == "url"), "") - handle = engine._table_handle(row, url_field) - bag = fdef["metric"] - series = ig_master.series_for({handle}) if handle else {} - value, rows = engine.metric_value(series.get(handle), bag.get("measure"), - bag.get("window"), bag.get("agg") or "") - return {"table": table_key, "field": field_key, "rowId": str(row_id), "handle": handle, - "measure": bag.get("measure"), "window": bag.get("window"), - "value": value, "rows": rows[:200], - "note": "" if value is not None else - "no master data answers this window — the cell is honestly blank"} - - -@router.get("/automations/{auto_id}/rows") -def run_rows(auto_id: str, session: Session = Depends(_GATE)): - """The rows the LAST run touched — the drill-down behind a run's counts. - - Every count in this product drills to the exact rows behind it ([[no-unverifiable-aggregates]]); - a run history that said "412 updated" and could not show which would be the thing that rule - exists to forbid. - """ - defn = engine.all_definitions(session.runtime).get(str(auto_id)) - if defn is None: - raise err(404, "unknown_automation", "no automation with that id") - last = (defn.get("runs") or [{}])[0] - table_key = (defn.get("config") or {}).get("targetTable") or "" - t = engine.ut_get(session.runtime, table_key) or {} - rows = t.get("rows") or {} - ids = [str(i) for i in (last.get("affected") or [])] - return {"table": table_key, "label": t.get("label") or table_key, - "fields": [{"key": f.get("key"), "label": f.get("label")} - for f in (t.get("fields") or [])], - "rows": [{"id": i, **(rows.get(i) or {})} for i in ids if i in rows], - "truncated": len(ids) >= 200} - - -# --- the in-process scheduler ------------------------------------------------------------ -# ⚠ OPT-IN (`AIOS_AUTOMATIONS=1`), and that is an AMENDMENT to the wave brief's "default-on", -# made on a measurement: `verify_api.py` runs for 196 s, i.e. longer than three tick intervals, -# and `tick_all` reaches `runtime.get_runtime()` — which builds tenants and moves the LRU cache -# that verify_api's own isolation checks read. Default-on would put a background thread inside -# the subject of another session's gate. `AIOS_PREWARM=1` at `main.py:353` is the same decision -# for the same reason, so this follows it rather than inventing a second convention. -# -# The loop also SLEEPS FIRST (`engine.scheduler_loop`) — defence in depth, proven in -# verify_automation.py section T rather than assumed. -# -# ⛔ THE DEPLOY MUST SET `AIOS_AUTOMATIONS=1` (with `AIOS_AUTOMATION_TICK_TOKEN`) or schedules -# only ever fire from the external cron POSTing /automations/tick. Both paths work; neither is -# implicit. Booked in the session-D mailbox. -engine.start_scheduler() - -# C3 (wave 22): register the trigger listener onto the platform's row-event seam. THIS module -# is where the registration belongs — it is the one place that already imports both sides, so -# neither the engine nor platform/core grows a dependency on the other. Idempotent: a reimport -# must not double-fire every trigger. -import core.user_tables as _ut_hooks # noqa: E402 - -if engine.grid_hook not in _ut_hooks.ROW_HOOKS: - _ut_hooks.ROW_HOOKS.append(engine.grid_hook) - -# ⭐⭐ W31 QA — DECLARE THE MACHINE-OWNED CHILD DATABASES, for the same reason and in the same -# place as the hook above: this module already imports both sides, so neither the engine nor -# `platform/core` grows a dependency on the other. Idempotent by construction (a set). -# -# ⛔ THE OWNER FOUND WHAT THIS FIXES, IN PRODUCTION, AFTER THE TICKET READ GREEN. W31-T32 locked -# the TikTok children by stamping `recordMode` at the two SPAWN sites, and proved the stamp -# arrives "on the next write". That is true and it is not the `done-when`: every TikTok database -# already sitting in a tenant kept offering "+ New record" until somebody re-ran a TikTok -# automation, and nobody had. Owner, verbatim (2026-08-13): *"the databases for Tiktok do not have -# the small lock icon as I asked"* — and the rule, restated: *"just like Instagram Post database -# (which is locked), only the IG Profile and TT Profile should be editable."* -# ⚠ A DECLARATION NEEDS NO WRITE, so it is true for EVERY tenant the moment the API boots — no -# migration, no boot ordering, no per-tenant walk, and nothing that a stale store can undo. The -# stored flag still locks a table nobody declares; the two are OR'd. -_ut_hooks.register_locked_records(engine.LOCKED_CHILD_TABLES) +"""routes_automation.py — wave-18 item 5 (contract C4-AUTO): the automation surface's API. + +The router is thin on purpose: everything that can be wrong about an automation — a malformed +cron, a URL the SSRF rail refuses, a key field that is not one of the mapped columns — is decided +in `automation_engine`, which is a pure-ish module a gate can drive without a server. This file +does auth, shape and status codes. + +⛔ THE TICK ENDPOINT IS THE ONE UNAUTHENTICATED ROUTE, AND IT IS FAIL-CLOSED TWICE OVER. It takes +no session (an external cron has no cookie), so it is gated on a shared secret in +`X-AIOS-TICK-TOKEN`; and when `AIOS_AUTOMATION_TICK_TOKEN` is UNSET the route refuses everything +rather than admitting everyone. A "no token configured means no check" default is how an internal +trigger becomes a public one — the same class of mistake as an empty-200 permission answer. +""" +import os +import time + +from fastapi import APIRouter, Body, Depends, Header, Request + +import automation_engine as engine +import oauth_connect +import routes_oauth +import scope_cache +from deps import Session, err, module_gate, require_session + +router = APIRouter(prefix="/api/v1") + +# C5 (wave 22): the OAuth connector surface rides INSIDE this router — main.py belongs to no +# session this wave, and this router is already mounted there. `/api/v1` + `/oauth/...`. +router.include_router(routes_oauth.router) +# ⭐ WAVE 23 (C11): the connectors directory rides this router for the same reason the OAuth +# routes do — it is mounted already, so E's page needs no main.py change to reach it. (C still +# owns main.py; this avoids an unnecessary ask across a fence.) +import routes_connectors # noqa: E402 +router.include_router(routes_connectors.router) + +#: The registry key this surface carries (C-AUTONAV — A adds the row; the gate is live now, so +#: the day the row lands the wall is already the one that was tested). +MODULE = "automation" + +_GATE = module_gate(MODULE) + + +def _wire(defn, tenant): + """One automation, as the client reads it. `running` is PROCESS state, never store state — + see the engine header on why a persisted 'running' is a permanent lock.""" + live = engine.running(tenant, defn.get("id")) + sched = defn.get("schedule") or {} + nxt = engine.next_fire(sched.get("cron")) if sched.get("enabled") else None + status = dict(defn.get("status") or {}) + if live: + status = {**status, "state": "running", "startedAt": live.get("startedAt"), + "step": live.get("step")} + return { + "id": defn.get("id"), "name": defn.get("name"), "kind": defn.get("kind"), + "config": defn.get("config") or {}, "schedule": sched, "status": status, + "runs": list(defn.get("runs") or [])[:engine.MAX_RUNS], + "created": defn.get("created"), "createdBy": defn.get("createdBy"), + "nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "", + "running": bool(live), + # ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the + # vendor". THE MONEY GUARD, and it was INERT for a whole wave because this line was + # missing: the client declared `awaitingResults` REQUIRED and read it in `runBlock()`, + # nothing ever sent it, so `undefined` was falsy and the Run button stayed armed. + # + # ⛔ WHY IT IS NOT `running`. `running` above is PROCESS state (`engine.running`) and is + # FALSE for the entire 20–30 MINUTE vendor wait, which IS the hazard window: the ticks + # are 3 minutes apart, so between them the automation is idle and the button re-arms. + # MEASURED the expensive way on 2026-08-06 (~$0.025): with no snapshot outstanding a + # Run-now press returned 200 and started a BRAND-NEW billable corpus search. + # + # ⛔ AND NOT A PERSISTED "running" FLAG EITHER — the engine header forbids one (it + # outlives the process and locks the automation forever). `state.pendingSnapshot` is the + # field that already exists, is already persisted, is already carried untouched through + # `clean_definition`, and is the SAME expression `pending_collect_ids()` uses to decide + # what to collect. One truth, two readers. + "awaitingResults": bool(str((defn.get("state") or {}).get("pendingSnapshot") + or "").strip()), + # C3 (wave 22): the trigger config rides whole — the webhook token included, because + # the person configuring the external caller has to be shown the URL somewhere, and + # this payload is session-gated behind the same wall as everything else here. + "trigger": defn.get("trigger") or None, + "statusNote": defn.get("statusNote") or "", + # The one-sentence summary (airtable-brief rec 7), composed from the definition so it + # cannot describe steps the engine does not run. + "sentence": engine.compose_sentence(defn), + # THE CANVAS TOPOLOGY (R5) rides with the automation rather than being rebuilt in the + # client, for the same reason `cronPresets` does: the engine that RUNS the steps is the + # only thing entitled to say what the steps are. + "graph": engine.graph(defn), + # ⭐ WAVE 23 (C4/C5) — THE BUILDER'S OWN STATE, and its absence here was a silent-drop + # bug caught in review rather than by a gate: `flow` was stored, patchable and validated, + # but never sent back. B would have saved a flow through PATCH, got a 200, and watched + # every action vanish on reload — the classic "it didn't save" with nothing red anywhere. + "flow": defn.get("flow") or {"actions": []}, + # ⭐⭐ WAVE 32 · T45 (owner item 10) — CONFIGURED / UNCONFIGURED, per action, on the wire. + # + # ⛔ THE SERVER SAYS IT, not the client, and that is the whole reason it is here: the same + # `engine.action_needs` answers this label, `engine.run_refusal`'s 400 and `run_now`'s own + # refusal, so a card cannot read "Configured" over an action the run will refuse. Three + # readers, one predicate — the alternative is a client-side rule that agrees with the + # server until somebody adds a key to one of them (`awaitingResults` above is this file's + # own record of what the other shape costs). + # ⚠ IT IS A LIST, KEYED BY ACTION ID, and it names the NESTED actions too — an unconfigured + # step inside an If / then branch is exactly the one a person cannot see. + # ⚠ COSTS NO STORE READ. `action_needs` is pure over `(kind, config)`; this route is the + # one W30-T12 took the `user_tables` deep copy off, and a label is not worth putting it + # back. What needs the target database's schema (an enrich binding resolved by the profile + # FLAG) stays a run-time refusal — see `ACTION_REQUIRED`'s note. + "unconfigured": engine.unconfigured_actions(defn), + } + + +def _triggers_vocab(session): + """C3's server-owned trigger list: `[{key, label, ready, needs, planned}]`, keys EXACTLY + `engine.TRIGGER_KEYS` + `engine.TRIGGER_PLANNED`. `ready:false` + `needs` renders as a + not-configured state — never a dead control, never a client-side union. + + ⭐ WAVE 23 (R2): the list is the WHOLE Airtable-parity vocabulary, and the two triggers we + have not built ride it with `planned: true`. That is the honest version of "show all, wire + eight": the picker paints them faded with a reason instead of a shorter list that quietly + implies the missing ones do not exist. `clean_trigger` refuses them, so the faded state is + enforced at the door and not merely in the client's `disabled` attribute. + """ + tick_on = _tick_state()["enabled"] + g = oauth_connect.status(session.runtime, session.uname).get("google") or {} + email_ready = bool(g.get("configured")) and bool(g.get("connected")) \ + and not g.get("reconnect") + email_needs = "" if email_ready else ( + "connect_gmail" if g.get("configured") else "configure_google") + per = { + "manual": (True, ""), + "schedule": (tick_on, "" if tick_on else "arm_tick"), + "event_field": (True, ""), + "record_updated": (True, ""), + "record_created": (True, ""), + "enters_view": (True, ""), + "webhook": (True, ""), + "email": (email_ready, email_needs), + "form_submitted": (True, ""), + # ⭐ WAVE 24 (C-TRIG) — Instagram discovery, now a trigger. Readiness is the vendor key, + # the same bit `paidReady` carries: with no key the search door is closed and the picker + # must say so rather than offering a control that silently finds nothing. `clean_trigger` + # still ACCEPTS it either way — readiness is a deployment fact, not a validity one, which + # is the same split `email` already makes. + "ig_profile_match": (engine.bd_ready(), "" if engine.bd_ready() else "configure_brightdata"), + # ⭐ WAVE 29 (D-9 / R1) — TikTok, live. ⛔ IT NEEDS ITS OWN ROW EVEN THOUGH THE ANSWER IS + # IDENTICAL, and the reason is the `.get(k, (True, ""))` default below: a trigger this dict + # forgets is reported READY, so a deployment with no vendor key would offer TikTok search + # as configured and the search would find nothing. Same key, same readiness bit, stated. + "tiktok_profile_match": (engine.bd_ready(), + "" if engine.bd_ready() else "configure_brightdata"), + } + #: ⭐ WAVE 24 — the server's own one-line description per trigger. C-TYPES: the picker renders + #: THIS under the option, because a CLIENT paraphrase of a server vocabulary is a second copy + #: of it, free to drift. Absent = the client shows nothing, never something invented. + detail = { + "manual": "It runs only when you press Run now", + "schedule": "It runs on a repeating schedule", + "event_field": "A record in the database starts matching a condition you set", + "record_updated": "Any of the columns you watch is changed", + "record_created": "A new record is added to the database", + "enters_view": "A record starts appearing in a saved view", + "webhook": "Something outside calls this automation's URL", + "email": "A message arrives in the connected mailbox", + "form_submitted": "Somebody submits one of this database's forms", + "ig_profile_match": "Search Instagram for profiles matching your filters, on a schedule", + "button_clicked": "Somebody presses a button on a record", + "comment_added": "Somebody comments on a record", + "web_page_changed": "A page you are watching is different from last time", + "tiktok_profile_match": "Search TikTok for profiles matching your filters, on a schedule", + } + + def _taxonomy(k): + """⭐ WAVE 25 · C2 — the four taxonomy keys, composed from the ENGINE's maps. + + ⛔ `.get(k) or FALLBACK`, NEVER `TRIGGER_GROUP_OF[k]`. The first draft of this indexed the + map on the reasoning that a default is how a Connector trigger quietly appears under + Database — and the gate rejected it, correctly, against the incident `per.get(k, ...)` + eight lines below records: a key added to `TRIGGER_KEYS` without remembering a dict beside + it raised KeyError and took `GET /automations` down, i.e. the whole surface, which polls + this every 2.5 s. A mis-grouped row is cosmetic; a 500 is not, and the ranking is not + close. + ⚠ THE CLASSIFICATION IS STILL MANDATORY — it is enforced at the GATE (no shipped trigger + may land in `other`) rather than at the request. Soft here, hard there. + """ + g = engine.TRIGGER_GROUP_OF.get(k) or engine.TRIGGER_GROUP_FALLBACK + return {"group": g, + "groupLabel": engine.TRIGGER_GROUPS[g]["label"], + "groupOrder": engine.TRIGGER_GROUPS[g]["order"], + # The SUB-group inside "Connector"; None everywhere else. ⚠ Group by this key, + # render its label — it is NOT a connector-directory slug (see the engine's note). + "connector": engine.TRIGGER_CONNECTOR.get(k), + # D-55: "the cron drives this one". Derived from the engine's schedule set MINUS + # `manual`, so the client's `CRON_DRIVEN_TRIGGERS` copy can be deleted. + "schedules": k in engine.TRIGGER_CRON_KEYS} + + out = [] + for k in engine.TRIGGER_KEYS: + # ⚠ `.get` WITH A DEFAULT, NOT `per[k]`. This loop walks the ENGINE's vocabulary and + # indexed a hand-maintained dict beside it: adding a key to `TRIGGER_KEYS` without + # remembering this dict raised KeyError and 500'd `GET /automations` — the payload the + # whole automation surface polls every 2.5 s — with every gate and `tsc` still green. + # Defaulting to "ready, needs nothing" is the honest fallback: a trigger the engine + # offers and this route has no readiness opinion about is simply available. + ready, needs = per.get(k, (True, "")) + row = {"key": k, "label": engine.TRIGGER_LABELS[k], "ready": ready, "needs": needs, + "planned": False, "detail": detail.get(k, ""), + # A3(3): the connect affordance is SERVER-COMPOSED — the client never maps a + # `needs` token to a route, so B's CONNECT_PROVIDERS shim deletes itself. + "connect": None, **_taxonomy(k)} + if not ready and needs == "connect_gmail": + row["connect"] = {"provider": "google", + "startUrl": "/api/v1/oauth/google/start"} + out.append(row) + for k in engine.TRIGGER_PLANNED: + out.append({"key": k, "label": engine.TRIGGER_LABELS[k], "ready": False, + "needs": "coming_soon", "planned": True, "connect": None, + "detail": detail.get(k, ""), **_taxonomy(k)}) + return out + + +def _tick_state(): + """⭐ WAVE 21 (C6 amendment A1) — can a SCHEDULE fire on this deployment? + + TWO independent paths can: the in-process scheduler (`AIOS_AUTOMATIONS=1`, + `automation_engine.py` module bottom) and an external cron POSTing `/automations/tick`, + gated on `AIOS_AUTOMATION_TICK_TOKEN` (AWS EventBridge in production). The Step-1 Trigger + card must be honest in both directions: "schedules won't fire" on a deployment where + EventBridge demonstrably fires them daily is the exact lie R9 forbids. `external` means + "the door is OPEN", never "the caller is alive" — the client's copy says so.""" + inproc = os.environ.get("AIOS_AUTOMATIONS") == "1" + ext = bool(os.environ.get("AIOS_AUTOMATION_TICK_TOKEN")) + return {"enabled": bool(inproc or ext), + "source": "in-process" if inproc else ("external" if ext else "")} + + +#: W29-T01 — the Board retirement ran, per tenant, this process. Same shape and same reasoning as +#: `_C8_MIGRATED` below: a one-way cleanup of data no living code path creates any more, whose +#: cost is a full walk of every row-cell in the tenant and whose result on pass 2..N is always +#: "nothing to do". ⚠ A module-global keyed by tenant is deliberately NOT reset by a table create +#: or delete — a new database cannot contain the legacy stage cells this retires. +_BOARD_RETIRED = set() + +#: ⭐⭐ WAVE 30 · T12 — THE PICKER DEFAULT, MEMOISED. `{tenant: (stamp, table_key)}`, the shape +#: `scope_cache` stores. +#: +#: ⛔ WHY THE DERIVED STRING AND NOT THE DOCUMENT. The obvious cache here is the `user_tables` +#: bucket itself, and it is the wrong one: that bucket is up to 35.8 MB per tenant and this box is +#: the HF free tier, so caching it would trade a latency problem for a memory one. What the warm +#: path actually needs is `discover_default_table`'s ANSWER — one short string. +#: +#: ⛔ AND WHY NOT A PLAIN `_BOARD_RETIRED`-STYLE ONCE-PER-PROCESS SET, which would have been less +#: code: the election reads which profile databases exist and which hold rows, and BOTH change +#: while the process lives (somebody creates a database, an automation writes the first row). A +#: once-per-process memo would pin the picker's default to whatever was true at boot and never +#: correct itself — a default that disagrees with the save door, which is exactly the wave-25 R2 +#: defect the `"table"` line's own comment records. A TTL bounds the staleness instead. +#: +#: ⚠ `scope_cache` rather than a hand-rolled dict, because it is the house pattern for precisely +#: this (`routes_customers`, `routes_products`, `pages` all use it) and it is stale-while-refresh: +#: once a copy exists NO request blocks on a rebuild. Automation was the one module importing it +#: nowhere, which the wave-30 scout named as the reason every other surface feels fast. +_DISCOVER_DEFAULT = {} +#: 5 minutes — the same order as `apiBridge.ts:CUSTOMERS_FRESH_MS` on the client. Overridable so a +#: gate can pin it rather than sleep. +_DISCOVER_DEFAULT_TTL = float(os.environ.get("AIOS_AUTOMATION_DEFAULT_TTL") or 300) + + +#: The one store key `_LentTables` intercepts. DERIVED from the engine's own constant rather than +#: written out here: `engine.ut_all` reads the bucket through it, so if that key ever moves, the +#: lend moves with it instead of silently becoming a pass-through that still looks correct. +_UT_STORE_KEY = engine.UT_STORE_KEY + + +def _LentTables(runtime, tables): + """⭐⭐ WAVE 30 · T13, NOW W31-C1 — a read-only `st` that serves ONE already-read `user_tables` + document and passes every other key straight through to the real runtime. + + ⭐⭐ WAVE 31 · C1 — THE BODY IS NOW `core.user_tables.lend`, AND THE CLASS THAT USED TO BE HERE + IS GONE. W30's version carried its own note that the general fix was unavailable at the time: + *"adding a `tables=` parameter to `may_open` would mean editing platform/core/user_tables.py, + which belongs to another lane this wave."* It is session B's lane THIS wave, they built the + generalisation (`user_tables.lend` + the `_LENDABLE` allow-list) for W31-T10, and this is the + second caller adopting it rather than becoming a third copy of the shape. + + ⭐ AND THE SWAP FIXES SOMETHING THIS FUNCTION NEVER COVERED. The old class intercepted exactly + one key, so `may_open`'s LAST branch — `shares.may_see` → `role_for` → `st.get('object_shares')` + — still took a fresh read PER TABLE. `_LENDABLE` covers both buckets, so a shared database no + longer costs a second whole-document read per row of the picker. The name is kept because + `verify_automation`'s NC66 pins this symbol as the thing it replaces to restore the `1 + N` + state; keeping it a function means that control still has exactly one seam to swap. + + ⛔ THE OBVIOUS FIX IS STILL THE FORBIDDEN ONE. Inlining the creator-or-admin test here would + remove the N reads and re-create the exact defect wave 20 fixed: this route USED to carry its + own wider rule (`createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a + database in the picker and was refused the moment they opened it. `may_open` stays THE one + resolver, unmodified and still called per table; it is simply no longer charged for a document + the caller is already holding. + """ + # ⚠ IMPORTED HERE, not at module scope — `core` is deliberately kept out of this module's + # import-time graph (the same reason `automation_tables` does it locally 300 lines down). + import core.user_tables as _ut + return _ut.lend(runtime, **{_UT_STORE_KEY: tables}) + + +def _discover_default(session, tables): + """The discovery picker's default table for this tenant — WITHOUT a bucket read on a warm call. + + ⚠ `tables` is the document the caller ALREADY holds on a cold call (the retirement pass reads + one). Passing it through means the cold path elects from the copy it has rather than taking a + second one, so this is never an extra read — only ever a saved one. + """ + if not session.runtime.available(): + return "" + if tables is not None: + # Cold call: the document is in hand. Elect from it and prime the memo in the same pass. + value = engine.discover_default_table(session.runtime, tables=tables) + _DISCOVER_DEFAULT[session.tenant] = (time.time(), value) + return value + return scope_cache.get(_DISCOVER_DEFAULT, session.tenant, _DISCOVER_DEFAULT_TTL, + lambda: engine.discover_default_table(session.runtime)) + + +def warm_default(rt): + """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT. + + ⛔ THE HALF THE MEMO CANNOT FIX, and it is why this exists rather than being a nicety. The memo + above makes calls 2..N free; **call 1 is still a full `user_tables` download**, and it lands on + whoever clicks Automation first after a deploy — the one visitor with no cache anywhere, + waiting on a document whose documented ceiling is 35.8 MB / ~1.4 s, taken under the store's + single lock. That is the owner's *"only automation has a loading screen"* on a cold Space, and + two waves of memoising the warm path could never touch it. Automation was the only module + importing `scope_cache` nowhere AND the only one absent from `_prewarm`. + + ⭐ IT ELECTS, IT DOES NOT CACHE THE DOCUMENT — deliberately, and this is the whole design. + `discover_default_table` reads the bucket once here, in the prewarm daemon thread where nobody + is waiting, and what survives is a DERIVED STRING. Holding the 35.8 MB document resident would + trade a latency the tenant notices for memory the HF free tier does not have, which is the + argument `_DISCOVER_DEFAULT`'s own note makes against caching it. + + ⚠ SAME TTL AS THE REQUEST PATH, not a permanent set: the election reads which databases exist + and which hold rows, and both change while the process lives. Priming by assignment is exactly + what the cold request path already does one function up, so there is one way this memo is + filled, not two. + + Returns the elected key (`""` when the store is unavailable), so a caller can log it rather + than guess whether the warm-up did anything. Called from `main.py:_prewarm` only. + """ + tenant = str(getattr(rt, "key", "") or "") + if not tenant or not rt.available(): + return "" + value = engine.discover_default_table(rt) + _DISCOVER_DEFAULT[tenant] = (time.time(), value) + return value + + +@router.get("/automations") +def list_automations(session: Session = Depends(_GATE)): + """`{automations: [...], kinds: [...], cronPresets: [...]}` — the rail's whole payload. + + The vocabularies ride WITH the list rather than sitting in a client constant: the cron + presets and the kind list are the server's, and a client copy of either is a thing that goes + stale silently (the editor would offer a preset the parser rejects).""" + # ⭐⭐ WAVE 29 (W29-T01, owner item 5: "Automation takes a while to appear"). THE COMPLAINT WAS + # THIS ROUTE, and the cost was never the automations: it read the whole `user_tables` bucket + # THREE TIMES per call — once for the stage-field retirement scan, twice more inside + # `discover_default_table`. That bucket's documented ceiling is 35.8 MB / ~1.4 s to serialize + # (`automation_engine.py` header) and `core/store.py` re-serializes on EVERY `.get()`, hit or + # miss, UNDER THE STORE LOCK — so the reads also serialize behind each other. ~4 s of deep + # copying to render a rail that shows a name and a toggle. + # + # Now: ONE read, lent to both callers. And the retirement SCAN — O(all row-cells in the + # tenant), which hits its own `if not stage_keys: return` only AFTER walking every row of + # every table — runs once per tenant per process, mirroring the `_C8_MIGRATED` guard below. + # + # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent": + # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s + # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can + # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either — + # wave 26's R6 deleted the board that made them. + # ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves). + # ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The + # line below used to be unconditional — every single request deep-copied the tenant's entire + # `user_tables` document (documented ceiling 35.8 MB / ~1.4 s, and `core/store.py:Store.get` + # re-serializes on EVERY `.get()`, hit or miss, UNDER THE STORE LOCK so the copies also queue + # behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two + # consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING. + # A whole-tenant document, per request, to render a rail showing a name and a toggle. + tables = None + if session.runtime.available() and session.tenant not in _BOARD_RETIRED: + tables = engine.ut_all(session.runtime) + _BOARD_RETIRED.add(session.tenant) + # Idempotent Board retirement removes only engine-marked stage fields and legacy Board + # state. User-created Status/Stage columns remain intact. + engine.retire_automation_board_state(session.runtime, tables=tables) + defs = engine.all_definitions(session.runtime) + items = [_wire(d, session.tenant) for _, d in + sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] + return {"automations": items, + # ⭐ WAVE 24 — DERIVED from the engine's own `KINDS`, not a hand-written trio. It was + # three literals that happened to match, which is a second copy of a server + # vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a + # hand list would now be wrong in three ways at once. `creatable` carries R6 onto the + # wire, so the ruling is a fact the client can read rather than one it must remember. + "kinds": [{"key": k, "label": engine.KIND_LABELS.get(k, k), + "creatable": k not in engine.RETIRED_KINDS} + for k in engine.KINDS], + "cronPresets": engine.CRON_PRESETS, + # ⚠ A BOOLEAN, NEVER THE KEY. The surface needs to say "the paid rung is not + # configured" honestly instead of offering a tier that will silently refuse — and + # that needs exactly one bit. Shipping the key itself to a browser would put a + # billable secret in every user's devtools. + "paidReady": engine.bd_ready(), + # THE SOURCE REGISTRY (D-9's seam), on the wire for the same reason `cronPresets` is: + # the module that RUNS a source is the only thing entitled to say what it can do, and + # a client copy of "Instagram can discover, TikTok cannot" goes stale in silence. + "sources": engine.source_status(), + # The discovery vocabulary, likewise server-owned: every name here is MEASURED- + # accepted by the vendor's own validator, so a client that invented one would build a + # query the API rejects. `lead` is the subset seen carrying VALUES on real rows. + # ⭐⭐ WAVE 32 · T46 (D-167) — THE VOCABULARY HAS A PLATFORM, AND `byKind` IS ADDITIVE + # ON PURPOSE. `fields`/`lead` keep INSTAGRAM's 21 and 3, so no stored automation and no + # client that has not adopted this changes behaviour today; `byKind` carries the per- + # corpus answer, and the SERVER already refuses a TikTok predicate naming one of the 16 + # fields TikTok's dataset does not have (`clean_predicates(..., kind)`). The door is + # closed either way — this is what lets the Find panel stop OFFERING them. + # ⚠ Derived through `engine.filter_fields`, the same accessor the validator uses, so + # the published vocabulary and the enforced one cannot drift — which is precisely what + # D-167 was: a route serving 21 names and a validator checking the same 21, both wrong + # about TikTok together, with nothing able to notice. + "discoverByKind": {k: {"fields": list(engine.filter_fields(k)[0]), + "lead": list(engine.filter_fields(k)[1])} + for k in engine.DISCOVERY_KINDS}, + "discover": {"fields": list(engine.BD_FILTER_FIELDS), + "lead": list(engine.BD_FILTER_LEAD), + "operators": list(engine.BD_FILTER_OPS), + "nullaryOperators": list(engine.BD_NULLARY_OPS), + "maxRecords": engine.BD_MAX_RECORDS, + # Wave 22 C4 — ADDITIVE: per-field flags for the toggle rows (the 3 + # PII fields stay structurally absent, R3) + the too-big guard's + # numbers, so the surface can say WHY a filter is refused before the + # server has to. + "filterMeta": engine.filter_meta(), + "guard": {"minNarrowing": engine.BD_MIN_NARROWING, + "maxRecords": engine.BD_MAX_RECORDS}, + # ⛔ D-59's `categoryOptions` IS DELIBERATELY NOT HERE — it is served by + # `GET /automations/discover/categories`. It sat on this payload for one + # commit and that was a real defect: THIS ROUTE IS POLLED EVERY 2.5 s by + # the whole automation surface, and deriving the options reads two user + # tables plus the PLATFORM MASTER, which is a different HF repo — i.e. a + # network round-trip per poll, per open tab. Caught by the gate's own + # output, which started carrying `store:get:master_snapshots` errors from + # a suite whose stated contract is that it touches no network. + # C6/R5's vocabulary, so the seed picker cannot offer a source the + # validator refuses. + "seedSources": list(engine.SEED_SOURCES), + "seedMaxRows": engine.SEED_MAX_ROWS, + # W29-T01: `tables` is the snapshot read once at the top of this handler. + # ⚠ IT WAS TAKEN BEFORE THE RETIREMENT WROTE, and that is deliberate and + # harmless: retirement only removes `stage_auto_*` FIELDS and pops those + # same keys off rows, and the election reads the preset-profile vocabulary + # (`handle` + N preset keys) and whether a table has ANY rows. A stage key + # is in neither set, and no row is ever deleted — so the pre-write + # snapshot and the post-write bucket cannot elect different tables. + # 2026-08-10 — the OFFER must name the table the SAVE will actually use. + # This was the bare `DISCOVER_TABLE` constant while `create`/`patch` now + # resolve a targetless discovery flow to the profile database the tenant + # already has, so the picker would have shown `ut_ig_candidates` and the + # save would have written somewhere else — a default that disagrees with + # itself across two panels, which is the shape wave 25's R2 fixed for + # `targetTable` vs the action's `table`. + # ⭐ WAVE 30 · T12 — through the per-tenant memo. The ELECTION rule and + # everything the comment above says about it are unchanged; what changed + # is that a warm request no longer re-reads a 35.8 MB document to + # recompute a string that did not move. + "table": (_discover_default(session, tables) + or engine.DISCOVER_TABLE)}, + "storeAvailable": bool(session.runtime.available()), + # Wave 22 C3 — the trigger vocabulary, session-scoped because email readiness is a + # per-USER fact (the poll runs through the creator's own Gmail connection). + "triggers": _triggers_vocab(session), + # ⭐ WAVE 23 C4 (R3) — the ACTION MENU, including what we have not built. Each row + # carries `ready`, so B paints "Send email" and "Run script" faded with the server's + # own reason instead of omitting them — the owner asked for Airtable's full menu, and + # a shorter list would imply those actions do not exist. `clean_actions` REFUSES an + # unready kind, so the faded state is a wall rather than a styling choice. + "actionsCatalog": engine.action_catalog(), + # The builder's own vocabulary: how deep a condition tree may nest, how deep groups + # may nest, and the ceilings. B reads these instead of hard-coding the same numbers + # into its "+ Add condition" affordance. + "flow": {"condOps": list(engine.LANE_OPS), + "nullaryCondOps": list(engine.LANE_NULLARY_OPS), + "maxCondDepth": engine.MAX_COND_DEPTH, + "maxCondChildren": engine.MAX_COND_CHILDREN, + "maxGroupDepth": engine.MAX_GROUP_DEPTH, + "maxActions": engine.MAX_ACTIONS, + # ⭐ W29-T09 (owner item 11) — THE POST-GROUP VOCABULARY, on the wire for the + # same reason `cronPresets` is: `clean_post_groups` REFUSES a type it does + # not know, so a client that invented one would build a config the save door + # rejects with a sentence about a word the person never typed. Both halves + # ride — the stored key and the label a person reads — because a client-side + # translation of `video` into "Reels" is a second copy of this list. + "postTypes": [{"key": t, "label": engine.POST_TYPE_LABELS.get(t, t)} + for t in engine.POST_TYPES], + # The ceiling a group's limit is judged against. `clean_post_groups` bounds a + # group by the action's OWN `maxPosts`, not by a constant, so the control can + # only warn honestly if it reads the same number the validator uses. + "maxPostsPerPull": engine.MAX_POSTS_PER_PULL}, + # Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see + # `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie. + "tick": _tick_state()} + + +@router.post("/automations/discover/estimate") +def discover_estimate(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """What would this search cost? Shown BEFORE the run, never after. + + ⚠ THE ANSWER IS AN ESTIMATE AND SAYS SO IN ITS OWN PAYLOAD (`basis: "SPEC"`). Bright Data + never returns a price before a run — the funds gate fires first and `price: 0` means "not + priced", not "free" — and this account's token cannot read a balance (`/customer/balance` + answers 403). A number presented as billed would be the invented measurement this whole + module refuses to make. + """ + return engine.discover_estimate((body or {}).get("recordsLimit")) + + +#: C8's migration ran, per tenant, this process. Once is the point: the sweep only writes when +#: an unbound bag exists, so after the first pass this is a read that finds nothing. +_C8_MIGRATED = set() + + +def _table_views(session, key): + """⭐ WAVE 24 (item 8) — the saved views on ONE user table as `[{id, label}]`, or **None** + when the workspace bucket could not be read at all. + + The client said "This server did not offer a view list" because the payload genuinely had + no `views` key. C-TYPES: **absent stays ABSENT.** `[]` is a MEASUREMENT ("this database has + no saved views") and `None` is a STATE ("nobody looked") — collapsing them is precisely how + an empty picker comes to read as a claim about the database. + + ⛔ SOURCED FROM `core.table_store`, WHICH IS THE READER `view_filter` ALREADY USES — not a + second one, the same one, listed instead of looked up. The brief said to source it the way + the grid does; the grid's projection (`aios_grid.views_from_defs`) is the wrong list HERE and + the difference matters: it INJECTS the system view ("All records") and PROJECTS cohorts as + views, and neither of those lives in the stored bucket — so `view_filter` answers "view no + longer exists" for every one of them. A picker built from that projection would offer options + the `enters_view` trigger cannot resolve, which is a worse bug than the missing key it fixes. + (An `enters_view` trigger on "All records" would also mean "fire on every record", so its + absence is correct rather than a gap.) + + ⚠ `consume_corrections=False`: this is a READ for a picker, and the default MUTATES — it + takes and clears the pending field-correction acknowledgement, so listing views would eat a + protocol message meant for the grid. + """ + try: + import core.table_store as table_store + # ⛔ THE REACHABILITY PROBE IS NOT REDUNDANT, and leaving it out was a real defect this + # gate's own NC caught: EVERY `TableStore` accessor wraps its store read in `try/except` + # and returns `{}` on failure. So a bucket that could not be read is indistinguishable + # from one with no views — which collapses exactly the ABSENT/EMPTY distinction this + # function exists to preserve, and the caller would ship `views: []` as a measurement + # nobody took. The one read that is allowed to raise has to be ours. + session.runtime.get(f"{key}_table_workspace") + tops = table_store.make(f"{key}_table_workspace", st=session.runtime) + own = (tops.workspace(session.uname, consume_corrections=False) or {}).get("views") or {} + shared = tops.shared_views(session.uname, session.admin) or {} + except Exception: # noqa: BLE001 + return None + merged = {**shared, **own} # a view lives in ONE home; the merge is belt-and-braces + return sorted( + ({"id": str(vid), "label": str((v or {}).get("name") or vid)} + for vid, v in merged.items() if isinstance(v, dict)), + key=lambda r: r["label"].lower()) + + +@router.get("/automations/tables") +def automation_tables(session: Session = Depends(_GATE)): + """The blank databases an automation can target, with their fields. + + ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the + two can never collide. It exists because the automation editor needs the table+field list to + build a config at all, and D must not block on A's route landing. When C3-UT is live this + keeps working (same bucket) — it is a duplicate reader, never a second writer. + + ⛔ WAVE 20, item 3 — IT NO LONGER MIRRORS THE WALL, IT CALLS IT. This route re-implemented + `may_open` and got it WIDER: it also admitted `createdBy in ('automation', 'scheduler')`, so + a non-admin saw automation-created databases here and was refused the moment they opened, + edited or deleted one. A duplicate READER is fine; a duplicate WALL is not, because the two + only disagree in front of a user. One resolver now, and the engine stamps a human owner so + the merge takes nothing legitimate away (`ut_ensure`, `MACHINE_OWNERS`). + """ + import core.user_tables as user_tables + + # C8 (wave 22): bind pre-law automation bags to the definitions that write them — once + # per tenant per process, and a no-op read after the first real pass. A write-on-read, + # stated out loud: this is the surface whose stale bags mislead (the column picker), so + # it is where the truth gets repaired. + if session.tenant not in _C8_MIGRATED and session.runtime.available(): + _C8_MIGRATED.add(session.tenant) + try: + engine.bind_unbound_fields(session.runtime) + except Exception: # noqa: BLE001 + pass + + out = [] + # ⭐⭐ WAVE 30 · T13 — ONE read of the tenant document, LENT to the wall for every table. + # It was `1 + N` full deep copies (this `ut_all`, then `may_open` → `get` → `all_tables` per + # table), each of a document with a 35.8 MB / ~370 ms ceiling, all of them queued behind + # `Store._lock`. `may_open` is unchanged and still asked about every table — see `_LentTables` + # on why re-implementing the wall here is the one fix that is NOT available. + _tables_doc = engine.ut_all(session.runtime) + _lent = _LentTables(session.runtime, _tables_doc) + for key, t in sorted(_tables_doc.items(), + key=lambda kv: (kv[1].get("label") or "").lower()): + if not user_tables.may_open(key, session.uname, session.admin, st=_lent): + continue + row = {"key": key, "label": t.get("label") or key, + "source": t.get("source") or "Blank", + "rowCount": len(t.get("rows") or {}), + "fields": [{"key": f.get("key"), "label": f.get("label"), + "type": f.get("type") or "text", + "automation": f.get("automation") or None} + for f in (t.get("fields") or [])]} + views = _table_views(session, key) + if views is not None: + row["views"] = views # absent when the bucket did not answer — never [] + out.append(row) + return {"tables": out} + + +@router.get("/automations/presets") +def automation_presets(table: str = "", session: Session = Depends(_GATE)): + """⭐ WAVE 25 · C1 / R2b — the Instagram preset columns, diffed against ONE database. + + `{fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` — the + "already in this database" / "will be created" split the owner asked for, so the Create record + action's configuration can SHOW what pointing it here does before anything is spent. + + ⛔ DECLARED ABOVE `/automations/{auto_id}`, AND THAT IS LOAD-BEARING RATHER THAN TIDY. + FastAPI matches routes in DECLARATION order, so a literal path registered after a sibling + path-parameter route is never reached — this handler would simply never run and + `get_automation` would answer "no automation with that id" for the id `presets`. A 404 with a + plausible sentence is the worst possible failure here, because it reads as "the endpoint is + fine, the data is missing". `/automations/tables` sits above the same route for the same + reason; this follows it rather than inventing a second convention. + + ⚠ WALLED BY `may_open`, LIKE EVERY OTHER TABLE READER. Without it, asking about a table you + cannot open would answer which of its columns exist — a small disclosure, and exactly the + duplicate-wall mistake `automation_tables` records at wave 20. + """ + import core.user_tables as user_tables + + key = str(table or "").strip() + if key and not user_tables.may_open(key, session.uname, session.admin, + st=session.runtime): + raise err(404, "unknown_table", "no such database") + return engine.preset_plan(session.runtime, key) + + +@router.get("/automations/discover/categories") +def automation_categories(session: Session = Depends(_GATE)): + """⭐ DEBT D-59 — the Category combobox's options: the values this deployment has ACTUALLY + SEEN, each with its observed count. `{options: [{value, count}]}`. + + ⛔ ITS OWN ROUTE, ON PURPOSE. This belongs to the Find surface and is asked for when that + panel opens — it must never ride `GET /automations`, which the whole automation surface polls + every 2.5 s: deriving these reads two user tables AND the platform master (a different HF + repo), so on the polled payload it is a network round-trip per tab per poll. + + ⚠ THE CONTROL STAYS A COMBOBOX. These are the values we have seen, not the values that exist. + D-59 is explicit that transcribing Instagram's published taxonomy would be worse than no + dropdown — a filter on a value the corpus does not use returns zero rows and looks exactly + like an honest "no such accounts exist", which misleads precisely when it looks authoritative. + """ + return {"options": engine.observed_categories(session.runtime)} + + +@router.post("/automations/seed/derive") +def automation_seed_derive(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """⭐ WAVE 25 · C6 / R5 — "find me more accounts like the ones in this view". + + Body `{source: "view"|"cohort", table: "", id: ""}` → + `{derived: [...], basis: {rows, fields:[{name,label,value,coverage}], related, note}}` + + ⛔ IT DERIVES AND RETURNS; IT SAVES NOTHING. R5: the derived conditions are "visible and + editable, never hidden" — so the client drops them into the ordinary condition rows, where the + user edits them like anything else, and the ordinary PATCH stores them. A door that both + derived and saved would make the suggestion feel like a decision. + + ⚠ `basis` IS NOT DECORATION AND MUST BE RENDERED. It carries how many rows were read and the + MEASURED coverage of each characteristic, which is the difference between "12 of 12 of these + bios say florist" and "7 of 12 do" — presented identically, the weak one reads as authority. + + ⚠ Declared ABOVE `/automations/{auto_id}` for the reason `/automations/presets` is (FastAPI + matches in declaration order); that ordering is gated. + """ + import core.user_tables as user_tables + + body = body or {} + table = str(body.get("table") or "").strip() + if not table: + raise err(400, "no_table", "name the database to read the seed records from") + if not user_tables.may_open(table, session.uname, session.admin, st=session.runtime): + raise err(404, "unknown_table", "no such database") + source = str(body.get("source") or "view").strip().lower() + if source not in engine.SEED_SOURCES: + raise err(400, "bad_seed_source", + f"a seed comes from one of: {', '.join(engine.SEED_SOURCES)}") + rows, problem = engine.seed_rows(session.runtime, table, str(body.get("id") or "")) + if problem: + raise err(400, "bad_seed", problem) + derived, basis = engine.seed_predicates(rows) + return {"source": source, "table": table, "id": str(body.get("id") or ""), + "derived": derived, "basis": basis} + + +@router.get("/automations/{auto_id}") +def get_automation(auto_id: str, session: Session = Depends(_GATE)): + defn = engine.all_definitions(session.runtime).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + return {"automation": _wire(defn, session.tenant)} + + +@router.post("/automations") +def create_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """Create an automation. ⭐ WAVE 23 (C2): the body may carry + `target: {mode: "existing"|"new"|"automated", table?, label?}` — the wizard's FIRST question, + answered before the kind. `new` mints the blank database in the same call, so the automation + is never saved pointing at a table that does not exist yet.""" + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") + defn, error = engine.create(session.runtime, body or {}, username=session.uname) + if error: + raise err(400, "invalid_automation", error) + return {"automation": _wire(defn, session.tenant)} + + +@router.patch("/automations/{auto_id}") +def patch_automation(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") + defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname) + if error: + raise err(400 if error != "no such automation" else 404, + "invalid_automation" if error != "no such automation" else "unknown_automation", + error) + return {"automation": _wire(defn, session.tenant)} + + +@router.delete("/automations/{auto_id}") +def delete_automation(auto_id: str, session: Session = Depends(_GATE)): + """Delete the DEFINITION. ⚠ The database it filled is NOT touched — an automation is the + thing that writes rows, not the thing that owns them, and deleting a job must never be a way + to lose data (the same rule the orphan count encodes).""" + if engine.running(session.tenant, auto_id): + raise err(409, "automation_running", "it is running — wait for it to finish") + engine.remove(session.runtime, auto_id) + return {"deleted": str(auto_id)} + + +@router.post("/automations/preview") +def preview_source(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """What does this URL actually offer? The field-map step — never writes anything.""" + body = body or {} + url = str(body.get("url") or "").strip() + if not url: + raise err(400, "no_url", "give a page URL to read") + try: + return engine.preview(url, str(body.get("extract") or "table"), + int(body.get("tableIndex") or 0)) + except engine.Refused as e: + raise err(400, "refused_url", str(e)) + except Exception as e: # noqa: BLE001 + raise err(502, "fetch_failed", f"could not read that page — {type(e).__name__}: " + f"{str(e)[:160]}") + + +@router.post("/automations/{auto_id}/run") +def run_automation(auto_id: str, session: Session = Depends(_GATE)): + """Start a run on a background thread. 409 when one is already in flight.""" + defn = engine.all_definitions(session.runtime).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + # ⛔ WAVE 32 · T45 (owner item 10) — REFUSED, NAMING THE ACTION. `run_now` refuses too, for the + # tick and the webhook; this one exists so the person who pressed the button reads the reason + # instead of watching a run start and end with nothing done. The two ask the SAME function, so + # they cannot come to disagree about what "configured" means. + refusal = engine.run_refusal(defn) + if refusal: + raise err(400, "action_unconfigured", refusal) + if not engine.run_async(session.runtime, session.tenant, auto_id, username=session.uname): + raise err(409, "automation_running", "that automation is already running") + return {"started": str(auto_id), "startedAt": time.strftime("%Y-%m-%dT%H:%M:%S")} + + +@router.post("/automations/{auto_id}/nodes/{node_id}/toggle") +def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depends(_GATE)): + """Flip one step on the canvas. The SERVER decides what a node's switch means (see + `engine.NODE_TOGGLES`) — the client only reports which node was clicked. + + A node with no switch answers 400 with the sentence saying why, rather than silently doing + nothing: a control that appears to work and does not is worse than one that refuses. + """ + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") + if engine.running(session.tenant, auto_id): + raise err(409, "automation_running", "it is running — wait for it to finish") + defn, error = engine.toggle_node(session.runtime, auto_id, node_id) + if error: + raise err(404 if error == "no such automation" else 400, + "unknown_automation" if error == "no such automation" else "node_not_toggleable", + error) + return {"automation": _wire(defn, session.tenant)} + + +@router.post("/automations/{auto_id}/hook/{token}") +async def automation_hook(auto_id: str, token: str, request: Request): + """C3's webhook trigger — the tick-endpoint pattern one level down: unauthenticated BY + DESIGN (the external caller has no cookie), gated on a per-automation token minted when the + trigger was configured, constant-time compared. The tenant is FOUND by the (id, token) + pair — a wrong token answers 403 for every tenant, so the route confirms nothing about + which slugs exist. + + ⭐ WAVE 24 (D-41) — THE BODY IS READ DEFENSIVELY AND IS NEVER A REASON TO REFUSE. + ⛔ It is deliberately NOT declared as `body: dict = Body(...)`, which is the obvious way to + write this and would be a live regression: FastAPI would then VALIDATE the payload, so an + existing caller posting text, form-encoding, an empty body or slightly malformed JSON would + start getting a 422 from a door that has accepted anything since wave 22. A webhook sender is + somebody else's system; we do not get to change what it must send in order to fire a flow. + An unreadable body simply maps nothing — the flow still fires, exactly as it did before. + + `run_in_threadpool` keeps the store I/O off the event loop: `hook_fire` walks every tenant + and may commit a row, and this handler had to become `async` only to read the request body. + """ + from starlette.concurrency import run_in_threadpool + from harness import runtime as _rt + + try: + payload_in = await request.json() + except Exception: # noqa: BLE001 + payload_in = None + + def _fire(): + last = (404, {"error": "unknown_automation", + "message": "no automation with that id and token"}) + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + status, payload = engine.hook_fire(rt, slug, auto_id, token, body=payload_in) + if status == 200: + return 200, payload + if status != 404: + last = (status, payload) + return last + + status, payload = await run_in_threadpool(_fire) + if status == 200: + return payload + raise err(status, str(payload.get("error") or "refused"), + str(payload.get("message") or "refused")) + + +@router.post("/automations/tick") +def tick(request: Request, x_aios_tick_token: str = Header(default="")): + """Fire every due schedule, for every tenant. The durable-cron entry point (R5). + + Unauthenticated BY DESIGN and gated on a shared secret instead — an EventBridge rule has no + cookie. Refuses when the secret is not configured (see the module header).""" + want = os.environ.get("AIOS_AUTOMATION_TICK_TOKEN") or "" + if not want: + raise err(403, "tick_disabled", + "AIOS_AUTOMATION_TICK_TOKEN is not configured — the tick endpoint is closed") + got = x_aios_tick_token or request.headers.get("X-AIOS-TICK-TOKEN") or "" + if got != want: + raise err(403, "bad_tick_token", "that token is not valid for this deployment") + started = engine.tick_all() + return {"started": started, "at": time.strftime("%Y-%m-%dT%H:%M:%S")} + + +@router.post("/automations/ig/purge") +def purge_ig_subject(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """D-24 (wave 22): the right-to-erasure door for ONE Instagram subject — walks the + tenant's four `ut_ig_*` tables AND the platform master (R2 pooled a copy there, so a purge + that skipped it would not be erasure). Admin-only: erasure is a compliance act, not a + grid gesture. Answers the per-table removal counts — the one aggregate whose drill is the + rows' ABSENCE.""" + if not session.admin: + raise err(403, "admin_only", "erasing a subject is an admin action") + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was " + "purged") + handle = str((body or {}).get("handle") or "").strip() + if not handle: + raise err(400, "no_handle", "name the Instagram handle to erase") + counts = engine.purge_subject(session.runtime, handle) + return {"handle": handle.lstrip("@").lower(), "removed": counts, + "total": sum(counts.values())} + + +@router.get("/automations/metrics/{table_key}/{field_key}/{row_id}/rows") +def metric_drill(table_key: str, field_key: str, row_id: str, + session: Session = Depends(_GATE)): + """C7's drill: the EXACT master snapshot rows behind one metric cell + ([[no-unverifiable-aggregates]]) — recomputed on ask with the same function that filled + the cell, so the drill can never disagree with the number by construction.""" + import core.user_tables as user_tables + + if not user_tables.may_open(table_key, session.uname, session.admin, st=session.runtime): + raise err(404, "unknown_table", "no such database") + t = engine.ut_get(session.runtime, table_key) or {} + fdef = next((f for f in (t.get("fields") or []) if f.get("key") == field_key), None) + if not fdef or not isinstance(fdef.get("metric"), dict): + raise err(404, "not_a_metric", "that column is not a metric field") + row = (t.get("rows") or {}).get(str(row_id)) + if row is None: + raise err(404, "unknown_row", "that record is not in the database") + import ig_master + url_field = next((f.get("key") for f in (t.get("fields") or []) + if f.get("type") == "url"), "") + handle = engine._table_handle(row, url_field) + bag = fdef["metric"] + series = ig_master.series_for({handle}) if handle else {} + value, rows = engine.metric_value(series.get(handle), bag.get("measure"), + bag.get("window"), bag.get("agg") or "") + return {"table": table_key, "field": field_key, "rowId": str(row_id), "handle": handle, + "measure": bag.get("measure"), "window": bag.get("window"), + "value": value, "rows": rows[:200], + "note": "" if value is not None else + "no master data answers this window — the cell is honestly blank"} + + +@router.get("/automations/{auto_id}/rows") +def run_rows(auto_id: str, session: Session = Depends(_GATE)): + """The rows the LAST run touched — the drill-down behind a run's counts. + + Every count in this product drills to the exact rows behind it ([[no-unverifiable-aggregates]]); + a run history that said "412 updated" and could not show which would be the thing that rule + exists to forbid. + """ + defn = engine.all_definitions(session.runtime).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + last = (defn.get("runs") or [{}])[0] + table_key = (defn.get("config") or {}).get("targetTable") or "" + t = engine.ut_get(session.runtime, table_key) or {} + rows = t.get("rows") or {} + ids = [str(i) for i in (last.get("affected") or [])] + return {"table": table_key, "label": t.get("label") or table_key, + "fields": [{"key": f.get("key"), "label": f.get("label")} + for f in (t.get("fields") or [])], + "rows": [{"id": i, **(rows.get(i) or {})} for i in ids if i in rows], + "truncated": len(ids) >= 200} + + +# --- the in-process scheduler ------------------------------------------------------------ +# ⚠ OPT-IN (`AIOS_AUTOMATIONS=1`), and that is an AMENDMENT to the wave brief's "default-on", +# made on a measurement: `verify_api.py` runs for 196 s, i.e. longer than three tick intervals, +# and `tick_all` reaches `runtime.get_runtime()` — which builds tenants and moves the LRU cache +# that verify_api's own isolation checks read. Default-on would put a background thread inside +# the subject of another session's gate. `AIOS_PREWARM=1` at `main.py:353` is the same decision +# for the same reason, so this follows it rather than inventing a second convention. +# +# The loop also SLEEPS FIRST (`engine.scheduler_loop`) — defence in depth, proven in +# verify_automation.py section T rather than assumed. +# +# ⛔ THE DEPLOY MUST SET `AIOS_AUTOMATIONS=1` (with `AIOS_AUTOMATION_TICK_TOKEN`) or schedules +# only ever fire from the external cron POSTing /automations/tick. Both paths work; neither is +# implicit. Booked in the session-D mailbox. +engine.start_scheduler() + +# C3 (wave 22): register the trigger listener onto the platform's row-event seam. THIS module +# is where the registration belongs — it is the one place that already imports both sides, so +# neither the engine nor platform/core grows a dependency on the other. Idempotent: a reimport +# must not double-fire every trigger. +import core.user_tables as _ut_hooks # noqa: E402 + +if engine.grid_hook not in _ut_hooks.ROW_HOOKS: + _ut_hooks.ROW_HOOKS.append(engine.grid_hook) + +# ⭐⭐ W31 QA — DECLARE THE MACHINE-OWNED CHILD DATABASES, for the same reason and in the same +# place as the hook above: this module already imports both sides, so neither the engine nor +# `platform/core` grows a dependency on the other. Idempotent by construction (a set). +# +# ⛔ THE OWNER FOUND WHAT THIS FIXES, IN PRODUCTION, AFTER THE TICKET READ GREEN. W31-T32 locked +# the TikTok children by stamping `recordMode` at the two SPAWN sites, and proved the stamp +# arrives "on the next write". That is true and it is not the `done-when`: every TikTok database +# already sitting in a tenant kept offering "+ New record" until somebody re-ran a TikTok +# automation, and nobody had. Owner, verbatim (2026-08-13): *"the databases for Tiktok do not have +# the small lock icon as I asked"* — and the rule, restated: *"just like Instagram Post database +# (which is locked), only the IG Profile and TT Profile should be editable."* +# ⚠ A DECLARATION NEEDS NO WRITE, so it is true for EVERY tenant the moment the API boots — no +# migration, no boot ordering, no per-tenant walk, and nothing that a stale store can undo. The +# stored flag still locks a table nobody declares; the two are OR'd. +_ut_hooks.register_locked_records(engine.LOCKED_CHILD_TABLES) diff --git a/api/routes_changes.py b/api/routes_changes.py index e3e439226e8697d03142aca0e830c510e1888c73..82de7d68cf1de8fcbd681de089bff07b4de73f0c 100644 --- a/api/routes_changes.py +++ b/api/routes_changes.py @@ -55,7 +55,7 @@ So: a valid session, a scope sanitised to a bucket-safe shape, and — for a `ut cached verdict that this session may open it. """ import time -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends import core.shared_overlay as shared_overlay import core.store as store @@ -224,87 +224,3 @@ def changes(scope: str = "", session: Session = Depends(require_session)): # a token frozen at 0 would promise liveness the backend is not delivering. tokens[label] = rev.get("token") if isinstance(rev, dict) else None return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()} - - -# ── ⭐⭐ D-144 (W33-T32) — A CAUGHT RENDER ERROR TELLS SOMEBODY ──────────────────────────────── -# -# ⛔ THE DEBT, in its own words: *"A caught render error tells nobody."* W30-T39 gave the app its -# first error boundary — measured at the time, `componentDidCatch|getDerivedStateFromError` was -# **0 hits** across all of `web/src`, so any throw anywhere blanked the whole product. The -# boundary CONTAINS the failure; it does not REPORT it, so the next user complaint still arrives -# as a sentence rather than as a stack. That is why owner item 2's crash could not be diagnosed -# and had to be fixed at both candidate causes. -# -# ⚠ IT LIVES HERE, IN THE CHANGE-TOKEN ROUTER, AND THE REASON IS WORTH ONE LINE. It is not about -# change tokens. It is here because this router is already mounted, already in one lane's fence, -# and a NEW `routes_*.py` costs a cross-fence mount (contract C2) that this wave has already spent -# three times — and a fourth finished-but-unmounted router is precisely the failure wave 23 shipped -# three of. Subject purity would have been bought with the exact risk the contract exists to -# prevent. ⚠ It performs no `store.get()`, so this file's zero-deep-copy claim (and -# `verify_live_workspace`'s count of it) is untouched: `telemetry.log` appends one line to a -# container-local JSONL. -# -# ⛔⛔ AND IT CARRIES NO ROW DATA, WHICH IS THE DEBT'S OWN HARD CONDITION. The body is rebuilt KEY -# BY KEY from exactly two strings; there is no `**body` anywhere below. A crash reporter that -# forwarded props or state would quietly ship a customer's cell values into a log — the one way -# this feature could do more harm than the silence it replaces. -_CLIENT_ERR_MAX = 4 * 1024 -_CLIENT_ERR_WINDOW_S = 60 -_CLIENT_ERR_PER_WINDOW = 12 -_CLIENT_ERR_HITS: dict = {} - - -@router.post("/client-error") -async def client_error(request: Request, session: Session = Depends(require_session)): - """One caught render error, forwarded to telemetry. `{message, componentStack}`. - - ⚠ SESSION-GATED, deliberately. A public crash sink is an unauthenticated append to a log file, - which is a disk-filling primitive with a URL. Only signed-in accounts can crash this app in a - way we need to hear about. - ⚠ RATE-LIMITED PER ACCOUNT, because the failure mode is a render LOOP: a boundary that catches, - resets and re-throws would otherwise post several times a second, for ever, from every open - tab. Beyond the window the report is DROPPED and the request still answers 200 — the client - must never learn to retry a report. - """ - now = time.time() - who = session.uname or "?" - seen = [t for t in _CLIENT_ERR_HITS.get(who, ()) if now - t < _CLIENT_ERR_WINDOW_S] - seen.append(now) - _CLIENT_ERR_HITS[who] = seen - if len(_CLIENT_ERR_HITS) > 2048: - for k in [k for k, v in _CLIENT_ERR_HITS.items() - if not v or now - v[-1] > _CLIENT_ERR_WINDOW_S]: - _CLIENT_ERR_HITS.pop(k, None) - if len(seen) > _CLIENT_ERR_PER_WINDOW: - return {"ok": True, "recorded": False} - - # The body, BOUNDED BY THE STREAM. `Body(...)` would already have been read and parsed before - # this line ran, and `content-length` is caller-supplied — the same argument `routes_forms` - # and `routes_publish` both carry. - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > _CLIENT_ERR_MAX: - raise err(413, "body_too_large", "that report is too large") - chunks.append(chunk) - import json as _json - try: - body = _json.loads(b"".join(chunks) or b"{}") - except ValueError: - body = {} - if not isinstance(body, dict): - body = {} - - # ⛔ TWO KEYS, BOTH TRUNCATED, NOTHING ELSE. See the block note above. - message = str(body.get("message") or "")[:500] - stack = str(body.get("componentStack") or "")[:4000] - if not message and not stack: - return {"ok": True, "recorded": False} - try: - import harness.telemetry as _tel - _tel.log("client_error", user=session.uname, tenant=session.tenant, - message=message, componentStack=stack) - except Exception: # noqa: BLE001 - # Telemetry may never break a request — `deps.py` carries the same rule at its own site. - return {"ok": True, "recorded": False} - return {"ok": True, "recorded": True} diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 25bafcd1158e54d207444bff1a56c86d08e7870b..309c29c1507de67daf16e5804144cb405d832fc4 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -30,7 +30,7 @@ from deps import Session, require_session #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a #: second correct one here would leave the same two-copies shape that produced the defect. -from routes_keychain import TENANT_WIDE_TYPES, env_odoo_available +from routes_keychain import env_odoo_available # ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit). # This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory` @@ -396,31 +396,15 @@ def directory(rt, uname, is_admin=False, incomplete=None): # reader. A token connector is the case R4 is really about: a member's own Stripe # key is theirs, and the card must say which kind of connection it is looking at. "scope": scopes.get(t["entry"]) or "business", - # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A - # WIDENED ONE. This route's own docstring has promised since wave 23 that a - # non-admin may *"connect their OWN per-user accounts"*, and the next sentence - # then handed them an empty `manage` — a contradiction inside one paragraph, and - # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from - # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing - # still missing was a way to reach it from the map. - # - # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid. - # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"` - # row, and its own comment says why in the right words: *"the line is TENANT-WIDE - # vs PER-USER, not admin vs everyone … what they must never be offered is the - # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion - # is CORRECT and stays green. A member's personal connection is a different door - # into the same room, so it gets its own word. - # - # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE. - # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so - # offering a personal door for one of those types would be offering a button that - # 400s — the exact failure R6 removed from the Google card. Today that means - # Stripe and Shopify open for a member and Meta Ads does not - # ([[one-question-two-normalizers]]). - "manage": ("keychain" if is_admin - else "" if t["entry"] in TENANT_WIDE_TYPES - else "keychain_personal"), + # ⚠ `manage` STAYS ADMIN-ONLY HERE, deliberately, and it is a smaller door than + # R4 allows. A member may now hold a PERSONAL connection — the room for it is + # Settings → Keychains, which W32-T11 opens to them — but this card's action word + # has been "the tenant-wide credential door" since wave 23 and a sibling gate + # asserts a member is offered none. Widening it is a one-word change once that + # assertion is retargeted; booked rather than done, because reddening another + # lane's gate to add a second door to a room that is already reachable is a bad + # trade. + "manage": "keychain" if is_admin else "", # Said once, on the row, rather than in a paragraph somewhere (R13): storing the # key is real and useful, and it is not the same as the data being on screen. "note": "Storing the key connects the account; reading its data lands in a later " @@ -582,22 +566,9 @@ def connectors_directory(session: Session = Depends(require_session)): """C11: every connector this platform knows about, with its state for THIS reader. Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the - workspace is connected to, and connect their OWN per-user accounts — which is the whole point - of R7's per-user slots. - - ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and - then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction - inside a single paragraph, which is how the promise survived three waves without a door behind - it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row: - - · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay - that way: it edits the key the whole workspace reads through. - · `keychain_personal` — the same room, for a connection that is this reader's own. Offered - to a non-admin **only** where a personal credential is actually - allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so - Odoo and Meta Ads never get it). - · `""` — nothing this reader can do here; the client renders no control - rather than a door that would 403. + workspace is connected to (and connect their OWN per-user accounts, which is the whole point + of R7's per-user slots). `manage` is what carries admin-ness — it is empty for a non-admin, + so the client renders the card without an action rather than offering a door that 403s. """ # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as # careful and is silently False for every admin on earth — the manage actions would simply diff --git a/api/routes_customers.py b/api/routes_customers.py index a8c69b255c7b87ca150dda49f227146fbe1aa732..f7cee1791ee0de4552c2bb0c634bbec16ce7d2b4 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -209,17 +209,6 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # The derived channel: cohort membership cells + measure column values, ONE dict — the # same read-only channel the embed host hands to rows_from_pool. derived = aios_grid.cohort_cells(lists) - # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried - # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id` - # as its join key — so the merge would have lost the one value every Odoo document joins on. - # - # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS - # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is - # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for - # one fact, which is the class of defect item 12 is about. This channel exists for exactly - # this — a value the host knows per render and the pool has no business storing. - for pid in pids: - derived.setdefault(pid, {})["partner_id"] = pid for pid, cells in measure_resolve.column_values( fields, team_id, pids, today, stamp, rt.measure_memo, on_error=_measure_err).items(): diff --git a/api/routes_keychain.py b/api/routes_keychain.py index a5df0eb8e6dd266b6cf191c4662285aaf453fcd5..aa051d5b975eeabd5ac67db60652ea04b47edc98 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -1,965 +1,865 @@ -"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). - -Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER -return a decrypted field — list rows carry a masked preview, and the decrypt function is a -connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's -env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** -count (rows holding overlay data whose pids the current pool no longer serves; counted and -drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly -in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with -the keychain cutover wave R3 staged. -""" -import os - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session -# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door -# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall -# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per -# entry instead of per endpoint. `session.admin` is still what business-wide requires. - -router = APIRouter(prefix="/api/v1") - -#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by -#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another -#: freezes nothing while reporting success, which is the shape of the bug D-10 books. -from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 - ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) -#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED -#: odoo connector is paused; read by the pool path while paused; survives a Space restart. -SNAPSHOT_KEY = "connector_snapshots" - -#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. -#: -#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves -#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and -#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's -#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's -#: connection, and handing it to another tenant is the leak this method exists to prevent."* -#: -#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and -#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value -#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, -#: the gate reds instead of the product quietly disagreeing with itself. -ENV_ODOO_TENANT = "royal-imports" - - -def env_odoo_available(rt): - """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) - - ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. - `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant - guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was - `connected` and offered "Manage keys" for a credential belonging to another company. This - module asked the same question correctly two functions below, which is the whole shape of - [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in - as the second tenant. - - ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which - source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, - wrong for "should the environment row be listed at all", which is what the connectors pane - needs in order to show an inactive env source beside an active keychain one. - - Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is - how another tenant's environment got reported as this tenant's connection in the first place. - """ - try: - if not (os.environ.get("ODOO_URL") or "").strip(): - return False - return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT - except Exception: # noqa: BLE001 - return False - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. -# Every keychain entry and connector carries a scope, selectable per connection; business-wide is -# admin-only and applies to every user in the tenant."* -# -# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file -# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is -# an access rule, not a secret, so it lives in the layer that already decides who may ask. -# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves -# cannot drift into two vocabularies. -SCOPES = ("business", "personal") -#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before -#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. -#: Materialising it would be a migration nobody authorised and would rewrite the store on the next -#: read [[a-migration-that-runs-on-the-next-write]]. -DEFAULT_SCOPE = "business" -#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. -#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean -#: the same thing and there is only one way to spell the common case. -SCOPE_KEY = "keychain_scopes" - -#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. -#: -#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* -#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is -#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they -#: would silently become the credential the entire workspace's databases are built from, which is -#: a credential elevation wearing a scope picker. -#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes -#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is -#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door -#: from the only side B owns, and the resolver-side guard is booked for A. -TENANT_WIDE_TYPES = ("odoo", "meta_ads") - - -def clean_scope(raw, default=DEFAULT_SCOPE): - """A scope word from the wire, or None when the caller said something we do not speak. - - Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole - reason this returns None rather than falling back: a typo'd `"personel"` silently becoming - business-wide is exactly the failure a scope picker exists to prevent. - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default - got = str(raw).strip().lower() - return got if got in SCOPES else None - - -def _scope_rows(rt): - try: - return dict(rt.get(SCOPE_KEY) or {}) - except Exception: # noqa: BLE001 - return {} - - -def entry_scope(rt, entry_id, rows=None): - """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a - census does not re-read the store once per entry.""" - r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) - if not isinstance(r, dict): - return DEFAULT_SCOPE, "" - return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) - - -def may_see(scope, owner, uname): - """R4's visibility rule: business-wide is everyone's, personal is its owner's. - - ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the - per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token - is identity, not infrastructure"*). An admin who could read every member's personal credential - would make "personal" a label rather than a boundary. - """ - return scope != "personal" or str(owner) == str(uname) - - -def visible_entries(rt, uname, is_admin=False): - """This USER's view of the keychain: every business entry plus their own personal ones, each - row carrying its `scope` and `owner` so no caller has to ask a second time. - - ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can - hold a connection of their own; it does not hand them four characters of the workspace's Odoo - key. So a business row a member did not create arrives WITHOUT `preview` — they can see that - the connection exists and is theirs to use, which is the whole of what R4 grants. The default - is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. - """ - rows = _scope_rows(rt) - out = [] - for e in _kc().list_entries(rt): - scope, owner = entry_scope(rt, e["id"], rows) - if not owner: - owner = str(e.get("createdBy") or "") - if not may_see(scope, owner, uname): - continue - row = {**e, "scope": scope, "owner": owner} - if not (is_admin or str(owner) == str(uname)): - row["preview"] = "" - out.append(row) - return out - - -def _write_scope(rt, entry_id, scope, owner): - """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the - default, so the store holds one spelling of the common case.""" - def _up(cur): - if scope == DEFAULT_SCOPE: - cur.pop(str(entry_id), None) - else: - cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} - return cur - - rt.update(SCOPE_KEY, _up, flush="sync") - return True - - -def _may_touch(session, row): - """May this session change or delete `row`? An admin owns the business-wide ones; a member - owns their own personal ones. Anything else is not theirs to move.""" - if row.get("scope") == "personal": - return str(row.get("owner") or "") == str(session.uname) - return bool(session.admin) - - -def _kc(): - import core.keychain as keychain - return keychain - - -def _resolved_odoo_key(rt): - """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the - shapes this module needs: the display string (`env` / `keychain:`) and the key the pause - flag is stored under. - - ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, - beside the `odoo_source()` it must agree with. This function had its own copy of the same - three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and - a second copy is exactly how a pause flag comes to be written against one resolution and read - against another, freezing nothing while the UI reports success. The two SHAPES stay here - because they are this module's presentation concern; the DECISION does not. - """ - flag_key = rt.odoo_flag_key() - if not flag_key: - return None, None - return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key - - -def odoo_paused(rt): - """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry - that is not the resolved source freezes nothing — it serves nothing. - - ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the - measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question - the customer pool does. This name stays because `routes_customers`, `routes_products` and - `verify_api` all call it — moving the logic without moving the door keeps one answer and - costs no caller a change. - """ - return bool(rt.odoo_paused()) - - -def _snap_scope_key(team_id, agent): - return f"t={team_id}|a={agent}" - - -def load_pool_snapshot(rt, team_id, agent): - """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider - scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" - try: - snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} - e = snap.get(_snap_scope_key(team_id, agent)) - if isinstance(e, dict) and isinstance(e.get("rows"), list): - return float(e.get("ts") or 0), e["rows"] - except Exception: - pass - return None - - -def save_pool_snapshots(rt, taken_by=""): - """Persist every currently-cached pool scope as the pause-time snapshot ('the last - successful sync', made concrete). Ensures the consolidated default scope exists first so - a pause on a cold process still captures something to serve.""" - import time as _time - import routes_customers as _rc - try: - _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on - except Exception: - pass # cold + Odoo down: persist whatever IS cached - pools = {} - for key, entry in list(rt.pool_cache.items()): - if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" - and isinstance(entry, tuple) and len(entry) == 2 - and isinstance(entry[1], list)): - pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} - if not pools: - return 0 - - def _up(cur): - cur["odoo_pool"] = pools - cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") - cur["takenBy"] = str(taken_by or "") - return cur - - rt.update(SNAPSHOT_KEY, _up, flush="sync") - return len(pools) - - -def _rel_reconnect(rt): - """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot - disagree about what "resume" means.""" - import odoo_relational as rel - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = False - cur.pop("frozenAt", None) - cur.pop("frozenBy", None) - return cur - - rt.update(rel.CONFIG_KEY, _up, flush="sync") - return True - - -def _own_row(session, entry_id): - """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the - caller cannot see: telling a member that somebody else's personal credential EXISTS is the - disclosure the scope is for.""" - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(entry_id)), None) - if row is None: - raise err(404, "no_entry", "no such key") - return row - - -@router.get("/admin/keychain") -def list_keychain(session: Session = Depends(require_session)): - """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. - R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would - ship the feature and no door to it. The wall moved INTO the payload: a member sees the - business-wide entries and their own, never anybody else's personal one.""" - kc = _kc() - return {"entries": visible_entries(session.runtime, session.uname, - bool(session.admin)), - "locked": not kc.unlocked(), - #: the vocabulary and the permission, so the client renders a picker it can honour - #: rather than offering an option the server will refuse (R4: business is admin-only). - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - "tenantWideTypes": list(TENANT_WIDE_TYPES)} - - -@router.post("/admin/keychain", status_code=201) -def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): - kc = _kc() - body = body or {} - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - scope = clean_scope(body.get("scope")) - if scope is None: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can create one. You can add it as a personal connection instead.") - etype = str(body.get("type") or "").strip().lower() - if scope == "personal" and etype in TENANT_WIDE_TYPES: - # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this - # credential is what the WHOLE workspace's databases are built from, so "personal" would - # be a label on a tenant-wide key rather than a boundary around it. - raise err(400, "scope_not_available", - f"a {etype} connection is what this whole workspace's databases are read " - f"through, so it is always business-wide — it cannot be a personal connection. " - f"An administrator can add it for everyone.") - try: - row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), - body.get("fields"), session.uname) - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", - f"the keychain is locked — {e}. A secret is never stored unencrypted.") - except ValueError as e: - raise err(400, "bad_entry", str(e)) - except Exception: - raise err(503, "store_unavailable", "the entry was not saved — try again") - # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode - # of a side bucket is to publish a credential, not to hide one. So the second write is not - # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was - # stored. `business` needs no row at all, so this branch is the only one that can be partial. - # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing - # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin - # find a second switch. The tables were never dropped, so "resume" is one flag. - if etype == "odoo": - try: - _rel_reconnect(session.runtime) - except Exception: # noqa: BLE001 - pass - if scope != DEFAULT_SCOPE: - try: - _write_scope(session.runtime, row["id"], scope, session.uname) - except Exception: - try: - kc.delete_entry(session.runtime, row["id"]) - except Exception: # noqa: BLE001 - pass - raise err(503, "store_unavailable", - "the key was not saved — its sharing setting could not be stored, so " - "nothing was kept. Try again.") - return {"entry": {**row, "scope": scope, "owner": session.uname}} - - -@router.put("/admin/keychain/{entry_id}") -def update_key(entry_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so - "edit this key" means "replace it" and that is `DELETE` + `POST`.""" - row = _own_row(session, entry_id) - scope = clean_scope((body or {}).get("scope"), default=None) - if scope is None: - raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can make one business-wide.") - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to change") - if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: - raise err(400, "scope_not_available", - f"a {row.get('type')} connection is what this whole workspace's databases are " - f"read through, so it is always business-wide.") - owner = row.get("owner") or session.uname - try: - _write_scope(session.runtime, entry_id, scope, owner) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} - - -@router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): - row = _own_row(session, entry_id) - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to delete") - try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} - - -@router.post("/admin/keychain/{entry_id}/test") -def test_key(entry_id: str, session: Session = Depends(require_session)): - _own_row(session, entry_id) # 404 for an entry this caller may not see - return _kc().test_entry(session.runtime, entry_id) - - -def _unsynced_customer_records(session): - """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no - longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a - tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the - answer is `known: False`, never a fabricated zero.""" - try: - import core.table_store as table_store - bucket = session.runtime.get("customer_table_workspace") or {} - overlay_pids = {} - for uname, ws in bucket.items(): - if uname == table_store.SHARED_KEY or not isinstance(ws, dict): - continue - for pid, cells in (ws.get("overlays") or {}).items(): - if isinstance(cells, dict) and cells: - overlay_pids.setdefault(str(pid), cells) - if not overlay_pids: - return {"known": True, "count": 0, "rows": []} - from routes_customers import allowed_pids - pool = {str(p) for p in allowed_pids(session)} - orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) - if str(x).isdigit() else 0) - rows = [] - for p in orphans[:50]: - cells = overlay_pids[p] - hint = next((str(v) for v in cells.values() if str(v).strip()), "") - rows.append({"pid": int(p) if str(p).isdigit() else p, - "fields": len(cells), "hint": hint[:80]}) - return {"known": True, "count": len(orphans), "rows": rows, - "shown": min(len(orphans), 50)} - except Exception as e: - return {"known": False, "count": None, "rows": [], - "note": f"pool unavailable — {type(e).__name__}"} - - -@router.get("/admin/connectors") -def connectors(session: Session = Depends(require_session)): - kc = _kc() - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same - # facts with a status column. Reading `kc.list_entries` here instead would have shown a member - # every colleague's personal connection on the screen next door to the one that hides them. - entries = visible_entries(session.runtime, session.uname, bool(session.admin)) - # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — - # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env - # for tenant #0 only, else nothing (fail closed — never another tenant's environment). - # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source - # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would - # make the answer depend on who opened the pane. Personal entries are excluded for the same - # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. - _scopes = _scope_rows(session.runtime) - first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) - if e["type"] == "odoo" - and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) - # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the - # connectors DIRECTORY answer "does the environment serve this tenant?" with one function - # instead of two spellings that agreed until a second tenant signed in. - if first_odoo and kc.unlocked(): - resolved = f"keychain:{first_odoo}" - elif env_odoo_available(session.runtime): - resolved = "env" - else: - resolved = None - rows = [] - if env_odoo_available(session.runtime): - rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", - "source": "env", "active": resolved == "env", - # the deployment's own credential — business-wide by construction, and it - # has no owner to be personal to. - "scope": DEFAULT_SCOPE, "owner": "", - "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) - for e in entries: - rows.append({"key": e["id"], "label": e["label"], "type": e["type"], - "source": "keychain", "preview": e["preview"], - "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", - "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), - "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) - out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. - # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the - # hole it left: "measures not already computed may still reach the source". D-10 - # closed that hole — `harness/datastore.py` (the mirror every measure column is - # answered from) refuses to sync while paused, and `routes_products._pool_for` got the - # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather - # than left standing, because a warning that outlives its defect teaches the reader to - # ignore warnings. - # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely - # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and - # collapsing them into "everything freezes" would be the kind of tidy summary that - # stops being true the first time one of them changes. - # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the - # correction is recorded here rather than silently applied. D-62 said this note - # "promises a behaviour on a dashboard measure path that has been dead since W16". - # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from - # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while - # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is - # `/api/v1/pages/{key}` (D-52), which this note never mentioned. - # - # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures - # stop moving rather than going blank". BOTH pool paths answer **503** when they have - # no copy to serve — the customer path for a scope with no snapshot - # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because - # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says - # so in as many words). So a paused connector plus a restarted server is exactly the - # blank screen this sentence promised could not happen. A warning that over-promises - # is worse than none: it is the sentence somebody quotes when the screen disagrees. - "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " - "views stay, and nothing reaches the source while it is paused. " - "Anything this server has already read keeps showing: the customer " - "workspace serves its pause-time snapshot, the product list serves " - "the last copy read since startup, and measure columns keep answering " - "from the mirror as it stood when you paused. What has NOT been read " - "cannot be shown — a scope with no snapshot, or the product list after " - "a restart, reports that the source is paused instead of showing " - "figures. Resume to start reading live again.")} - # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to - # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on - # the payload that is NOT about connectors: it unions overlays across EVERY user of the - # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user - # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member - # would have read colleagues' typed notes off the Connectors pane. It also filters against - # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and - # the number itself becomes wrong for them as well as private. - # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already - # returned, and the audit has to walk the payload, not the entry list I was thinking about. - if session.tenant == "royal-imports" and session.admin: - out["unsynced"] = _unsynced_customer_records(session) - return out - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database -# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to -# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a -# switch flipped here is a switch the sync path obeys — a config the route knows and the sync -# path does not is a control that does nothing and reports success. -def _rel(): - import odoo_relational as rel - return rel - - -def _odoo_entry(session): - """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing - is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" - kc = _kc() - scopes = _scope_rows(session.runtime) - for e in kc.list_entries(session.runtime): - if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": - return e - return None - - -def _odoo_source_fields(session, entry): - """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. - - ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no - route returns its output; this returns the three fields that identify WHICH server, and the - api key is not among them. The masked preview is the entry's own and was computed at write. - ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by - the process, and an admin editing it from a tenant screen would be editing the container. - """ - if entry is None: - return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), - os.environ.get("ODOO_USER", ""), False) - try: - f = _kc().read_fields(session.runtime, entry["id"]) or {} - except Exception: # noqa: BLE001 - return ("", "", "", True) # locked keychain: honest blanks, still editable - return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) - - -def _odoo_admin(session): - """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the - whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide - type cannot be personal in the first place.""" - if not session.admin: - raise err(403, "not_admin", - "the Odoo connection serves this whole workspace, so only an administrator can " - "configure it") - - -@router.get("/admin/connectors/odoo/config") -def odoo_config(session: Session = Depends(require_session)): - """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a - person needs to see before changing any of it.""" - _odoo_admin(session) - rel = _rel() - entry = _odoo_entry(session) - server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) - cfg = rel.read_config(session.runtime) - return { - "applicable": bool(rel.is_royal(session.tenant)), - "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) - else "none"), - "entryId": (entry or {}).get("id", ""), - "label": (entry or {}).get("label", "Odoo (environment)"), - "preview": (entry or {}).get("preview", ""), - "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, - "serverDbEditable": editable, - "grids": rel.grid_choices(session.runtime), - "syncEvery": cfg["syncEvery"], - "syncOptions": list(rel.SYNC_PRESETS), - "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, - "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], - # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: - # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders - # no button rather than one that 400s. - "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) - and not cfg["frozen"]), - } - - -@router.put("/admin/connectors/odoo/config") -def odoo_config_put(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED - back rather than silently applied.""" - _odoo_admin(session) - rel = _rel() - body = body or {} - notes = [] - - grids = body.get("grids") - known = {c["key"] for c in rel.grid_choices(session.runtime)} - clean_grids = None - if isinstance(grids, dict): - unknown = sorted(str(k) for k in grids if str(k) not in known) - if unknown: - # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid - # this connector does not have, and swallowing it makes the two disagree quietly. - raise err(400, "unknown_grid", - f"this connector has no grid called {', '.join(unknown)}") - clean_grids = {str(k): bool(v) for k, v in grids.items()} - if clean_grids and not any(clean_grids.get(k, True) for k in known): - notes.append("every grid is switched off — nothing will be materialised on the next " - "sync, and the databases you already have are left untouched") - - every = body.get("syncEvery") - clean_every = None - if every is not None: - clean_every = str(every).strip().lower() - if clean_every not in rel.SYNC_PRESETS: - # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. - # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, - # and never silently ignored either, because a control that discards your answer - # without a word is how a limit becomes invisible. - clean_every = rel.DEFAULT_SYNC - notes.append(f"{every!r} is not an interval this connector offers, and anything under " - f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " - f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") - - server_db = body.get("serverDb") - if server_db is not None: - server_db = " ".join(str(server_db).split())[:80] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if clean_grids is not None: - cur.setdefault("grids", {}).update(clean_grids) - if clean_every is not None: - cur["syncEvery"] = clean_every - return cur - - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - - if server_db: - notes.append(_rewrite_server_db(session, server_db)) - - # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE - # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: - # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( - # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So - # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value - # is stored, clamped, displayed and never read. - # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a - # default 1800 s and syncs anyway. "Only when I ask" asks all the same. - # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not - # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it - # is REPORTED here, at the moment of the change, with what it really controls. Delete these - # notes when the loop becomes per-tenant, not before. - if clean_every is not None: - if not rel.is_royal(session.tenant): - notes.append("this interval is saved, but the sync loop currently reads its schedule " - "from one workspace for the whole deployment — so it will not change how " - "often YOUR data refreshes until per-workspace scheduling ships") - else: - notes.append("this interval is saved and it is the one the deployment's sync loop " - "uses — it changes the refresh rate for every workspace on this " - "deployment, not only this one") - if clean_every == "manual": - notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " - "manual-only branch, so data still refreshes on the default interval") - - out = odoo_config(session) - return {**out, "notes": [n for n in notes if n]} - - -def _rewrite_server_db(session, server_db): - """Point the stored Odoo credential at a different server database (R9's first reading). - - ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of - how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. - So this is add-then-delete through the module's own doors, with the side rows (pause flag, - scope) carried across because they are keyed by ENTRY ID. - ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the - delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a - resync landing inside that window could read the OLD database. The alternative order can - leave the workspace with no credential at all, which is worse than one stale read. Milliseconds - of ambiguity beats a lost key. - """ - kc = _kc() - entry = _odoo_entry(session) - if entry is None: - return ("the server database is set on this deployment's environment, not in the " - "keychain, so it was not changed here") - try: - fields = kc.read_fields(session.runtime, entry["id"]) or {} - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", f"the keychain is locked — {e}") - if not fields: - raise err(400, "bad_entry", "this credential could not be read back to be changed") - if str(fields.get("db") or "") == server_db: - return "" - fields["db"] = server_db - try: - new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) - except Exception: - raise err(503, "store_unavailable", - "the server database was not changed — the existing connection is untouched") - # carry the side rows across, then retire the old entry - try: - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - if entry["id"] in flags: - def _mv(cur): - cur[new["id"]] = cur.pop(entry["id"], {}) - return cur - session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) - kc.delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - except Exception: # noqa: BLE001 - return (f"the connection now points at {server_db}, but the previous credential could " - f"not be removed — delete it under Keychains") - return f"the connection now points at the {server_db} database" - - -@router.post("/admin/connectors/odoo/disconnect") -def odoo_disconnect(session: Session = Depends(require_session)): - """R10 — remove the credential and FREEZE the grids as static data. - - ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the - key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — - including the boot rebuild and the resync loop, which for tenant #0 would otherwise - re-materialise from the process ENVIRONMENT and quietly undo the disconnect. - ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and - every FIELD DEFINITION stays, user-added columns included, because a field a person added is - the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes - one flag in a different bucket, which is what makes that guarantee structural rather than - careful. - """ - _odoo_admin(session) - rel = _rel() - import datetime as _dt - entry = _odoo_entry(session) - if not entry and not env_odoo_available(session.runtime): - raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = True - cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - cur["frozenBy"] = str(session.uname) - return cur - - # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT - # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which - # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, - # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read - # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, - # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected - # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's - # own sentence promised *"nothing is being refreshed"*. - # - # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause - # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` - # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed - # from `pause_connector` because it learned them the hard way: - # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no - # resolved source left to name, and the flag would land under a key nothing reads (D-10). - # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to - # serve; a failed capture leaves the connector live rather than paused-with-nothing. - _, flag_key = _resolved_odoo_key(session.runtime) - snapshots = 0 - if flag_key: - try: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - except Exception: # noqa: BLE001 - # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. - snapshots = 0 - - # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and - # the connector is still live; if the delete failed AFTER the flag landed, the tenant is - # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order - # can leave a workspace with no key and a connector that still tries to sync. - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "nothing was disconnected — try again") - - paused_mirror = False - if flag_key: - def _pause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": True} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") - paused_mirror = True - except Exception: # noqa: BLE001 - paused_mirror = False - - removed = "" - if entry is not None: - try: - _kc().delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - removed = entry["id"] - except Exception: - raise err(503, "store_unavailable", - "the databases are frozen but the stored credential was not removed — " - "delete it under Keychains") - - # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the - # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT - # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a - # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: - # the limit that cannot be removed is REPORTED, with what to do instead. - note = ("Your Odoo databases are frozen: every row and every column you had is still there and " - "still readable, and nothing is being refreshed.") - if not paused_mirror and flag_key: - note += (" ⚠ The live mirror could not be paused, so the two read-through databases " - "(order lines and GL lines) may keep advancing — pause the Odoo connector under " - "Keychains to stop them.") - note += (" Reconnecting adds the key back and resumes into the same databases." if entry - else " This workspace's Odoo credential comes from the deployment environment, so " - "there was no stored key to remove — the databases are frozen and Reconnect " - "resumes them into the same tables.") - return {"frozen": True, "removedEntry": removed, - # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean - # nobody returns is a guarantee nobody can check. - "pausedMirror": paused_mirror, "snapshots": snapshots, - "source": "keychain" if entry else "env", - "note": note} - - -@router.post("/admin/connectors/odoo/reconnect") -def odoo_reconnect(session: Session = Depends(require_session)): - """R10's second sentence — *"Reconnecting resumes into the same tables."* - - It clears the freeze and nothing else: the tables were never dropped, so there is nothing to - recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this - is the switch that lets the sync path see it again. - """ - _odoo_admin(session) - try: - _rel_reconnect(session.runtime) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only - # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the - # two read-through grids would sit at the disconnect date while the panel said "connected - # again" — the same disagreement between the halves of "frozen", pointing the other way. - # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the - # resolved source only exists again by this point. - _, flag_key = _resolved_odoo_key(session.runtime) - resumed = False - if flag_key: - def _unpause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": False} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") - resumed = True - except Exception: # noqa: BLE001 - resumed = False - connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) - return {"frozen": False, "connected": connected, "resumedMirror": resumed, - "note": ("Odoo is connected again and the databases you already had will refresh in " - "place." if connected else - "The freeze is lifted, but there is no Odoo credential yet — add one under " - "Keychains and the databases resume into the same tables.")} - - -@router.post("/admin/connectors/{key}/pause") -def pause_connector(key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - paused = bool((body or {}).get("paused")) - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an - # admin act; pausing your own personal one is yours. The env source has no keychain row and - # is business-wide by construction, hence the admin fallthrough. - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(key)), None) - if row is not None: - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to pause") - elif not session.admin: - raise err(403, "forbidden", "administrators only") - - # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a - # "last successful sync" to serve before the freeze takes effect. Capturing before the - # flag flips means a failed capture leaves the connector live (never paused-with-nothing). - snapshots = 0 - _, flag_key = _resolved_odoo_key(session.runtime) - if paused and flag_key and str(key) == flag_key: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - - def _up(cur): - cur[str(key)] = {"paused": paused} - return cur - - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"key": key, "paused": paused, "snapshots": snapshots} +"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). + +Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER +return a decrypted field — list rows carry a masked preview, and the decrypt function is a +connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's +env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** +count (rows holding overlay data whose pids the current pool no longer serves; counted and +drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly +in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with +the keychain cutover wave R3 staged. +""" +import os + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session +# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door +# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall +# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per +# entry instead of per endpoint. `session.admin` is still what business-wide requires. + +router = APIRouter(prefix="/api/v1") + +#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by +#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another +#: freezes nothing while reporting success, which is the shape of the bug D-10 books. +from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 + ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) +#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED +#: odoo connector is paused; read by the pool path while paused; survives a Space restart. +SNAPSHOT_KEY = "connector_snapshots" + +#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. +#: +#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves +#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and +#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's +#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's +#: connection, and handing it to another tenant is the leak this method exists to prevent."* +#: +#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and +#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value +#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, +#: the gate reds instead of the product quietly disagreeing with itself. +ENV_ODOO_TENANT = "royal-imports" + + +def env_odoo_available(rt): + """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) + + ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. + `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant + guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was + `connected` and offered "Manage keys" for a credential belonging to another company. This + module asked the same question correctly two functions below, which is the whole shape of + [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in + as the second tenant. + + ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which + source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, + wrong for "should the environment row be listed at all", which is what the connectors pane + needs in order to show an inactive env source beside an active keychain one. + + Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is + how another tenant's environment got reported as this tenant's connection in the first place. + """ + try: + if not (os.environ.get("ODOO_URL") or "").strip(): + return False + return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT + except Exception: # noqa: BLE001 + return False + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. +# Every keychain entry and connector carries a scope, selectable per connection; business-wide is +# admin-only and applies to every user in the tenant."* +# +# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file +# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is +# an access rule, not a secret, so it lives in the layer that already decides who may ask. +# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves +# cannot drift into two vocabularies. +SCOPES = ("business", "personal") +#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before +#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. +#: Materialising it would be a migration nobody authorised and would rewrite the store on the next +#: read [[a-migration-that-runs-on-the-next-write]]. +DEFAULT_SCOPE = "business" +#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. +#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean +#: the same thing and there is only one way to spell the common case. +SCOPE_KEY = "keychain_scopes" + +#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. +#: +#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* +#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is +#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they +#: would silently become the credential the entire workspace's databases are built from, which is +#: a credential elevation wearing a scope picker. +#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes +#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is +#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door +#: from the only side B owns, and the resolver-side guard is booked for A. +TENANT_WIDE_TYPES = ("odoo", "meta_ads") + + +def clean_scope(raw, default=DEFAULT_SCOPE): + """A scope word from the wire, or None when the caller said something we do not speak. + + Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole + reason this returns None rather than falling back: a typo'd `"personel"` silently becoming + business-wide is exactly the failure a scope picker exists to prevent. + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + got = str(raw).strip().lower() + return got if got in SCOPES else None + + +def _scope_rows(rt): + try: + return dict(rt.get(SCOPE_KEY) or {}) + except Exception: # noqa: BLE001 + return {} + + +def entry_scope(rt, entry_id, rows=None): + """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a + census does not re-read the store once per entry.""" + r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) + if not isinstance(r, dict): + return DEFAULT_SCOPE, "" + return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) + + +def may_see(scope, owner, uname): + """R4's visibility rule: business-wide is everyone's, personal is its owner's. + + ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the + per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token + is identity, not infrastructure"*). An admin who could read every member's personal credential + would make "personal" a label rather than a boundary. + """ + return scope != "personal" or str(owner) == str(uname) + + +def visible_entries(rt, uname, is_admin=False): + """This USER's view of the keychain: every business entry plus their own personal ones, each + row carrying its `scope` and `owner` so no caller has to ask a second time. + + ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can + hold a connection of their own; it does not hand them four characters of the workspace's Odoo + key. So a business row a member did not create arrives WITHOUT `preview` — they can see that + the connection exists and is theirs to use, which is the whole of what R4 grants. The default + is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. + """ + rows = _scope_rows(rt) + out = [] + for e in _kc().list_entries(rt): + scope, owner = entry_scope(rt, e["id"], rows) + if not owner: + owner = str(e.get("createdBy") or "") + if not may_see(scope, owner, uname): + continue + row = {**e, "scope": scope, "owner": owner} + if not (is_admin or str(owner) == str(uname)): + row["preview"] = "" + out.append(row) + return out + + +def _write_scope(rt, entry_id, scope, owner): + """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the + default, so the store holds one spelling of the common case.""" + def _up(cur): + if scope == DEFAULT_SCOPE: + cur.pop(str(entry_id), None) + else: + cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} + return cur + + rt.update(SCOPE_KEY, _up, flush="sync") + return True + + +def _may_touch(session, row): + """May this session change or delete `row`? An admin owns the business-wide ones; a member + owns their own personal ones. Anything else is not theirs to move.""" + if row.get("scope") == "personal": + return str(row.get("owner") or "") == str(session.uname) + return bool(session.admin) + + +def _kc(): + import core.keychain as keychain + return keychain + + +def _resolved_odoo_key(rt): + """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the + shapes this module needs: the display string (`env` / `keychain:`) and the key the pause + flag is stored under. + + ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, + beside the `odoo_source()` it must agree with. This function had its own copy of the same + three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and + a second copy is exactly how a pause flag comes to be written against one resolution and read + against another, freezing nothing while the UI reports success. The two SHAPES stay here + because they are this module's presentation concern; the DECISION does not. + """ + flag_key = rt.odoo_flag_key() + if not flag_key: + return None, None + return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key + + +def odoo_paused(rt): + """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry + that is not the resolved source freezes nothing — it serves nothing. + + ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the + measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question + the customer pool does. This name stays because `routes_customers`, `routes_products` and + `verify_api` all call it — moving the logic without moving the door keeps one answer and + costs no caller a change. + """ + return bool(rt.odoo_paused()) + + +def _snap_scope_key(team_id, agent): + return f"t={team_id}|a={agent}" + + +def load_pool_snapshot(rt, team_id, agent): + """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider + scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" + try: + snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} + e = snap.get(_snap_scope_key(team_id, agent)) + if isinstance(e, dict) and isinstance(e.get("rows"), list): + return float(e.get("ts") or 0), e["rows"] + except Exception: + pass + return None + + +def save_pool_snapshots(rt, taken_by=""): + """Persist every currently-cached pool scope as the pause-time snapshot ('the last + successful sync', made concrete). Ensures the consolidated default scope exists first so + a pause on a cold process still captures something to serve.""" + import time as _time + import routes_customers as _rc + try: + _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on + except Exception: + pass # cold + Odoo down: persist whatever IS cached + pools = {} + for key, entry in list(rt.pool_cache.items()): + if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" + and isinstance(entry, tuple) and len(entry) == 2 + and isinstance(entry[1], list)): + pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} + if not pools: + return 0 + + def _up(cur): + cur["odoo_pool"] = pools + cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") + cur["takenBy"] = str(taken_by or "") + return cur + + rt.update(SNAPSHOT_KEY, _up, flush="sync") + return len(pools) + + +def _rel_reconnect(rt): + """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot + disagree about what "resume" means.""" + import odoo_relational as rel + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = False + cur.pop("frozenAt", None) + cur.pop("frozenBy", None) + return cur + + rt.update(rel.CONFIG_KEY, _up, flush="sync") + return True + + +def _own_row(session, entry_id): + """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the + caller cannot see: telling a member that somebody else's personal credential EXISTS is the + disclosure the scope is for.""" + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(entry_id)), None) + if row is None: + raise err(404, "no_entry", "no such key") + return row + + +@router.get("/admin/keychain") +def list_keychain(session: Session = Depends(require_session)): + """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. + R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would + ship the feature and no door to it. The wall moved INTO the payload: a member sees the + business-wide entries and their own, never anybody else's personal one.""" + kc = _kc() + return {"entries": visible_entries(session.runtime, session.uname, + bool(session.admin)), + "locked": not kc.unlocked(), + #: the vocabulary and the permission, so the client renders a picker it can honour + #: rather than offering an option the server will refuse (R4: business is admin-only). + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + "tenantWideTypes": list(TENANT_WIDE_TYPES)} + + +@router.post("/admin/keychain", status_code=201) +def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): + kc = _kc() + body = body or {} + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + scope = clean_scope(body.get("scope")) + if scope is None: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can create one. You can add it as a personal connection instead.") + etype = str(body.get("type") or "").strip().lower() + if scope == "personal" and etype in TENANT_WIDE_TYPES: + # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this + # credential is what the WHOLE workspace's databases are built from, so "personal" would + # be a label on a tenant-wide key rather than a boundary around it. + raise err(400, "scope_not_available", + f"a {etype} connection is what this whole workspace's databases are read " + f"through, so it is always business-wide — it cannot be a personal connection. " + f"An administrator can add it for everyone.") + try: + row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), + body.get("fields"), session.uname) + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", + f"the keychain is locked — {e}. A secret is never stored unencrypted.") + except ValueError as e: + raise err(400, "bad_entry", str(e)) + except Exception: + raise err(503, "store_unavailable", "the entry was not saved — try again") + # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode + # of a side bucket is to publish a credential, not to hide one. So the second write is not + # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was + # stored. `business` needs no row at all, so this branch is the only one that can be partial. + # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing + # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin + # find a second switch. The tables were never dropped, so "resume" is one flag. + if etype == "odoo": + try: + _rel_reconnect(session.runtime) + except Exception: # noqa: BLE001 + pass + if scope != DEFAULT_SCOPE: + try: + _write_scope(session.runtime, row["id"], scope, session.uname) + except Exception: + try: + kc.delete_entry(session.runtime, row["id"]) + except Exception: # noqa: BLE001 + pass + raise err(503, "store_unavailable", + "the key was not saved — its sharing setting could not be stored, so " + "nothing was kept. Try again.") + return {"entry": {**row, "scope": scope, "owner": session.uname}} + + +@router.put("/admin/keychain/{entry_id}") +def update_key(entry_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so + "edit this key" means "replace it" and that is `DELETE` + `POST`.""" + row = _own_row(session, entry_id) + scope = clean_scope((body or {}).get("scope"), default=None) + if scope is None: + raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can make one business-wide.") + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to change") + if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: + raise err(400, "scope_not_available", + f"a {row.get('type')} connection is what this whole workspace's databases are " + f"read through, so it is always business-wide.") + owner = row.get("owner") or session.uname + try: + _write_scope(session.runtime, entry_id, scope, owner) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} + + +@router.delete("/admin/keychain/{entry_id}") +def delete_key(entry_id: str, session: Session = Depends(require_session)): + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + try: + _kc().delete_entry(session.runtime, entry_id) + _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + return {"ok": True} + + +@router.post("/admin/keychain/{entry_id}/test") +def test_key(entry_id: str, session: Session = Depends(require_session)): + _own_row(session, entry_id) # 404 for an entry this caller may not see + return _kc().test_entry(session.runtime, entry_id) + + +def _unsynced_customer_records(session): + """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no + longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a + tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the + answer is `known: False`, never a fabricated zero.""" + try: + import core.table_store as table_store + bucket = session.runtime.get("customer_table_workspace") or {} + overlay_pids = {} + for uname, ws in bucket.items(): + if uname == table_store.SHARED_KEY or not isinstance(ws, dict): + continue + for pid, cells in (ws.get("overlays") or {}).items(): + if isinstance(cells, dict) and cells: + overlay_pids.setdefault(str(pid), cells) + if not overlay_pids: + return {"known": True, "count": 0, "rows": []} + from routes_customers import allowed_pids + pool = {str(p) for p in allowed_pids(session)} + orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) + if str(x).isdigit() else 0) + rows = [] + for p in orphans[:50]: + cells = overlay_pids[p] + hint = next((str(v) for v in cells.values() if str(v).strip()), "") + rows.append({"pid": int(p) if str(p).isdigit() else p, + "fields": len(cells), "hint": hint[:80]}) + return {"known": True, "count": len(orphans), "rows": rows, + "shown": min(len(orphans), 50)} + except Exception as e: + return {"known": False, "count": None, "rows": [], + "note": f"pool unavailable — {type(e).__name__}"} + + +@router.get("/admin/connectors") +def connectors(session: Session = Depends(require_session)): + kc = _kc() + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same + # facts with a status column. Reading `kc.list_entries` here instead would have shown a member + # every colleague's personal connection on the screen next door to the one that hides them. + entries = visible_entries(session.runtime, session.uname, bool(session.admin)) + # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — + # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env + # for tenant #0 only, else nothing (fail closed — never another tenant's environment). + # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source + # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would + # make the answer depend on who opened the pane. Personal entries are excluded for the same + # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. + _scopes = _scope_rows(session.runtime) + first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) + if e["type"] == "odoo" + and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) + # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the + # connectors DIRECTORY answer "does the environment serve this tenant?" with one function + # instead of two spellings that agreed until a second tenant signed in. + if first_odoo and kc.unlocked(): + resolved = f"keychain:{first_odoo}" + elif env_odoo_available(session.runtime): + resolved = "env" + else: + resolved = None + rows = [] + if env_odoo_available(session.runtime): + rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", + "source": "env", "active": resolved == "env", + # the deployment's own credential — business-wide by construction, and it + # has no owner to be personal to. + "scope": DEFAULT_SCOPE, "owner": "", + "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) + for e in entries: + rows.append({"key": e["id"], "label": e["label"], "type": e["type"], + "source": "keychain", "preview": e["preview"], + "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", + "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), + "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) + out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. + # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the + # hole it left: "measures not already computed may still reach the source". D-10 + # closed that hole — `harness/datastore.py` (the mirror every measure column is + # answered from) refuses to sync while paused, and `routes_products._pool_for` got the + # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather + # than left standing, because a warning that outlives its defect teaches the reader to + # ignore warnings. + # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely + # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and + # collapsing them into "everything freezes" would be the kind of tidy summary that + # stops being true the first time one of them changes. + # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the + # correction is recorded here rather than silently applied. D-62 said this note + # "promises a behaviour on a dashboard measure path that has been dead since W16". + # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from + # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while + # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is + # `/api/v1/pages/{key}` (D-52), which this note never mentioned. + # + # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures + # stop moving rather than going blank". BOTH pool paths answer **503** when they have + # no copy to serve — the customer path for a scope with no snapshot + # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because + # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says + # so in as many words). So a paused connector plus a restarted server is exactly the + # blank screen this sentence promised could not happen. A warning that over-promises + # is worse than none: it is the sentence somebody quotes when the screen disagrees. + "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " + "views stay, and nothing reaches the source while it is paused. " + "Anything this server has already read keeps showing: the customer " + "workspace serves its pause-time snapshot, the product list serves " + "the last copy read since startup, and measure columns keep answering " + "from the mirror as it stood when you paused. What has NOT been read " + "cannot be shown — a scope with no snapshot, or the product list after " + "a restart, reports that the source is paused instead of showing " + "figures. Resume to start reading live again.")} + # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to + # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on + # the payload that is NOT about connectors: it unions overlays across EVERY user of the + # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user + # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member + # would have read colleagues' typed notes off the Connectors pane. It also filters against + # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and + # the number itself becomes wrong for them as well as private. + # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already + # returned, and the audit has to walk the payload, not the entry list I was thinking about. + if session.tenant == "royal-imports" and session.admin: + out["unsynced"] = _unsynced_customer_records(session) + return out + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database +# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to +# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a +# switch flipped here is a switch the sync path obeys — a config the route knows and the sync +# path does not is a control that does nothing and reports success. +def _rel(): + import odoo_relational as rel + return rel + + +def _odoo_entry(session): + """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing + is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" + kc = _kc() + scopes = _scope_rows(session.runtime) + for e in kc.list_entries(session.runtime): + if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": + return e + return None + + +def _odoo_source_fields(session, entry): + """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. + + ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no + route returns its output; this returns the three fields that identify WHICH server, and the + api key is not among them. The masked preview is the entry's own and was computed at write. + ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by + the process, and an admin editing it from a tenant screen would be editing the container. + """ + if entry is None: + return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), + os.environ.get("ODOO_USER", ""), False) + try: + f = _kc().read_fields(session.runtime, entry["id"]) or {} + except Exception: # noqa: BLE001 + return ("", "", "", True) # locked keychain: honest blanks, still editable + return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) + + +def _odoo_admin(session): + """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the + whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide + type cannot be personal in the first place.""" + if not session.admin: + raise err(403, "not_admin", + "the Odoo connection serves this whole workspace, so only an administrator can " + "configure it") + + +@router.get("/admin/connectors/odoo/config") +def odoo_config(session: Session = Depends(require_session)): + """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a + person needs to see before changing any of it.""" + _odoo_admin(session) + rel = _rel() + entry = _odoo_entry(session) + server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) + cfg = rel.read_config(session.runtime) + return { + "applicable": bool(rel.is_royal(session.tenant)), + "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) + else "none"), + "entryId": (entry or {}).get("id", ""), + "label": (entry or {}).get("label", "Odoo (environment)"), + "preview": (entry or {}).get("preview", ""), + "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, + "serverDbEditable": editable, + "grids": rel.grid_choices(session.runtime), + "syncEvery": cfg["syncEvery"], + "syncOptions": list(rel.SYNC_PRESETS), + "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, + "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], + # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: + # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders + # no button rather than one that 400s. + "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) + and not cfg["frozen"]), + } + + +@router.put("/admin/connectors/odoo/config") +def odoo_config_put(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED + back rather than silently applied.""" + _odoo_admin(session) + rel = _rel() + body = body or {} + notes = [] + + grids = body.get("grids") + known = {c["key"] for c in rel.grid_choices(session.runtime)} + clean_grids = None + if isinstance(grids, dict): + unknown = sorted(str(k) for k in grids if str(k) not in known) + if unknown: + # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid + # this connector does not have, and swallowing it makes the two disagree quietly. + raise err(400, "unknown_grid", + f"this connector has no grid called {', '.join(unknown)}") + clean_grids = {str(k): bool(v) for k, v in grids.items()} + if clean_grids and not any(clean_grids.get(k, True) for k in known): + notes.append("every grid is switched off — nothing will be materialised on the next " + "sync, and the databases you already have are left untouched") + + every = body.get("syncEvery") + clean_every = None + if every is not None: + clean_every = str(every).strip().lower() + if clean_every not in rel.SYNC_PRESETS: + # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. + # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, + # and never silently ignored either, because a control that discards your answer + # without a word is how a limit becomes invisible. + clean_every = rel.DEFAULT_SYNC + notes.append(f"{every!r} is not an interval this connector offers, and anything under " + f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " + f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") + + server_db = body.get("serverDb") + if server_db is not None: + server_db = " ".join(str(server_db).split())[:80] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if clean_grids is not None: + cur.setdefault("grids", {}).update(clean_grids) + if clean_every is not None: + cur["syncEvery"] = clean_every + return cur + + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + + if server_db: + notes.append(_rewrite_server_db(session, server_db)) + + out = odoo_config(session) + return {**out, "notes": [n for n in notes if n]} + + +def _rewrite_server_db(session, server_db): + """Point the stored Odoo credential at a different server database (R9's first reading). + + ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of + how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. + So this is add-then-delete through the module's own doors, with the side rows (pause flag, + scope) carried across because they are keyed by ENTRY ID. + ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the + delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a + resync landing inside that window could read the OLD database. The alternative order can + leave the workspace with no credential at all, which is worse than one stale read. Milliseconds + of ambiguity beats a lost key. + """ + kc = _kc() + entry = _odoo_entry(session) + if entry is None: + return ("the server database is set on this deployment's environment, not in the " + "keychain, so it was not changed here") + try: + fields = kc.read_fields(session.runtime, entry["id"]) or {} + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", f"the keychain is locked — {e}") + if not fields: + raise err(400, "bad_entry", "this credential could not be read back to be changed") + if str(fields.get("db") or "") == server_db: + return "" + fields["db"] = server_db + try: + new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) + except Exception: + raise err(503, "store_unavailable", + "the server database was not changed — the existing connection is untouched") + # carry the side rows across, then retire the old entry + try: + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + if entry["id"] in flags: + def _mv(cur): + cur[new["id"]] = cur.pop(entry["id"], {}) + return cur + session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) + kc.delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + except Exception: # noqa: BLE001 + return (f"the connection now points at {server_db}, but the previous credential could " + f"not be removed — delete it under Keychains") + return f"the connection now points at the {server_db} database" + + +@router.post("/admin/connectors/odoo/disconnect") +def odoo_disconnect(session: Session = Depends(require_session)): + """R10 — remove the credential and FREEZE the grids as static data. + + ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the + key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — + including the boot rebuild and the resync loop, which for tenant #0 would otherwise + re-materialise from the process ENVIRONMENT and quietly undo the disconnect. + ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and + every FIELD DEFINITION stays, user-added columns included, because a field a person added is + the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes + one flag in a different bucket, which is what makes that guarantee structural rather than + careful. + """ + _odoo_admin(session) + rel = _rel() + import datetime as _dt + entry = _odoo_entry(session) + if not entry and not env_odoo_available(session.runtime): + raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = True + cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + cur["frozenBy"] = str(session.uname) + return cur + + # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and + # the connector is still live; if the delete failed AFTER the flag landed, the tenant is + # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order + # can leave a workspace with no key and a connector that still tries to sync. + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "nothing was disconnected — try again") + removed = "" + if entry is not None: + try: + _kc().delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + removed = entry["id"] + except Exception: + raise err(503, "store_unavailable", + "the databases are frozen but the stored credential was not removed — " + "delete it under Keychains") + return {"frozen": True, "removedEntry": removed, + "note": "Your Odoo databases are frozen: every row and every column you had is still " + "there and still readable, and nothing is being refreshed. Reconnecting adds " + "the key back and resumes into the same databases."} + + +@router.post("/admin/connectors/odoo/reconnect") +def odoo_reconnect(session: Session = Depends(require_session)): + """R10's second sentence — *"Reconnecting resumes into the same tables."* + + It clears the freeze and nothing else: the tables were never dropped, so there is nothing to + recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this + is the switch that lets the sync path see it again. + """ + _odoo_admin(session) + try: + _rel_reconnect(session.runtime) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) + return {"frozen": False, "connected": connected, + "note": ("Odoo is connected again and the databases you already had will refresh in " + "place." if connected else + "The freeze is lifted, but there is no Odoo credential yet — add one under " + "Keychains and the databases resume into the same tables.")} + + +@router.post("/admin/connectors/{key}/pause") +def pause_connector(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + paused = bool((body or {}).get("paused")) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an + # admin act; pausing your own personal one is yours. The env source has no keychain row and + # is business-wide by construction, hence the admin fallthrough. + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(key)), None) + if row is not None: + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to pause") + elif not session.admin: + raise err(403, "forbidden", "administrators only") + + # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a + # "last successful sync" to serve before the freeze takes effect. Capturing before the + # flag flips means a failed capture leaves the connector live (never paused-with-nothing). + snapshots = 0 + _, flag_key = _resolved_odoo_key(session.runtime) + if paused and flag_key and str(key) == flag_key: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + + def _up(cur): + cur[str(key)] = {"paused": paused} + return cur + + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"key": key, "paused": paused, "snapshots": snapshots} diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 7795caf496a8598b3c52b4cf5b0663939012d5c8..8e49c41134aafde8a9738e53d63775c2d76fea88 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -1,114 +1,114 @@ -"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). - -Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, -shapes and status codes here; every decision that could be wrong lives in the module a gate -can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, -so the day a second provider lands here is the day nothing in this file changes. - -MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session -`main.py`, and `routes_automation` is already included there — so this router rides inside it -(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that -alters no path. - -⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent -screen** — it is a top-level navigation the client reaches by ``, never JSON. The -callback 302s BACK to the return path the `state` carried (relative-only, sanitised by -`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever -went wrong rides in the query string; a dead-end error page where the app used to be reads as -"the product broke", not "the connect failed". -""" -import os - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import RedirectResponse - -import oauth_connect -from deps import Session, err, require_session - -router = APIRouter(prefix="/oauth") - - -def _redirect_uri(request: Request, provider: str) -> str: - """The redirect URI this deployment registers at the provider — env-pinned when the - container sits behind a proxy that rewrites the scheme (the HF Space), else derived from - the request. MUST match a console-registered URI verbatim, so it is computed in exactly - one place. - - ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to - the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and - the request-derived fallback below is effectively dev-only. - ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host - this returns is where the provider sends the user BACK, and the session cookie is host-only - (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host - the user actually browsed plants the session on the wrong hostname and they return logged - out. Moving the app to a new hostname means moving this value AND re-registering the - resulting URI in the provider console; one without the other fails closed. - Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" - base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") - if not base: - base = f"{request.url.scheme}://{request.url.netloc}" - return f"{base}/api/v1/oauth/{provider}/callback" - - -@router.get("/status") -def oauth_status(session: Session = Depends(require_session)): - """C5's status shape for the SESSION user, one entry per registry provider: - `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's - `ready` reads through.""" - return oauth_connect.status(session.runtime, session.uname) - - -def _offered_or_404(provider: str): - """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. - - The owner pasted the failure this replaces: clicking Connect on the Google card answered - `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is - not offered at all, so the honest status is the one for a URL that does not exist. `404` - rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not - coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). - - ⚠ Every door in this router goes through it, START included, because the JSON the owner saw - came from the start route and a guard on one leg is a guard on one leg. - """ - if not oauth_connect.offered(provider): - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - - -@router.get("/{provider}/start") -def oauth_start(provider: str, request: Request, next: str = "", - session: Session = Depends(require_session)): - """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the - RELATIVE path the callback returns the browser to; it rides inside the single-use state, - sanitised, so the round trip cannot be steered off-origin.""" - _offered_or_404(provider) - url, problem = oauth_connect.start(provider, session.uname, - _redirect_uri(request, provider), next_path=next) - if problem: - raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) - return RedirectResponse(url, status_code=302) - - -@router.get("/{provider}/callback") -def oauth_callback(provider: str, request: Request, - session: Session = Depends(require_session), - state: str = "", code: str = "", error: str = ""): - """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends - the browser back to the state's return path — connected or not (see module header).""" - _offered_or_404(provider) - if error: - home = "/#/" - return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) - email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) - sep = "&" if "?" in home else "?" - if problem: - return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) - return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) - - -@router.post("/{provider}/disconnect") -def oauth_disconnect(provider: str, session: Session = Depends(require_session)): - _offered_or_404(provider) - if oauth_connect.provider_def(provider) is None: - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - oauth_connect.disconnect(session.runtime, session.uname, provider) - return {"disconnected": provider} +"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). + +Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, +shapes and status codes here; every decision that could be wrong lives in the module a gate +can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, +so the day a second provider lands here is the day nothing in this file changes. + +MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session +`main.py`, and `routes_automation` is already included there — so this router rides inside it +(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that +alters no path. + +⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent +screen** — it is a top-level navigation the client reaches by ``, never JSON. The +callback 302s BACK to the return path the `state` carried (relative-only, sanitised by +`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever +went wrong rides in the query string; a dead-end error page where the app used to be reads as +"the product broke", not "the connect failed". +""" +import os + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import RedirectResponse + +import oauth_connect +from deps import Session, err, require_session + +router = APIRouter(prefix="/oauth") + + +def _redirect_uri(request: Request, provider: str) -> str: + """The redirect URI this deployment registers at the provider — env-pinned when the + container sits behind a proxy that rewrites the scheme (the HF Space), else derived from + the request. MUST match a console-registered URI verbatim, so it is computed in exactly + one place. + + ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to + the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and + the request-derived fallback below is effectively dev-only. + ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host + this returns is where the provider sends the user BACK, and the session cookie is host-only + (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host + the user actually browsed plants the session on the wrong hostname and they return logged + out. Moving the app to a new hostname means moving this value AND re-registering the + resulting URI in the provider console; one without the other fails closed. + Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" + base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") + if not base: + base = f"{request.url.scheme}://{request.url.netloc}" + return f"{base}/api/v1/oauth/{provider}/callback" + + +@router.get("/status") +def oauth_status(session: Session = Depends(require_session)): + """C5's status shape for the SESSION user, one entry per registry provider: + `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's + `ready` reads through.""" + return oauth_connect.status(session.runtime, session.uname) + + +def _offered_or_404(provider: str): + """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. + + The owner pasted the failure this replaces: clicking Connect on the Google card answered + `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is + not offered at all, so the honest status is the one for a URL that does not exist. `404` + rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not + coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). + + ⚠ Every door in this router goes through it, START included, because the JSON the owner saw + came from the start route and a guard on one leg is a guard on one leg. + """ + if not oauth_connect.offered(provider): + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + + +@router.get("/{provider}/start") +def oauth_start(provider: str, request: Request, next: str = "", + session: Session = Depends(require_session)): + """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the + RELATIVE path the callback returns the browser to; it rides inside the single-use state, + sanitised, so the round trip cannot be steered off-origin.""" + _offered_or_404(provider) + url, problem = oauth_connect.start(provider, session.uname, + _redirect_uri(request, provider), next_path=next) + if problem: + raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) + return RedirectResponse(url, status_code=302) + + +@router.get("/{provider}/callback") +def oauth_callback(provider: str, request: Request, + session: Session = Depends(require_session), + state: str = "", code: str = "", error: str = ""): + """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends + the browser back to the state's return path — connected or not (see module header).""" + _offered_or_404(provider) + if error: + home = "/#/" + return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) + email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) + sep = "&" if "?" in home else "?" + if problem: + return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) + return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) + + +@router.post("/{provider}/disconnect") +def oauth_disconnect(provider: str, session: Session = Depends(require_session)): + _offered_or_404(provider) + if oauth_connect.provider_def(provider) is None: + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + oauth_connect.disconnect(session.runtime, session.uname, provider) + return {"disconnected": provider} diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index bc613d0fa1f00b318cf522e13160df7e9f110881..6792dc58921b7e7674e769665985d87a44453d53 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -1,1221 +1,1210 @@ -"""routes_odoo_tables.py — the door to the Odoo relational spawn (wave 27 item 17, contract C8). - -Two endpoints and no cleverness: refresh the locked Odoo databases, and report their freshness. -The work itself lives in `odoo_relational.py`; this file only decides WHO may ask and turns a -refusal into a status code. - -⛔ THE MOUNT IS DONE (`main.py` includes this router) and the wave-23 scar it was written against -— three finished routers shipping 404-dead behind green gates — is covered by `verify_api` -enumerating `main.app.routes`. ⚠ BUT WAVE 28 PROVED THAT CONTROL IS ONLY HALF OF THE QUESTION: -being mounted is not being CALLABLE. This router was mounted, enumerated, green, and answered a -plain-text 500 to every request for a day because of an attribute typo in the admin check -(`session.is_admin`, which does not exist). A route-existence check cannot see that; only calling -it can. `verify_api` now does both. - -ADMIN-GATED, and not for tidiness: a refresh REWRITES four locked databases for the whole tenant -and deletes the rows that left the population. That is an operator action. -⭐ WAVE 30 (R6/R7, contract C2) ADDS A THIRD ENDPOINT AND IT IS A DIFFERENT KIND OF THING: the -READ-THROUGH WINDOW. The two above operate on the materialised copy; `/{table_key}/rows` does not -read the copy at all. See the block above `GRID_SOURCES` for why that had to change. -""" -import json - -from fastapi import Depends -from fastapi import APIRouter, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - - -def _rel(): - import odoo_relational - return odoo_relational - - -def _rt(): - from harness import runtime - return runtime - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W31-T45 / D-169 — THE CROSS-TENANT MIRROR GUARD, AND WHY EVERY DOOR BELOW GOES THROUGH IT. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# `harness/datastore` holds ONE process-wide DuckDB connection over a process-global `DB_PATH`, -# and ONE Space process serves EVERY tenant. `TenantRuntime.assert_datastore_matches` was written -# for exactly the failure that arrangement produces — its own docstring names it: *"an analytical -# read served from another tenant's file returns real, plausible, wrong rows … and the number -# reconciles against the wrong book"* — and until this wave its only caller anywhere was -# `verify_api.py`. The guard was correct, tested, and governed nothing. -# -# ⛔ WHAT ACTUALLY HELD THE BOUNDARY WAS THE CUSTOMER LIST, NOT THE CODE. Three of the four doors -# below are reachable only by a tenant whose own `user_tables` document declares the table -# (`_defn_or_refuse`) or whose slug passes `is_royal` — so no second tenant could reach the mirror -# because no second tenant had mirror databases. R2 puts Meta Ads on this same mirror for GTM Lab, -# which ends that fact. A boundary that holds because of who our customers are is not a boundary. -# -# ⚠ AND ONE DOOR WAS ALREADY THROUGH IT TODAY — measured, not theorised: `/odoo-tables/status` -# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable` -# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row -# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes. -def _mirror_cur(session): - """A mirror cursor for THIS session's tenant, or a refusal that says which failure it is. - - Returns `(cursor, mismatch_sentence)` — exactly one of the two is None, so a caller cannot - accidentally treat "refused" as "empty". - - ⛔ IT CATCHES ONLY THE MISMATCH AND LETS THE OTHER `RuntimeError` THROUGH, because the two mean - opposite actions and every caller already has a policy for the second one: - - * `DatastoreMismatch` — this process has ANOTHER tenant's file open. It never fixes itself by - waiting; the deployment is mis-pinned. Returned as a SENTENCE so each door can choose its - own status (the row doors raise, the status door reports — R6's second sentence). - * a plain `RuntimeError` — the mirror is completing its first sync, and retrying works. It - PROPAGATES, so `odoo_table_rows` keeps answering `503 store_not_ready` and the status door - keeps degrading to "no size half", exactly as each did before this guard existed. - - ⚠ Collapsing them was the tempting simplification and it is the defect: `DatastoreMismatch` - SUBCLASSES `RuntimeError`, so a single `except RuntimeError` here would render every - cross-tenant refusal as a retry banner. Catching the subclass first is a correctness rule. - """ - rtm = _rt() - try: - return rtm.mirror_cursor(session.runtime), None - except rtm.DatastoreMismatch as e: - return None, str(e) - - -@router.post("/odoo-tables/refresh") -def refresh_odoo_tables(session: Session = Depends(require_session)): - """Rebuild all four locked Odoo databases from the tenant's mirror — customers, products, - invoices and orders (it was invoices + customers until 2026-08-09). - - Idempotent: row ids are Odoo ids, so a re-run updates in place. Returns what MOVED, because - "refreshed" with no counts is indistinguishable from a no-op over an empty mirror. - """ - # ⛔ `deps.err()` RETURNS an HTTPException, it does not raise one — so it must be `raise - # err(...)`. `return err(...)` serialises the exception object with a **200**, which is the - # wave-23 shape exactly: a finished route, green everything, wrong on the wire. The house - # convention is 243 `raise err` against 6 strays; this file uses `raise`. - rel = _rel() - # ⛔⛔ THIS LINE WAS `session.is_admin` AND IT IS WHY D-107 LOOKED LIKE A STORE PROBLEM FOR A - # DAY. `Session` is a plain dataclass with ONE admin accessor, the `.admin` property; there is - # no `is_admin` and no `__getattr__`, so the attribute lookup raised `AttributeError` — SEVEN - # LINES ABOVE the `try:` below, for every caller, admin or not. That is the whole explanation - # for the measured signature: a bare `500` carrying Starlette's default PLAIN-TEXT body - # instead of our JSON envelope, on a route whose own handler had just been taught to name the - # exception. The refresh never ran, never reached `plan()`, never touched the mirror. - # ⚠ The tell was in the diagnosis all along and was read as evidence about the STORE: "it is - # not a timeout, the same operation takes 16.8 s from a laptop". Correct, and the reason was - # that the request never got as far as doing any work at all. - # ⭐ It was the SOLE `session.is_admin` in the repo against ~40 `session.admin` — an - # unmounted-shaped defect that no gate could see, because `verify_api` pinned these two routes - # as MOUNTED and never CALLED them. That gate now calls them; see `verify_api`'s odoo-tables - # section and its negative control. - if not session.admin: - raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.") - if not rel.is_royal(session.tenant): - # A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty - # locked databases it can never fill would be worse than refusing. - raise err(400, "not_applicable", - "These databases are built from the Royal Imports Odoo mirror (R1).") - try: - out = rel.refresh(session.runtime, session.tenant, username=session.user) - # ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN. - # - # The spawn writes every bucket's rows straight into the tenant document by mutating it - # inside its own updater, so no guard in `core.user_tables` is on that path (measured, not - # assumed: `_ensure_table_inplace` never calls a function there). Stripping AFTER the write - # is what makes "a read-through grid stores no rows" true rather than intended — and the - # refresh is the only moment a stripped table can come back. - # ⚠ Reported in the response, because a silent 7 MB moving in or out of a tenant's - # document is exactly the kind of thing an operator should be able to see happening. - sync_read_through() - moved = _ut().strip_materialised(st=session.runtime) - return {"ok": True, **out, "unmaterialised": moved} - except rel.Refused as e: - # A refusal is the ANSWER, not a crash: the caller must see WHY nothing was written. - raise err(409, "refused", str(e)) - except RuntimeError as e: - # `datastore.ro_con()` raises this while the store is completing its first sync. Relaying - # it beats writing a partial table from a half-synced mirror. - raise err(503, "store_not_ready", str(e)) - except Exception as e: # noqa: BLE001 - # ⛔ AN UNEXPECTED FAILURE MUST STILL SAY WHAT IT WAS. Measured live 2026-08-09: this - # route answered a bare `500` and the operator had no way to learn why — the reason lived - # only in a container log nobody can reach from the product. A spawn that rewrites four - # locked databases is exactly the operation whose failure needs a sentence. - # ⚠ The TYPE is included deliberately: `BinderException: Referenced column "agent_id" not - # found` is a different action (wait for the column backfill) from a timeout or an auth - # failure, and "500" cannot tell them apart. - raise err(500, "refresh_failed", f"{type(e).__name__}: {e}") - - -@router.get("/odoo-tables/status") -def odoo_tables_status(session: Session = Depends(require_session)): - """Row counts + the newest `refreshed` stamp per table. - - ⚠ The stamp is the whole point while the resync wiring is outstanding: a locked collections - worklist that quietly stopped updating is a worklist that lies, so its age must be readable - without anybody running a refresh to find out. - """ - rel, out = _rel(), {} - import core.user_tables as user_tables - # ⭐ W30-T31 — THE ELIGIBILITY PASS RUNS HERE TOO, and the mirror cursor is taken best-effort: - # the freshness surface must answer on a box with no mirror (that is what it is FOR), so a - # store that is not ready costs the size half of the question, never the whole endpoint. - # ⭐⭐ W31-T45 / D-169 — THIS IS THE DOOR THAT WAS ALREADY LEAKING, and it leaked as COUNTS - # rather than as rows, which is why nothing caught it. There is no tenant gate above this - # line: `rel.is_royal(session.tenant)` appears once, as an `applicable` FIELD in the response - # 60 lines below. So a nurilab or gtmlab session took a cursor on tenant #0's `royal.duckdb`, - # `_source_for` read its columns and `_ds.window` counted its rows — real, plausible numbers - # belonging to another customer, returned 200 OK. - # ⛔ IT REPORTS RATHER THAN RAISING, deliberately, and the distinction is R6's second sentence: - # "which Odoo databases exist for me" is a legitimate question for any tenant and its honest - # answer here is *none* — a 409 would be refusing the question instead of the leak. So the - # mirror half is refused, `rowsFrom` says so per table, and the refusal rides the payload with - # its cause. A count that silently became `-1` would be the silent truncation R6 bans. - cur, mirror_refused = None, None - try: - cur, mirror_refused = _mirror_cur(session) - except Exception: # noqa: BLE001 - pass # mid-first-sync (or no mirror at all) — the document half answers - eligible, why_not = sync_read_through(cur) - # ⚠ ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)` - # while those were the only two; the 2026-08-09 widening added products and orders, and a - # hard-coded list here would have reported "everything is fine" over two databases it had - # stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count). - for _bucket, key, _label, _fields in rel.TABLES: - table = user_tables.get(key, st=session.runtime) or {} - rows = (table.get("rows") or {}) - stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()] - # ⛔⛔ A FRESHNESS SURFACE THAT LIES IS WORSE THAN NO FRESHNESS SURFACE, and this endpoint - # was one step from becoming one. Both numbers below are derived from `table["rows"]`, so - # the moment a grid stops materialising they would read `rows: 0, refreshed: ""` — an - # operator would see a database that looks EMPTY and STALE on a route whose own docstring - # says a worklist that quietly stopped updating must be legible without a refresh. So a - # read-through table is counted from the MIRROR and says where its count came from. - materialised = user_tables.materialises(key, st=session.runtime) - if not materialised: - try: - rows = {} - stamps = [] - spec = _source_for(cur, key) if cur is not None else None - if spec is not None: - from harness import datastore as _ds - frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") - else {"table": spec["table"]}) - n = _ds.window(select="1", where=spec.get("where") or "", - order_by=spec["id"], limit=1, cur=cur, **frm)["total"] - else: - n = -1 # unknown, and never reported as zero - except Exception: # noqa: BLE001 - n = -1 - out[key] = { - "exists": bool(table), - "rows": (len(rows) if materialised else n), - # ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from - # nowhere must not be labelled with the source it did not come from. - "rowsFrom": ("document" if materialised - else ("refused" if mirror_refused else "mirror")), - "materialised": materialised, - # why this grid still keeps its rows in the tenant document, when it does not have to - "materialisedBecause": (why_not.get(key, "") if materialised else ""), - "refreshed": (max([s for s in stamps if s], default="") if materialised - else "live — read through the mirror"), - "locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE, - # W30/R7: whether this database is served THROUGH the mirror rather than from the copy - # above. Reported per table, because they convert one at a time and an operator - # reading `rows: 0` needs to know whether that means "empty" or "not stored here". - "readThrough": key in _sources(), - } - # ⭐ W30-T32 / R6's SECOND SENTENCE, APPLIED TO OUR OWN HALF-BUILT STATE. A read-through - # binding whose FIELD DECLARATION has not landed yet answers 404 on the rows route, and an - # operator would read that as "the grid does not exist" rather than as "half of it shipped". - # These are the two line grains: bound here, declared in `odoo_relational` by W30-T35. - # ⚠ It reports the KEYS, never a field list — inventing a contract here is exactly the second - # source of truth the ticket forbids. - pending = {k: {"readThrough": True, "declared": False, - "cause": "this connected grid has a read-through binding but no field " - "declaration in odoo_relational yet, so it cannot be opened", - "recommendation": "declare its fields + a TABLES row (W30-T35); the binding " - "and the window are already live"} - for k in _sources() if k not in out} - return {"ok": True, "applicable": rel.is_royal(session.tenant), "tables": out, - "bound_not_declared": pending, - # ⭐ W31-T45 / D-169 — R6's SECOND SENTENCE, ON A GUARD RATHER THAN ON A ROW CAP. A - # limit that genuinely cannot be removed must be REPORTED with its cause and a - # recommended fix; silence is the violation. `null` when the mirror answered. - "mirrorRefused": ( - {"cause": mirror_refused, - "effect": "sizes_unavailable", - "recommendation": "this deployment has another tenant's analytical store open; " - "sizes are withheld rather than read from it. Pin the process " - "to this tenant (AIOS_DUCKDB_PATH / datastore.use_path) to " - "restore them."} - if mirror_refused else None), - # ⭐ W31-T46 / D-160 — WHY THERE IS NO MIRROR, when there is none. The last time this - # was silent it read as a pinned tag and a store problem for days. `null` when the - # container has one; a sentence with a cause and a fix when it does not. - "mirrorSeed": _seed_state()} - - -def _seed_state(): - """`main.MIRROR_SEED` as a reportable block, or None when the mirror is present. - - ⚠ IMPORTED LAZILY AND FAIL-QUIET: `main` imports this router, so a module-level import here - would be a cycle, and a freshness surface must never 500 because a diagnostic was unavailable. - """ - try: - import main as _main - from harness import datastore as _ds - if _ds.DB_PATH.exists(): - return None - state = dict(_main.MIRROR_SEED) - if not state.get("cause"): - state["cause"] = "this container has no analytical mirror yet" - state["recommendation"] = ("the boot seed runs independently of AIOS_PREWARM; if this " - "persists, check HF_TOKEN on the deployment") - return state - except Exception: # noqa: BLE001 - return None - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ THE READ-THROUGH WINDOW — owner ruling R6 ("no cap on connected-source data") via R7. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# THE PROBLEM IT REPLACES, in the owner's numbers. `core.user_tables.MAX_ROWS = 60_000` bounds a -# `ut_*` table because ALL of a tenant's tables live in ONE JSON document that `Store.get` -# deep-copies per request (D-87: four Odoo databases = 20.7 MB, ~370 ms a copy). Orders at -# **32,826** is already 55% of that ceiling; order lines (**255,286**) are 4.3× over it and GL -# lines (**971,034**) 16× over, so those two could never be grids at all — `odoo_relational.plan` -# refuses them rather than truncating, which is the correct refusal of the wrong architecture. -# -# ⭐ "NO CAP" IS NOT A BIGGER NUMBER. Raising `MAX_ROWS` would make every request slower for every -# tenant, including the ones who never open an Odoo grid. The mirror ALREADY holds all of it -# uncapped — measured on this box: 963,783 GL lines counted in ~24 ms — so the fix is to stop -# copying and serve a WINDOW: the requested slice, plus a `SELECT count(*)` that tells the truth -# about the whole. -# -# ⛔⛔ `total` IS NEVER `len(rows)`. It comes from `datastore.window`'s own count statement over -# the SAME predicate. A window whose count is its own length is a fabricated aggregate wearing an -# authoritative face ([[no-unverifiable-aggregates]]). -# -# ⛔ PREDICATES PUSH DOWN, and this is the half a client cannot be trusted with. A filter chip -# evaluated over the 200 rows that happen to be in memory would report "40 matches" out of -# 971,034 — [[one-question-two-normalizers]] at scale. `harness/filter_sql.py` compiles OUR filter -# vocabulary to SQL and is the same evaluator the TS engine is held in step with, so the fold and -# the display answer one question. -# -# ⭐ THE SPEC BELOW IS A SECOND STATEMENT OF SOMETHING `odoo_relational`'s READER ALREADY SAYS, -# AND THAT IS THE REAL RISK HERE — not SQL injection. It is gated rather than trusted: -# `verify_scopes.section_read_through` runs the REAL reader over the REAL mirror as an ORACLE and -# asserts the windowed path agrees ROW FOR ROW and TOTAL FOR TOTAL. A binding that drifts from its -# reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as -# though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]). -# -# ⚠ ONE BUCKET IS BOUND HERE (orders). The other seven still serve from the materialised copy and -# say so on the wire (`readThrough: false`), because R6's second sentence — *"if there is lag or -# it can't be done, you need to explicitly tell me why and recommend a fix"* — makes an unconverted -# grid something to REPORT, not something to leave looking converted. Adding one is a spec row plus -# a green oracle check. - -#: `ut_*` key -> how to read that grid straight out of the mirror. -#: -#: `where`/`select` are SQL WE author (never a request value); every request value is bound through -#: `params` by `filter_sql`. `cols` maps a FIELD KEY (what `odoo_relational._fields()` declares, -#: and what a saved view's filters name) to `(sql expression, coercion)`. The coercion reproduces -#: the reader's own python cast, by CALLING the reader's helpers where one exists — `_as_date` and -#: `_in_scope` are imported, not re-implemented. -#: Keys a reader does not produce (link columns, `refreshed`) are absent here on purpose: they are -#: filled by the grid at render time exactly as they are on the materialised path. -GRID_SOURCES = {} - -#: ⭐⭐ W31-T49 / C4 — THE SOURCE PROVIDERS. Odoo builds its specs below; ANY OTHER connector adds -#: its own by registering a builder here, and `_sources()` folds them into the one registry every -#: door already reads. -#: -#: ⛔ THIS IS THE SEAM THAT KEEPS "ONE REGISTRY" TRUE WHILE THE FILE STAYS ODOO-NAMED. The -#: alternative — a second `META_GRID_SOURCES` consulted beside this one — would give the platform -#: two answers to *"how is this connected grid read?"*, and every consumer (`_source_for`, -#: `whole_pool`, `population`, `odoo_tables_status`, `routes_tables.scoped_pids`) would have to -#: learn both or silently serve one. That is [[one-question-two-normalizers]] on the read path. -#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first -#: read — the same lazy rule `_sources` already follows for `odoo_relational`. -_SOURCE_PROVIDERS = [] - - -def register_source_provider(fn): - """Add a `() -> {table_key: spec}` builder to the connected-grid registry. - - Additive and idempotent by identity, so a re-import cannot double-register. Returns the number - of providers now known — a caller that wants to assert its registration took has a number. - """ - if callable(fn) and fn not in _SOURCE_PROVIDERS: - _SOURCE_PROVIDERS.append(fn) - return len(_SOURCE_PROVIDERS) - - -def _sources(): - """Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them. - - ⛔⛔ ODOO GOES THROUGH THE SEAM TOO, AND THAT IS THE POINT OF W31-T49 RATHER THAN A FLOURISH. - The first version built Odoo's specs inline here and folded other providers in afterwards, - which quietly says "Odoo is the registry and everyone else is an addendum" — two mechanisms - for one question, with the second one exercised by nobody until a connector arrives. It also - left `register_source_provider` with no production caller at all, which `verify_reachability` - correctly reddened as a capability behind no door ([[artifact-with-no-importer]]). Registering - the incumbent through its own seam makes the seam load-bearing from the first request. - - ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and - importing that module at file-import time would drag the Odoo layer into every process that - mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read. - """ - if GRID_SOURCES: - return GRID_SOURCES - for build in list(_SOURCE_PROVIDERS): - try: - extra = build() or {} - except Exception: # noqa: BLE001 - # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must - # not go dark because a newer connector's module is unhappy — and vice versa. - continue - if extra: - GRID_SOURCES.update(extra) - _ut().register_connected(*extra) - return GRID_SOURCES - - -def _odoo_sources(): - """Odoo's read-through bindings — `{table_key: spec}`. Registered as a provider below. - - ⚠ IT RETURNS a dict rather than mutating the module global: `_sources` owns the merge, so a - provider that half-built its specs and raised cannot leave a partial registry behind. - """ - rel = _rel() - # ⭐ R6 / W30-T29 — TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops - # being a fact about them. `core` never imports up, so the declaration goes this way round. - # ⚠ ALL EIGHT, not just the read-through-bound ones — every row in these tables comes from - # Odoo, which is what R6 is about; being served from the stored copy today is our conversion - # state, not a property of the data. `_sources`' own `register_connected` covers only the keys - # a provider RETURNS, so this call is not redundant with it and must not be folded into it. - _ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES]) - GRID_SOURCES = {} # the LOCAL registry this builder fills and returns - _s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)), - (lambda v: float(v or 0.0))) - GRID_SOURCES[rel.ORDERS_KEY] = { - "table": "sale_order", - # ⛔ IMPORTED, NOT RETYPED. `_CONFIRMED` is the fixed wholesale scope; if it ever changes, - # this window changes with it and the oracle check proves it did. - "where": f"{rel._CONFIRMED} AND partner_id IS NOT NULL", - "id": "id", - "needs_excluded": True, # `wholesale_scope` is resolved against the excluded set - "cols": { - "order_no": ("name", _s), - "odoo_id": ("id", _i), - "customer": ("partner_name", _s), - rel.JOIN_KEY: ("partner_id", _i), - "order_date": ("date_order", rel._as_date), - "amount_untaxed": ("amount_untaxed", _n), - "team": ("team_name", _s), - "state": ("state", _s), - "invoice_status": ("invoice_status", _s), - # ⭐ THE SCOPE COLUMN BECOMES REAL SQL, which is the point. On the materialised path it - # is `_in_scope(pid, excluded)` — a python set test, and a filter on it therefore could - # not push down. Inlining the ids (ints, from our own query) makes it a column the - # mirror can filter and sort on, so the R6 "limit" it would otherwise have earned does - # not exist. `{excluded}` is substituted by `_source_for` below. - "wholesale_scope": ("CASE WHEN partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), - }, - } - - # ═════════════════════════════════════════════════════════════════════════════════════════ - # ⭐⭐ W30-T32 — THE TWO LINE GRAINS. These are the grids R6 exists for: they have never had a - # `ut_*` table and never can, at any cap. MEASURED on this box's mirror, warm: - # sale_order_line 254,189 in the confirmed scope (256,810 unscoped) — 63.9 MB as JSON - # account_move_line 963,783 — ~240 MB as JSON - # Against `MAX_ROWS = 60_000` that is 4.2x and 16x, and against the 32 MB per-table document - # budget it is 2x and 7.5x. Read THROUGH, both serve a page in 134–166 ms. - # - # ⛔ THE FIELD KEYS BELOW ARE HALF OF A CONTRACT AND `odoo_relational` OWNS THE OTHER HALF - # (W30-T35, session E). `cols` binds a field key to SQL; the field's label, type and order are - # DECLARED THERE, once. Until that declaration lands the route answers 404 for these two keys - # (`rel.TABLES` has no entry), and `odoo_tables_status` REPORTS them as bound-not-declared - # rather than leaving them invisible — R6's second sentence applied to our own conversion. - # ⚠ A key here with no declaration there is a silent NO-CELL (`rows_from_pool` projects - # strictly); a key there with no binding here is an INACTIVE filter leaf, which WIDENS. The - # two lists are checked against each other by `verify_scopes.section_line_grids`. - ol_key = getattr(rel, "ORDER_LINES_KEY", "ut_odoo_order_lines") - gl_key = getattr(rel, "GL_LINES_KEY", "ut_odoo_gl_lines") - _ut().register_connected(ol_key, gl_key) - # ⛔ THE SCOPE IS THE ORDER'S, AND THE LINE TABLE CANNOT ANSWER IT ALONE: `sale_order_line` - # carries no `state` (12 columns, measured), so the confirmed-order scope — and `order_date`, - # and the order NAME a person reads the grid by — only exist across the join. MEASURED, warm, - # best of two: the JOIN beats `order_id IN (SELECT id FROM sale_order WHERE …)` at both depths - # (166 / 483 ms against 237 / 565 ms at offset 0 / 200,000), so the shape is chosen on a - # number rather than on taste. - # ⚠ `_CONFIRMED` is IMPORTED and QUALIFIED, never retyped — it opens with the bare column - # `state`, which `sale_order_line` does not have, so the prefix is what keeps it unambiguous - # if that table ever gains one. `section_line_grids` counts the same population a second way - # (a subquery, not a join) and the two must agree, which is what catches a mis-qualification. - GRID_SOURCES[ol_key] = { - "from_sql": "(sale_order_line sol JOIN sale_order so ON so.id = sol.order_id)", - "tables": {"sol": "sale_order_line", "so": "sale_order"}, - "where": f"so.{rel._CONFIRMED}", - "id": "sol.id", - "needs_excluded": True, - "cols": { - "odoo_id": ("sol.id", _i), - "order_no": ("so.name", _s), - "order_id": ("sol.order_id", _i), - "customer": ("sol.order_partner_name", _s), - rel.JOIN_KEY: ("sol.order_partner_id", _i), - "product": ("sol.product_name", _s), - rel.PRODUCT_JOIN_KEY: ("sol.product_id", _i), - "qty": ("sol.product_uom_qty", _n), - "price_subtotal": ("sol.price_subtotal", _n), - "margin": ("sol.margin", _n), - "purchase_price": ("sol.purchase_price", _n), - "order_date": ("so.date_order", rel._as_date), - "state": ("so.state", _s), - "wholesale_scope": ( - "CASE WHEN sol.order_partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), - }, - } - # ⚠ UNSCOPED ON PURPOSE, and it is a decision rather than an omission: every other grid here - # carries a fixed scope, but a GENERAL LEDGER whose draft and cancelled entries are invisible - # is a ledger that cannot be reconciled. `parent_state` rides as a column so a person filters - # in SQL over all 963,783 rows instead of us choosing for them. (Posted-only is 944,846.) - # The join to `account_account` is what makes `account_code` — the key `ut_odoo_accounts` is - # linked on — available at all; the mirror flattens `account_id`/`account_name` onto the line - # but not the CODE, and 192 accounts hash-join for free. - GRID_SOURCES[gl_key] = { - "from_sql": ("(account_move_line aml LEFT JOIN account_account aa " - "ON aa.id = aml.account_id)"), - "tables": {"aml": "account_move_line", "aa": "account_account"}, - "where": "", - "id": "aml.id", - "needs_excluded": True, - "cols": { - "odoo_id": ("aml.id", _i), - "entry": ("aml.move_name", _s), - "move_id": ("aml.move_id", _i), - "account": ("aml.account_name", _s), - rel.ACCOUNT_JOIN_KEY: ("aa.code", _s), - "customer": ("aml.partner_name", _s), - rel.JOIN_KEY: ("aml.partner_id", _i), - "date": ("aml.date", rel._as_date), - "debit": ("aml.debit", _n), - "credit": ("aml.credit", _n), - "balance": ("aml.balance", _n), - "line_type": ("aml.display_type", _s), - "move_type": ("aml.move_type", _s), - "parent_state": ("aml.parent_state", _s), - "wholesale_scope": ( - "CASE WHEN aml.partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), - }, - } - # ⭐ W30-T31 — THE GL ACCOUNT REGISTRY, BOUND BECAUSE IT IS THE ONE GRID THAT CAN ACTUALLY - # STOP MATERIALISING TODAY. 192 rows, nothing folds it, nothing links at it, and it fits - # inside one window — the three conditions `_unmaterialisable` checks. It is small, and that - # is the point: it is the first shipped database whose rows are NOT in the tenant document, - # so the stratum is proven on a real table instead of on a mechanism with no subject. - # ⚠ `account_fields()` declares six columns and `read_accounts` is one `cur.execute` over - # `account_account`; `section_read_through`'s differential oracle holds this binding to it. - # ⚠ `is_expense` REPRODUCES `read_accounts`' predicate IN SQL rather than inventing one, and - # that predicate is itself a copy of the semantic layer's `gl_lines` scope. Three statements of - # one rule is two too many, but the reader's own comment explains why it is copied rather than - # imported, and `section_read_through`'s differential oracle is what keeps this one honest. - GRID_SOURCES[rel.ACCOUNTS_KEY] = { - "table": "account_account", - "where": "", - "id": "id", - "cols": { - rel.ACCOUNT_JOIN_KEY: ("code", _s), - "account_name": ("name", _s), - "odoo_id": ("id", _i), - "account_type": ("account_type", _s), - "is_expense": ("CASE WHEN account_type IN ('expense','expense_depreciation') " - "THEN '1' ELSE '' END", _s), - }, - } - return GRID_SOURCES - - -#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers -#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`, -#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid. -register_source_provider(_odoo_sources) - - -def sync_read_through(cur=None): - """Register every grid that may stop storing rows, and REPORT why the rest may not. - - Returns `({key: eligible}, {key: reason})`. Idempotent, cheap, and safe to call from any door: - registration is additive and `strip_materialised` is a no-op once a table is empty. - - ⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility - question needs BOTH halves — `odoo_relational`'s field declarations (for the fold matrix) and - the mirror (for the size) — and `core` may import neither. - """ - eligible, reasons = _unmaterialisable(cur) - keys = [k for k, ok in eligible.items() if ok] - if keys: - _ut().register_read_through(*keys) - return eligible, reasons - - -#: ⚠ A mirror can be `ready()` and still lack a column (`ready()` reads entity PHASES; column -#: backfills checkpoint separately) — the gap that already cost a live 500. Every projected -#: expression is checked against the real column list and degraded to a literal, exactly as -#: `odoo_relational._col` does for the reader, so a fresh Space serves a blank cell rather than a -#: DuckDB Binder error. -#: -#: ⛔ W30-T32 — IT TAKES AN ALIAS MAP NOW, AND WITHOUT THAT THE GUARD WAS ABOUT TO GO BLIND. The -#: line-grain grids project `sol.price_subtotal` / `aml.parent_state`, and a dotted string is not -#: `isalnum()`, so the old single-table version returned EVERY qualified expression unchecked — -#: the same "expression, nothing to check" branch that correctly skips a CASE. Both of those -#: columns are 2026-07-28 backfills that a mirror can genuinely be missing, so the blind spot -#: would have surfaced as a bare DuckDB Binder error on a fresh Space, which is precisely the -#: failure this helper exists to prevent ([[gate-answers-the-wrong-question]]). -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W30-T31 / D-87 — WHICH CONNECTED GRIDS MAY STOP MATERIALISING, AND WHY MOST MAY NOT YET. -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The prize is real: all of a tenant's `ut_*` tables live in ONE document that `Store.get` -# deep-copies on EVERY call, hit or miss, and the four original Odoo grids are 20.7 MB / ~370 ms -# of it. Every permission check in the app pays that. -# -# ⛔ AND YOU CANNOT SIMPLY DELETE THE ROWS, WHICH IS THE FINDING THIS FUNCTION ENCODES. Measured -# mechanically across the eight field declarations, not by eye: -# * `ut_odoo_invoices` and `ut_odoo_orders` — 16.2 of those 20.7 MB — are folded by SIX link -# rollups on `ut_odoo_customers` (`ar_outstanding`, `open_invoices`, `oldest_due`, -# `invoiced_all_time`, `order_count`, `last_order`). `automation_engine.compute_relation_cells` -# answers those by reading the RAW document, so with the rows gone it writes zeros — silently. -# * every table with a `link` pointing AT it (agents, vendors, bills, customers) has its link -# CELLS materialised the same way, from the target's stored rows. -# * `ut_odoo_customers` and `ut_odoo_products` carry SOURCE rollups, and `rollup_sql.compute` -# writes those cells INTO their own stored rows: no rows, no cells. -# * and while the grid client still asks for a whole table (F's W30-T42 is what changes this), -# a population larger than one window could only be served by TRUNCATING it — which R6 -# forbids more strongly than it forbids a cap. -# -# ⭐ SO THE PREDICATE IS DERIVED FROM THE DECLARATIONS RATHER THAN LISTED. The day a lane converts -# those six link rollups to SOURCE rollups (`orders_ytd` on that same table is the precedent), or -# the day the client pages, the affected grids become eligible here with NO code change — and -# until then each one's reason is reported per table on the status door, which is R6's second -# sentence applied to our own conversion state. -def _fold_reasons(rel): - """`{table_key: "why its stored rows are still read by something else"}`. - - Read out of the FIELD DECLARATIONS themselves — one pass over `rel.TABLES`. A table absent - from this map is folded by nothing. - """ - reasons = {} - - def _add(key, why): - reasons.setdefault(str(key), []).append(why) - - for _bucket, key, _label, mk in rel.TABLES: - try: - fields = mk() - except Exception: # noqa: BLE001 - continue - by_key = {f.get("key"): f for f in fields if isinstance(f, dict)} - short = str(key).replace("ut_odoo_", "") - for f in fields: - if not isinstance(f, dict): - continue - if f.get("type") == "link" and (f.get("link") or {}).get("table"): - _add(f["link"]["table"], f"{short}.{f['key']} is a link whose cells are built " - f"from these rows") - if f.get("type") != "rollup": - continue - bag = f.get("rollup") or {} - if isinstance(bag.get("source"), dict): - _add(key, f"{short}.{f['key']} is a source rollup and its cells are written " - f"into these rows") - continue - tgt = ((by_key.get(str(bag.get("link") or "")) or {}).get("link") or {}).get("table") - if tgt: - _add(tgt, f"{short}.{f['key']} folds these rows") - return {k: "; ".join(v) for k, v in reasons.items()} - - -def _unmaterialisable(cur=None): - """`({key: eligible}, {key: reason})` — who may stop storing rows, and why the rest may not. - - ⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in - the first cut of this function. A table too big for the tenant document has NO materialised - option at all, so making it ineligible would have handed `odoo_relational.plan` a `row_limit` - of None and invited it to build 963,783 python dicts. The window ceiling is a different, much - softer thing: it only limits what the whole-table CLIENT door can serve today. - - rows > MAX_ROWS -> read-through REQUIRED (the document cannot hold it) - else if something folds it -> stays materialised (a fold over no rows writes ZEROS) - else if rows > WINDOW_MAX -> stays materialised until the client pages (W30-T42) - else -> eligible - - ⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never - part of the materialised spawn is eligible — a table is never freed by a question we skipped. - """ - rel = _rel() - folds, eligible, reasons = _fold_reasons(rel), {}, {} - # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT. - # W30-T35 declared the two line grains, which have no python row builder anywhere — their - # absence from `_READERS` IS that statement — so a table the spawn could never materialise - # started reading as spawned, and on a mirror-less deployment fell through to "its size could - # not be read" and came back INELIGIBLE. That inverts this function's own first law (size - # forces read-through; it never blocks it). Ask the question through the reader map, which is - # what actually decides whether a row could ever have been built. - spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})} - from harness import datastore - ut = _ut() - for key, spec in _sources().items(): - total = None - if cur is not None: - try: - frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") - else {"table": spec["table"]}) - total = datastore.window(select="1", where=spec.get("where") or "", - order_by=spec["id"], limit=1, cur=cur, **frm)["total"] - except Exception: # noqa: BLE001 - total = None - if total is not None and total > ut.MAX_ROWS: - eligible[key] = True # no other home exists; this is not a choice - continue - if total is None and key not in spawned: - eligible[key] = True # never materialised, so the mirror is its home - continue - why = [] - if total is None: - why.append("its size could not be read from the mirror on this deployment") - if key in folds: - why.append(folds[key]) - if total is not None and total > datastore.WINDOW_MAX: - why.append(f"its {total:,} rows exceed the {datastore.WINDOW_MAX:,}-row window and " - f"the grid client still asks for whole tables, so un-materialising it " - f"today could only truncate") - eligible[key] = not why - if why: - reasons[key] = "; ".join(why) - return eligible, reasons - - -def _degrade(expr, have_by_alias, default_alias=""): - bare = expr.strip() - alias, _, col = bare.partition(".") - if col and alias.replace("_", "").isalnum() and col.replace("_", "").isalnum(): - have = have_by_alias.get(alias) - if have is None: - return expr # an alias we do not own — leave it to the caller's SQL - return expr if col.lower() in have else "NULL" - if not bare.replace("_", "").isalnum(): - return expr # an expression, not a bare column — nothing to check - # ⚠ A BARE COLUMN IN A JOINED SPEC RESOLVES THE WAY SQL RESOLVES IT — against every table in - # the FROM, not against nothing. Checking it against `have_by_alias[""]`, which a joined spec - # does not have, would degrade every such column to NULL: a silent blank cell, which is the - # failure this helper exists to avoid rather than to cause. - have = have_by_alias.get(default_alias) - if have is None: - have = set().union(*have_by_alias.values()) if have_by_alias else set() - return expr if bare.lower() in have else "NULL" - - -def _source_for(cur, table_key): - """The resolved spec for one connected grid, or None when this grid is not read-through yet.""" - spec = _sources().get(str(table_key or "")) - if not spec: - return None - from harness import datastore - # ⚠ `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table - # spec declares none and its columns are bare, so it degrades against `table` as before. - aliases = dict(spec.get("tables") or {}) - if spec.get("table"): - aliases.setdefault("", spec["table"]) - have_by_alias = {} - for alias, tname in aliases.items(): - cols = datastore.columns_of(tname, cur=cur) - if not cols: - return None # the mirror has no such table on this deployment - have_by_alias[alias] = cols - excluded = "" - if spec.get("needs_excluded"): - ids = sorted(int(p) for p in _rel().excluded_ids(cur)) - # `-1` keeps the IN-list non-empty and matches no Odoo id, so the SQL shape is constant - # whether or not this tenant excludes a channel. - excluded = ", ".join(str(i) for i in ids) or "-1" - cols = {} - for key, (expr, cast) in spec["cols"].items(): - cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr, - have_by_alias), cast) - return {**spec, "cols": cols} - - -#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in -#: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of -#: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of -#: the person BEFORE the wait rather than after it. -_DEEP_PAGE = 100_000 - - -#: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our -#: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid -#: cannot disagree about whether a column is text or a number. -_FILTER_TYPE = {"select": "status", "checkbox": "text"} - - -def _filter_columns(spec, fields): - """`{colId: {sql, type, aggregate}}` — what `filter_sql` needs to compile a predicate. - - Only columns with a real mirror expression are offered. An omitted column is UNKNOWN to the - compiler, which skips it — and that is the one behaviour that must be REPORTED rather than - accepted, because an ignored condition WIDENS (the tri-state engine's inactive-leaf rule). - `_unpushable` below turns every such skip into an R6 sentence. - """ - by_key = {f["key"]: f for f in fields} - out = {} - for key, (expr, _cast) in spec["cols"].items(): - ftype = str((by_key.get(key) or {}).get("type") or "text") - out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False} - return out - - -def _leaf_cols(nodes): - """Every `colId` a filter tree names, at any depth.""" - seen = set() - for n in (nodes or []): - if not isinstance(n, dict): - continue - if n.get("children") is not None: - seen |= _leaf_cols(n.get("children")) - elif n.get("colId"): - seen.add(str(n["colId"])) - rhs = n.get("rhs") - if isinstance(rhs, dict) and rhs.get("colId"): - seen.add(str(rhs["colId"])) - return seen - - -def _json_arg(raw, what): - """Decode a JSON query argument, or refuse. ⛔ NEVER degrade to "no filter": a filter that - silently fails to parse WIDENS the answer, and the caller sees a plausible bigger number.""" - if raw in (None, ""): - return None - try: - val = json.loads(raw) - except Exception: # noqa: BLE001 - raise err(400, "bad_argument", f"{what} must be JSON") - return val - - -@router.get("/odoo-tables/{table_key}/rows") -def odoo_table_rows(table_key: str, - offset: int = Query(default=0, ge=0), - limit: int = Query(default=0), - filters: str = Query(default=None), - filterConj: str = Query(default="and"), - sorts: str = Query(default=None), - search: str = Query(default=None), - session: Session = Depends(require_session)): - """ONE WINDOW over a connected grid, read straight from the mirror (contract C2). - - `{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` — - `rows` is the requested slice and `total` is the count of everything the CURRENT PREDICATE - matches, from its own `SELECT count(*)`. `rows.length < total` is the normal case. - - ⛔ `total` is never `len(rows)`; `totalUnfiltered` is the population with the predicate - dropped, so a client can render "N of M" without inventing either number. - ⛔ The filter, the sort and the search all resolve in SQL against the whole table. Anything - that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in - `limits` with its cause and a recommendation — R6's second sentence — and never silently - dropped, because an ignored condition widens. - """ - import aios_grid - from harness import datastore - from routes_tables import _defn_or_refuse - - rel = _rel() - # THE WALL FIRST, and it is the SAME one the materialised path uses — 404 for a key that does - # not exist, 403 for one this session may not open. Reused rather than re-stated: a second - # idea of "may this session open this database" is a permission bug waiting to happen. - # - # ⭐⭐ W33-T02 / D-213 — `defs_only=True`, AND THIS ROUTE IS THE CLEANEST OPT-IN ON THE BOARD. - # `defn` is read EXACTLY ONCE below, for `recordMode`; the fields come from `rel.TABLES`'s - # `mk_fields()` and the rows from `datastore.window`, so nothing here has ever touched - # `defn["rows"]`. That is what makes it safe by inspection rather than by argument — and it is - # why the same change must NOT be swept across this file: `odoo_tables_status` a few hundred - # lines down reads `table["rows"]` and its per-row `refreshed` stamps for the MATERIALISED - # grids, and a projection there raises `KeyError` on every one of them. - # ⚠ `mirror_stamp`'s bare `except: return ""` would convert exactly that raise into a silently - # BLANK `refreshed` column rather than a red — which is why the boundary is drawn here, at the - # one function that provably needs no row, instead of at the file. - defn = _defn_or_refuse(session, table_key, defs_only=True) - fields = None - for _bucket, key, _label, mk_fields in rel.TABLES: - if key == table_key: - fields = mk_fields() - break - if fields is None: - raise err(404, "not_connected", "that database is not a connected Odoo grid") - - # ⭐⭐ W31-T45 / D-169 — THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot. - # That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process - # has open the one this TENANT's rows live in". A definition wall is satisfied the moment a - # tenant's own document declares a connected table — which is exactly what R2's Meta Ads spawn - # gives GTM Lab — and it would then serve that session a window over whatever DuckDB file the - # process happens to be pinned to. Rows, not counts. So this door RAISES. - # ⛔ 409, NOT 503, and the two were one line from being confused: `DatastoreMismatch` subclasses - # `RuntimeError`, and the handler directly below turns a `RuntimeError` into - # `503 store_not_ready` — "the store is completing its first sync, retry in a few minutes". A - # mis-pinned process never becomes un-mis-pinned by waiting, so relaying it as a retry would - # turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the - # try, not inside it. - try: - cur, mismatch = _mirror_cur(session) - except RuntimeError as e: - # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the - # mismatch subclass, which is what makes this broad clause safe to keep here. - raise err(503, "store_not_ready", str(e)) - if mismatch: - raise err(409, "cross_tenant_store", mismatch) - - spec = _source_for(cur, table_key) - if spec is None: - raise err(409, "not_read_through", - "this connected database is still served from its stored copy; it has no " - "read-through binding on this deployment yet") - - cols = _filter_columns(spec, fields) - limits, tree = [], _json_arg(filters, "filters") - sort_spec = _json_arg(sorts, "sorts") or [] - - # ── the predicate, pushed down ──────────────────────────────────────────────────────────── - named = _leaf_cols(tree) - missing = sorted(named - set(cols)) - if missing: - limits.append({ - "subject": ", ".join(missing), "effect": "filter_ignored", - "cause": "these columns have no expression in the mirror (a link, a rollup, or a " - "column this deployment's mirror has not backfilled), so a condition on " - "them cannot be answered in SQL", - "recommendation": "filter on the id column the link is built from, or open the " - "linked database directly"}) - # ⭐ R6, MEASURED, AND IT IS THE KIND OF LIMIT THE RULING EXISTS FOR — a difference in the - # ANSWER, not in the speed. - # - # `filter_sql._value_sql` compiles every numeric comparison as `round_even(x, 0)`, and says - # why: *"reproduces `aios_grid._round` … The grid displays rounded values; filters must agree - # with what is on screen."* That is true of `source: "odoo"` columns, which `rows_from_pool` - # rounds. It is FALSE here — these columns are `source: "overlay"` (a storage choice, not a - # display one) so `rows_from_pool` passes the exact value through, and the TS engine - # (`useVisibleRows.toNum`) does not round either. So the pushdown compares at whole units while - # the cell beside it carries cents. - # - # MEASURED on the live orders mirror (32,700 rows, 47.9% with a non-integer amount): - # > 1000 python 4,476 vs SQL 4,473 (-3) - # > 173.6 python 24,711 vs SQL 24,726 (+15) - # > 500.25 python 11,028 vs SQL 11,026 (-2) - # Small, and NOT nothing. Rounding the wire to match would have been the other fix and it was - # rejected on measurement: it changes 96 of every 200 money cells (173.55 -> 174) to remove a - # 0.05% counting difference — a visible product regression traded for an invisible one. - # ⛔ SO IT IS REPORTED INSTEAD. Booked for the owner of `harness/filter_sql.py`, which is not - # this fence; see mailbox/D.md. - numeric = sorted(k for k in (named & set(cols)) - if cols[k]["type"] in ("currency", "int", "pct")) - if numeric: - limits.append({ - "subject": ", ".join(numeric), "effect": "precision", - "cause": "a number condition is evaluated in SQL at whole-unit precision " - "(`filter_sql` rounds to match the grids whose values the server rounds), " - "while these cells carry their exact value — so a row within half a unit of " - "the threshold can fall on the other side of it", - "recommendation": "compare against a whole number, or use a range that does not sit " - "on a fractional boundary"}) - - where, params = spec["where"], [] - try: - pred = _fs().compile_filter_tree( - tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols, - today=_today()) - except ValueError as e: - # ⛔ A REFUSAL IS AN ANSWER, NOT A CRASH — and it must not become "no filter". Ranking ops - # ("top 10") have no SQL form in this compiler; saying so beats returning every row. - raise err(400, "filter_unsupported", str(e)) - if pred is not None: - if pred.uses_aggregate: - raise err(400, "filter_unsupported", - "a condition on an aggregate column belongs in HAVING, and that path is " - "deliberately not built for windowed grids") - where = f"({where}) AND {pred.sql}" if where else pred.sql - params.extend(pred.params) - if str(search or "").strip(): - got = _fs().compile_search(search.strip(), cols) - if got is not None: - where = f"({where}) AND {got.sql}" if where else got.sql - params.extend(got.params) - - # ── the order, made TOTAL ───────────────────────────────────────────────────────────────── - # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one - # row on two pages and drop another entirely — a duplicate the user sees with no error - # anywhere. `compile_order_by`'s own docstring says so. - order = _fs().compile_order_by(sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"] - - select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) - # ⚠ EXACTLY ONE of `table`/`from_sql` — `window` raises if both or neither arrive, so the - # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM. - frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', - where=where, params=tuple(params), order_by=order, - offset=offset, limit=(limit or None), cur=cur, **frm) - if win["clamped"]: - limits.append({ - "subject": "limit", "effect": "window_clamped", - "cause": f"one response carries at most {datastore.WINDOW_MAX} rows so a single " - f"request cannot exhaust memory for every other tenant on the process", - "recommendation": "page with `offset`; `total` already reports the whole population, " - "and no row is unreachable"}) - # ⭐ R6's SECOND SENTENCE, ON THE ONE LIMIT THAT SURVIVES THE CONVERSION. Removing the row cap - # does not make every row equally cheap: `LIMIT/OFFSET` WALKS the offset, so the deeper the - # page the longer the scan. MEASURED warm on 963,783 GL lines — offset 0: 134 ms · 100,000: - # 71 ms · 900,000: **1,450 ms**; sorted by date rather than by id, offset 500,000: 1,645 ms. - # It is a real cost, it is nobody's mistake, and the owner asked to be told rather than to - # discover it: say so with the fix, which is a CURSOR the client has to send. - if offset >= _DEEP_PAGE: - limits.append({ - "subject": "offset", "effect": "slow", - "cause": f"a page {offset:,} rows deep is reached by walking every row before it " - f"(SQL OFFSET has no other meaning), which costs about a second past " - f"half a million rows", - "recommendation": "jump with a filter or a sort instead of scrolling, or ask for " - "keyset paging (`id > `), which is O(page) at any depth " - "and needs the client to send the last row it holds"}) - - # ── the wire rows, through the SAME serialiser the materialised path uses ────────────────── - # ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the - # `refreshed` column identically instead of one of them leaving it blank. - rows_src = _pool_from_window(spec, win, - stamp=mirror_stamp(spec, cur, table_key, - session.runtime)) - overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src} - - # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID. - # - # A user-added column on a connected grid has nowhere per-user to live: there is no `ut_*` row - # to hang it on any more, and `table_store`'s per-user strata would make a SHARED view name a - # column other accounts do not have — an unknown column is an INACTIVE condition in the - # tri-state engine, which WIDENS. `core/shared_overlay.py` was built for exactly this and has - # had no product door since it shipped (W29-T62). - # - # ⛔ `cells(table_key, pids)` TAKES THE PIDS AND THERE IS NO "EVERYTHING" CALL — and here that - # is an asset rather than a chore: **the window IS the scoped row set**, already narrowed by - # the wall and the predicate, so the argument it demands is the list we just fetched. - # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already - # answered "may this session open this surface". - shared_defs = _so().fields(table_key, st=session.runtime) - if shared_defs: - fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()] - for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src], - st=session.runtime).items(): - overlays.setdefault(pid, {}).update(cells) - rows = aios_grid.rows_from_pool(rows_src, fields, overlays) - - unfiltered = win["total"] if where == spec["where"] else datastore.window( - select="1", where=spec["where"], order_by=spec["id"], limit=1, cur=cur, **frm)["total"] - - return {"ok": True, "fields": fields, "rows": rows, - # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`. - "total": win["total"], "totalUnfiltered": unfiltered, - "offset": win["offset"], "limit": win["limit"], "limits": limits, - "today": _today(), "identity": {"pid": "pid"}, - "scope": {"table": table_key, "readThrough": True}, - "recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)} - - -def mirror_stamp(spec, cur=None, table_key="", rt=None): - """⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date. - - ⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table - declares `refreshed` (`odoo_relational._refreshed_field`), and a MATERIALISED grid gets a - per-row stamp written by the spawn. A read-through grid has no stored row to carry one, so - `_pool_from_window` produced no `refreshed` key at all and `rows_from_pool` projected it as - `""` — a freshness column that used to say something and now silently says nothing, on the two - biggest grids in the product. Nothing goes red for that; it just quietly stops being true. - - ⭐ THE HONEST VALUE ALREADY EXISTS — the mirror keeps a per-entity sync stamp in `_sync_state`, - which is the same fact one level up: these rows are as fresh as the last sync of the table - they are read through. It is per-TABLE, not per-row, and that is not a downgrade — for a grid - that stores no rows, "when did this table last sync" IS the per-row answer. - - ⚠ READ ON THE CALLER'S CURSOR, never `datastore.status()`. That opens a SECOND connection to a - file DuckDB holds exclusively, so a freshness column would turn a working grid into an - IOException — a cosmetic fix taking out the feature it decorates. - ⚠ Date only (`YYYY-MM-DD`), because `refreshed` is declared `type: "date"` and the grid's date - renderer is what reads it; handing it a full timestamp renders the ISO string a user should - never see (the wave-26 `copyData` scar, one column over). - - ⛔⛔ THE TABLE COMES FROM THE **ID COLUMN'S ALIAS**, AND THE FIRST DRAFT GOT THIS WRONG IN THE - ONLY WAY THAT MATTERS. It read `spec["table"]` with a `tables[""]` fallback — and **neither of - the two grids D-155 is about has a bare `table` key**: `ut_odoo_order_lines` and - `ut_odoo_gl_lines` are join-shaped (`from_sql` + `tables: {"sol": …, "so": …}` / - `{"aml": …, "aa": …}`), so the lookup returned `""` and the fix would have shipped doing - nothing on exactly the two grids it was written for — green gate, unchanged product. It was - caught only because the gate's fixture used `ut_odoo_orders`, a MATERIALISED grid that never - had the defect [[gate-answers-the-wrong-question]]. - ⭐ The id column IS the grain: `sol.id` means this grid is one row per `sale_order_line`, and - the freshness of a joined lookup table (`sale_order`, `account_account`) is not this grid's - freshness. So the alias is read off `spec["id"]`, and a bare `id` degrades to `table` — which - is exactly the single-table case. - - ⛔ AND IT ANSWERS `""` FOR A **MATERIALISED** GRID, ON PURPOSE. Those eight tables store a - per-ROW `refreshed` written by the spawn, which is strictly better than one table-level date — - and this value arrives as an OVERLAY, so returning a stamp here would quietly overwrite eight - working grids' per-row stamps while fixing two blank ones. D-155's subject is the grid that has - NOWHERE to keep a row; a fix that also rewrites the grids that do is a different, unrequested - change [[reuse-and-delete-are-hypotheses]]. - """ - # the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot - # tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank), - # never a stamp that might overwrite a per-row one. - if table_key: - try: - if _ut().materialises(str(table_key), st=rt): - return "" - except Exception: # noqa: BLE001 - return "" - ident = str(spec.get("id") or "") - alias = ident.split(".", 1)[0] if "." in ident else "" - table = str((spec.get("tables") or {}).get(alias) or spec.get("table") - or (spec.get("tables") or {}).get("") or "") - if not table or cur is None: - return "" - try: - row = cur.execute("SELECT updated_at FROM _sync_state WHERE entity = ?", - [table]).fetchone() - except Exception: # noqa: BLE001 - return "" # no mirror bookkeeping ⇒ blank, exactly as before - return str((row or [""])[0] or "")[:10] - - -def _pool_from_window(spec, win, stamp=""): - """`[{pid, **cells}]` — THE one place a mirror window becomes product rows. - - ⛔ ONE BUILDER, TWO CALLERS, and that is deliberate rather than tidy: the windowed route and - the whole-table read-through below would otherwise each cast the same columns their own way, - and a cell that renders differently depending on which door served it is this repo's recorded - defect class ([[one-question-two-normalizers]]). - - ⚠ `stamp` (D-155) rides here for that same reason: both doors must produce the same - `refreshed` cell, and a caller that forgot it would give one door a freshness column and the - other a blank one. Empty when the caller cannot cheaply know — blank is what shipped, so the - degradation is the previous behaviour rather than a new wrong value. - """ - keys = list(spec["cols"]) - extra = {"refreshed": stamp} if stamp else {} - return [{"pid": int(r[0]), **extra, - **{k: spec["cols"][k][1](v) for k, v in zip(keys, r[1:])}} for r in win["rows"]] - - -class TooBigToMaterialise(Exception): - """A read-through table asked for WHOLE exceeds one window — refuse, never truncate.""" - - -def population(table_key, cur=None, rt=None): - """How many rows a read-through grid HAS, without fetching one — or None if it is not bound. - - ⭐⭐ W31 / B's ask (mailbox/B.md B-2). `routes_tables.scoped_pids` learned "this grid is too big - to list" the only way that existed: call `whole_pool()` and catch `TooBigToMaterialise`. That - pulls a full 5,000-row window out of DuckDB and throws it away on every `/workspace` for - `ut_odoo_gl_lines` and `ut_odoo_order_lines` — MEASURED by B at ~1,600 ms of a ~3,460 ms - in-proc envelope. This asks the SIZE instead: one `SELECT count(*)` over the same predicate. - - ⛔ IT ANSWERS A DIFFERENT QUESTION, NOT THE SAME ONE MORE CHEAPLY, and that distinction is what - keeps it safe. B's constraint is that the pid set must stay IDENTICAL to `scoped_pool`'s BY - CONSTRUCTION rather than by two queries that agree today ([[one-question-two-normalizers]]) — - so this returns a COUNT and never a pid. A caller uses it to decide whether to ask for pids at - all; when it does ask, the pids still come from the one `whole_pool` fetch they always did. - - ⚠ `None` means "no read-through binding on this deployment", which is NOT `0`. A caller that - collapses them reports an empty grid where it should report an unbound one — the exact - distinction `odoo_tables_status` already reports as `bound_not_declared`. - ⚠ `total` is the count over the spec's OWN `where`, i.e. the same population `whole_pool` - would return — the fixed scope is not dropped for being cheaper. - """ - from harness import datastore - if rt is not None: - rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry - cur = cur if cur is not None else datastore.ro_con() - spec = _source_for(cur, table_key) - if spec is None: - return None - frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"], - limit=1, cur=cur, **frm)["total"] - - -def whole_pool(table_key, cur=None, rt=None): - """Every row of a read-through grid, in `scoped_pool`'s shape — or a refusal. - - ⚠ THIS EXISTS FOR THE CLIENT WE HAVE, NOT THE ONE WE WANT. The grid still fetches whole - tables (`GET /tables/{key}/rows`); paging is F's W30-T42. So a database whose rows have left - this tenant's document has to be servable to that client somehow, and the honest answer for a - small one is "read all of it from the mirror". `_unmaterialisable` only ever registers a table - that fits, so the refusal below is a guard against the population GROWING past the window - later — at which point R6 requires a sentence, not a quietly shorter grid. - - ⭐⭐ W31-T45 / D-169 — `rt` IS THE TENANT RUNTIME, AND IT IS OPTIONAL FOR A STATED REASON - RATHER THAN A LAZY ONE. This function serves ROWS, so it is the door where a cross-tenant read - would be worst — but its only production caller is `routes_tables.py::_read_through_rows`, - which is in another lane's fence THIS WAVE (B's W31-T20 is rewriting that exact path) and does - not thread a session down to here. Making `rt` required would break that caller on import; a - second predicate invented locally would be one question with two normalizers, a recorded - defect class here. So the guard fires when a caller passes the runtime it already holds, and - the ask to thread `session.runtime` through `_read_through_rows` is posted to B in - `mailbox/D.md`. ⛔ Until that lands this door is guarded only by `_defn_or_refuse` upstream — - stated here rather than left for a reader to discover, because an unguarded row door is - precisely what D-169 is about. - """ - from harness import datastore - if rt is not None: - cur = cur if cur is not None else _rt().mirror_cursor(rt) - rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent - cur = cur if cur is not None else datastore.ro_con() - spec = _source_for(cur, table_key) - if spec is None: - raise TooBigToMaterialise( - f"{table_key}: this database is served through the mirror but has no read-through " - f"binding on this deployment, so its rows cannot be read at all") - frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} - select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) - win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=spec.get("where"), - order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm) - if win["total"] > len(win["rows"]): - # ⛔ REFUSE, NEVER TRIM. A short grid that says nothing is exactly the silent truncation - # R6's second sentence is about — and the caller turns this into a message with a cause. - raise TooBigToMaterialise( - f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row " - f"window, and this database no longer stores rows in the tenant document; it can " - f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)") - return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt)) - - -def _fs(): - from harness import filter_sql - return filter_sql - - -def _so(): - import core.shared_overlay as shared_overlay - return shared_overlay - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _today(): - import time - return time.strftime("%Y-%m-%d") +"""routes_odoo_tables.py — the door to the Odoo relational spawn (wave 27 item 17, contract C8). + +Two endpoints and no cleverness: refresh the locked Odoo databases, and report their freshness. +The work itself lives in `odoo_relational.py`; this file only decides WHO may ask and turns a +refusal into a status code. + +⛔ THE MOUNT IS DONE (`main.py` includes this router) and the wave-23 scar it was written against +— three finished routers shipping 404-dead behind green gates — is covered by `verify_api` +enumerating `main.app.routes`. ⚠ BUT WAVE 28 PROVED THAT CONTROL IS ONLY HALF OF THE QUESTION: +being mounted is not being CALLABLE. This router was mounted, enumerated, green, and answered a +plain-text 500 to every request for a day because of an attribute typo in the admin check +(`session.is_admin`, which does not exist). A route-existence check cannot see that; only calling +it can. `verify_api` now does both. + +ADMIN-GATED, and not for tidiness: a refresh REWRITES four locked databases for the whole tenant +and deletes the rows that left the population. That is an operator action. +⭐ WAVE 30 (R6/R7, contract C2) ADDS A THIRD ENDPOINT AND IT IS A DIFFERENT KIND OF THING: the +READ-THROUGH WINDOW. The two above operate on the materialised copy; `/{table_key}/rows` does not +read the copy at all. See the block above `GRID_SOURCES` for why that had to change. +""" +import json + +from fastapi import Depends +from fastapi import APIRouter, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +def _rel(): + import odoo_relational + return odoo_relational + + +def _rt(): + from harness import runtime + return runtime + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W31-T45 / D-169 — THE CROSS-TENANT MIRROR GUARD, AND WHY EVERY DOOR BELOW GOES THROUGH IT. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# `harness/datastore` holds ONE process-wide DuckDB connection over a process-global `DB_PATH`, +# and ONE Space process serves EVERY tenant. `TenantRuntime.assert_datastore_matches` was written +# for exactly the failure that arrangement produces — its own docstring names it: *"an analytical +# read served from another tenant's file returns real, plausible, wrong rows … and the number +# reconciles against the wrong book"* — and until this wave its only caller anywhere was +# `verify_api.py`. The guard was correct, tested, and governed nothing. +# +# ⛔ WHAT ACTUALLY HELD THE BOUNDARY WAS THE CUSTOMER LIST, NOT THE CODE. Three of the four doors +# below are reachable only by a tenant whose own `user_tables` document declares the table +# (`_defn_or_refuse`) or whose slug passes `is_royal` — so no second tenant could reach the mirror +# because no second tenant had mirror databases. R2 puts Meta Ads on this same mirror for GTM Lab, +# which ends that fact. A boundary that holds because of who our customers are is not a boundary. +# +# ⚠ AND ONE DOOR WAS ALREADY THROUGH IT TODAY — measured, not theorised: `/odoo-tables/status` +# takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable` +# FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row +# counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes. +def _mirror_cur(session): + """A mirror cursor for THIS session's tenant, or a refusal that says which failure it is. + + Returns `(cursor, mismatch_sentence)` — exactly one of the two is None, so a caller cannot + accidentally treat "refused" as "empty". + + ⛔ IT CATCHES ONLY THE MISMATCH AND LETS THE OTHER `RuntimeError` THROUGH, because the two mean + opposite actions and every caller already has a policy for the second one: + + * `DatastoreMismatch` — this process has ANOTHER tenant's file open. It never fixes itself by + waiting; the deployment is mis-pinned. Returned as a SENTENCE so each door can choose its + own status (the row doors raise, the status door reports — R6's second sentence). + * a plain `RuntimeError` — the mirror is completing its first sync, and retrying works. It + PROPAGATES, so `odoo_table_rows` keeps answering `503 store_not_ready` and the status door + keeps degrading to "no size half", exactly as each did before this guard existed. + + ⚠ Collapsing them was the tempting simplification and it is the defect: `DatastoreMismatch` + SUBCLASSES `RuntimeError`, so a single `except RuntimeError` here would render every + cross-tenant refusal as a retry banner. Catching the subclass first is a correctness rule. + """ + rtm = _rt() + try: + return rtm.mirror_cursor(session.runtime), None + except rtm.DatastoreMismatch as e: + return None, str(e) + + +@router.post("/odoo-tables/refresh") +def refresh_odoo_tables(session: Session = Depends(require_session)): + """Rebuild all four locked Odoo databases from the tenant's mirror — customers, products, + invoices and orders (it was invoices + customers until 2026-08-09). + + Idempotent: row ids are Odoo ids, so a re-run updates in place. Returns what MOVED, because + "refreshed" with no counts is indistinguishable from a no-op over an empty mirror. + """ + # ⛔ `deps.err()` RETURNS an HTTPException, it does not raise one — so it must be `raise + # err(...)`. `return err(...)` serialises the exception object with a **200**, which is the + # wave-23 shape exactly: a finished route, green everything, wrong on the wire. The house + # convention is 243 `raise err` against 6 strays; this file uses `raise`. + rel = _rel() + # ⛔⛔ THIS LINE WAS `session.is_admin` AND IT IS WHY D-107 LOOKED LIKE A STORE PROBLEM FOR A + # DAY. `Session` is a plain dataclass with ONE admin accessor, the `.admin` property; there is + # no `is_admin` and no `__getattr__`, so the attribute lookup raised `AttributeError` — SEVEN + # LINES ABOVE the `try:` below, for every caller, admin or not. That is the whole explanation + # for the measured signature: a bare `500` carrying Starlette's default PLAIN-TEXT body + # instead of our JSON envelope, on a route whose own handler had just been taught to name the + # exception. The refresh never ran, never reached `plan()`, never touched the mirror. + # ⚠ The tell was in the diagnosis all along and was read as evidence about the STORE: "it is + # not a timeout, the same operation takes 16.8 s from a laptop". Correct, and the reason was + # that the request never got as far as doing any work at all. + # ⭐ It was the SOLE `session.is_admin` in the repo against ~40 `session.admin` — an + # unmounted-shaped defect that no gate could see, because `verify_api` pinned these two routes + # as MOUNTED and never CALLED them. That gate now calls them; see `verify_api`'s odoo-tables + # section and its negative control. + if not session.admin: + raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.") + if not rel.is_royal(session.tenant): + # A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty + # locked databases it can never fill would be worse than refusing. + raise err(400, "not_applicable", + "These databases are built from the Royal Imports Odoo mirror (R1).") + try: + out = rel.refresh(session.runtime, session.tenant, username=session.user) + # ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN. + # + # The spawn writes every bucket's rows straight into the tenant document by mutating it + # inside its own updater, so no guard in `core.user_tables` is on that path (measured, not + # assumed: `_ensure_table_inplace` never calls a function there). Stripping AFTER the write + # is what makes "a read-through grid stores no rows" true rather than intended — and the + # refresh is the only moment a stripped table can come back. + # ⚠ Reported in the response, because a silent 7 MB moving in or out of a tenant's + # document is exactly the kind of thing an operator should be able to see happening. + sync_read_through() + moved = _ut().strip_materialised(st=session.runtime) + return {"ok": True, **out, "unmaterialised": moved} + except rel.Refused as e: + # A refusal is the ANSWER, not a crash: the caller must see WHY nothing was written. + raise err(409, "refused", str(e)) + except RuntimeError as e: + # `datastore.ro_con()` raises this while the store is completing its first sync. Relaying + # it beats writing a partial table from a half-synced mirror. + raise err(503, "store_not_ready", str(e)) + except Exception as e: # noqa: BLE001 + # ⛔ AN UNEXPECTED FAILURE MUST STILL SAY WHAT IT WAS. Measured live 2026-08-09: this + # route answered a bare `500` and the operator had no way to learn why — the reason lived + # only in a container log nobody can reach from the product. A spawn that rewrites four + # locked databases is exactly the operation whose failure needs a sentence. + # ⚠ The TYPE is included deliberately: `BinderException: Referenced column "agent_id" not + # found` is a different action (wait for the column backfill) from a timeout or an auth + # failure, and "500" cannot tell them apart. + raise err(500, "refresh_failed", f"{type(e).__name__}: {e}") + + +@router.get("/odoo-tables/status") +def odoo_tables_status(session: Session = Depends(require_session)): + """Row counts + the newest `refreshed` stamp per table. + + ⚠ The stamp is the whole point while the resync wiring is outstanding: a locked collections + worklist that quietly stopped updating is a worklist that lies, so its age must be readable + without anybody running a refresh to find out. + """ + rel, out = _rel(), {} + import core.user_tables as user_tables + # ⭐ W30-T31 — THE ELIGIBILITY PASS RUNS HERE TOO, and the mirror cursor is taken best-effort: + # the freshness surface must answer on a box with no mirror (that is what it is FOR), so a + # store that is not ready costs the size half of the question, never the whole endpoint. + # ⭐⭐ W31-T45 / D-169 — THIS IS THE DOOR THAT WAS ALREADY LEAKING, and it leaked as COUNTS + # rather than as rows, which is why nothing caught it. There is no tenant gate above this + # line: `rel.is_royal(session.tenant)` appears once, as an `applicable` FIELD in the response + # 60 lines below. So a nurilab or gtmlab session took a cursor on tenant #0's `royal.duckdb`, + # `_source_for` read its columns and `_ds.window` counted its rows — real, plausible numbers + # belonging to another customer, returned 200 OK. + # ⛔ IT REPORTS RATHER THAN RAISING, deliberately, and the distinction is R6's second sentence: + # "which Odoo databases exist for me" is a legitimate question for any tenant and its honest + # answer here is *none* — a 409 would be refusing the question instead of the leak. So the + # mirror half is refused, `rowsFrom` says so per table, and the refusal rides the payload with + # its cause. A count that silently became `-1` would be the silent truncation R6 bans. + cur, mirror_refused = None, None + try: + cur, mirror_refused = _mirror_cur(session) + except Exception: # noqa: BLE001 + pass # mid-first-sync (or no mirror at all) — the document half answers + eligible, why_not = sync_read_through(cur) + # ⚠ ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)` + # while those were the only two; the 2026-08-09 widening added products and orders, and a + # hard-coded list here would have reported "everything is fine" over two databases it had + # stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count). + for _bucket, key, _label, _fields in rel.TABLES: + table = user_tables.get(key, st=session.runtime) or {} + rows = (table.get("rows") or {}) + stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()] + # ⛔⛔ A FRESHNESS SURFACE THAT LIES IS WORSE THAN NO FRESHNESS SURFACE, and this endpoint + # was one step from becoming one. Both numbers below are derived from `table["rows"]`, so + # the moment a grid stops materialising they would read `rows: 0, refreshed: ""` — an + # operator would see a database that looks EMPTY and STALE on a route whose own docstring + # says a worklist that quietly stopped updating must be legible without a refresh. So a + # read-through table is counted from the MIRROR and says where its count came from. + materialised = user_tables.materialises(key, st=session.runtime) + if not materialised: + try: + rows = {} + stamps = [] + spec = _source_for(cur, key) if cur is not None else None + if spec is not None: + from harness import datastore as _ds + frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") + else {"table": spec["table"]}) + n = _ds.window(select="1", where=spec.get("where") or "", + order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + else: + n = -1 # unknown, and never reported as zero + except Exception: # noqa: BLE001 + n = -1 + out[key] = { + "exists": bool(table), + "rows": (len(rows) if materialised else n), + # ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from + # nowhere must not be labelled with the source it did not come from. + "rowsFrom": ("document" if materialised + else ("refused" if mirror_refused else "mirror")), + "materialised": materialised, + # why this grid still keeps its rows in the tenant document, when it does not have to + "materialisedBecause": (why_not.get(key, "") if materialised else ""), + "refreshed": (max([s for s in stamps if s], default="") if materialised + else "live — read through the mirror"), + "locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE, + # W30/R7: whether this database is served THROUGH the mirror rather than from the copy + # above. Reported per table, because they convert one at a time and an operator + # reading `rows: 0` needs to know whether that means "empty" or "not stored here". + "readThrough": key in _sources(), + } + # ⭐ W30-T32 / R6's SECOND SENTENCE, APPLIED TO OUR OWN HALF-BUILT STATE. A read-through + # binding whose FIELD DECLARATION has not landed yet answers 404 on the rows route, and an + # operator would read that as "the grid does not exist" rather than as "half of it shipped". + # These are the two line grains: bound here, declared in `odoo_relational` by W30-T35. + # ⚠ It reports the KEYS, never a field list — inventing a contract here is exactly the second + # source of truth the ticket forbids. + pending = {k: {"readThrough": True, "declared": False, + "cause": "this connected grid has a read-through binding but no field " + "declaration in odoo_relational yet, so it cannot be opened", + "recommendation": "declare its fields + a TABLES row (W30-T35); the binding " + "and the window are already live"} + for k in _sources() if k not in out} + return {"ok": True, "applicable": rel.is_royal(session.tenant), "tables": out, + "bound_not_declared": pending, + # ⭐ W31-T45 / D-169 — R6's SECOND SENTENCE, ON A GUARD RATHER THAN ON A ROW CAP. A + # limit that genuinely cannot be removed must be REPORTED with its cause and a + # recommended fix; silence is the violation. `null` when the mirror answered. + "mirrorRefused": ( + {"cause": mirror_refused, + "effect": "sizes_unavailable", + "recommendation": "this deployment has another tenant's analytical store open; " + "sizes are withheld rather than read from it. Pin the process " + "to this tenant (AIOS_DUCKDB_PATH / datastore.use_path) to " + "restore them."} + if mirror_refused else None), + # ⭐ W31-T46 / D-160 — WHY THERE IS NO MIRROR, when there is none. The last time this + # was silent it read as a pinned tag and a store problem for days. `null` when the + # container has one; a sentence with a cause and a fix when it does not. + "mirrorSeed": _seed_state()} + + +def _seed_state(): + """`main.MIRROR_SEED` as a reportable block, or None when the mirror is present. + + ⚠ IMPORTED LAZILY AND FAIL-QUIET: `main` imports this router, so a module-level import here + would be a cycle, and a freshness surface must never 500 because a diagnostic was unavailable. + """ + try: + import main as _main + from harness import datastore as _ds + if _ds.DB_PATH.exists(): + return None + state = dict(_main.MIRROR_SEED) + if not state.get("cause"): + state["cause"] = "this container has no analytical mirror yet" + state["recommendation"] = ("the boot seed runs independently of AIOS_PREWARM; if this " + "persists, check HF_TOKEN on the deployment") + return state + except Exception: # noqa: BLE001 + return None + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ THE READ-THROUGH WINDOW — owner ruling R6 ("no cap on connected-source data") via R7. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# THE PROBLEM IT REPLACES, in the owner's numbers. `core.user_tables.MAX_ROWS = 60_000` bounds a +# `ut_*` table because ALL of a tenant's tables live in ONE JSON document that `Store.get` +# deep-copies per request (D-87: four Odoo databases = 20.7 MB, ~370 ms a copy). Orders at +# **32,826** is already 55% of that ceiling; order lines (**255,286**) are 4.3× over it and GL +# lines (**971,034**) 16× over, so those two could never be grids at all — `odoo_relational.plan` +# refuses them rather than truncating, which is the correct refusal of the wrong architecture. +# +# ⭐ "NO CAP" IS NOT A BIGGER NUMBER. Raising `MAX_ROWS` would make every request slower for every +# tenant, including the ones who never open an Odoo grid. The mirror ALREADY holds all of it +# uncapped — measured on this box: 963,783 GL lines counted in ~24 ms — so the fix is to stop +# copying and serve a WINDOW: the requested slice, plus a `SELECT count(*)` that tells the truth +# about the whole. +# +# ⛔⛔ `total` IS NEVER `len(rows)`. It comes from `datastore.window`'s own count statement over +# the SAME predicate. A window whose count is its own length is a fabricated aggregate wearing an +# authoritative face ([[no-unverifiable-aggregates]]). +# +# ⛔ PREDICATES PUSH DOWN, and this is the half a client cannot be trusted with. A filter chip +# evaluated over the 200 rows that happen to be in memory would report "40 matches" out of +# 971,034 — [[one-question-two-normalizers]] at scale. `harness/filter_sql.py` compiles OUR filter +# vocabulary to SQL and is the same evaluator the TS engine is held in step with, so the fold and +# the display answer one question. +# +# ⭐ THE SPEC BELOW IS A SECOND STATEMENT OF SOMETHING `odoo_relational`'s READER ALREADY SAYS, +# AND THAT IS THE REAL RISK HERE — not SQL injection. It is gated rather than trusted: +# `verify_scopes.section_read_through` runs the REAL reader over the REAL mirror as an ORACLE and +# asserts the windowed path agrees ROW FOR ROW and TOTAL FOR TOTAL. A binding that drifts from its +# reader goes red; a bucket with no binding is REPORTED, never silently served from the copy as +# though it were read-through ([[one-evaluator-per-question]], [[gate-answers-the-wrong-question]]). +# +# ⚠ ONE BUCKET IS BOUND HERE (orders). The other seven still serve from the materialised copy and +# say so on the wire (`readThrough: false`), because R6's second sentence — *"if there is lag or +# it can't be done, you need to explicitly tell me why and recommend a fix"* — makes an unconverted +# grid something to REPORT, not something to leave looking converted. Adding one is a spec row plus +# a green oracle check. + +#: `ut_*` key -> how to read that grid straight out of the mirror. +#: +#: `where`/`select` are SQL WE author (never a request value); every request value is bound through +#: `params` by `filter_sql`. `cols` maps a FIELD KEY (what `odoo_relational._fields()` declares, +#: and what a saved view's filters name) to `(sql expression, coercion)`. The coercion reproduces +#: the reader's own python cast, by CALLING the reader's helpers where one exists — `_as_date` and +#: `_in_scope` are imported, not re-implemented. +#: Keys a reader does not produce (link columns, `refreshed`) are absent here on purpose: they are +#: filled by the grid at render time exactly as they are on the materialised path. +GRID_SOURCES = {} + +#: ⭐⭐ W31-T49 / C4 — THE SOURCE PROVIDERS. Odoo builds its specs below; ANY OTHER connector adds +#: its own by registering a builder here, and `_sources()` folds them into the one registry every +#: door already reads. +#: +#: ⛔ THIS IS THE SEAM THAT KEEPS "ONE REGISTRY" TRUE WHILE THE FILE STAYS ODOO-NAMED. The +#: alternative — a second `META_GRID_SOURCES` consulted beside this one — would give the platform +#: two answers to *"how is this connected grid read?"*, and every consumer (`_source_for`, +#: `whole_pool`, `population`, `odoo_tables_status`, `routes_tables.scoped_pids`) would have to +#: learn both or silently serve one. That is [[one-question-two-normalizers]] on the read path. +#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first +#: read — the same lazy rule `_sources` already follows for `odoo_relational`. +_SOURCE_PROVIDERS = [] + + +def register_source_provider(fn): + """Add a `() -> {table_key: spec}` builder to the connected-grid registry. + + Additive and idempotent by identity, so a re-import cannot double-register. Returns the number + of providers now known — a caller that wants to assert its registration took has a number. + """ + if callable(fn) and fn not in _SOURCE_PROVIDERS: + _SOURCE_PROVIDERS.append(fn) + return len(_SOURCE_PROVIDERS) + + +def _sources(): + """Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them. + + ⛔⛔ ODOO GOES THROUGH THE SEAM TOO, AND THAT IS THE POINT OF W31-T49 RATHER THAN A FLOURISH. + The first version built Odoo's specs inline here and folded other providers in afterwards, + which quietly says "Odoo is the registry and everyone else is an addendum" — two mechanisms + for one question, with the second one exercised by nobody until a connector arrives. It also + left `register_source_provider` with no production caller at all, which `verify_reachability` + correctly reddened as a capability behind no door ([[artifact-with-no-importer]]). Registering + the incumbent through its own seam makes the seam load-bearing from the first request. + + ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and + importing that module at file-import time would drag the Odoo layer into every process that + mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read. + """ + if GRID_SOURCES: + return GRID_SOURCES + for build in list(_SOURCE_PROVIDERS): + try: + extra = build() or {} + except Exception: # noqa: BLE001 + # ⛔ ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must + # not go dark because a newer connector's module is unhappy — and vice versa. + continue + if extra: + GRID_SOURCES.update(extra) + _ut().register_connected(*extra) + return GRID_SOURCES + + +def _odoo_sources(): + """Odoo's read-through bindings — `{table_key: spec}`. Registered as a provider below. + + ⚠ IT RETURNS a dict rather than mutating the module global: `_sources` owns the merge, so a + provider that half-built its specs and raised cannot leave a partial registry behind. + """ + rel = _rel() + # ⭐ R6 / W30-T29 — TELL THE STORE LAYER WHICH DATABASES ARE CONNECTED, so `MAX_ROWS` stops + # being a fact about them. `core` never imports up, so the declaration goes this way round. + # ⚠ ALL EIGHT, not just the read-through-bound ones — every row in these tables comes from + # Odoo, which is what R6 is about; being served from the stored copy today is our conversion + # state, not a property of the data. `_sources`' own `register_connected` covers only the keys + # a provider RETURNS, so this call is not redundant with it and must not be folded into it. + _ut().register_connected(*[key for _b, key, _l, _f in rel.TABLES]) + GRID_SOURCES = {} # the LOCAL registry this builder fills and returns + _s, _i, _n = ((lambda v: str(v or "")), (lambda v: int(v or 0)), + (lambda v: float(v or 0.0))) + GRID_SOURCES[rel.ORDERS_KEY] = { + "table": "sale_order", + # ⛔ IMPORTED, NOT RETYPED. `_CONFIRMED` is the fixed wholesale scope; if it ever changes, + # this window changes with it and the oracle check proves it did. + "where": f"{rel._CONFIRMED} AND partner_id IS NOT NULL", + "id": "id", + "needs_excluded": True, # `wholesale_scope` is resolved against the excluded set + "cols": { + "order_no": ("name", _s), + "odoo_id": ("id", _i), + "customer": ("partner_name", _s), + rel.JOIN_KEY: ("partner_id", _i), + "order_date": ("date_order", rel._as_date), + "amount_untaxed": ("amount_untaxed", _n), + "team": ("team_name", _s), + "state": ("state", _s), + "invoice_status": ("invoice_status", _s), + # ⭐ THE SCOPE COLUMN BECOMES REAL SQL, which is the point. On the materialised path it + # is `_in_scope(pid, excluded)` — a python set test, and a filter on it therefore could + # not push down. Inlining the ids (ints, from our own query) makes it a column the + # mirror can filter and sort on, so the R6 "limit" it would otherwise have earned does + # not exist. `{excluded}` is substituted by `_source_for` below. + "wholesale_scope": ("CASE WHEN partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + } + + # ═════════════════════════════════════════════════════════════════════════════════════════ + # ⭐⭐ W30-T32 — THE TWO LINE GRAINS. These are the grids R6 exists for: they have never had a + # `ut_*` table and never can, at any cap. MEASURED on this box's mirror, warm: + # sale_order_line 254,189 in the confirmed scope (256,810 unscoped) — 63.9 MB as JSON + # account_move_line 963,783 — ~240 MB as JSON + # Against `MAX_ROWS = 60_000` that is 4.2x and 16x, and against the 32 MB per-table document + # budget it is 2x and 7.5x. Read THROUGH, both serve a page in 134–166 ms. + # + # ⛔ THE FIELD KEYS BELOW ARE HALF OF A CONTRACT AND `odoo_relational` OWNS THE OTHER HALF + # (W30-T35, session E). `cols` binds a field key to SQL; the field's label, type and order are + # DECLARED THERE, once. Until that declaration lands the route answers 404 for these two keys + # (`rel.TABLES` has no entry), and `odoo_tables_status` REPORTS them as bound-not-declared + # rather than leaving them invisible — R6's second sentence applied to our own conversion. + # ⚠ A key here with no declaration there is a silent NO-CELL (`rows_from_pool` projects + # strictly); a key there with no binding here is an INACTIVE filter leaf, which WIDENS. The + # two lists are checked against each other by `verify_scopes.section_line_grids`. + ol_key = getattr(rel, "ORDER_LINES_KEY", "ut_odoo_order_lines") + gl_key = getattr(rel, "GL_LINES_KEY", "ut_odoo_gl_lines") + _ut().register_connected(ol_key, gl_key) + # ⛔ THE SCOPE IS THE ORDER'S, AND THE LINE TABLE CANNOT ANSWER IT ALONE: `sale_order_line` + # carries no `state` (12 columns, measured), so the confirmed-order scope — and `order_date`, + # and the order NAME a person reads the grid by — only exist across the join. MEASURED, warm, + # best of two: the JOIN beats `order_id IN (SELECT id FROM sale_order WHERE …)` at both depths + # (166 / 483 ms against 237 / 565 ms at offset 0 / 200,000), so the shape is chosen on a + # number rather than on taste. + # ⚠ `_CONFIRMED` is IMPORTED and QUALIFIED, never retyped — it opens with the bare column + # `state`, which `sale_order_line` does not have, so the prefix is what keeps it unambiguous + # if that table ever gains one. `section_line_grids` counts the same population a second way + # (a subquery, not a join) and the two must agree, which is what catches a mis-qualification. + GRID_SOURCES[ol_key] = { + "from_sql": "(sale_order_line sol JOIN sale_order so ON so.id = sol.order_id)", + "tables": {"sol": "sale_order_line", "so": "sale_order"}, + "where": f"so.{rel._CONFIRMED}", + "id": "sol.id", + "needs_excluded": True, + "cols": { + "odoo_id": ("sol.id", _i), + "order_no": ("so.name", _s), + "order_id": ("sol.order_id", _i), + "customer": ("sol.order_partner_name", _s), + rel.JOIN_KEY: ("sol.order_partner_id", _i), + "product": ("sol.product_name", _s), + rel.PRODUCT_JOIN_KEY: ("sol.product_id", _i), + "qty": ("sol.product_uom_qty", _n), + "price_subtotal": ("sol.price_subtotal", _n), + "margin": ("sol.margin", _n), + "purchase_price": ("sol.purchase_price", _n), + "order_date": ("so.date_order", rel._as_date), + "state": ("so.state", _s), + "wholesale_scope": ( + "CASE WHEN sol.order_partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + } + # ⚠ UNSCOPED ON PURPOSE, and it is a decision rather than an omission: every other grid here + # carries a fixed scope, but a GENERAL LEDGER whose draft and cancelled entries are invisible + # is a ledger that cannot be reconciled. `parent_state` rides as a column so a person filters + # in SQL over all 963,783 rows instead of us choosing for them. (Posted-only is 944,846.) + # The join to `account_account` is what makes `account_code` — the key `ut_odoo_accounts` is + # linked on — available at all; the mirror flattens `account_id`/`account_name` onto the line + # but not the CODE, and 192 accounts hash-join for free. + GRID_SOURCES[gl_key] = { + "from_sql": ("(account_move_line aml LEFT JOIN account_account aa " + "ON aa.id = aml.account_id)"), + "tables": {"aml": "account_move_line", "aa": "account_account"}, + "where": "", + "id": "aml.id", + "needs_excluded": True, + "cols": { + "odoo_id": ("aml.id", _i), + "entry": ("aml.move_name", _s), + "move_id": ("aml.move_id", _i), + "account": ("aml.account_name", _s), + rel.ACCOUNT_JOIN_KEY: ("aa.code", _s), + "customer": ("aml.partner_name", _s), + rel.JOIN_KEY: ("aml.partner_id", _i), + "date": ("aml.date", rel._as_date), + "debit": ("aml.debit", _n), + "credit": ("aml.credit", _n), + "balance": ("aml.balance", _n), + "line_type": ("aml.display_type", _s), + "move_type": ("aml.move_type", _s), + "parent_state": ("aml.parent_state", _s), + "wholesale_scope": ( + "CASE WHEN aml.partner_id IN ({excluded}) THEN '' ELSE '1' END", _s), + }, + } + # ⭐ W30-T31 — THE GL ACCOUNT REGISTRY, BOUND BECAUSE IT IS THE ONE GRID THAT CAN ACTUALLY + # STOP MATERIALISING TODAY. 192 rows, nothing folds it, nothing links at it, and it fits + # inside one window — the three conditions `_unmaterialisable` checks. It is small, and that + # is the point: it is the first shipped database whose rows are NOT in the tenant document, + # so the stratum is proven on a real table instead of on a mechanism with no subject. + # ⚠ `account_fields()` declares six columns and `read_accounts` is one `cur.execute` over + # `account_account`; `section_read_through`'s differential oracle holds this binding to it. + # ⚠ `is_expense` REPRODUCES `read_accounts`' predicate IN SQL rather than inventing one, and + # that predicate is itself a copy of the semantic layer's `gl_lines` scope. Three statements of + # one rule is two too many, but the reader's own comment explains why it is copied rather than + # imported, and `section_read_through`'s differential oracle is what keeps this one honest. + GRID_SOURCES[rel.ACCOUNTS_KEY] = { + "table": "account_account", + "where": "", + "id": "id", + "cols": { + rel.ACCOUNT_JOIN_KEY: ("code", _s), + "account_name": ("name", _s), + "odoo_id": ("id", _i), + "account_type": ("account_type", _s), + "is_expense": ("CASE WHEN account_type IN ('expense','expense_depreciation') " + "THEN '1' ELSE '' END", _s), + }, + } + return GRID_SOURCES + + +#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers +#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`, +#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid. +register_source_provider(_odoo_sources) + + +def sync_read_through(cur=None): + """Register every grid that may stop storing rows, and REPORT why the rest may not. + + Returns `({key: eligible}, {key: reason})`. Idempotent, cheap, and safe to call from any door: + registration is additive and `strip_materialised` is a no-op once a table is empty. + + ⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility + question needs BOTH halves — `odoo_relational`'s field declarations (for the fold matrix) and + the mirror (for the size) — and `core` may import neither. + """ + eligible, reasons = _unmaterialisable(cur) + keys = [k for k, ok in eligible.items() if ok] + if keys: + _ut().register_read_through(*keys) + return eligible, reasons + + +#: ⚠ A mirror can be `ready()` and still lack a column (`ready()` reads entity PHASES; column +#: backfills checkpoint separately) — the gap that already cost a live 500. Every projected +#: expression is checked against the real column list and degraded to a literal, exactly as +#: `odoo_relational._col` does for the reader, so a fresh Space serves a blank cell rather than a +#: DuckDB Binder error. +#: +#: ⛔ W30-T32 — IT TAKES AN ALIAS MAP NOW, AND WITHOUT THAT THE GUARD WAS ABOUT TO GO BLIND. The +#: line-grain grids project `sol.price_subtotal` / `aml.parent_state`, and a dotted string is not +#: `isalnum()`, so the old single-table version returned EVERY qualified expression unchecked — +#: the same "expression, nothing to check" branch that correctly skips a CASE. Both of those +#: columns are 2026-07-28 backfills that a mirror can genuinely be missing, so the blind spot +#: would have surfaced as a bare DuckDB Binder error on a fresh Space, which is precisely the +#: failure this helper exists to prevent ([[gate-answers-the-wrong-question]]). +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W30-T31 / D-87 — WHICH CONNECTED GRIDS MAY STOP MATERIALISING, AND WHY MOST MAY NOT YET. +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The prize is real: all of a tenant's `ut_*` tables live in ONE document that `Store.get` +# deep-copies on EVERY call, hit or miss, and the four original Odoo grids are 20.7 MB / ~370 ms +# of it. Every permission check in the app pays that. +# +# ⛔ AND YOU CANNOT SIMPLY DELETE THE ROWS, WHICH IS THE FINDING THIS FUNCTION ENCODES. Measured +# mechanically across the eight field declarations, not by eye: +# * `ut_odoo_invoices` and `ut_odoo_orders` — 16.2 of those 20.7 MB — are folded by SIX link +# rollups on `ut_odoo_customers` (`ar_outstanding`, `open_invoices`, `oldest_due`, +# `invoiced_all_time`, `order_count`, `last_order`). `automation_engine.compute_relation_cells` +# answers those by reading the RAW document, so with the rows gone it writes zeros — silently. +# * every table with a `link` pointing AT it (agents, vendors, bills, customers) has its link +# CELLS materialised the same way, from the target's stored rows. +# * `ut_odoo_customers` and `ut_odoo_products` carry SOURCE rollups, and `rollup_sql.compute` +# writes those cells INTO their own stored rows: no rows, no cells. +# * and while the grid client still asks for a whole table (F's W30-T42 is what changes this), +# a population larger than one window could only be served by TRUNCATING it — which R6 +# forbids more strongly than it forbids a cap. +# +# ⭐ SO THE PREDICATE IS DERIVED FROM THE DECLARATIONS RATHER THAN LISTED. The day a lane converts +# those six link rollups to SOURCE rollups (`orders_ytd` on that same table is the precedent), or +# the day the client pages, the affected grids become eligible here with NO code change — and +# until then each one's reason is reported per table on the status door, which is R6's second +# sentence applied to our own conversion state. +def _fold_reasons(rel): + """`{table_key: "why its stored rows are still read by something else"}`. + + Read out of the FIELD DECLARATIONS themselves — one pass over `rel.TABLES`. A table absent + from this map is folded by nothing. + """ + reasons = {} + + def _add(key, why): + reasons.setdefault(str(key), []).append(why) + + for _bucket, key, _label, mk in rel.TABLES: + try: + fields = mk() + except Exception: # noqa: BLE001 + continue + by_key = {f.get("key"): f for f in fields if isinstance(f, dict)} + short = str(key).replace("ut_odoo_", "") + for f in fields: + if not isinstance(f, dict): + continue + if f.get("type") == "link" and (f.get("link") or {}).get("table"): + _add(f["link"]["table"], f"{short}.{f['key']} is a link whose cells are built " + f"from these rows") + if f.get("type") != "rollup": + continue + bag = f.get("rollup") or {} + if isinstance(bag.get("source"), dict): + _add(key, f"{short}.{f['key']} is a source rollup and its cells are written " + f"into these rows") + continue + tgt = ((by_key.get(str(bag.get("link") or "")) or {}).get("link") or {}).get("table") + if tgt: + _add(tgt, f"{short}.{f['key']} folds these rows") + return {k: "; ".join(v) for k, v in reasons.items()} + + +def _unmaterialisable(cur=None): + """`({key: eligible}, {key: reason})` — who may stop storing rows, and why the rest may not. + + ⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in + the first cut of this function. A table too big for the tenant document has NO materialised + option at all, so making it ineligible would have handed `odoo_relational.plan` a `row_limit` + of None and invited it to build 963,783 python dicts. The window ceiling is a different, much + softer thing: it only limits what the whole-table CLIENT door can serve today. + + rows > MAX_ROWS -> read-through REQUIRED (the document cannot hold it) + else if something folds it -> stays materialised (a fold over no rows writes ZEROS) + else if rows > WINDOW_MAX -> stays materialised until the client pages (W30-T42) + else -> eligible + + ⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never + part of the materialised spawn is eligible — a table is never freed by a question we skipped. + """ + rel = _rel() + folds, eligible, reasons = _fold_reasons(rel), {}, {} + # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT. + # W30-T35 declared the two line grains, which have no python row builder anywhere — their + # absence from `_READERS` IS that statement — so a table the spawn could never materialise + # started reading as spawned, and on a mirror-less deployment fell through to "its size could + # not be read" and came back INELIGIBLE. That inverts this function's own first law (size + # forces read-through; it never blocks it). Ask the question through the reader map, which is + # what actually decides whether a row could ever have been built. + spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})} + from harness import datastore + ut = _ut() + for key, spec in _sources().items(): + total = None + if cur is not None: + try: + frm = ({"from_sql": spec["from_sql"]} if spec.get("from_sql") + else {"table": spec["table"]}) + total = datastore.window(select="1", where=spec.get("where") or "", + order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + except Exception: # noqa: BLE001 + total = None + if total is not None and total > ut.MAX_ROWS: + eligible[key] = True # no other home exists; this is not a choice + continue + if total is None and key not in spawned: + eligible[key] = True # never materialised, so the mirror is its home + continue + why = [] + if total is None: + why.append("its size could not be read from the mirror on this deployment") + if key in folds: + why.append(folds[key]) + if total is not None and total > datastore.WINDOW_MAX: + why.append(f"its {total:,} rows exceed the {datastore.WINDOW_MAX:,}-row window and " + f"the grid client still asks for whole tables, so un-materialising it " + f"today could only truncate") + eligible[key] = not why + if why: + reasons[key] = "; ".join(why) + return eligible, reasons + + +def _degrade(expr, have_by_alias, default_alias=""): + bare = expr.strip() + alias, _, col = bare.partition(".") + if col and alias.replace("_", "").isalnum() and col.replace("_", "").isalnum(): + have = have_by_alias.get(alias) + if have is None: + return expr # an alias we do not own — leave it to the caller's SQL + return expr if col.lower() in have else "NULL" + if not bare.replace("_", "").isalnum(): + return expr # an expression, not a bare column — nothing to check + # ⚠ A BARE COLUMN IN A JOINED SPEC RESOLVES THE WAY SQL RESOLVES IT — against every table in + # the FROM, not against nothing. Checking it against `have_by_alias[""]`, which a joined spec + # does not have, would degrade every such column to NULL: a silent blank cell, which is the + # failure this helper exists to avoid rather than to cause. + have = have_by_alias.get(default_alias) + if have is None: + have = set().union(*have_by_alias.values()) if have_by_alias else set() + return expr if bare.lower() in have else "NULL" + + +def _source_for(cur, table_key): + """The resolved spec for one connected grid, or None when this grid is not read-through yet.""" + spec = _sources().get(str(table_key or "")) + if not spec: + return None + from harness import datastore + # ⚠ `tables` maps the SQL alias a projection uses to the mirror table behind it. A single-table + # spec declares none and its columns are bare, so it degrades against `table` as before. + aliases = dict(spec.get("tables") or {}) + if spec.get("table"): + aliases.setdefault("", spec["table"]) + have_by_alias = {} + for alias, tname in aliases.items(): + cols = datastore.columns_of(tname, cur=cur) + if not cols: + return None # the mirror has no such table on this deployment + have_by_alias[alias] = cols + excluded = "" + if spec.get("needs_excluded"): + ids = sorted(int(p) for p in _rel().excluded_ids(cur)) + # `-1` keeps the IN-list non-empty and matches no Odoo id, so the SQL shape is constant + # whether or not this tenant excludes a channel. + excluded = ", ".join(str(i) for i in ids) or "-1" + cols = {} + for key, (expr, cast) in spec["cols"].items(): + cols[key] = (_degrade(expr.format(excluded=excluded) if "{excluded}" in expr else expr, + have_by_alias), cast) + return {**spec, "cols": cols} + + +#: How deep a page has to be before the walk is worth a sentence. Derived from the measurement in +#: the route below, not chosen: 100,000 is still 71 ms, 200,000 is 483 ms, and the second half of +#: the GL table is where it passes a second. Reporting from 100,000 puts the sentence in front of +#: the person BEFORE the wait rather than after it. +_DEEP_PAGE = 100_000 + + +#: `filter_sql` speaks the CLIENT's type vocabulary (`types.ts isNumericType`), and two of our +#: field kinds are spelled differently there. Mapped in one place so the pushdown and the grid +#: cannot disagree about whether a column is text or a number. +_FILTER_TYPE = {"select": "status", "checkbox": "text"} + + +def _filter_columns(spec, fields): + """`{colId: {sql, type, aggregate}}` — what `filter_sql` needs to compile a predicate. + + Only columns with a real mirror expression are offered. An omitted column is UNKNOWN to the + compiler, which skips it — and that is the one behaviour that must be REPORTED rather than + accepted, because an ignored condition WIDENS (the tri-state engine's inactive-leaf rule). + `_unpushable` below turns every such skip into an R6 sentence. + """ + by_key = {f["key"]: f for f in fields} + out = {} + for key, (expr, _cast) in spec["cols"].items(): + ftype = str((by_key.get(key) or {}).get("type") or "text") + out[key] = {"sql": expr, "type": _FILTER_TYPE.get(ftype, ftype), "aggregate": False} + return out + + +def _leaf_cols(nodes): + """Every `colId` a filter tree names, at any depth.""" + seen = set() + for n in (nodes or []): + if not isinstance(n, dict): + continue + if n.get("children") is not None: + seen |= _leaf_cols(n.get("children")) + elif n.get("colId"): + seen.add(str(n["colId"])) + rhs = n.get("rhs") + if isinstance(rhs, dict) and rhs.get("colId"): + seen.add(str(rhs["colId"])) + return seen + + +def _json_arg(raw, what): + """Decode a JSON query argument, or refuse. ⛔ NEVER degrade to "no filter": a filter that + silently fails to parse WIDENS the answer, and the caller sees a plausible bigger number.""" + if raw in (None, ""): + return None + try: + val = json.loads(raw) + except Exception: # noqa: BLE001 + raise err(400, "bad_argument", f"{what} must be JSON") + return val + + +@router.get("/odoo-tables/{table_key}/rows") +def odoo_table_rows(table_key: str, + offset: int = Query(default=0, ge=0), + limit: int = Query(default=0), + filters: str = Query(default=None), + filterConj: str = Query(default="and"), + sorts: str = Query(default=None), + search: str = Query(default=None), + session: Session = Depends(require_session)): + """ONE WINDOW over a connected grid, read straight from the mirror (contract C2). + + `{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` — + `rows` is the requested slice and `total` is the count of everything the CURRENT PREDICATE + matches, from its own `SELECT count(*)`. `rows.length < total` is the normal case. + + ⛔ `total` is never `len(rows)`; `totalUnfiltered` is the population with the predicate + dropped, so a client can render "N of M" without inventing either number. + ⛔ The filter, the sort and the search all resolve in SQL against the whole table. Anything + that CANNOT (a column with no mirror expression, an op the compiler refuses) is reported in + `limits` with its cause and a recommendation — R6's second sentence — and never silently + dropped, because an ignored condition widens. + """ + import aios_grid + from harness import datastore + from routes_tables import _defn_or_refuse + + rel = _rel() + # THE WALL FIRST, and it is the SAME one the materialised path uses — 404 for a key that does + # not exist, 403 for one this session may not open. Reused rather than re-stated: a second + # idea of "may this session open this database" is a permission bug waiting to happen. + defn = _defn_or_refuse(session, table_key) + fields = None + for _bucket, key, _label, mk_fields in rel.TABLES: + if key == table_key: + fields = mk_fields() + break + if fields is None: + raise err(404, "not_connected", "that database is not a connected Odoo grid") + + # ⭐⭐ W31-T45 / D-169 — THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot. + # That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process + # has open the one this TENANT's rows live in". A definition wall is satisfied the moment a + # tenant's own document declares a connected table — which is exactly what R2's Meta Ads spawn + # gives GTM Lab — and it would then serve that session a window over whatever DuckDB file the + # process happens to be pinned to. Rows, not counts. So this door RAISES. + # ⛔ 409, NOT 503, and the two were one line from being confused: `DatastoreMismatch` subclasses + # `RuntimeError`, and the handler directly below turns a `RuntimeError` into + # `503 store_not_ready` — "the store is completing its first sync, retry in a few minutes". A + # mis-pinned process never becomes un-mis-pinned by waiting, so relaying it as a retry would + # turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the + # try, not inside it. + try: + cur, mismatch = _mirror_cur(session) + except RuntimeError as e: + # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the + # mismatch subclass, which is what makes this broad clause safe to keep here. + raise err(503, "store_not_ready", str(e)) + if mismatch: + raise err(409, "cross_tenant_store", mismatch) + + spec = _source_for(cur, table_key) + if spec is None: + raise err(409, "not_read_through", + "this connected database is still served from its stored copy; it has no " + "read-through binding on this deployment yet") + + cols = _filter_columns(spec, fields) + limits, tree = [], _json_arg(filters, "filters") + sort_spec = _json_arg(sorts, "sorts") or [] + + # ── the predicate, pushed down ──────────────────────────────────────────────────────────── + named = _leaf_cols(tree) + missing = sorted(named - set(cols)) + if missing: + limits.append({ + "subject": ", ".join(missing), "effect": "filter_ignored", + "cause": "these columns have no expression in the mirror (a link, a rollup, or a " + "column this deployment's mirror has not backfilled), so a condition on " + "them cannot be answered in SQL", + "recommendation": "filter on the id column the link is built from, or open the " + "linked database directly"}) + # ⭐ R6, MEASURED, AND IT IS THE KIND OF LIMIT THE RULING EXISTS FOR — a difference in the + # ANSWER, not in the speed. + # + # `filter_sql._value_sql` compiles every numeric comparison as `round_even(x, 0)`, and says + # why: *"reproduces `aios_grid._round` … The grid displays rounded values; filters must agree + # with what is on screen."* That is true of `source: "odoo"` columns, which `rows_from_pool` + # rounds. It is FALSE here — these columns are `source: "overlay"` (a storage choice, not a + # display one) so `rows_from_pool` passes the exact value through, and the TS engine + # (`useVisibleRows.toNum`) does not round either. So the pushdown compares at whole units while + # the cell beside it carries cents. + # + # MEASURED on the live orders mirror (32,700 rows, 47.9% with a non-integer amount): + # > 1000 python 4,476 vs SQL 4,473 (-3) + # > 173.6 python 24,711 vs SQL 24,726 (+15) + # > 500.25 python 11,028 vs SQL 11,026 (-2) + # Small, and NOT nothing. Rounding the wire to match would have been the other fix and it was + # rejected on measurement: it changes 96 of every 200 money cells (173.55 -> 174) to remove a + # 0.05% counting difference — a visible product regression traded for an invisible one. + # ⛔ SO IT IS REPORTED INSTEAD. Booked for the owner of `harness/filter_sql.py`, which is not + # this fence; see mailbox/D.md. + numeric = sorted(k for k in (named & set(cols)) + if cols[k]["type"] in ("currency", "int", "pct")) + if numeric: + limits.append({ + "subject": ", ".join(numeric), "effect": "precision", + "cause": "a number condition is evaluated in SQL at whole-unit precision " + "(`filter_sql` rounds to match the grids whose values the server rounds), " + "while these cells carry their exact value — so a row within half a unit of " + "the threshold can fall on the other side of it", + "recommendation": "compare against a whole number, or use a range that does not sit " + "on a fractional boundary"}) + + where, params = spec["where"], [] + try: + pred = _fs().compile_filter_tree( + tree, conj=(filterConj if filterConj in ("and", "or") else "and"), columns=cols, + today=_today()) + except ValueError as e: + # ⛔ A REFUSAL IS AN ANSWER, NOT A CRASH — and it must not become "no filter". Ranking ops + # ("top 10") have no SQL form in this compiler; saying so beats returning every row. + raise err(400, "filter_unsupported", str(e)) + if pred is not None: + if pred.uses_aggregate: + raise err(400, "filter_unsupported", + "a condition on an aggregate column belongs in HAVING, and that path is " + "deliberately not built for windowed grids") + where = f"({where}) AND {pred.sql}" if where else pred.sql + params.extend(pred.params) + if str(search or "").strip(): + got = _fs().compile_search(search.strip(), cols) + if got is not None: + where = f"({where}) AND {got.sql}" if where else got.sql + params.extend(got.params) + + # ── the order, made TOTAL ───────────────────────────────────────────────────────────────── + # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one + # row on two pages and drop another entirely — a duplicate the user sees with no error + # anywhere. `compile_order_by`'s own docstring says so. + order = _fs().compile_order_by(sort_spec, cols, tiebreak_sql=spec["id"]) or spec["id"] + + select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) + # ⚠ EXACTLY ONE of `table`/`from_sql` — `window` raises if both or neither arrive, so the + # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM. + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', + where=where, params=tuple(params), order_by=order, + offset=offset, limit=(limit or None), cur=cur, **frm) + if win["clamped"]: + limits.append({ + "subject": "limit", "effect": "window_clamped", + "cause": f"one response carries at most {datastore.WINDOW_MAX} rows so a single " + f"request cannot exhaust memory for every other tenant on the process", + "recommendation": "page with `offset`; `total` already reports the whole population, " + "and no row is unreachable"}) + # ⭐ R6's SECOND SENTENCE, ON THE ONE LIMIT THAT SURVIVES THE CONVERSION. Removing the row cap + # does not make every row equally cheap: `LIMIT/OFFSET` WALKS the offset, so the deeper the + # page the longer the scan. MEASURED warm on 963,783 GL lines — offset 0: 134 ms · 100,000: + # 71 ms · 900,000: **1,450 ms**; sorted by date rather than by id, offset 500,000: 1,645 ms. + # It is a real cost, it is nobody's mistake, and the owner asked to be told rather than to + # discover it: say so with the fix, which is a CURSOR the client has to send. + if offset >= _DEEP_PAGE: + limits.append({ + "subject": "offset", "effect": "slow", + "cause": f"a page {offset:,} rows deep is reached by walking every row before it " + f"(SQL OFFSET has no other meaning), which costs about a second past " + f"half a million rows", + "recommendation": "jump with a filter or a sort instead of scrolling, or ask for " + "keyset paging (`id > `), which is O(page) at any depth " + "and needs the client to send the last row it holds"}) + + # ── the wire rows, through the SAME serialiser the materialised path uses ────────────────── + # ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the + # `refreshed` column identically instead of one of them leaving it blank. + rows_src = _pool_from_window(spec, win, + stamp=mirror_stamp(spec, cur, table_key, + session.runtime)) + overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src} + + # ⭐⭐ W30-T28 — THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID. + # + # A user-added column on a connected grid has nowhere per-user to live: there is no `ut_*` row + # to hang it on any more, and `table_store`'s per-user strata would make a SHARED view name a + # column other accounts do not have — an unknown column is an INACTIVE condition in the + # tri-state engine, which WIDENS. `core/shared_overlay.py` was built for exactly this and has + # had no product door since it shipped (W29-T62). + # + # ⛔ `cells(table_key, pids)` TAKES THE PIDS AND THERE IS NO "EVERYTHING" CALL — and here that + # is an asset rather than a chore: **the window IS the scoped row set**, already narrowed by + # the wall and the predicate, so the argument it demands is the list we just fetched. + # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already + # answered "may this session open this surface". + shared_defs = _so().fields(table_key, st=session.runtime) + if shared_defs: + fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()] + for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src], + st=session.runtime).items(): + overlays.setdefault(pid, {}).update(cells) + rows = aios_grid.rows_from_pool(rows_src, fields, overlays) + + unfiltered = win["total"] if where == spec["where"] else datastore.window( + select="1", where=spec["where"], order_by=spec["id"], limit=1, cur=cur, **frm)["total"] + + return {"ok": True, "fields": fields, "rows": rows, + # ⛔ FROM THE COUNT STATEMENT. Never `len(rows)`. + "total": win["total"], "totalUnfiltered": unfiltered, + "offset": win["offset"], "limit": win["limit"], "limits": limits, + "today": _today(), "identity": {"pid": "pid"}, + "scope": {"table": table_key, "readThrough": True}, + "recordsMutable": bool(defn.get("recordMode") != _ut().AUTOMATION_RECORD_MODE)} + + +def mirror_stamp(spec, cur=None, table_key="", rt=None): + """⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date. + + ⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table + declares `refreshed` (`odoo_relational._refreshed_field`), and a MATERIALISED grid gets a + per-row stamp written by the spawn. A read-through grid has no stored row to carry one, so + `_pool_from_window` produced no `refreshed` key at all and `rows_from_pool` projected it as + `""` — a freshness column that used to say something and now silently says nothing, on the two + biggest grids in the product. Nothing goes red for that; it just quietly stops being true. + + ⭐ THE HONEST VALUE ALREADY EXISTS — the mirror keeps a per-entity sync stamp in `_sync_state`, + which is the same fact one level up: these rows are as fresh as the last sync of the table + they are read through. It is per-TABLE, not per-row, and that is not a downgrade — for a grid + that stores no rows, "when did this table last sync" IS the per-row answer. + + ⚠ READ ON THE CALLER'S CURSOR, never `datastore.status()`. That opens a SECOND connection to a + file DuckDB holds exclusively, so a freshness column would turn a working grid into an + IOException — a cosmetic fix taking out the feature it decorates. + ⚠ Date only (`YYYY-MM-DD`), because `refreshed` is declared `type: "date"` and the grid's date + renderer is what reads it; handing it a full timestamp renders the ISO string a user should + never see (the wave-26 `copyData` scar, one column over). + + ⛔⛔ THE TABLE COMES FROM THE **ID COLUMN'S ALIAS**, AND THE FIRST DRAFT GOT THIS WRONG IN THE + ONLY WAY THAT MATTERS. It read `spec["table"]` with a `tables[""]` fallback — and **neither of + the two grids D-155 is about has a bare `table` key**: `ut_odoo_order_lines` and + `ut_odoo_gl_lines` are join-shaped (`from_sql` + `tables: {"sol": …, "so": …}` / + `{"aml": …, "aa": …}`), so the lookup returned `""` and the fix would have shipped doing + nothing on exactly the two grids it was written for — green gate, unchanged product. It was + caught only because the gate's fixture used `ut_odoo_orders`, a MATERIALISED grid that never + had the defect [[gate-answers-the-wrong-question]]. + ⭐ The id column IS the grain: `sol.id` means this grid is one row per `sale_order_line`, and + the freshness of a joined lookup table (`sale_order`, `account_account`) is not this grid's + freshness. So the alias is read off `spec["id"]`, and a bare `id` degrades to `table` — which + is exactly the single-table case. + + ⛔ AND IT ANSWERS `""` FOR A **MATERIALISED** GRID, ON PURPOSE. Those eight tables store a + per-ROW `refreshed` written by the spawn, which is strictly better than one table-level date — + and this value arrives as an OVERLAY, so returning a stamp here would quietly overwrite eight + working grids' per-row stamps while fixing two blank ones. D-155's subject is the grid that has + NOWHERE to keep a row; a fix that also rewrites the grids that do is a different, unrequested + change [[reuse-and-delete-are-hypotheses]]. + """ + # the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot + # tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank), + # never a stamp that might overwrite a per-row one. + if table_key: + try: + if _ut().materialises(str(table_key), st=rt): + return "" + except Exception: # noqa: BLE001 + return "" + ident = str(spec.get("id") or "") + alias = ident.split(".", 1)[0] if "." in ident else "" + table = str((spec.get("tables") or {}).get(alias) or spec.get("table") + or (spec.get("tables") or {}).get("") or "") + if not table or cur is None: + return "" + try: + row = cur.execute("SELECT updated_at FROM _sync_state WHERE entity = ?", + [table]).fetchone() + except Exception: # noqa: BLE001 + return "" # no mirror bookkeeping ⇒ blank, exactly as before + return str((row or [""])[0] or "")[:10] + + +def _pool_from_window(spec, win, stamp=""): + """`[{pid, **cells}]` — THE one place a mirror window becomes product rows. + + ⛔ ONE BUILDER, TWO CALLERS, and that is deliberate rather than tidy: the windowed route and + the whole-table read-through below would otherwise each cast the same columns their own way, + and a cell that renders differently depending on which door served it is this repo's recorded + defect class ([[one-question-two-normalizers]]). + + ⚠ `stamp` (D-155) rides here for that same reason: both doors must produce the same + `refreshed` cell, and a caller that forgot it would give one door a freshness column and the + other a blank one. Empty when the caller cannot cheaply know — blank is what shipped, so the + degradation is the previous behaviour rather than a new wrong value. + """ + keys = list(spec["cols"]) + extra = {"refreshed": stamp} if stamp else {} + return [{"pid": int(r[0]), **extra, + **{k: spec["cols"][k][1](v) for k, v in zip(keys, r[1:])}} for r in win["rows"]] + + +class TooBigToMaterialise(Exception): + """A read-through table asked for WHOLE exceeds one window — refuse, never truncate.""" + + +def population(table_key, cur=None, rt=None): + """How many rows a read-through grid HAS, without fetching one — or None if it is not bound. + + ⭐⭐ W31 / B's ask (mailbox/B.md B-2). `routes_tables.scoped_pids` learned "this grid is too big + to list" the only way that existed: call `whole_pool()` and catch `TooBigToMaterialise`. That + pulls a full 5,000-row window out of DuckDB and throws it away on every `/workspace` for + `ut_odoo_gl_lines` and `ut_odoo_order_lines` — MEASURED by B at ~1,600 ms of a ~3,460 ms + in-proc envelope. This asks the SIZE instead: one `SELECT count(*)` over the same predicate. + + ⛔ IT ANSWERS A DIFFERENT QUESTION, NOT THE SAME ONE MORE CHEAPLY, and that distinction is what + keeps it safe. B's constraint is that the pid set must stay IDENTICAL to `scoped_pool`'s BY + CONSTRUCTION rather than by two queries that agree today ([[one-question-two-normalizers]]) — + so this returns a COUNT and never a pid. A caller uses it to decide whether to ask for pids at + all; when it does ask, the pids still come from the one `whole_pool` fetch they always did. + + ⚠ `None` means "no read-through binding on this deployment", which is NOT `0`. A caller that + collapses them reports an empty grid where it should report an unbound one — the exact + distinction `odoo_tables_status` already reports as `bound_not_declared`. + ⚠ `total` is the count over the spec's OWN `where`, i.e. the same population `whole_pool` + would return — the fixed scope is not dropped for being cheaper. + """ + from harness import datastore + if rt is not None: + rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry + cur = cur if cur is not None else datastore.ro_con() + spec = _source_for(cur, table_key) + if spec is None: + return None + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + return datastore.window(select="1", where=spec.get("where") or "", order_by=spec["id"], + limit=1, cur=cur, **frm)["total"] + + +def whole_pool(table_key, cur=None, rt=None): + """Every row of a read-through grid, in `scoped_pool`'s shape — or a refusal. + + ⚠ THIS EXISTS FOR THE CLIENT WE HAVE, NOT THE ONE WE WANT. The grid still fetches whole + tables (`GET /tables/{key}/rows`); paging is F's W30-T42. So a database whose rows have left + this tenant's document has to be servable to that client somehow, and the honest answer for a + small one is "read all of it from the mirror". `_unmaterialisable` only ever registers a table + that fits, so the refusal below is a guard against the population GROWING past the window + later — at which point R6 requires a sentence, not a quietly shorter grid. + + ⭐⭐ W31-T45 / D-169 — `rt` IS THE TENANT RUNTIME, AND IT IS OPTIONAL FOR A STATED REASON + RATHER THAN A LAZY ONE. This function serves ROWS, so it is the door where a cross-tenant read + would be worst — but its only production caller is `routes_tables.py::_read_through_rows`, + which is in another lane's fence THIS WAVE (B's W31-T20 is rewriting that exact path) and does + not thread a session down to here. Making `rt` required would break that caller on import; a + second predicate invented locally would be one question with two normalizers, a recorded + defect class here. So the guard fires when a caller passes the runtime it already holds, and + the ask to thread `session.runtime` through `_read_through_rows` is posted to B in + `mailbox/D.md`. ⛔ Until that lands this door is guarded only by `_defn_or_refuse` upstream — + stated here rather than left for a reader to discover, because an unguarded row door is + precisely what D-169 is about. + """ + from harness import datastore + if rt is not None: + cur = cur if cur is not None else _rt().mirror_cursor(rt) + rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent + cur = cur if cur is not None else datastore.ro_con() + spec = _source_for(cur, table_key) + if spec is None: + raise TooBigToMaterialise( + f"{table_key}: this database is served through the mirror but has no read-through " + f"binding on this deployment, so its rows cannot be read at all") + frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]} + select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items()) + win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=spec.get("where"), + order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm) + if win["total"] > len(win["rows"]): + # ⛔ REFUSE, NEVER TRIM. A short grid that says nothing is exactly the silent truncation + # R6's second sentence is about — and the caller turns this into a message with a cause. + raise TooBigToMaterialise( + f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row " + f"window, and this database no longer stores rows in the tenant document; it can " + f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)") + return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt)) + + +def _fs(): + from harness import filter_sql + return filter_sql + + +def _so(): + import core.shared_overlay as shared_overlay + return shared_overlay + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _today(): + import time + return time.strftime("%Y-%m-%d") diff --git a/api/routes_query.py b/api/routes_query.py index d01c08d0277e4c673439218de39cb6c247c00e58..2cb64ab478432b3a3a7e3c962f632dc713065775 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -1,713 +1,693 @@ -"""routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors. - - POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING - POST /api/v1/query/save {question, scope, spec} -> the saved view - GET /api/v1/query -> the AI-built views this caller may open - DELETE /api/v1/query/{qid} -> forget one - -**The AI turns a question about ONE granted database into a view SPEC. It may only build what the -backend already supports** (R1), and the wall on which database it may name is the EXISTING one. - -⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50 -(`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored -YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers -`ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its -own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid -cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model -never writes SQL and never invents a field name — it only picks registry keys.* - -The real foundation is the view stack that already ships, and this module is `routes_templates.py` -with a model where the curated registry used to be: - - · the wall `routes_templates._target_or_refuse` (imported, never re-implemented) - · the column contract `core.view_templates.columns_named` / `missing_columns` - · the view skeleton `core.view_templates._view` (one definition of what a view is) - · the validators `aios_grid._clean_display` / `clean_filter_tree` - · the writer `core.table_store.make(ws).save_view(..., shared=False)` - -⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns -None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a -leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a -name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So -membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are -the vocabulary ORACLE, never the refusal. - -⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than -inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions -(`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions -with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec -answering a different question ("Revenue by Creator" against a database holding no revenue). -Changing refusal from an enum value to its own tool changed cerebras not at all and made groq -worse. -⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated -here because the first draft of this header claimed the opposite. A door that dies whenever one -free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator -and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is -unreachable the answer is a sentence and never a view. Which provider answered rides back in the -response so a wrong answer stays diagnosable (`_call_model`'s note). - -⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments -against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real -output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP -error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this -file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument -object), and a 400 body is READ before it is treated as transport. - -⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column -is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT -QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own -shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the -validated spec (never asked of the model, which would let it flatter its own output), and saving is -a second, explicit call. The person is the last validator, by design. -""" -import datetime as _dt -import hashlib -import json -import os -import re - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself -#: is a real saved view in its target's own workspace — this is only the index, so listing the -#: Query module never has to read every granted table's workspace to find them. Same split -#: `core/alerts.py` draws between definitions and the inbox. -QUERY_KEY = 'query_views' - -MAX_QUESTION = 500 -MAX_VISIBLE = 24 -MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule) - -#: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a -#: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's -#: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces -#: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that -#: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the -#: `swipe`/`form` failure ("whole, correct and unreachable") in reverse. -QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list') - -#: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived -#: from a question about existing records. A reason per member, because "excluded" with no cause -#: is how a hold outlives its own justification. -QUERY_EXCLUDED = { - 'form': "a form is a door records come IN through and shows none — a question about existing " - "records cannot describe one, and its spec carries emails and a public token", - 'catalog': "a catalog is a published artifact needing page authoring and product codes " - "(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows", - 'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two " - "product decisions a question does not contain", -} - -#: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON -#: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client, -#: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct -#: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked -#: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract -#: with ZERO select columns, and every validator passed it. -_MODE_REFS = { - 'kanban': [('stackField', ('select',), True)], - 'calendar': [('dateField', ('date',), True)], - 'timeseries': [('dateField', ('date',), True)], - 'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)], -} - - -def _now_iso(): - """UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule).""" - return _dt.datetime.now(_dt.timezone.utc).isoformat() - - -def _grid(): - import aios_grid - return aios_grid - - -def _templates(): - import core.view_templates as view_templates - return view_templates - - -def _target(session, scope): - """`(fields, source)` for a database this session may open — or a refusal. - - ⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same - predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module - grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's - instruction is explicit that the wall is re-used, never re-implemented beside itself. The - private name is deliberate: an alias would mean editing a file no lane's fence covers. - - It returns field KEYS; this door also needs each field's TYPE (for the prompt and for - `_MODE_REFS`), so the types are read from the same source the wall consulted — never from a - list assembled here. - """ - import routes_templates - keys, source = routes_templates._target_or_refuse(session, scope) - key = str(scope or '').strip() - defs = [] - if key.startswith('ut_'): - import core.user_tables as user_tables - defn = user_tables.get(key, st=session.runtime) or {} - by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')} - else: - g = _grid() - raw = g.product_fields() if key == 'product_data' else g.FIELDS - by_key = {str(f.get('key')): f for f in raw if f.get('key')} - for k in keys: - f = by_key.get(k) or {} - defs.append({'key': k, 'label': str(f.get('label') or k)[:80], - 'type': str(f.get('type') or 'text')}) - return defs, source - - -# ── the model ─────────────────────────────────────────────────────────────────────────────── - -#: This module's OWN provider order — cerebras first. See the header: the refusal is a model -#: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one. -#: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place -#: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too. -QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter') - - -def _providers(): - import harness.analyst as analyst - pin = os.environ.get("AIOS_QUERY_PROVIDER") - by_name = {p["name"]: p for p in analyst.PROVIDERS} - order = [pin] if pin else list(QUERY_PROVIDER_ORDER) - return [by_name[n] for n in order - if n in by_name and os.environ.get(by_name[n]["env"])] - - -def _spec_schema(field_keys): - """The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a - refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads - exactly like a transport failure.""" - ops = sorted(_grid().FILTER_OPS) - col = {"type": "string", "enum": sorted(field_keys)} - return { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"], - "description": "refused = this database cannot answer the question"}, - "name": {"type": "string", "description": "a short title for the view"}, - "refusal": {"type": "string", - "description": "when kind=refused: ONE plain sentence naming what is " - "missing"}, - "visible": {"type": "array", "items": col}, - "filters": {"type": "array", "items": { - "type": "object", - "properties": {"colId": col, "op": {"type": "string", "enum": ops}, - "value": {"type": "string"}}, - "required": ["colId", "op"]}}, - "filterConj": {"type": "string", "enum": ["and", "or"]}, - "sorts": {"type": "array", "items": { - "type": "object", - "properties": {"colId": col, - "dir": {"type": "string", "enum": ["asc", "desc"]}}, - "required": ["colId", "dir"]}}, - "groupBy": col, - "stackField": col, - "dateField": col, - "colorField": col, - "sizeField": col, - }, - "required": ["kind"], - } - - -def _system_prompt(fields): - """The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the - target's own columns) — the model is never told about a key this server cannot honour.""" - cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]} - for f in fields], separators=(",", ":")) - kinds = ", ".join(QUERY_KINDS) - ops = ", ".join(sorted(_grid().FILTER_OPS)) - selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"] - dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"] - return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you -never invent a column or a kind of view. - -THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys: -{cols} - -VIEW KINDS you may name, and nothing else: {kinds} -FILTER OPS you may name, and nothing else: {ops} -The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)} -The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)} - -RULES: -1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref - MUST be one of the keys above. -2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or - counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED, - and it must be a select column); something over time -> calendar or timeseries (dateField - REQUIRED, and it must be a date column); geography -> map. -3. `visible` is the columns the question is about, most important first — not every column. -4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column - this database does not have, a number it does not store, or a picture that is not in the kind - list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one - asked is the worst possible answer — worse than saying you cannot build it. When in doubt, - refuse. -5. Every `value` is a string.""" - - -def _call_model(question, fields, chat=None): - """`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder. - - `chat` is injectable so `verify_query.py` proves this door end to end with no key and no - tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline). - - ⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the - FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer - unanswerable questions with plausible specs. Falling through on a 429 is still the right call — - a Query module that dies whenever one free tier rate-limits is worse than one that answers with - a weaker model behind the same validator and the same human read-back — but *which* model - answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is - what happens when the whole ladder is unreachable, not when the best rung is. - """ - tools = [{"type": "function", "function": { - "name": "build_view", - "description": "Emit the view spec, or refuse.", - "parameters": _spec_schema([f["key"] for f in fields])}}] - messages = [{"role": "system", "content": _system_prompt(fields)}, - {"role": "user", "content": question}] - if chat is not None: - return chat(messages, tools), None, "injected" - - provs = _providers() - if not provs: - # No key at all: say so. An AI feature that silently does nothing is indistinguishable - # from one that was never built ([[flag-shipped-without-its-writer]]). - return None, ("the assistant is not configured on this deployment, so a view cannot be " - "built from a question yet"), None - import requests - last = None - for p in provs: - try: - r = requests.post(p["url"], timeout=60, - headers={"Authorization": f"Bearer {os.environ[p['env']]}"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1200}) - except Exception as e: # network blip — try the next provider - last = f"{p['name']}: {type(e).__name__}" - continue - if r.status_code == 400: - # ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the - # schema and returns the generation it rejected. Reading it turns a "transport - # failure" back into the refusal it actually was. - spec = _spec_from_400(r.text) - if spec is not None: - return spec, None, p["name"] - last = f"{p['name']}: 400" - continue - if r.status_code != 200: - last = f"{p['name']}: HTTP {r.status_code}" - continue # 429 included: the NEXT provider, not a sleep - try: - msg = r.json()["choices"][0]["message"] - calls = msg.get("tool_calls") or [] - if not calls: - last = f"{p['name']}: no tool call" - continue - return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"] - except Exception as e: - last = f"{p['name']}: unreadable answer ({type(e).__name__})" - continue - # ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it - # is rate-limited and the rest are too, the answer is a sentence, never a fabricated view. - return None, ("the assistant could not be reached just now (" + (last or "no provider") + - ") — try the question again in a moment"), None - - -_FAILED_GEN = re.compile(r'(\{.*?\})\s*', re.S) - - -def _spec_from_400(body): - """The spec out of a `tool_use_failed` body's `failed_generation`, or None.""" - try: - err_obj = (json.loads(body) or {}).get("error") or {} - except Exception: - return None - gen = str(err_obj.get("failed_generation") or "") - if not gen: - return None - m = _FAILED_GEN.search(gen) - raw = m.group(1) if m else gen.strip() - try: - spec = json.loads(raw) - except Exception: - return None - return spec if isinstance(spec, dict) else None - - -# ── validation: the REFUSAL this contract promises ─────────────────────────────────────────── - -def _validate(spec, fields): - """`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set. - - Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners - as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and - never a broken view. - """ - g = _grid() - if not isinstance(spec, dict): - return None, "the assistant did not answer with a view", "no_spec" - by_key = {f['key']: f['type'] for f in fields} - keys = set(by_key) - kind = spec.get('kind') - - if kind == 'refused': - return None, (str(spec.get('refusal') or '').strip() - or "this database cannot answer that question"), "model_refused" - if kind in QUERY_EXCLUDED: - return None, (f"a {kind} view cannot be built from a question — " - + QUERY_EXCLUDED[kind]), "unsupported_kind" - if kind not in QUERY_KINDS: - # ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name - # promising something else. - return None, (f"{kind!r} is not a kind of view this product can build" - if kind else "the assistant did not name a kind of view"), "unsupported_kind" - - named = set(spec.get('visible') or ()) - for s in spec.get('sorts') or (): - if isinstance(s, dict) and s.get('colId'): - named.add(str(s['colId'])) - for f in spec.get('filters') or (): - if isinstance(f, dict) and f.get('colId'): - named.add(str(f['colId'])) - if spec.get('groupBy'): - named.add(str(spec['groupBy'])) - for ref in ('stackField', 'dateField', 'colorField', 'sizeField'): - if spec.get(ref): - named.add(str(spec[ref])) - missing = sorted(named - keys) - if missing: - # NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader - # this is the wrong database for this question. - return None, ("this database does not have the columns that answer needs: " - + ", ".join(missing)), "unknown_columns" - - visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE] - if not visible: - # An empty projection is not a view. `_default_view_config` would fill one in, which is - # how a question about two columns becomes a 285-column wall. - return None, ("that question did not name anything to show — try naming the columns you " - "want to see"), "no_columns" - - # ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX - # column the way the grid does" — a set row lands in several buckets, so a sum means something - # else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket - # keying, which is in no lane's fence this wave (booked, with that reason). What IS in this - # lane's power is refusing to GENERATE the broken case: the AI must only build what the backend - # and the renderer already support, which is R1's own sentence. Delete this guard when D-137 - # closes — the refusal names the debt so the reader knows it is a limit, not a rule. - if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'): - return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds " - f"several values at once, or a tick box, does not bucket the way a chart " - f"needs. Try a table grouped by it instead"), "chart_key_unsupported" - - for ref, families, required in _MODE_REFS.get(kind, ()): - val = spec.get(ref) - if val and by_key.get(val) not in families: - return None, (f"a {kind} view needs {ref[:-5]} to be a " - f"{' or '.join(families)} column, and {val!r} is a " - f"{by_key.get(val)} column"), "wrong_ref_type" - if required and not val: - return None, (f"a {kind} view needs a {' or '.join(families)} column and this " - f"database has none that fits"), "missing_ref" - - raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)] - filters = g.clean_filter_tree(raw_filters, keys) - if len(filters) != len(raw_filters): - # The belt caught something membership did not — an unknown op. Refuse rather than save a - # view whose condition was silently dropped (the widening this file exists to prevent). - return None, ("part of that filter is not something this product can express — try " - "asking it more simply"), "filter_dropped" - - display = {'mode': kind} - for ref, _families, _req in _MODE_REFS.get(kind, ()): - if spec.get(ref): - display[ref] = spec[ref] - cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None - if kind != 'grid' and not cleaned_display: - return None, (f"this product could not build a {kind} view from that question" - ), "display_dropped" - - clean = { - 'kind': kind, - 'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query', - 'visible': visible, - 'filters': filters, - 'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and', - 'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'} - for s in (spec.get('sorts') or ()) if isinstance(s, dict) - and s.get('colId') in keys][:3], - 'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None, - 'display': cleaned_display, - } - return clean, None, None - - -def _explain(clean, fields): - """A plain-language read-back of what the view WILL do — DERIVED from the validated spec. - - ⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which - is precisely the failure this sentence exists to expose: a spec whose every column is real and - which answers a different question is invisible to validation, so the only thing that catches - it is a person reading an honest description of the spec that was actually accepted. - """ - label = {f['key']: f['label'] for f in fields} - ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least', - 'lt': 'is under', 'lte': 'is at most', 'contains': 'contains', - 'doesNotContain': 'does not contain', 'isEmpty': 'is empty', - 'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within', - 'topN': 'is in the top', 'bottomN': 'is in the bottom'} - kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board', - 'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'} - parts = [f"{kinds.get(clean['kind'], clean['kind'])} of " - + ", ".join(label.get(k, k) for k in clean['visible'][:6]) - + (f" and {len(clean['visible']) - 6} more columns" - if len(clean['visible']) > 6 else "")] - if clean['filters']: - joiner = " or " if clean['filterConj'] == 'or' else " and " - parts.append("showing only records where " + joiner.join( - f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}" - + (f" {f['value']}" if f.get('value') else "") - for f in clean['filters'] if isinstance(f, dict) and f.get('colId'))) - else: - parts.append("showing every record") - if clean['groupBy']: - parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}") - if clean['sorts']: - s = clean['sorts'][0] - parts.append(f"sorted by {label.get(s['colId'], s['colId'])}" - + (" (highest first)" if s['dir'] == 'desc' else " (lowest first)")) - for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'), - ('colorField', 'coloured by'), ('sizeField', 'sized by')): - val = (clean.get('display') or {}).get(ref) - if val: - parts.append(f"{word} {label.get(val, val)}") - return ", ".join(parts) + "." - - -# ── the registry ──────────────────────────────────────────────────────────────────────────── - -def _qid(scope, question): - """PINNED per (database, question), the `view_templates` discipline: asking the same thing - twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by - appending a number, so an unpinned id fills the store with numbered near-duplicates.""" - h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8")) - return "q_" + h.hexdigest()[:12] - - -def _registry(session): - try: - recs = session.runtime.get(QUERY_KEY) or {} - except Exception: - return {} - return recs if isinstance(recs, dict) else {} - - -def _mine(session, rec): - return session.admin or str(rec.get('createdBy') or '') == session.uname - - -#: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE -#: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving -#: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set -#: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the -#: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door -#: would then refuse is a control that lies" — and it is the same predicate twice, published once. -BUILTIN_SCOPES = ('customer_data', 'product_data') - - -@router.get("/query") -def list_queries(session: Session = Depends(require_session)): - """The AI-built views this caller may open: their own, still-granted, still-existing. - - ⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn - and a database deleted after a view was saved, and the Query module is chrome: it may render - nothing the server would not grant today (`nav.ts`'s chrome law). - """ - import routes_templates - out = [] - for qid, rec in sorted(_registry(session).items(), - key=lambda kv: str((kv[1] or {}).get('createdAt') or ''), - reverse=True): - if not isinstance(rec, dict) or not _mine(session, rec): - continue - try: - routes_templates._target_or_refuse(session, rec.get('scope')) - except Exception: - continue # no longer granted, or gone - out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'), - "name": rec.get('name'), "kind": rec.get('kind'), - "question": rec.get('question'), "explain": rec.get('explain'), - "createdAt": rec.get('createdAt')}) - return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)} - - -@router.post("/query/build") -def build_query(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.** - - A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a - successful answer to the person asking, and an error envelope would paint the error page R1 - forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an - over-long question, a database this session may not open. - - ⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took - `chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument - as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and - `POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on - `"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly — - which is exactly the shape of a test seam that becomes an input. The seam now lives on the - module-level `_build` below: the gate calls that, and the wire cannot. - """ - body = body if isinstance(body, dict) else {} - return _build(body.get("question"), body.get("scope"), session) - - -def _build(question, scope, session, chat=None): - """`build_query`'s body, with the model injectable — see that route's note on why the seam is - here and not on the signature FastAPI reads.""" - question = ' '.join(str(question or "").split()) - if not question: - raise err(400, "bad_request", "no question was asked") - if len(question) > MAX_QUESTION: - raise err(400, "question_too_long", - f"a question must be under {MAX_QUESTION} characters") - fields, _source = _target(session, scope) - if not fields: - raise err(400, "no_columns", "that database has no columns to build a view from") - - spec, transport, provider = _call_model(question, fields, chat=chat) - if transport: - return {"spec": None, "explain": None, "refused": transport, "provider": provider} - clean, refusal, code = _validate(spec, fields) - if refusal: - return {"spec": None, "explain": None, "refused": refusal, "reason": code, - "provider": provider} - return {"spec": clean, "explain": _explain(clean, fields), - "refused": None, "scope": scope, - "id": _qid(scope, question), "provider": provider, - "fallback": _is_fallback(provider)} - - -def _is_fallback(provider): - """True when the answer came from a rung BELOW the top of this module's ladder. - - ⛔ THIS IS THE HALF THAT WAS MISSING, AND IT IS A READER, NOT A WRITER. `_call_model` has - always returned WHICH provider answered, and its own docstring says exactly why that matters: - *"Only the FIRST provider is measured to refuse honestly … the ones below it answer - unanswerable questions with plausible specs."* So on a 429 the ladder falls through — correctly, - a Query module that dies when one free tier rate-limits is worse — and the person then reads a - confident, well-formed view spec built by a model that does not know how to say "this database - cannot answer that". **Nothing on the wire told them.** The value existed and had no consumer. - ⚠ `injected` (the gate's stubbed chat) is NOT a fallback: it is the test seam, and reporting it - as a degraded answer would make every gate assertion about this flag meaningless. - """ - if not provider or provider == "injected": - return False - order = [p["name"] for p in _providers()] - return bool(order) and provider != order[0] - - -@router.post("/query/save") -def save_query(body: dict = Body(default=None), session: Session = Depends(require_session)): - """Save an accepted spec as the CALLING USER's own view, and index it under Query. - - ⛔ THE SPEC IS RE-VALIDATED HERE, against the target's contract as it is NOW. The build door - wrote nothing, so this is the first write and it is the only place authorisation and validity - have to hold together — and a client is not a wall. An unsupported kind or an unknown column - is a **400 with a named code** on this door (the caller sent it), where the same fact is a - plain sentence on `build` (the model produced it). - """ - body = body if isinstance(body, dict) else {} - scope = str(body.get("scope") or '').strip() - question = ' '.join(str(body.get("question") or "").split())[:MAX_QUESTION] - fields, _source = _target(session, scope) - clean, refusal, code = _validate(body.get("spec"), fields) - if refusal: - raise err(400, code or "bad_spec", refusal) - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - - vt = _templates() - ws_key = vt.workspace_key(scope) - if not ws_key: - raise err(404, "unknown_table", "that database does not exist") - qid = _qid(scope, question) if question else _qid(scope, clean['name']) - - # ⛔ THE SKELETON IS `view_templates._view`, NOT A SECOND ONE. "A view assembled from a - # DIFFERENT skeleton is a second definition of what a view is — and the two drift on the day - # one of them gains a key." It also puts the mode at `config.display.mode`, which is the key - # the grid actually reads (`config.displayMode` was the first draft, and nothing reads it). - payload = vt._view(qid, clean['name'], f"Built by the assistant from: {question}"[:300], - clean['visible'], mode=clean['kind'], - sorts=clean['sorts'], filters=clean['filters'], - group_by=clean['groupBy'], conj=clean['filterConj']) - if clean.get('display'): - payload['config']['display'] = dict(clean['display']) - payload['createdBy'] = session.uname - - # ⛔ THE CAP REFUSES; IT DOES NOT EVICT — and the first draft evicted. Dropping the oldest entry - # would leave its saved VIEW behind in the table's own workspace: "stranded in the target's view - # list with no way back to the question that made it", which is the thing `delete_query` below - # exists to prevent. And an eviction nobody was told about is W30/R6's second sentence broken — - # a limit that cannot be removed must be REPORTED with its cause. So the 201st is a refusal - # naming the number and the way out, checked BEFORE the view is written. - existing = _registry(session) - if qid not in existing and len(existing) >= MAX_REGISTRY: - raise err(400, "query_limit", - f"you already have {len(existing)} saved questions, which is the limit " - f"({MAX_REGISTRY}). Delete one you no longer need and save this again.") - - import core.table_store as table_store - ops = table_store.make(ws_key, st=session.runtime) - saved = ops.save_view(session.uname, payload, shared=False) - - entry = {"scope": scope, "viewId": qid, "name": saved.get('name') or clean['name'], - "kind": clean['kind'], "question": question, - "explain": _explain(clean, fields), "createdBy": session.uname, - "createdAt": _now_iso()} - - def _up(data): - data = data if isinstance(data, dict) else {} - data[qid] = entry - return data - - session.runtime.update(QUERY_KEY, _up, flush='async') - return {"id": qid, **entry} - - -@router.delete("/query/{qid}") -def delete_query(qid: str, session: Session = Depends(require_session)): - """Forget an AI-built view — the index entry AND the view itself. - - Leaving the view behind would strand it in the target's view list with no way back to the - question that made it ([[a-record-can-outlive-its-subject]]). - """ - rec = _registry(session).get(str(qid)) - if not isinstance(rec, dict) or not _mine(session, rec): - raise err(404, "unknown_query", "that view does not exist") - _target(session, rec.get('scope')) # the wall, again, on the write path - vt = _templates() - ws_key = vt.workspace_key(rec.get('scope')) - if ws_key and session.runtime.available(): - import core.table_store as table_store - table_store.make(ws_key, st=session.runtime).delete_view( - session.uname, rec.get('viewId') or str(qid)) - - def _up(data): - data = data if isinstance(data, dict) else {} - data.pop(str(qid), None) - return data - - session.runtime.update(QUERY_KEY, _up, flush='async') - return {"deleted": str(qid)} +"""routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors. + + POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING + POST /api/v1/query/save {question, scope, spec} -> the saved view + GET /api/v1/query -> the AI-built views this caller may open + DELETE /api/v1/query/{qid} -> forget one + +**The AI turns a question about ONE granted database into a view SPEC. It may only build what the +backend already supports** (R1), and the wall on which database it may name is the EXISTING one. + +⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50 +(`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored +YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers +`ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its +own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid +cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model +never writes SQL and never invents a field name — it only picks registry keys.* + +The real foundation is the view stack that already ships, and this module is `routes_templates.py` +with a model where the curated registry used to be: + + · the wall `routes_templates._target_or_refuse` (imported, never re-implemented) + · the column contract `core.view_templates.columns_named` / `missing_columns` + · the view skeleton `core.view_templates._view` (one definition of what a view is) + · the validators `aios_grid._clean_display` / `clean_filter_tree` + · the writer `core.table_store.make(ws).save_view(..., shared=False)` + +⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns +None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a +leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a +name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So +membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are +the vocabulary ORACLE, never the refusal. + +⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than +inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions +(`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions +with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec +answering a different question ("Revenue by Creator" against a database holding no revenue). +Changing refusal from an enum value to its own tool changed cerebras not at all and made groq +worse. +⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated +here because the first draft of this header claimed the opposite. A door that dies whenever one +free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator +and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is +unreachable the answer is a sentence and never a view. Which provider answered rides back in the +response so a wrong answer stays diagnosable (`_call_model`'s note). + +⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments +against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real +output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP +error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this +file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument +object), and a 400 body is READ before it is treated as transport. + +⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column +is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT +QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own +shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the +validated spec (never asked of the model, which would let it flatter its own output), and saving is +a second, explicit call. The person is the last validator, by design. +""" +import datetime as _dt +import hashlib +import json +import os +import re + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself +#: is a real saved view in its target's own workspace — this is only the index, so listing the +#: Query module never has to read every granted table's workspace to find them. Same split +#: `core/alerts.py` draws between definitions and the inbox. +QUERY_KEY = 'query_views' + +MAX_QUESTION = 500 +MAX_VISIBLE = 24 +MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule) + +#: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a +#: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's +#: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces +#: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that +#: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the +#: `swipe`/`form` failure ("whole, correct and unreachable") in reverse. +QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list') + +#: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived +#: from a question about existing records. A reason per member, because "excluded" with no cause +#: is how a hold outlives its own justification. +QUERY_EXCLUDED = { + 'form': "a form is a door records come IN through and shows none — a question about existing " + "records cannot describe one, and its spec carries emails and a public token", + 'catalog': "a catalog is a published artifact needing page authoring and product codes " + "(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows", + 'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two " + "product decisions a question does not contain", +} + +#: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON +#: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client, +#: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct +#: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked +#: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract +#: with ZERO select columns, and every validator passed it. +_MODE_REFS = { + 'kanban': [('stackField', ('select',), True)], + 'calendar': [('dateField', ('date',), True)], + 'timeseries': [('dateField', ('date',), True)], + 'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)], +} + + +def _now_iso(): + """UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule).""" + return _dt.datetime.now(_dt.timezone.utc).isoformat() + + +def _grid(): + import aios_grid + return aios_grid + + +def _templates(): + import core.view_templates as view_templates + return view_templates + + +def _target(session, scope): + """`(fields, source)` for a database this session may open — or a refusal. + + ⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same + predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module + grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's + instruction is explicit that the wall is re-used, never re-implemented beside itself. The + private name is deliberate: an alias would mean editing a file no lane's fence covers. + + It returns field KEYS; this door also needs each field's TYPE (for the prompt and for + `_MODE_REFS`), so the types are read from the same source the wall consulted — never from a + list assembled here. + """ + import routes_templates + keys, source = routes_templates._target_or_refuse(session, scope) + key = str(scope or '').strip() + defs = [] + if key.startswith('ut_'): + import core.user_tables as user_tables + defn = user_tables.get(key, st=session.runtime) or {} + by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')} + else: + g = _grid() + raw = g.product_fields() if key == 'product_data' else g.FIELDS + by_key = {str(f.get('key')): f for f in raw if f.get('key')} + for k in keys: + f = by_key.get(k) or {} + defs.append({'key': k, 'label': str(f.get('label') or k)[:80], + 'type': str(f.get('type') or 'text')}) + return defs, source + + +# ── the model ─────────────────────────────────────────────────────────────────────────────── + +#: This module's OWN provider order — cerebras first. See the header: the refusal is a model +#: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one. +#: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place +#: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too. +QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter') + + +def _providers(): + import harness.analyst as analyst + pin = os.environ.get("AIOS_QUERY_PROVIDER") + by_name = {p["name"]: p for p in analyst.PROVIDERS} + order = [pin] if pin else list(QUERY_PROVIDER_ORDER) + return [by_name[n] for n in order + if n in by_name and os.environ.get(by_name[n]["env"])] + + +def _spec_schema(field_keys): + """The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a + refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads + exactly like a transport failure.""" + ops = sorted(_grid().FILTER_OPS) + col = {"type": "string", "enum": sorted(field_keys)} + return { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"], + "description": "refused = this database cannot answer the question"}, + "name": {"type": "string", "description": "a short title for the view"}, + "refusal": {"type": "string", + "description": "when kind=refused: ONE plain sentence naming what is " + "missing"}, + "visible": {"type": "array", "items": col}, + "filters": {"type": "array", "items": { + "type": "object", + "properties": {"colId": col, "op": {"type": "string", "enum": ops}, + "value": {"type": "string"}}, + "required": ["colId", "op"]}}, + "filterConj": {"type": "string", "enum": ["and", "or"]}, + "sorts": {"type": "array", "items": { + "type": "object", + "properties": {"colId": col, + "dir": {"type": "string", "enum": ["asc", "desc"]}}, + "required": ["colId", "dir"]}}, + "groupBy": col, + "stackField": col, + "dateField": col, + "colorField": col, + "sizeField": col, + }, + "required": ["kind"], + } + + +def _system_prompt(fields): + """The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the + target's own columns) — the model is never told about a key this server cannot honour.""" + cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]} + for f in fields], separators=(",", ":")) + kinds = ", ".join(QUERY_KINDS) + ops = ", ".join(sorted(_grid().FILTER_OPS)) + selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"] + dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"] + return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you +never invent a column or a kind of view. + +THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys: +{cols} + +VIEW KINDS you may name, and nothing else: {kinds} +FILTER OPS you may name, and nothing else: {ops} +The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)} +The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)} + +RULES: +1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref + MUST be one of the keys above. +2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or + counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED, + and it must be a select column); something over time -> calendar or timeseries (dateField + REQUIRED, and it must be a date column); geography -> map. +3. `visible` is the columns the question is about, most important first — not every column. +4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column + this database does not have, a number it does not store, or a picture that is not in the kind + list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one + asked is the worst possible answer — worse than saying you cannot build it. When in doubt, + refuse. +5. Every `value` is a string.""" + + +def _call_model(question, fields, chat=None): + """`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder. + + `chat` is injectable so `verify_query.py` proves this door end to end with no key and no + tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline). + + ⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the + FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer + unanswerable questions with plausible specs. Falling through on a 429 is still the right call — + a Query module that dies whenever one free tier rate-limits is worse than one that answers with + a weaker model behind the same validator and the same human read-back — but *which* model + answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is + what happens when the whole ladder is unreachable, not when the best rung is. + """ + tools = [{"type": "function", "function": { + "name": "build_view", + "description": "Emit the view spec, or refuse.", + "parameters": _spec_schema([f["key"] for f in fields])}}] + messages = [{"role": "system", "content": _system_prompt(fields)}, + {"role": "user", "content": question}] + if chat is not None: + return chat(messages, tools), None, "injected" + + provs = _providers() + if not provs: + # No key at all: say so. An AI feature that silently does nothing is indistinguishable + # from one that was never built ([[flag-shipped-without-its-writer]]). + return None, ("the assistant is not configured on this deployment, so a view cannot be " + "built from a question yet"), None + import requests + last = None + for p in provs: + try: + r = requests.post(p["url"], timeout=60, + headers={"Authorization": f"Bearer {os.environ[p['env']]}"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1200}) + except Exception as e: # network blip — try the next provider + last = f"{p['name']}: {type(e).__name__}" + continue + if r.status_code == 400: + # ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the + # schema and returns the generation it rejected. Reading it turns a "transport + # failure" back into the refusal it actually was. + spec = _spec_from_400(r.text) + if spec is not None: + return spec, None, p["name"] + last = f"{p['name']}: 400" + continue + if r.status_code != 200: + last = f"{p['name']}: HTTP {r.status_code}" + continue # 429 included: the NEXT provider, not a sleep + try: + msg = r.json()["choices"][0]["message"] + calls = msg.get("tool_calls") or [] + if not calls: + last = f"{p['name']}: no tool call" + continue + return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"] + except Exception as e: + last = f"{p['name']}: unreadable answer ({type(e).__name__})" + continue + # ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it + # is rate-limited and the rest are too, the answer is a sentence, never a fabricated view. + return None, ("the assistant could not be reached just now (" + (last or "no provider") + + ") — try the question again in a moment"), None + + +_FAILED_GEN = re.compile(r'(\{.*?\})\s*', re.S) + + +def _spec_from_400(body): + """The spec out of a `tool_use_failed` body's `failed_generation`, or None.""" + try: + err_obj = (json.loads(body) or {}).get("error") or {} + except Exception: + return None + gen = str(err_obj.get("failed_generation") or "") + if not gen: + return None + m = _FAILED_GEN.search(gen) + raw = m.group(1) if m else gen.strip() + try: + spec = json.loads(raw) + except Exception: + return None + return spec if isinstance(spec, dict) else None + + +# ── validation: the REFUSAL this contract promises ─────────────────────────────────────────── + +def _validate(spec, fields): + """`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set. + + Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners + as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and + never a broken view. + """ + g = _grid() + if not isinstance(spec, dict): + return None, "the assistant did not answer with a view", "no_spec" + by_key = {f['key']: f['type'] for f in fields} + keys = set(by_key) + kind = spec.get('kind') + + if kind == 'refused': + return None, (str(spec.get('refusal') or '').strip() + or "this database cannot answer that question"), "model_refused" + if kind in QUERY_EXCLUDED: + return None, (f"a {kind} view cannot be built from a question — " + + QUERY_EXCLUDED[kind]), "unsupported_kind" + if kind not in QUERY_KINDS: + # ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name + # promising something else. + return None, (f"{kind!r} is not a kind of view this product can build" + if kind else "the assistant did not name a kind of view"), "unsupported_kind" + + named = set(spec.get('visible') or ()) + for s in spec.get('sorts') or (): + if isinstance(s, dict) and s.get('colId'): + named.add(str(s['colId'])) + for f in spec.get('filters') or (): + if isinstance(f, dict) and f.get('colId'): + named.add(str(f['colId'])) + if spec.get('groupBy'): + named.add(str(spec['groupBy'])) + for ref in ('stackField', 'dateField', 'colorField', 'sizeField'): + if spec.get(ref): + named.add(str(spec[ref])) + missing = sorted(named - keys) + if missing: + # NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader + # this is the wrong database for this question. + return None, ("this database does not have the columns that answer needs: " + + ", ".join(missing)), "unknown_columns" + + visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE] + if not visible: + # An empty projection is not a view. `_default_view_config` would fill one in, which is + # how a question about two columns becomes a 285-column wall. + return None, ("that question did not name anything to show — try naming the columns you " + "want to see"), "no_columns" + + # ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX + # column the way the grid does" — a set row lands in several buckets, so a sum means something + # else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket + # keying, which is in no lane's fence this wave (booked, with that reason). What IS in this + # lane's power is refusing to GENERATE the broken case: the AI must only build what the backend + # and the renderer already support, which is R1's own sentence. Delete this guard when D-137 + # closes — the refusal names the debt so the reader knows it is a limit, not a rule. + if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'): + return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds " + f"several values at once, or a tick box, does not bucket the way a chart " + f"needs. Try a table grouped by it instead"), "chart_key_unsupported" + + for ref, families, required in _MODE_REFS.get(kind, ()): + val = spec.get(ref) + if val and by_key.get(val) not in families: + return None, (f"a {kind} view needs {ref[:-5]} to be a " + f"{' or '.join(families)} column, and {val!r} is a " + f"{by_key.get(val)} column"), "wrong_ref_type" + if required and not val: + return None, (f"a {kind} view needs a {' or '.join(families)} column and this " + f"database has none that fits"), "missing_ref" + + raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)] + filters = g.clean_filter_tree(raw_filters, keys) + if len(filters) != len(raw_filters): + # The belt caught something membership did not — an unknown op. Refuse rather than save a + # view whose condition was silently dropped (the widening this file exists to prevent). + return None, ("part of that filter is not something this product can express — try " + "asking it more simply"), "filter_dropped" + + display = {'mode': kind} + for ref, _families, _req in _MODE_REFS.get(kind, ()): + if spec.get(ref): + display[ref] = spec[ref] + cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None + if kind != 'grid' and not cleaned_display: + return None, (f"this product could not build a {kind} view from that question" + ), "display_dropped" + + clean = { + 'kind': kind, + 'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query', + 'visible': visible, + 'filters': filters, + 'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and', + 'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'} + for s in (spec.get('sorts') or ()) if isinstance(s, dict) + and s.get('colId') in keys][:3], + 'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None, + 'display': cleaned_display, + } + return clean, None, None + + +def _explain(clean, fields): + """A plain-language read-back of what the view WILL do — DERIVED from the validated spec. + + ⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which + is precisely the failure this sentence exists to expose: a spec whose every column is real and + which answers a different question is invisible to validation, so the only thing that catches + it is a person reading an honest description of the spec that was actually accepted. + """ + label = {f['key']: f['label'] for f in fields} + ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least', + 'lt': 'is under', 'lte': 'is at most', 'contains': 'contains', + 'doesNotContain': 'does not contain', 'isEmpty': 'is empty', + 'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within', + 'topN': 'is in the top', 'bottomN': 'is in the bottom'} + kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board', + 'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'} + parts = [f"{kinds.get(clean['kind'], clean['kind'])} of " + + ", ".join(label.get(k, k) for k in clean['visible'][:6]) + + (f" and {len(clean['visible']) - 6} more columns" + if len(clean['visible']) > 6 else "")] + if clean['filters']: + joiner = " or " if clean['filterConj'] == 'or' else " and " + parts.append("showing only records where " + joiner.join( + f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}" + + (f" {f['value']}" if f.get('value') else "") + for f in clean['filters'] if isinstance(f, dict) and f.get('colId'))) + else: + parts.append("showing every record") + if clean['groupBy']: + parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}") + if clean['sorts']: + s = clean['sorts'][0] + parts.append(f"sorted by {label.get(s['colId'], s['colId'])}" + + (" (highest first)" if s['dir'] == 'desc' else " (lowest first)")) + for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'), + ('colorField', 'coloured by'), ('sizeField', 'sized by')): + val = (clean.get('display') or {}).get(ref) + if val: + parts.append(f"{word} {label.get(val, val)}") + return ", ".join(parts) + "." + + +# ── the registry ──────────────────────────────────────────────────────────────────────────── + +def _qid(scope, question): + """PINNED per (database, question), the `view_templates` discipline: asking the same thing + twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by + appending a number, so an unpinned id fills the store with numbered near-duplicates.""" + h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8")) + return "q_" + h.hexdigest()[:12] + + +def _registry(session): + try: + recs = session.runtime.get(QUERY_KEY) or {} + except Exception: + return {} + return recs if isinstance(recs, dict) else {} + + +def _mine(session, rec): + return session.admin or str(rec.get('createdBy') or '') == session.uname + + +#: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE +#: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving +#: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set +#: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the +#: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door +#: would then refuse is a control that lies" — and it is the same predicate twice, published once. +BUILTIN_SCOPES = ('customer_data', 'product_data') + + +@router.get("/query") +def list_queries(session: Session = Depends(require_session)): + """The AI-built views this caller may open: their own, still-granted, still-existing. + + ⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn + and a database deleted after a view was saved, and the Query module is chrome: it may render + nothing the server would not grant today (`nav.ts`'s chrome law). + """ + import routes_templates + out = [] + for qid, rec in sorted(_registry(session).items(), + key=lambda kv: str((kv[1] or {}).get('createdAt') or ''), + reverse=True): + if not isinstance(rec, dict) or not _mine(session, rec): + continue + try: + routes_templates._target_or_refuse(session, rec.get('scope')) + except Exception: + continue # no longer granted, or gone + out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'), + "name": rec.get('name'), "kind": rec.get('kind'), + "question": rec.get('question'), "explain": rec.get('explain'), + "createdAt": rec.get('createdAt')}) + return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)} + + +@router.post("/query/build") +def build_query(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.** + + A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a + successful answer to the person asking, and an error envelope would paint the error page R1 + forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an + over-long question, a database this session may not open. + + ⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took + `chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument + as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and + `POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on + `"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly — + which is exactly the shape of a test seam that becomes an input. The seam now lives on the + module-level `_build` below: the gate calls that, and the wire cannot. + """ + body = body if isinstance(body, dict) else {} + return _build(body.get("question"), body.get("scope"), session) + + +def _build(question, scope, session, chat=None): + """`build_query`'s body, with the model injectable — see that route's note on why the seam is + here and not on the signature FastAPI reads.""" + question = ' '.join(str(question or "").split()) + if not question: + raise err(400, "bad_request", "no question was asked") + if len(question) > MAX_QUESTION: + raise err(400, "question_too_long", + f"a question must be under {MAX_QUESTION} characters") + fields, _source = _target(session, scope) + if not fields: + raise err(400, "no_columns", "that database has no columns to build a view from") + + spec, transport, provider = _call_model(question, fields, chat=chat) + if transport: + return {"spec": None, "explain": None, "refused": transport, "provider": provider} + clean, refusal, code = _validate(spec, fields) + if refusal: + return {"spec": None, "explain": None, "refused": refusal, "reason": code, + "provider": provider} + return {"spec": clean, "explain": _explain(clean, fields), + "refused": None, "scope": scope, + "id": _qid(scope, question), "provider": provider} + + +@router.post("/query/save") +def save_query(body: dict = Body(default=None), session: Session = Depends(require_session)): + """Save an accepted spec as the CALLING USER's own view, and index it under Query. + + ⛔ THE SPEC IS RE-VALIDATED HERE, against the target's contract as it is NOW. The build door + wrote nothing, so this is the first write and it is the only place authorisation and validity + have to hold together — and a client is not a wall. An unsupported kind or an unknown column + is a **400 with a named code** on this door (the caller sent it), where the same fact is a + plain sentence on `build` (the model produced it). + """ + body = body if isinstance(body, dict) else {} + scope = str(body.get("scope") or '').strip() + question = ' '.join(str(body.get("question") or "").split())[:MAX_QUESTION] + fields, _source = _target(session, scope) + clean, refusal, code = _validate(body.get("spec"), fields) + if refusal: + raise err(400, code or "bad_spec", refusal) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") + + vt = _templates() + ws_key = vt.workspace_key(scope) + if not ws_key: + raise err(404, "unknown_table", "that database does not exist") + qid = _qid(scope, question) if question else _qid(scope, clean['name']) + + # ⛔ THE SKELETON IS `view_templates._view`, NOT A SECOND ONE. "A view assembled from a + # DIFFERENT skeleton is a second definition of what a view is — and the two drift on the day + # one of them gains a key." It also puts the mode at `config.display.mode`, which is the key + # the grid actually reads (`config.displayMode` was the first draft, and nothing reads it). + payload = vt._view(qid, clean['name'], f"Built by the assistant from: {question}"[:300], + clean['visible'], mode=clean['kind'], + sorts=clean['sorts'], filters=clean['filters'], + group_by=clean['groupBy'], conj=clean['filterConj']) + if clean.get('display'): + payload['config']['display'] = dict(clean['display']) + payload['createdBy'] = session.uname + + # ⛔ THE CAP REFUSES; IT DOES NOT EVICT — and the first draft evicted. Dropping the oldest entry + # would leave its saved VIEW behind in the table's own workspace: "stranded in the target's view + # list with no way back to the question that made it", which is the thing `delete_query` below + # exists to prevent. And an eviction nobody was told about is W30/R6's second sentence broken — + # a limit that cannot be removed must be REPORTED with its cause. So the 201st is a refusal + # naming the number and the way out, checked BEFORE the view is written. + existing = _registry(session) + if qid not in existing and len(existing) >= MAX_REGISTRY: + raise err(400, "query_limit", + f"you already have {len(existing)} saved questions, which is the limit " + f"({MAX_REGISTRY}). Delete one you no longer need and save this again.") + + import core.table_store as table_store + ops = table_store.make(ws_key, st=session.runtime) + saved = ops.save_view(session.uname, payload, shared=False) + + entry = {"scope": scope, "viewId": qid, "name": saved.get('name') or clean['name'], + "kind": clean['kind'], "question": question, + "explain": _explain(clean, fields), "createdBy": session.uname, + "createdAt": _now_iso()} + + def _up(data): + data = data if isinstance(data, dict) else {} + data[qid] = entry + return data + + session.runtime.update(QUERY_KEY, _up, flush='async') + return {"id": qid, **entry} + + +@router.delete("/query/{qid}") +def delete_query(qid: str, session: Session = Depends(require_session)): + """Forget an AI-built view — the index entry AND the view itself. + + Leaving the view behind would strand it in the target's view list with no way back to the + question that made it ([[a-record-can-outlive-its-subject]]). + """ + rec = _registry(session).get(str(qid)) + if not isinstance(rec, dict) or not _mine(session, rec): + raise err(404, "unknown_query", "that view does not exist") + _target(session, rec.get('scope')) # the wall, again, on the write path + vt = _templates() + ws_key = vt.workspace_key(rec.get('scope')) + if ws_key and session.runtime.available(): + import core.table_store as table_store + table_store.make(ws_key, st=session.runtime).delete_view( + session.uname, rec.get('viewId') or str(qid)) + + def _up(data): + data = data if isinstance(data, dict) else {} + data.pop(str(qid), None) + return data + + session.runtime.update(QUERY_KEY, _up, flush='async') + return {"deleted": str(qid)} diff --git a/api/routes_records.py b/api/routes_records.py index c37f74b261d63bb390db45660363e8d94ff796d4..b30aa3103388460dd961a26609faa6f8a6af2bfc 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -1,211 +1,164 @@ -"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. - -⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a -customer-shaped wall bolted to the module import line: `_in_book` asked -`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and -typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the -panel answered "that customer is not in your book" — the owner's report. The dangerous half is -the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment -is filed against somebody's customer where the whole team can read it. - -THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the -events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall -per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving -one here: - - customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant - product `routes_products.scoped_pool` behind the `product_data` grant - ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall - (404 unknown / 403 not yours — a user table has no module grant). - ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every - ROW to derive a pid set this module discards, and RAISES 409 on a - read-through grid past one window, which is why the record drawer painted - an error page on `ut_odoo_gl_lines`. - -⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a -section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning -another session's gate mid-wave — the WALL is the query parameter, not the word. - -⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every -shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a -400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for -the same reason: a typo served as `customer` answers a question nobody asked). -""" -from fastapi import APIRouter, Body, Depends, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table -#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. -_CUSTOMER_SCOPES = ("", "customer", "cohort") - - -def _scope_or_400(raw): - scope = str(raw or "customer").strip().lower() - if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): - return "customer" if scope in _CUSTOMER_SCOPES else scope - raise err(400, "bad_scope", - "scope must be customer, cohort, product or a ut_ database — refusing to guess") - - -def _pool_or_refuse(session: Session, scope: str): - """The pids this session may attach comments to ON THIS DATABASE — grant wall included. - - Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, - so a caller who may not open the surface never learns anything about the row they asked about. - - ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past - one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns - `frozenset()` for both — see the branch below. - ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that - do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` - that REPLACES this function with its own lambda. A gate's test double is a caller - ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a - bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no - failing name. Change the shape here and that double changes with it. - """ - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - session.require(PRODUCT_MODULE) - pids, _team, _rows, _fields = scoped_pool(session) - # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes - # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised - # a `TypeError` — a 500 on every product comment — while both other branches worked. A - # return shape is a contract even when the function is private. - return (pids, False) - if scope.startswith("ut_"): - # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS - # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the - # rows routes do. - # - # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. - # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on - # a read-through grid larger than one window it RAISES `409 window_required`. So opening the - # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel - # that renders comments about ONE row it already has. `scoped_pids` answers the identical - # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes - # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` - # used to become openable on those two grids. - from routes_tables import scoped_pids - - limits = [] - pids, _fields, _defn = scoped_pids(session, scope, limits=limits) - # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is - # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a - # database with no rows and for a read-through grid too big to enumerate — it distinguishes - # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move - # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which - # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). - # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database - # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the - # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside - # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. - return (pids, bool(limits)) - from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids - - session.require(CUSTOMER_MODULE) - return (frozenset(allowed_pids(session)), False) - - -def _in_book(pid, session, scope): - pids, unbounded = _pool_or_refuse(session, scope) - if unbounded: - # The table wall passed and the row set is larger than this process will enumerate. Said - # out loud rather than silently admitting: R6's second sentence is that a limit which - # cannot be removed gets REPORTED, and this is the one place the report has no envelope to - # ride in. - print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " - f"admitting on the table wall alone, per D-72") - return - if pid not in pids: - # 403, not 404: the record may exist, but this session may not inspect it. - raise err(403, "out_of_scope", "that record is not in your book") - - -def _unavailable(): - return err( - 503, - "store_unavailable", - "record comments are temporarily unavailable — no change was saved", - ) - - -# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD -# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the -# wall was always the query param). The old path stays as an ALIAS because the shipped client -# still calls it; verify_api pins the canonical path AND that the alias answers, so removing -# the alias later is a decision, never an accident. -@router.get("/records/{pid}/comments") -@router.get("/customers/{pid}/comments") -def comments(pid: int, scope: str = Query(default="customer"), - session: Session = Depends(require_session)): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - rows = record_comments.list_comments(session.runtime, pid, scope=scope) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comments": rows} - - -@router.post("/records/{pid}/comments", status_code=201) -@router.post("/customers/{pid}/comments", status_code=201) -def create_comment( - pid: int, - body: dict = Body(default=None), - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - comment = record_comments.add_comment( - session.runtime, - pid, - (body or {}).get("body"), - session.uname, - session.user.get("name") or session.uname, - scope=scope, - ) - except ValueError as exc: - raise err(400, "bad_comment", str(exc)) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comment": comment} - - -@router.delete("/records/{pid}/comments/{comment_id}") -@router.delete("/customers/{pid}/comments/{comment_id}") -def remove_comment( - pid: int, - comment_id: str, - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - deleted = record_comments.delete_comment( - session.runtime, - pid, - comment_id, - session.uname, - admin=session.admin, - scope=scope, - ) - except record_comments.CommentForbidden: - raise err(403, "comment_forbidden", "only the author may delete this comment") - except record_comments.CommentsUnavailable: - raise _unavailable() - if not deleted: - raise err(404, "comment_not_found", "that comment no longer exists") - return {"ok": True, "id": comment_id} +"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. + +⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a +customer-shaped wall bolted to the module import line: `_in_book` asked +`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and +typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the +panel answered "that customer is not in your book" — the owner's report. The dangerous half is +the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment +is filed against somebody's customer where the whole team can read it. + +THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the +events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall +per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving +one here: + + customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant + product `routes_products.scoped_pool` behind the `product_data` grant + ut_ `routes_tables.scoped_pool`, whose `_defn_or_refuse` IS the wall + (404 unknown / 403 not yours — a user table has no module grant) + +⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a +section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning +another session's gate mid-wave — the WALL is the query parameter, not the word. + +⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every +shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a +400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for +the same reason: a typo served as `customer` answers a question nobody asked). +""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table +#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. +_CUSTOMER_SCOPES = ("", "customer", "cohort") + + +def _scope_or_400(raw): + scope = str(raw or "customer").strip().lower() + if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): + return "customer" if scope in _CUSTOMER_SCOPES else scope + raise err(400, "bad_scope", + "scope must be customer, cohort, product or a ut_ database — refusing to guess") + + +def _pool_or_refuse(session: Session, scope: str): + """The pids this session may attach comments to ON THIS DATABASE — grant wall included. + + Returns the frozenset. Raises the topic's own 403/404/503, so a caller who may not open the + surface never learns anything about the row they asked about. + """ + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + session.require(PRODUCT_MODULE) + pids, _team, _rows, _fields = scoped_pool(session) + return pids + if scope.startswith("ut_"): + # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pool` IS + # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the + # rows routes do. + from routes_tables import scoped_pool + + pids, _rows, _fields, _defn = scoped_pool(session, scope) + return pids + from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids + + session.require(CUSTOMER_MODULE) + return frozenset(allowed_pids(session)) + + +def _in_book(pid, session, scope): + if pid not in _pool_or_refuse(session, scope): + # 403, not 404: the record may exist, but this session may not inspect it. + raise err(403, "out_of_scope", "that record is not in your book") + + +def _unavailable(): + return err( + 503, + "store_unavailable", + "record comments are temporarily unavailable — no change was saved", + ) + + +# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD +# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the +# wall was always the query param). The old path stays as an ALIAS because the shipped client +# still calls it; verify_api pins the canonical path AND that the alias answers, so removing +# the alias later is a decision, never an accident. +@router.get("/records/{pid}/comments") +@router.get("/customers/{pid}/comments") +def comments(pid: int, scope: str = Query(default="customer"), + session: Session = Depends(require_session)): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + rows = record_comments.list_comments(session.runtime, pid, scope=scope) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comments": rows} + + +@router.post("/records/{pid}/comments", status_code=201) +@router.post("/customers/{pid}/comments", status_code=201) +def create_comment( + pid: int, + body: dict = Body(default=None), + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + comment = record_comments.add_comment( + session.runtime, + pid, + (body or {}).get("body"), + session.uname, + session.user.get("name") or session.uname, + scope=scope, + ) + except ValueError as exc: + raise err(400, "bad_comment", str(exc)) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comment": comment} + + +@router.delete("/records/{pid}/comments/{comment_id}") +@router.delete("/customers/{pid}/comments/{comment_id}") +def remove_comment( + pid: int, + comment_id: str, + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + deleted = record_comments.delete_comment( + session.runtime, + pid, + comment_id, + session.uname, + admin=session.admin, + scope=scope, + ) + except record_comments.CommentForbidden: + raise err(403, "comment_forbidden", "only the author may delete this comment") + except record_comments.CommentsUnavailable: + raise _unavailable() + if not deleted: + raise err(404, "comment_not_found", "that comment no longer exists") + return {"ok": True, "id": comment_id} diff --git a/api/routes_shares.py b/api/routes_shares.py index 0579302b5319d2a751a05174d98e94438b6f3b14..7a96392eba03033ca450a2eb4a970663603fcc9c 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -1,393 +1,385 @@ -"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). - - GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} - PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) - GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} - -`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail -already speaks, now extended to folders and databases so there is ONE vocabulary in the UI -(R10: "the same picker views use"). - -⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires -`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's -CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could -widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor -for non-administrators; that is a courtesy, and this check is the wall. - -⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means -every account that can already open the surface: `require_session` plus the topic's own gate run -first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field -closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that -could already reach the data ([[aios-permissioning]]). - -⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED -MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` -and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be -DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes -`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded -by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every -column — and an `*` database grant admits every account in the tenant to all of it. -That is a real capability, deliberately kept; what was wrong was a docstring promising a second -wall that does not exist for this kind. Scoping user tables is booked, not done -(`waves/wave32/sharing-audit.md` S-8). - -⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS -registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` -— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here -whose object is invisible under that one is a row in a list that opens a refusal, which is what -made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but -the two vocabularies are still two. -""" -from fastapi import APIRouter, Body, Depends - -import core.shares as shares -import core.users as users -from deps import Session, err, require_session -# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES -# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the -# reader agreeing about one string is the whole difference between an Inbox row that opens the -# shared database and one that is quietly unclickable. -from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC - -router = APIRouter(prefix="/api/v1") - - -def _kind_or_400(raw): - try: - return shares._check_kind(raw) - except ValueError as e: - raise err(400, "bad_kind", str(e)) - - -# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── -# -# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all -# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free -# strings off the URL and the only dependency was `require_session`, so any signed-in account -# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, -# an object with no grant record skipped the check entirely and the caller was stamped OWNER — -# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not -# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. -# -# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own -# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the -# victim next opens the dialog. - -#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share -#: route is not told which topic — so resolving one means asking each. -_BUILTIN_TOPICS = ("customer", "product") - - -def _topics(session): - """Every topic whose workspace could hold a view or folder for this tenant. - - ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on - tenant #0) to answer a question about KEYS (D-185). - """ - try: - import core.user_tables as ut - return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) - except Exception: # noqa: BLE001 - return _BUILTIN_TOPICS - - -def _owns_object(session, kind, oid): - """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? - - ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking - each topic's workspace in turn, which is N store reads; making every share call pay that - would put a loop on a route the manage-access dialog opens. The dangerous path is the one - where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs - exactly there, and the common path (a record exists, `may_administer` decides) is untouched. - """ - if session.admin: - return True - if kind == "database": - # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already - # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" - # here would be the second definition this wave keeps finding. - try: - import core.user_tables as ut - return bool(ut.may_open(oid, session.uname, is_admin=session.admin, - st=session.runtime)) - except Exception: # noqa: BLE001 - return False - try: - import core.table_store as table_store - except Exception: # noqa: BLE001 - return False - for topic in _topics(session): - try: - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - except Exception: # noqa: BLE001 - continue - if hit: - # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the - # person whose personal stratum holds it — anybody else reaching this line is - # exactly the case S-1 describes. - return str(hit[0]) == str(session.uname) - return False - - -def _can_see_object(session, kind, oid): - """May this caller READ an object's grant list — i.e. can they reach the object at all? - - ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A - REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the - ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches - PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with - `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit - in the grid** — answered `404` when bob opened its manage-access dialog. Measured before - fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said - `404 no_object`. - ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", - and the wall consulted the grant registry (system A) plus stratum ownership, never the view's - `permissions` (system B) — which is the one that actually decides who may OPEN it. - ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not - gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — - 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a - legitimate collaborator their view does not exist. - - ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its - owner — that is S-1, and widening this predicate onto `put_share` would re-open it. - """ - if _owns_object(session, kind, oid): - return True - if kind != "view": - # A folder carries no per-object visibility flag of its own, and a database's `may_open` - # (inside `_owns_object`) already admits grantees. Nothing wider to ask. - return False - try: - import core.table_store as table_store - for topic in _topics(session): - hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) - if hit: - return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, - session.uname, is_admin=session.admin)) - except Exception: # noqa: BLE001 - return False - return False - - -def _entries_or_400(session, entries): - """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. - - ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: - a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE - of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, - so a typo'd name is stored, reported as a successful save, and never reaches anybody. The - sharer believes the person has access. That is item 18's plain reading. - ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker - (`_people`). One route, two populations, and the write door was the permissive one. - ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for - "every account that can already open the surface". - """ - known = {p["username"].strip().lower() for p in _people(session.tenant)} - unknown = [] - for e in entries or (): - if not isinstance(e, dict): - continue - user = str(e.get("user") or "").strip().lower() - if user and user != shares.EVERYONE and user not in known: - unknown.append(user) - if unknown: - raise err(400, "unknown_people", - "no active account in this workspace is named " - + ", ".join(sorted(set(unknown))) - + " — nothing was shared. Pick people from the list rather than typing a name.") - - -@router.get("/share/mine") -def my_shares(session: Session = Depends(require_session)): - """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). - - Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves - in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL - that is not malformed at all. - """ - return shares.shared_with(session.uname, st=session.runtime) - - -@router.get("/share/{kind}/{oid}") -def get_share(kind: str, oid: str, session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - rec = shares.grants(kind, oid, st=session.runtime) - role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) - may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime) - # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: - # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — - # to any signed-in session, about objects it cannot open. Now a caller with no role on an - # object must prove they can reach it, and gets a 404 otherwise: the same answer a - # non-existent id gives, so the route cannot be used to probe which ids are real. - # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a - # caller who has no relationship with the object at all. - if role is None and not _can_see_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - return { - **rec, - "role": role, - "mayAdminister": may_admin, - # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry - # them. `assignable_people` serves bare display names because `user`-kind CELLS store - # display names — that list's shape cannot change without migrating cell values — so - # this route serves objects of its own. Existing grants that were written as lowercased - # display names are normalised by the wave-21 cleanup script. - # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller - # who may open the editor. A read-only grantee gets the grant list (their fair question is - # "who else has this?") and not a directory of every account in the workspace. - "people": _people(session.tenant) if may_admin else [], - } - - -def _people(tenant): - """[{username, name}] for this tenant — same population as `assignable_people`, with the - BINDING identity alongside the display one.""" - try: - reg = users.registry() or {} - except Exception: - return [] - want = str(tenant or '').strip().lower() - out = [] - for uname, u in reg.items(): - if not isinstance(u, dict) or u.get('active') is False: - continue - if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: - continue - out.append({"username": str(uname), "name": str(u.get('name') or uname)}) - return sorted(out, key=lambda p: p["name"].lower()) - - -@router.put("/share/{kind}/{oid}") -def put_share(kind: str, oid: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - kind = _kind_or_400(kind) - body = body or {} - rec = shares.grants(kind, oid, st=session.runtime) - # An object with NO grant record yet has no owner — the first person to share it claims it. - # That is safe because reaching this route at all means passing the surface's own wall, and - # the alternative (refusing until somebody seeds an owner) would make a brand-new folder - # unshareable by the person who just made it. - if rec["owner"]: - if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, - st=session.runtime): - raise err(403, "not_owner", - "only the owner of this item (or an administrator) can change who it is " - "shared with") - # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record - # is still claimed by the first person to share it — that rule is right, and refusing until - # somebody seeds an owner would make a brand-new folder unshareable by the person who just - # made it. What was missing is the half the old comment ASSERTED and the code never did: the - # claimant has to be able to reach the object. Without this, any signed-in account could - # stamp itself owner of an id it had never seen and lock the real creator out for good. - elif not _owns_object(session, kind, oid): - raise err(404, "no_object", "no such item, or it is not shared with this account") - entries = body.get("entries") - if not isinstance(entries, list): - raise err(400, "bad_entries", - "entries must be a list of {user, role} — send [] to un-share, which is how " - "revoking is expressed") - _entries_or_400(session, entries) - out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, - st=session.runtime) - _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) - return out - - -def _notify_new_grantees(session, kind, oid, before, after): - """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. - - Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was - silent: the grant landed in a rail section the receiver had to notice on their own, which is - why "I shared it with you" and "I never saw it" were both true. - - ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read - because an alert is a live question about rows; a share is an EVENT that happened once, and - polling for it would mean re-deriving "was this new?" on every inbox open — the diff below - only exists here, at the moment the set changes. - - ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), - so every save re-sends everyone who was already there. Diffing against `before` is what stops - a rename or a role change from ringing the bell for people whose access did not change. - ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in - the tenant on a single click is a broadcast nobody asked for. The rail still shows it. - ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the - grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. - """ - try: - was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} - fresh = [str(e.get("user")) for e in (after or ()) - if isinstance(e, dict) and e.get("user") not in was - and e.get("user") != shares.EVERYONE] - if not fresh: - return - import core.alerts as alerts - - label, route, view_id = _object_ref(session, kind, oid) - if not route: - # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and - # `notification_view` would have to invent a target. Silence is the honest answer - # here; the rail still shows the grant under "Shared with me". - return - sharer = str(session.user.get("name") or session.uname) - for user in fresh: - # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must - # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes - # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED - # from there rather than typed again — one vocabulary, one owner. - # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY - # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and - # are honestly named by their machine; a SHARE has a real person, and only this call - # site knows who. ⛔ It is passed as its OWN field rather than recovered from the - # `detail` prose below: a sender parsed out of " shared this with you" breaks - # the first time the sentence is reworded, silently, in the header - # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. - alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, - detail=f"{sharer} shared this with you", actor=sharer, - st=session.runtime) - except Exception: # noqa: BLE001 - return - - -def _object_ref(session, kind, oid): - """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. - - ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT - WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced - `target: {module: "database", id: "view_42"}` — an instruction to open a database named - `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not - addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to - travel. Caught by looking at the notification the driver actually produced, not by reading - the code back. - - ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the - receiver nothing they can act on, and the id is already in the target. - ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than - a row that opens nowhere. - """ - try: - if kind == "database": - import core.user_tables as ut - defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} - # A user table IS its own route key in both vocabularies (`route_for_topic`). - return (str(defn.get("label") or "").strip() or "A database", str(oid), "") - import core.table_store as table_store - from routes_alerts import route_for_topic - for topic in _topics(session): - ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) - hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) - if not hit: - continue - route = route_for_topic(topic) - if not route: - break - row = hit[1] if len(hit) > 1 else {} - name = str((row or {}).get("name") or "").strip() - # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens - # the grid and stops there rather than naming a view the receiver did not get. - return (name or ("A view" if kind == "view" else "A folder"), - route, str(oid) if kind == "view" else "") - except Exception: # noqa: BLE001 - pass - return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "") +"""routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE). + + GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people} + PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) + GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} + +`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail +already speaks, now extended to folders and databases so there is ONE vocabulary in the UI +(R10: "the same picker views use"). + +⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires +`shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's +CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could +widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor +for non-administrators; that is a courtesy, and this check is the wall. + +⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means +every account that can already open the surface: `require_session` plus the topic's own gate run +first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field +closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that +could already reach the data ([[aios-permissioning]]). + +⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED +MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")` +and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be +DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes +`hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded +by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every +column — and an `*` database grant admits every account in the tenant to all of it. +That is a real capability, deliberately kept; what was wrong was a docstring promising a second +wall that does not exist for this kind. Scoping user tables is booked, not done +(`waves/wave32/sharing-audit.md` S-8). + +⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS +registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared` +— the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here +whose object is invisible under that one is a row in a list that opens a refusal, which is what +made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but +the two vocabularies are still two. +""" +from fastapi import APIRouter, Body, Depends + +import core.shares as shares +import core.users as users +from deps import Session, err, require_session +# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES +# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the +# reader agreeing about one string is the whole difference between an Inbox row that opens the +# shared database and one that is quietly unclickable. +from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC + +router = APIRouter(prefix="/api/v1") + + +def _kind_or_400(raw): + try: + return shares._check_kind(raw) + except ValueError as e: + raise err(400, "bad_kind", str(e)) + + +# ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ───────────── +# +# `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all +# means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free +# strings off the URL and the only dependency was `require_session`, so any signed-in account +# could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`, +# an object with no grant record skipped the check entirely and the caller was stamped OWNER — +# sticky, so **the real creator was then refused on their own view, permanently.** Driven, not +# argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript. +# +# ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own +# "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the +# victim next opens the dialog. + +#: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share +#: route is not told which topic — so resolving one means asking each. +_BUILTIN_TOPICS = ("customer", "product") + + +def _topics(session): + """Every topic whose workspace could hold a view or folder for this tenant. + + ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on + tenant #0) to answer a question about KEYS (D-185). + """ + try: + import core.user_tables as ut + return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {})) + except Exception: # noqa: BLE001 + return _BUILTIN_TOPICS + + +def _owns_object(session, kind, oid): + """May this caller CLAIM an object that has no grant record yet — i.e. do they own it? + + ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking + each topic's workspace in turn, which is N store reads; making every share call pay that + would put a loop on a route the manage-access dialog opens. The dangerous path is the one + where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs + exactly there, and the common path (a record exists, `may_administer` decides) is untouched. + """ + if session.admin: + return True + if kind == "database": + # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already + # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" + # here would be the second definition this wave keeps finding. + try: + import core.user_tables as ut + return bool(ut.may_open(oid, session.uname, is_admin=session.admin, + st=session.runtime)) + except Exception: # noqa: BLE001 + return False + try: + import core.table_store as table_store + except Exception: # noqa: BLE001 + return False + for topic in _topics(session): + try: + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + except Exception: # noqa: BLE001 + continue + if hit: + # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the + # person whose personal stratum holds it — anybody else reaching this line is + # exactly the case S-1 describes. + return str(hit[0]) == str(session.uname) + return False + + +def _can_see_object(session, kind, oid): + """May this caller READ an object's grant list — i.e. can they reach the object at all? + + ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A + REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the + ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches + PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with + `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit + in the grid** — answered `404` when bob opened its manage-access dialog. Measured before + fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said + `404 no_object`. + ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared", + and the wall consulted the grant registry (system A) plus stratum ownership, never the view's + `permissions` (system B) — which is the one that actually decides who may OPEN it. + ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not + gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too — + 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a + legitimate collaborator their view does not exist. + + ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its + owner — that is S-1, and widening this predicate onto `put_share` would re-open it. + """ + if _owns_object(session, kind, oid): + return True + if kind != "view": + # A folder carries no per-object visibility flag of its own, and a database's `may_open` + # (inside `_owns_object`) already admits grantees. Nothing wider to ask. + return False + try: + import core.table_store as table_store + for topic in _topics(session): + hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) + if hit: + return bool(table_store._may_see(hit[1] if len(hit) > 1 else {}, + session.uname, is_admin=session.admin)) + except Exception: # noqa: BLE001 + return False + return False + + +def _entries_or_400(session, entries): + """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME. + + ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly: + a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE + of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**, + so a typo'd name is stored, reported as a successful save, and never reaches anybody. The + sharer believes the person has access. That is item 18's plain reading. + ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker + (`_people`). One route, two populations, and the write door was the permissive one. + ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for + "every account that can already open the surface". + """ + known = {p["username"].strip().lower() for p in _people(session.tenant)} + unknown = [] + for e in entries or (): + if not isinstance(e, dict): + continue + user = str(e.get("user") or "").strip().lower() + if user and user != shares.EVERYONE and user not in known: + unknown.append(user) + if unknown: + raise err(400, "unknown_people", + "no active account in this workspace is named " + + ", ".join(sorted(set(unknown))) + + " — nothing was shared. Pick people from the list rather than typing a name.") + + +@router.get("/share/mine") +def my_shares(session: Session = Depends(require_session)): + """Everything shared WITH me, by kind — the "Shared with me" rail section (R10). + + Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves + in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL + that is not malformed at all. + """ + return shares.shared_with(session.uname, st=session.runtime) + + +@router.get("/share/{kind}/{oid}") +def get_share(kind: str, oid: str, session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + rec = shares.grants(kind, oid, st=session.runtime) + role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime) + # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: + # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — + # to any signed-in session, about objects it cannot open. Now a caller with no role on an + # object must prove they can reach it, and gets a 404 otherwise: the same answer a + # non-existent id gives, so the route cannot be used to probe which ids are real. + # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a + # caller who has no relationship with the object at all. + if role is None and not _can_see_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + return { + **rec, + "role": role, + "mayAdminister": may_admin, + # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry + # them. `assignable_people` serves bare display names because `user`-kind CELLS store + # display names — that list's shape cannot change without migrating cell values — so + # this route serves objects of its own. Existing grants that were written as lowercased + # display names are normalised by the wave-21 cleanup script. + # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller + # who may open the editor. A read-only grantee gets the grant list (their fair question is + # "who else has this?") and not a directory of every account in the workspace. + "people": _people(session.tenant) if may_admin else [], + } + + +def _people(tenant): + """[{username, name}] for this tenant — same population as `assignable_people`, with the + BINDING identity alongside the display one.""" + try: + reg = users.registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + out = [] + for uname, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append({"username": str(uname), "name": str(u.get('name') or uname)}) + return sorted(out, key=lambda p: p["name"].lower()) + + +@router.put("/share/{kind}/{oid}") +def put_share(kind: str, oid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + kind = _kind_or_400(kind) + body = body or {} + rec = shares.grants(kind, oid, st=session.runtime) + # An object with NO grant record yet has no owner — the first person to share it claims it. + # That is safe because reaching this route at all means passing the surface's own wall, and + # the alternative (refusing until somebody seeds an owner) would make a brand-new folder + # unshareable by the person who just made it. + if rec["owner"]: + if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin, + st=session.runtime): + raise err(403, "not_owner", + "only the owner of this item (or an administrator) can change who it is " + "shared with") + # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record + # is still claimed by the first person to share it — that rule is right, and refusing until + # somebody seeds an owner would make a brand-new folder unshareable by the person who just + # made it. What was missing is the half the old comment ASSERTED and the code never did: the + # claimant has to be able to reach the object. Without this, any signed-in account could + # stamp itself owner of an id it had never seen and lock the real creator out for good. + elif not _owns_object(session, kind, oid): + raise err(404, "no_object", "no such item, or it is not shared with this account") + entries = body.get("entries") + if not isinstance(entries, list): + raise err(400, "bad_entries", + "entries must be a list of {user, role} — send [] to un-share, which is how " + "revoking is expressed") + _entries_or_400(session, entries) + out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, + st=session.runtime) + _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) + return out + + +def _notify_new_grantees(session, kind, oid, before, after): + """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox. + + Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was + silent: the grant landed in a rail section the receiver had to notice on their own, which is + why "I shared it with you" and "I never saw it" were both true. + + ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read + because an alert is a live question about rows; a share is an EVENT that happened once, and + polling for it would mean re-deriving "was this new?" on every inbox open — the diff below + only exists here, at the moment the set changes. + + ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence), + so every save re-sends everyone who was already there. Diffing against `before` is what stops + a rename or a role change from ringing the bell for people whose access did not change. + ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in + the tenant on a single click is a broadcast nobody asked for. The rail still shows it. + ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the + grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`. + """ + try: + was = {e.get("user") for e in (before or ()) if isinstance(e, dict)} + fresh = [str(e.get("user")) for e in (after or ()) + if isinstance(e, dict) and e.get("user") not in was + and e.get("user") != shares.EVERYONE] + if not fresh: + return + import core.alerts as alerts + + label, route, view_id = _object_ref(session, kind, oid) + if not route: + # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and + # `notification_view` would have to invent a target. Silence is the honest answer + # here; the rail still shows the grant under "Shared with me". + return + sharer = str(session.user.get("name") or session.uname) + for user in fresh: + # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must + # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes + # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED + # from there rather than typed again — one vocabulary, one owner. + alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, + detail=f"{sharer} shared this with you", st=session.runtime) + except Exception: # noqa: BLE001 + return + + +def _object_ref(session, kind, oid): + """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS. + + ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT + WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced + `target: {module: "database", id: "view_42"}` — an instruction to open a database named + `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not + addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to + travel. Caught by looking at the notification the driver actually produced, not by reading + the code back. + + ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the + receiver nothing they can act on, and the id is already in the target. + ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than + a row that opens nowhere. + """ + try: + if kind == "database": + import core.user_tables as ut + defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} + # A user table IS its own route key in both vocabularies (`route_for_topic`). + return (str(defn.get("label") or "").strip() or "A database", str(oid), "") + import core.table_store as table_store + from routes_alerts import route_for_topic + for topic in _topics(session): + ops = table_store.make(f"{topic}_table_workspace", st=session.runtime) + hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid) + if not hit: + continue + route = route_for_topic(topic) + if not route: + break + row = hit[1] if len(hit) > 1 else {} + name = str((row or {}).get("name") or "").strip() + # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens + # the grid and stops there rather than naming a view the receiver did not get. + return (name or ("A view" if kind == "view" else "A folder"), + route, str(oid) if kind == "view" else "") + except Exception: # noqa: BLE001 + pass + return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "") diff --git a/api/routes_tables.py b/api/routes_tables.py index fae644151f61e70702cdae1a93e5266b7300b136..9c10a00bcb6bebef5a419438c739891c9817f849 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -1,1455 +1,1360 @@ -"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). - -The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served -through the SAME grid machinery every other topic rides — `table_store` for the per-user -workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every -durable write except rows. Rows are the one genuinely new channel: the events seam has no row -event types (the user_tables docstring's `row_add` gate was described, never built), so row -add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + -`user_tables.may_open` — a connector-backed table can never accept an invented row. - -TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a -Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 -covers them for free. - -VISIBILITY is `user_tables.may_open` — creator or admin, fail-closed. There is no module grant -to check because a user table is not a module; the per-table wall is the whole wall, and it is -applied in `_defn_or_refuse` before any payload is built. -""" -import threading -import json -import time - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - - -def _ut(): - import core.user_tables as user_tables - return user_tables - - -def _ops(session, table_key): - import core.table_store as table_store - return table_store.make(f"{table_key}_table_workspace", st=session.runtime) - - -#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF -#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. -_REL_LOCK = threading.Lock() -_REL_DIRTY = {} -_REL_RUNNING = {} - - -def _refresh_relations(session): - """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. - - ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's - item 2 was "adding a new record visually takes too long, I need to be able to spam it", and - this function was the largest single cost inside `POST /tables/{key}/rows`: - `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can - decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something - does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the - default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit - before the 201 came back. - - ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty - whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam - the item asks us to support is exactly the load that would melt it. So this COALESCES: at - most one pass runs, and at most one is queued behind it. - - ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in - flight must still get a pass afterwards, because the in-flight one snapshotted before that - write existed. Without the flag the LAST add in a burst is precisely the one whose rollups - never update — the failure nobody would notice until a total was quietly wrong. - - Eventual consistency is the accepted trade and was already the documented posture: the tick - repairs materialised cells regardless, and answering 503 here would invite the browser to - repeat a mutation that already succeeded. - """ - tenant = str(getattr(session, "tenant", "") or "") - rt = session.runtime - with _REL_LOCK: - _REL_DIRTY[tenant] = True - if _REL_RUNNING.get(tenant): - return # a worker is live; it will see the flag and loop - _REL_RUNNING[tenant] = True - - def _worker(): - import automation_engine as engine - try: - while True: - with _REL_LOCK: - if not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - return - _REL_DIRTY.pop(tenant, None) - try: - engine.refresh_relations(rt, log=lambda *_args: None) - except Exception as exc: # noqa: BLE001 - # The source write already landed and the response is already sent. Log and - # let the tick repair it; never retry in a tight loop. - print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") - finally: - # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked - # RUNNING would never refresh again for the life of the process. - with _REL_LOCK: - if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): - _REL_RUNNING.pop(tenant, None) - - threading.Thread(target=_worker, daemon=True, - name=f"rel-refresh:{tenant or 'default'}").start() - - -def _defn_or_refuse(session, table_key, st=None, defs_only=False): - """The per-table wall: 404 for a key that does not exist, 403 for one this session may not - open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but - not yours' is exactly what may_open is for. - - ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the - nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked - about the same table — it is simply not re-reading a 28.5 MB document to ask it. - - ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL - SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open - it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE - DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and - `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the - CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated - with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). - - ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, - `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass - it: a projected snapshot must never reach a post-write read-back (contract C5), and the - pre-write lend note below is the same argument one layer down. - - ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION - DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" - proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate - asserts the fast path was TAKEN. - """ - ut = _ut() - # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. - # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* - # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was - # the route count, not the caller count) and cost TWO whole-document deep copies at every one - # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions - # about ONE table. Lending here fixes every caller at once, including the eight routes the - # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · - # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). - # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The - # guard runs before any mutation and returns only the DEFINITION, so the snapshot never - # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes - # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which - # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it - # replaced. The remaining in-route reads are deliberately untouched and booked instead. - if st is None: - st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) - defn = ut.get(table_key, st=st) - if not defn: - raise err(404, "unknown_table", "that database does not exist") - if not ut.may_open(table_key, session.uname, session.admin, st=st): - raise err(403, "forbidden", "that database belongs to another user") - return defn - - -def _records_or_refuse(session, table_key, st=None): - """The human record-write wall for a database the automation engine owns.""" - # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — - # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. - st = st if st is not None else _ut().lend(session.runtime) - defn = _defn_or_refuse(session, table_key, st=st) - if not _ut().records_mutable(table_key, st=st): - raise err(403, "records_read_only", - "records in this automation-owned database are read-only — add Instagram " - "handles in a Profile database and let enrichment populate this database") - return defn - - -#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT -#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. -#: -#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity -#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never -#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the -#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, -#: and the wall then hid the survivor from the person who just found it. They searched, they -#: paid, and the screen said nothing arrived. -#: -#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named -#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores -#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the -#: write target is whatever database the user points the Create-record action at. So the wall -#: governed a table nobody writes, and fixing EITHER half alone would have armed the other -#: ([[defects-that-mask-each-other]]). -#: -#: The register offered two exits and R4 already implied this one. Restoring per-user visibility -#: instead would have required R4's merge to stop crossing users — a bigger change, against the -#: ruling, to bring back a wall that never governed anything real. -#: -#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every -#: automation-written table has one (a scraped row says `automation`), so inference would hide -#: every scraped row from every non-admin — the same disappearance defect, one table wide instead -#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. - - -def _too_big(): - """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" - import routes_odoo_tables - return routes_odoo_tables.TooBigToMaterialise - - -def _read_through_rows(table_key, field_keys, rt=None): - """The mirror's rows for one read-through grid, projected to this table's declared columns. - - ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, - `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; - the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this - process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, - which is exactly the shape that satisfies the first wall while failing the second. - `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every - tenant, so the two walls are not substitutes for each other. - - ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which - owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror - through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's - by construction rather than by a second query that agrees today. A pid-only `SELECT` would be - cheaper and would also be a SECOND statement of what a row of this table is - ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, - never in how they ask. - - Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides - whether that is a 409 or an unresolved pid scope. - """ - import routes_odoo_tables - - rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} - for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] - rows_src.sort(key=lambda r: r["pid"]) - return rows_src - - -#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. -#: -#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= -#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the -#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. -#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a -#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). -#: -#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is -#: "membership is unresolved on this grid", which is a different claim and has to be made out -#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are -#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` -#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. -_PID_SCOPE_LIMIT = { - "subject": "pids", "effect": "unresolved", - "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " - "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " - "every total is a SQL count over the whole table", -} - - -def scoped_pool(session: Session, table_key: str, st=None): - """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. - - ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as - `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere - else, so the permission question is answered by the same code against the same document. - ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then - reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). - - `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a - caller that only needs "which rows of this database may this session touch" — record comments - — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second - idea of what a user table's pool is. - """ - defn = _defn_or_refuse(session, table_key, st=st) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - field_keys = {f["key"] for f in fields_base} - # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. - # - # This is the one function that turns "what is stored" into "what this session may see", which - # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events - # route, the comments wall and the assembly all reach rows through it, so they all convert - # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve - # an EMPTY GRID — correct-looking, wrong, and silent. - # ⚠ The wall above has already run. This adds no scope of its own and takes none away. - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) - except _too_big() as e: - # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. - # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not - # this): a caller that asked for every row of a 963,783-row grid cannot be served a - # short one. `scoped_pids` below takes the same refusal and answers a different - # question with it, because an envelope needs no row. - raise err(409, "window_required", str(e)) - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - rows_src = [] - for rid, row in (defn.get("rows") or {}).items(): - if not str(rid).isdigit(): - continue - # WAVE 27 / D-72: no per-row owner filter. The table-level wall above - # (`_defn_or_refuse` -> `may_open`) is the WHOLE wall — see the retirement note on - # PER_USER_TABLES' former home. A row this session can reach is a row the tenant owns. - r = {k: v for k, v in (row or {}).items() if k in field_keys} - r["pid"] = int(rid) - rows_src.append(r) - rows_src.sort(key=lambda r: r["pid"]) - return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn - - -@router.patch("/tables/{table_key}/shared/{pid}") -def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has - been waiting for since it shipped (W29-T62, wave 30 T28). - - ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not - "sharing would be nice": a user-created column and its values live PER USER, so a shared view - filtering on one names a column other accounts do not have — and an unknown column is an - INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy - list would silently show the whole catalogue to everyone but its author. A column whose value - is the same for every reader is the precondition for editing it at all. - - ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no - reader and no writer by design; "may this session open this surface" is answered HERE, where - the session is. Do not push the question down there. - ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. - That is the point, and it is why this door declares the column too — a value with no - definition is a cell nobody can find. - """ - body = body if isinstance(body, dict) else {} - key = str(body.get("field") or "").strip() - if not key: - raise err(400, "bad_request", "a field key is required") - _defn_or_refuse(session, table_key) - from core import shared_overlay - if not shared_overlay.is_shared(table_key, key, st=session.runtime): - shared_overlay.put_field(table_key, key, { - "key": key, "label": str(body.get("label") or key), "source": "overlay", - "type": str(body.get("type") or "text"), "shared": True, - "createdBy": session.uname}, st=session.runtime) - try: - # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is - # the API that says so. It delegates to the plural, so both stay reachable through the one - # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 - # named it (the same lens that found `drop_field` had no door either). - stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), - st=session.runtime)} - except ValueError as e: - # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. - raise err(400, "bad_value", str(e)) - return {"ok": True, "pid": pid, "cells": stored, - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -@router.delete("/tables/{table_key}/shared/fields/{field_key}") -def delete_shared_field(table_key: str, field_key: str, - session: Session = Depends(require_session)): - """Remove a TENANT-WIDE column and every value in it. - - ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column - and none that removes one, so a column anybody added was permanent for the whole tenant. The - reachability gate found it from the other end — `shared_overlay.drop_field` was complete, - correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). - - ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a - value; dropping the column deletes that value for EVERY account at once, so it is the - destructive-op wall this repo already uses for a database delete — not `editRole`, which - governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). - ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed - is admin-only, which is the safe direction. - """ - _defn_or_refuse(session, table_key) - from core import shared_overlay - defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) - if not defn: - raise err(404, "unknown_field", "that column is not a shared column on this database") - owner = str(defn.get("createdBy") or "") - if not session.admin and owner != session.uname: - raise err(403, "forbidden", - f"a tenant-wide column can be removed by its creator or an admin — this one " - f"was added by {owner or 'somebody else'}, and dropping it would delete the " - f"value for every account") - dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) - return {"ok": True, "dropped": bool(dropped), - "fields": list(shared_overlay.fields(table_key, st=session.runtime))} - - -def scoped_pids(session: Session, table_key: str, limits=None): - """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without - building a row. - - ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write - (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate - it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on - every toggle — and the six keys the events route actually reads from that assembly - (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. - `rows_src` was computed and discarded. - - ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: - `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the - row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the - middle. The row WALL is unchanged — a narrower or wider set here would be a permission - change, and this is a performance change. - - ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document - read, which is D-87 and W30-T31. This removes the row pass. - ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then - `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions - have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a - PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. - Booked because a stale performance note is how a wave re-fixes something twice - ([[stale-baseline-unreadable-deltas]]). - - ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a - list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a - read-through grid whose population exceeds one window). The pid set is then EMPTY, and every - consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the - silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a - write is expected to carry the sentence through. Omitting the list means the caller accepts an - unannounced empty scope, which is only ever right for a caller that does not use the pids. - """ - # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. - # - # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` - # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a - # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document - # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` - # off the definition — no row — so the WALL can be answered from the 0.1% projection. - # - # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED - # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. - # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads - # `readThrough` — a definition key, safe under the projection, and already lent the defn so it - # costs no read of its own. - defn = _defn_or_refuse(session, table_key, defs_only=True) - fields_base = [dict(f) for f in (defn.get("fields") or [])] - # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the - # comprehension below would answer an EMPTY pid set — and the promise this function makes is - # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror - # through the SAME fetch that function uses rather than growing a second idea of the row set; - # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. - if not _ut().materialises(table_key, st=session.runtime, defn=defn): - try: - rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, - rt=session.runtime) - return frozenset(r["pid"] for r in rows), fields_base, defn - except _too_big() as e: - # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning - # this into a 409 is what made both line grids unopenable: the envelope was refused - # over rows it never renders. The scope is empty and SAID to be empty. - if limits is not None: - limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) - return frozenset(), fields_base, defn - except RuntimeError as e: - raise err(503, "store_not_ready", str(e)) - # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection - # above cannot serve it and would raise rather than answer `{}` (that is the whole design of - # `_Projected`). The wall has already passed on the projected document, so this re-reads the - # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped - # answer to a question already answered, which is how two ideas of ownership got into this file - # once before (see `may_open`'s own note in `core/user_tables.py`). - # - # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection - # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the - # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten - # read-through databases (every one of them, which is why the switch was slow) skip the big - # read entirely. - whole = _ut().get(table_key, st=session.runtime) - if whole is None: - # Between the wall and here the table was deleted by another request. Same refusal the - # wall gives, rather than an empty pid set nobody can distinguish from an empty table. - raise err(404, "unknown_table", "that database does not exist") - pids = frozenset(int(rid) for rid in (whole.get("rows") or {}) if str(rid).isdigit()) - return pids, fields_base, whole - - -def ut_write_ctx(session: Session, table_key: str): - """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. - - Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this - or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller - that starts needing rows should fail on an empty list it can see, not on a KeyError. - """ - import aios_grid - from core import grid_events - - limits = [] - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(), - admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, - scope_key=table_key, table=_ops(session, table_key)) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", - fields_base=fields_base) - return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), - # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather - # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. - "limits": limits, "defn": defn} - - -def ut_assembly(session: Session, table_key: str, storage_key: str = "", - consume_corrections: bool = True, with_rows: bool = True, st=None): - """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so - `/workspace` and the events route consume any of the three interchangeably. - - Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is - customer-grain, so there is nothing to offer over user rows. - - ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets - cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its - own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and - wrong the moment the store learned about topics. - - ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the - caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from - `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: - * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over - 963,783 rows nobody was going to look at, which is D-174 in one sentence; - * every materialised `ut_*` database stops allocating a dict per row and sorting them on a - route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per - database switch (owner item 7). - ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers - the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved - on the write door. - """ - import aios_grid - from core import grid_events - - limits = [] - if with_rows: - # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only - # a READ route passes one — see `scoped_pool`'s own note and contract C5. - pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) - else: - pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) - rows_src = [] - - ops = _ops(session, table_key) - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(), - admin=session.admin, fallback_ws=None, seen_ids={}, - # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones - # that happen to carry a scoped `table`. - st=session.runtime, - scope_key=table_key, table=ops) - ws = grid_events.table_workspace(ctx, allowed_pids=pids, - consume_corrections=consume_corrections) - workspace, fields, views, lists = aios_grid.workspace_wire( - ws, session.uname, set(pids), defs={}, scope_key=table_key, - storage_key=storage_key, fields_base=fields_base) - - return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, - "fields": fields, "views": views, "lists": lists, - # R9: the Cohorts column's cells from this table's own lists. - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), - # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test - # for is a key a consumer forgets to test for, and this one carries a refusal. - "limits": limits, "defn": defn} - - -def ut_label(defn, key, meta=None): - """THE name of a user table, resolved ONCE (wave 20, item 6a). - - `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that - shows a database name reads through here, because the alternative is what wave 20 found: the - rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker - showed the original (it reads the definition), and neither looked broken. - - ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver - stays because it makes every row already stored — renamed before that fix landed — read - correctly today, without a migration. - """ - return ((meta or {}).get(key, {}).get("name") - or (defn or {}).get("label") or key) - - -def nav_meta(session): - """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" - try: - got = session.runtime.get("nav_meta") - return got if isinstance(got, dict) else {} - except Exception: # noqa: BLE001 - return {} - - -@router.get("/tables") -def list_tables(session: Session = Depends(require_session)): - """This session's user tables — the list the '+ New database' surface renders. - - ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This - route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: - `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on - tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant - with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked - about every table; it is handed the document this function already holds. See - `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not - available. - """ - ut = _ut() - meta = nav_meta(session) - out = [] - tables = ut.all_tables(st=session.runtime) - lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) - for key, t in sorted(tables.items(), - key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): - if not ut.may_open(key, session.uname, session.admin, st=lent): - continue - out.append({"key": key, "label": ut_label(t, key, meta), - "source": t.get("source") or "Blank", - "recordsMutable": ut.records_mutable(key, st=lent), - "createdBy": t.get("createdBy") or "", - "created": t.get("created") or "", - "fields": [dict(f) for f in (t.get("fields") or [])], - "rowCount": len(t.get("rows") or {})}) - return {"tables": out} - - -#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a -#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* -#: -#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since -#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product -#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client -#: could make one, so the whole read-through path was reachable by editing Python — the -#: [[artifact-with-no-importer]] shape, twice burned in this repo already. -#: -#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and -#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that -#: topic can group by; a second list here would be a second definition of the same fact, and the -#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding -#: to a metric KEY instead of carrying SQL. -_ROLLUP_CACHE = {} - - -def _rollup_source_offer(): - """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. - - ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE - time refuses silently — the cells are simply left blank, hours later, on a column that looks - configured. Two exclusions do real work: - * a topic with NO dims cannot be grouped at all, so it can never key a parent row; - * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose - inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: - *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only - ever error. - Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what - the user is matching their own column against, and `payment_state` (a value) and - `partner` (an id) are matched to very different columns. - """ - if _ROLLUP_CACHE.get("offer"): - return _ROLLUP_CACHE["offer"] - from harness import semantic as sem - from harness import windows as W - ut = _ut() - - topics, metrics = sem.topics(), sem.metrics() - by_topic = {} - for key, m in metrics.items(): - # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. - if m.get("agg") in ("ratio", "derived"): - continue - by_topic.setdefault(m["topic"], []).append( - {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", - "description": m.get("description") or ""}) - - out = [] - for tkey, t in sorted(topics.items()): - dims = ((t.get("store") or {}).get("dims") or {}) - measures = by_topic.get(tkey) or [] - if not dims or not measures: - continue - out.append({ - "key": tkey, - "label": t.get("label") or tkey, - "grain": t.get("grain") or "", - "dims": [{"key": dkey, - "label": d.get("label") or dkey, - # `store_query` emits `_id` only when the dim carries a display name - # alongside the key; otherwise the value IS the key. `rollup_sql` handles - # both, and the editor says which so the user matches the right column. - "keyedBy": "id" if d.get("name_col") else "value"} - for dkey, d in dims.items()], - "measures": sorted(measures, key=lambda m: m["label"].lower()), - }) - - # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` - # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds - # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the - # validator refuses would let the editor build a field the save door rejects. - windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} - for k in ut.ROLLUP_SOURCE_WINDOWS] - offer = {"topics": out, "windows": windows} - _ROLLUP_CACHE["offer"] = offer - return offer - - -@router.get("/tables/rollup-sources") -def rollup_sources(session: Session = Depends(require_session)): - """The topic → metric → dim → window offer the rollup field editor renders. - - ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI - matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below - this line it still resolves; added ABOVE it, this endpoint would silently start arriving as - `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — - this is the cheap ordering that keeps it from mattering. - - ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model - files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live - Odoo data" in the field editor and been able to build a column that can only ever be blank, - because there is no mirror behind it. Three comments (here, in `apiBridge` and on the - `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, - and the mode switch's "render only when there is a choice" guard is built on that promise. - ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already - what `refresh` consults to decide whether these tables may exist at all, and a second copy of - the rule would be a second answer the day a tenant gains a mirror. - """ - import odoo_relational - if not odoo_relational.is_royal(session.tenant): - return {"topics": [], "windows": []} - return _rollup_source_offer() - - -@router.post("/tables", status_code=201) -def create_table(body: dict = Body(default=None), - session: Session = Depends(require_session)): - ut = _ut() - body = body or {} - label = str(body.get("label") or "").strip() - if not label: - raise err(400, "bad_label", "give the database a name") - if not session.runtime.available(): - raise err(503, "store_unavailable", - "the tenant store is unavailable — nothing was created") - source = body.get("source") - try: - key = ut.create(label, session.uname, fields=body.get("fields"), - source=source, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", - "the tenant store refused the write — nothing was created") - if not key: - raise err(400, "refused", - f"could not create it — the name may be empty or this tenant already has " - f"{ut.MAX_TABLES} databases") - return {"key": key} - - -@router.patch("/tables/{table_key}") -def patch_table(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Rename a database — IN ITS DEFINITION (wave 20, item 6a). - - ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the - nav's display layer; the definition is what the automation editor's database picker, the - schema drawer and every future reader see. A rename that lands in only one of them leaves a - picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an - amendment for whoever owns that door. - """ - defn = _defn_or_refuse(session, table_key) - ut = _ut() - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can rename it") - label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) - if not label: - raise err(400, "bad_label", "give the database a name") - return {"key": table_key, "label": label} - - -@router.get("/tables/{table_key}/footprint") -def table_footprint(table_key: str, session: Session = Depends(require_session)): - """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). - - Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories - without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled - like the delete itself: only someone who could delete may case the joint. - - ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row - COUNT needs the whole document, and only on a materialised table.""" - defn = _defn_or_refuse(session, table_key, defs_only=True) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - s = session.runtime - views, fields = set(), len(defn.get("fields") or []) - try: - bucket = s.get(f"{table_key}_table_workspace") or {} - for _u, ws in bucket.items(): - if isinstance(ws, dict): - views |= set((ws.get("views") or {}).keys()) - fields += len(ws.get("fields") or {}) # per-user custom/measure strata - except Exception: - pass - import core.shares as shares - g = shares.grants("database", table_key, st=s) - auto = [] - try: - import automation_engine as engine - for aid, d in (engine.all_definitions(s) or {}).items(): - if (d.get("config") or {}).get("targetTable") == str(table_key): - auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) - except Exception: - pass - # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where - # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, - # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the - # projection is not what makes it wrong. - # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 - # in a dialog headed "what dies with this database"). Booked rather than fixed here: this - # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` - # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. - if _ut().materialises(table_key, st=s, defn=defn): - rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) - else: - rows = 0 - return {"rows": rows, "fields": fields, "views": len(views), - "sharedUsers": len(g.get("entries") or []), - "automations": sorted(auto, key=lambda a: a["name"].lower())} - - -@router.delete("/tables/{table_key}") -def delete_table(table_key: str, session: Session = Depends(require_session)): - """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). - - ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the - creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role - grantee could reach this route and delete the database somebody shared with them. The wall - is now the definition's own `createdBy`, the same check the rename route always had. - - Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and - DISABLES bound automations with a status note — never deletes them. The client's confirm - dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the - dialog is a product requirement, not a formality.""" - defn = _defn_or_refuse(session, table_key) - if not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can delete it") - try: - import automation_engine as engine - engine.disable_for_table(session.runtime, table_key) - except Exception: - pass - try: - _ut().delete(table_key, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} - - -#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. -_IG_FORWARDED = set() - - -def _ig_forward(session): - """Bring this tenant's Instagram tables onto the current schema, at most once per process. - - ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE - path. `ut_ensure` calling it is right for a schema an automation is about to append to, and - useless for a change a PERSON is waiting to see: the owner's report was *"the first field is - still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. - The write path stays exactly as it was — this is a second door to the same idempotent call, - not a replacement for it. - - ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per - tenant per process; `migrate_ig_tables` returns without a write when every table is already - current (the common case after the first read); and a failure is SWALLOWED — a migration must - never be the reason a database will not open. - - ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be - retried on every subsequent read of every table for the life of the process — the write path is - still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing - store call on the hot path of a grid that is trying to render. - """ - tenant = str(getattr(session, "tenant", "") or "") - if tenant in _IG_FORWARDED: - return - _IG_FORWARDED.add(tenant) - try: - import automation_engine as engine - engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") - - - -#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized -#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is -#: not — the shape this exists for is one already-paid provider payload per row. -JSON_LIST_MAX = 400 - - -def _thin_json(fields, merged, table_key=""): - """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. - - ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads - `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something - large rather than something empty. `_truncated` is what the viewer keys its fetch on. - - ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the - viewer needs no table key, no record id and no new props threaded down through three - components to find the document — the route that removed the value says where it went. One - writer of that address instead of a server rule and a client rule that must agree forever. - """ - json_keys = [str(f.get("key")) for f in (fields or []) - if str(f.get("type") or "") == "json"] - if not json_keys: - return merged - out = {} - for pid, cells in (merged or {}).items(): - row = cells - for key in json_keys: - raw = cells.get(key) - if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: - if row is cells: - row = dict(cells) - row[key] = json.dumps({ - "_truncated": True, "bytes": len(raw), - "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) - out[pid] = row - return out - - -@router.get("/tables/{table_key}/rows") -def table_rows(table_key: str, session: Session = Depends(require_session)): - """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, - identity}` — so the client's generic topic fetch consumes it with zero new parsing. - - ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell - from the OVERLAY stratum only (that is what makes a custom column render standalone) — a - user table's base values live in its DEFINITION rows, so they are layered UNDER the user's - overlay edits here: base first, overlay wins. Without this every base cell reads empty - (found by this route's own gate check, not by luck).""" - import aios_grid - - _ig_forward(session) - # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. - # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE - # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), - # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask - # about the SAME document in the SAME request. The lend is the fix W31 QA already built for - # `_defn_or_refuse`; this threads it through the three sites that never got it. - # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a - # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the - # value it replaced (contract C5). - _lent = _ut().lend(session.runtime) - g = ut_assembly(session, table_key, st=_lent) - # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge - # this line does not perform since the route was written (owner item 3, 2026-08-09: - # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). - # - # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` - # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table - # is overlay-typed by construction, so it looked up a key that could not be there and every - # such cell rendered blank. - # - # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds - # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed - # in the store exactly as they should. Only the READ-BACK dropped them, which is why the - # value survived the gesture, vanished on reload, and looked like "it reset itself" — and - # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. - # - # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition - # value must not shadow an edit the user has made on top of it. - # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this - # widens what a caller can SEE by exactly their own edits and nothing else. - _overlays = (g.get("ws") or {}).get("overlays") or {} - merged = {} - for _r in g["rows_src"]: - _pid = str(_r["pid"]) - _cells = {k: v for k, v in _r.items() if k != "pid"} - _ov = _overlays.get(_pid) - if isinstance(_ov, dict): - _cells.update(_ov) - merged[_pid] = _cells - # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE - # JSON DOCUMENTS DO NOT RIDE THE LIST. - # - # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% - # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB - # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The - # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole - # vendor response crossed the wire, was parsed by the browser and held in memory purely so a - # clipped first line could be drawn. - # - # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by - # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — - # the one place a person actually reads it, for the one row they opened. The cell that rides - # the list is a VALID small document saying what it stands for, so the preview renders - # honestly instead of showing half a truncated brace. - # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, - # so a tenant using `json` for a short config sees no change at all. - rows = aios_grid.rows_from_pool( - g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) - # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you - # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this - # database says so here, with its cause and the recommendation, rather than waiting to be - # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the - # honest answer for a table nothing limits — never an absent key, which a client cannot tell - # apart from an older server. - _report = _ut().limit_report(table_key, st=_lent) - # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the - # only thing that ever set this key; the write door never stopped working and every client - # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a - # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the - # customer scope — `grid_events.docs_for` — never a twin here. - from core import grid_events as _ge - return {"fields": g["fields"], "rows": rows, "today": g["today"], - "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, - admin=session.admin, st=session.runtime), - "pulled_at": time.strftime("%Y-%m-%d %H:%M"), - "identity": {"pid": "pid"}, - "scope": {"table": table_key, "rowCount": len(rows)}, - "limits": [_report] if _report else [], - "recordsMutable": _ut().records_mutable(table_key, st=_lent)} - - -@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") -def table_cell(table_key: str, pid: str, fkey: str, - session: Session = Depends(require_session)): - """ONE cell, whole — the other half of `_thin_json`. - - ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor - for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON - viewer opens this for the one row a person is actually reading. - - ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's - view of the table), so this cannot become a side door onto a table the caller may not open — - which is the failure a "just fetch the raw cell" helper invites. - ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must - read back what they typed, not the definition value underneath it. - """ - g = ut_assembly(session, table_key) - field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) - if field is None: - raise err(404, "unknown field", f"{fkey!r} is not a column on this database") - row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) - if row is None: - raise err(404, "unknown record", f"no record {pid!r} in this database") - overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} - value = overlay.get(fkey, row.get(fkey)) - return {"table": table_key, "pid": str(pid), "field": str(fkey), - "value": "" if value is None else str(value)} - - -@router.post("/tables/{table_key}/rows", status_code=201) -def add_row(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). - - ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo - that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the - response rather than assume; the client re-anchors on what came back. - """ - _records_or_refuse(session, table_key) - ut = _ut() - values = (body or {}).get("values") or {} - if not isinstance(values, dict): - raise err(400, "bad_values", "values must be an object of {fieldKey: value}") - try: - rid = ut.add_row(table_key, values, session.uname, st=session.runtime, - rid=(body or {}).get("rid")) - except Exception: - raise err(503, "store_unavailable", "the row was not saved — the store refused") - if rid is None: - # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap - # sentence alone would misdirect — the reader would go and count rows. Ask the same - # validator the law used rather than re-deciding here (one rule, two voices). - pf = ut.profile_field(table_key, st=session.runtime) - if pf and pf["key"] in values: - _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) - if not ok: - raise err(400, "refused", - f"{str(values[pf['key']])[:80]!r} is not an Instagram profile — " - f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " - f"profile link (instagram.com/name)") - raise err(400, "refused", - f"row refused — the table may be at its {ut.MAX_ROWS}-row cap") - _refresh_relations(session) - return {"rid": rid, "pid": int(rid)} - - -@router.post("/tables/{table_key}/rows/import", status_code=201) -def import_rows(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. - - Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a - spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, - nothing is matched or overwritten, and the dialog says so before the button is pressed. - - ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate - the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in - an import is not a stray to be tidied away — it means the client offered a target it should - not have, and silently dropping it would leave the user looking for a column of values that - never arrived. The refusal names the column. - - ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every - profile cell validates, because a half-imported file is the worst outcome available: the user - cannot tell which rows landed without reconciling the spreadsheet by hand. - """ - _records_or_refuse(session, table_key) - ut = _ut() - rows_in = (body or {}).get("rows") - if not isinstance(rows_in, list) or not rows_in: - raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") - if any(not isinstance(r, dict) for r in rows_in): - raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") - defn = _defn_or_refuse(session, table_key) - by_key = {f["key"]: f for f in (defn.get("fields") or [])} - asked = {k for r in rows_in for k in r} - unknown = sorted(k for k in asked if k not in by_key) - if unknown: - raise err(400, "unknown_field", - f"this database has no column {unknown[0]!r}") - computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) - if computed: - label = by_key[computed[0]].get("label") or computed[0] - raise err(400, "computed_field", - f"{label!r} is worked out from other columns, so it cannot be imported into") - # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. - # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a - # second client, or a future importer met no wall at all and the string landed verbatim in a - # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE - # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every - # rule on this route exists to prevent. The sentence names the row and the column, because - # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. - for index, row in enumerate(rows_in): - for key, value in row.items(): - why = ut.cell_type_refusal(by_key[key], value) - if why: - raise err(400, "bad_value", f"row {index + 1}: {why} — nothing was imported") - try: - made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "nothing was imported — the store refused") - if made is None: - raise err(400, "refused", - f"nothing was imported — {len(rows_in)} rows would take this database past " - f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") - _refresh_relations(session) - return {"imported": len(made), "pids": [int(r) for r in made]} - - -# --------------------------------------------------------------------------------------------- -# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) -# --------------------------------------------------------------------------------------------- -# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same -# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's -# definition to everyone without handing over the table. This supersedes wave 17's "fields are -# per-user" law for this path only; the connector scopes keep their own model. -# -# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the -# per-user workspace overlay, which is why the automation editor's "Automation column" picker -# could not see a column the user had just created — it reads the DEFINITION. Same defect shape -# as the rename (item 6a): two places to look, and the surfaces disagreed silently. - -def _field_or_refuse(session, table_key, fkey=""): - """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. - - ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and - `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they - write through the `user_tables` write doors), so no projected snapshot survives into a write. - ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 - draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" - defn = _defn_or_refuse(session, table_key, defs_only=True) - ut = _ut() - if not ut.is_user_table(table_key, st=session.runtime): - raise err(400, "not_a_user_table", - "only a user-created database has an editable schema — a connected source " - "owns its own columns") - if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, - st=session.runtime): - field = next((f for f in (defn.get("fields") or []) - if f.get("key") == str(fkey)), None) - if isinstance((field or {}).get("automation"), dict) \ - and field["automation"].get("preset") is True: - # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not - # ut.may_edit_field` above is the wall). Said out loud because the sentence itself - # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a - # blanket "pre-set fields are locked" would now be the server explaining a refusal - # it did not make — `preset_editable` is the one predicate that answers this. - raise err(403, "preset_field_locked", - "this is a pre-set column, so its name and type are fixed; you may sort, " - "filter or hide it, edit any Rollup column, and add your own columns") - raise err(403, "forbidden", "that column can only be changed by the database's creator " - "or an admin") - if not fkey and not (session.admin or defn.get("createdBy") == session.uname): - raise err(403, "forbidden", "only the database's creator or an admin can add a column") - return defn - - -@router.post("/tables/{table_key}/fields", status_code=201) -def add_field(table_key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - _field_or_refuse(session, table_key) - ut = _ut() - field = ut.add_field(table_key, body or {}, st=session.runtime) - if not field: - # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers - # None for every refusal, so this route said "check the name and type" to somebody whose - # name and type were fine and whose automation column named a flow that does not exist. - # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the - # one thing that was never wrong. Checked HERE, in the route's own words, because the - # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never - # re-implements it (a second copy of the rule is how two doors start disagreeing). - raise err(400, "refused", - _refusal_sentence(ut, session, body or {}, table_key=table_key)) - if field.get("type") == "link": - synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) - field = synced.get("field") or field - # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route - # became reachable for one. It was gated on `link` alone, while `patch_field` and - # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first - # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). - # The user would have created the column, watched a 201 come back, and read a blank cell for - # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. - # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in - # the tenant, and adding a text column has nothing to fold. The condition is now "is this - # field relational", which is the question that was always meant. - if field.get("type") in ("link", "rollup"): - _refresh_relations(session) - return {"field": field} - - -def _refusal_sentence(ut, session, body, table_key="", fkey=""): - """Why was this column refused? The specific reason when we can name one, the general list - otherwise — never a specific-sounding guess.""" - # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is - # what the contract asks for and what makes it actionable — "at most one" sends the reader - # hunting through a 40-column schema for a flag they cannot see from the header. - if isinstance((body or {}).get("profile"), dict): - # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at - # once (an `int` profile column on a table that already has a profile column), and the - # TYPE is the one that is wrong about what the caller just sent — unconditionally, no - # matter what else is on the table. Answering "you already have one" to somebody whose - # real mistake was the column type sends them to fix the wrong thing, which is the - # misdirection D-46 closed one door over. - if str((body or {}).get("type") or "text").strip().lower() != "text": - return ("a profile column is a flag on an ordinary TEXT column — it validates what " - "is typed into it, which it can only do for text") - existing = ut.profile_field(table_key, st=session.runtime) if table_key else None - if existing and existing.get("key") != str(fkey): - return (f"this database already has a profile column — " - f"{existing.get('label') or existing.get('key')!r}. A database has at most " - f"one, so the automation knows which handle to enrich; edit that column, or " - f"take the flag off it first") - bag = (body or {}).get("automation") - if isinstance(bag, dict): - flow = str(bag.get("flowId") or "").strip() - if not flow: - return ("an automation column has to name the automation that fills it — pick a " - "flow, or make this an ordinary column") - if not ut.flow_bound(bag, st=session.runtime): - return (f"this column names automation {flow!r}, which does not exist in this " - f"workspace — it may have been deleted; pick a flow that is still there") - kind = str((body or {}).get("type") or "").strip() - if kind and kind not in ut.UT_FIELD_TYPES: - return (f"{kind!r} is not a column type here (types: " - f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") - return (f"the column was refused — check the name and type, or the table may be at its " - f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") - - -@router.patch("/tables/{table_key}/fields/{fkey}") -def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Edit one column's definition, and MIGRATE its values when options are renamed. - - ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: - [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed - Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and - every saved view that filtered on it. - """ - _field_or_refuse(session, table_key, fkey) - ut = _ut() - body = body or {} - migrated = None - renames = body.get("renames") - if renames: - try: - migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) - except Exception: - raise err(503, "store_unavailable", "the rename did not land — try again") - # The per-user workspace strata and any view filter naming the old value are the OTHER - # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an - # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a - # missing counterpart must not lose the half that DID land. - try: - import core.table_store as table_store - fn = getattr(table_store, "rename_choice_values", None) - if callable(fn): - fn(table_key, fkey, renames, st=session.runtime) - migrated = dict(migrated or {}, workspace=True) - except Exception: # noqa: BLE001 - migrated = dict(migrated or {}, workspace=False) - field = ut.patch_field(table_key, fkey, body, st=session.runtime) - if not field: - # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as - # adding one, so it must get the same sentence naming the column that already holds the - # flag — a patch that answered "check the name and type" would send the reader to the - # one thing that was never wrong (the D-46 lesson, one door over). - raise err(400, "refused", - _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) - synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) - field = synced.get("field") or field - _refresh_relations(session) - out = {"field": field} - if migrated is not None: - out["migrated"] = migrated - return out - - -@router.delete("/tables/{table_key}/fields/{fkey}") -def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): - _field_or_refuse(session, table_key, fkey) - if not _ut().delete_field(table_key, fkey, st=session.runtime): - raise err(400, "refused", - "that column could not be removed — a database must keep at least one") - _refresh_relations(session) - return {"deleted": fkey} - - -@router.delete("/tables/{table_key}/rows/{rid}") -def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): - # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. - # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* - # A single DELETE was FIVE whole-document deep copies before the commit even began — three in - # the guard (`get` + `may_open` + `records_mutable`) and two more inside - # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads - # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** - # (D-185), so the guard alone was seconds of copying to answer questions about one row. - # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` - # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document - # under the store lock, so a lent snapshot can never be the thing written back. - # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually - # consistent delete is indistinguishable from one that did not work. This makes the guard - # cheap; it does not make the commit optimistic. - lent = _ut().lend(session.runtime) - _records_or_refuse(session, table_key, st=lent) - try: - ok = _ut().delete_row(table_key, rid, st=lent) - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - if not ok: - raise err(400, "refused", "rows can only be deleted from user-created databases") - _refresh_relations(session) - return {"ok": True} - - -@router.patch("/tables/{table_key}/rows/{pid}") -def patch_row(table_key: str, pid: int, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Cell edits — the products PATCH on the user-table ctx. Routed through - `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; - the accepted values are read BACK from the bucket, never echoed from the request.""" - from core import grid_events - - updates = dict(body or {}) - if not updates: - raise err(400, "empty_patch", "no fields to update") - _records_or_refuse(session, table_key) - g = ut_assembly(session, table_key, consume_corrections=False) - if pid not in g["pids"]: - raise err(403, "out_of_scope", "that row is not in this database") - ctx = grid_events.EventCtx( - uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], - admin=session.admin, fallback_ws=None, seen_ids={}, hidden_keys=frozenset(), - st=session.runtime, # R6b (D-16) - scope_key=table_key, table=_ops(session, table_key)) - try: - grid_events.handle_one( - {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", - "pid": pid, "updates": updates}, ctx) - except grid_events.StoreUnavailable: - raise err(503, "store_unavailable", - "the tenant store is unavailable — your change was not saved") - # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. - # - # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH - # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. - # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's - # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began - # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` - # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring - # claiming otherwise is what makes the next reader look for a merge bug that is not here. - # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY - # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` - # asks whether THIS write landed, and this write goes to the definition. - stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) - .get(str(pid)) or {}) - accepted = {k: stored.get(k) for k in updates if k in stored} - - def _took(k): - """Did the cell TAKE this write? Normally that is "stored == asked". - - ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` - and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell - the client to roll back a write that landed. But it cannot simply be exempted either — - a junk handle leaves the OLD value sitting in `stored`, which would then read as - accepted. So the question asked is the exact one: **is what is stored the canonical form - of what was asked?** Anything else is a genuine refusal. - """ - if k not in accepted: - return False - want = str(updates[k]) - if stored.get(k) == want: - return True - pf = _ut().profile_field(table_key, st=session.runtime) - if pf and pf["key"] == k: - handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) - return bool(ok) and stored.get(k) == handle - return False - - refused = sorted(k for k in updates if not _took(k)) - out = {"pid": pid, "updates": accepted} - if refused: - out["refused"] = refused - # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a - # profile blank cleared. Without this the grid keeps painting a stale follower count under - # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other - # half: the client may keep only what the server actually took, and must be TOLD what else - # moved. Derived by diffing this row against what was asked for, so it cannot drift from - # whatever the clear rule decides to touch. - also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} - cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) - if cleared: - out["cleared"] = cleared - # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, - # amendment A5; SESSION B built and gated the function and correctly declared it INERT until - # this line existed, citing [[flag-shipped-without-its-writer]]). - # - # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead - # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the - # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we - # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try - # it again, the account is back". Without this call that person has no way back at all, and - # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. - # - # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this - # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a - # human touched the handle cell. - # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- - # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must - # be readable in the log rather than degrade into "the re-arm quietly stopped working". - _pf = _ut().profile_field(table_key, st=session.runtime) - if _pf and _pf["key"] in updates: - try: - import automation_engine as _engine - _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") - _refresh_relations(session) - return out +"""routes_tables.py — USER TABLES over the wire (wave 18, contract C3-UT). + +The runtime-database primitive: `core/user_tables.py` (wave-9 C6, host-only until now) served +through the SAME grid machinery every other topic rides — `table_store` for the per-user +workspace strata, `aios_grid.workspace_wire` for the wire shape, `core.grid_events` for every +durable write except rows. Rows are the one genuinely new channel: the events seam has no row +event types (the user_tables docstring's `row_add` gate was described, never built), so row +add/delete/patch are REST endpoints here, walled by `user_tables.is_user_table` + +`user_tables.may_open` — a connector-backed table can never accept an invented row. + +TENANCY: every store touch goes through `session.runtime` (the tenant's store handle), so a +Nurilab admin's tables live under Nurilab's prefix/repo, and the isolation gate's proof #3 +covers them for free. + +VISIBILITY is `user_tables.may_open` — creator or admin, fail-closed. There is no module grant +to check because a user table is not a module; the per-table wall is the whole wall, and it is +applied in `_defn_or_refuse` before any payload is built. +""" +import threading +import json +import time + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +def _ut(): + import core.user_tables as user_tables + return user_tables + + +def _ops(session, table_key): + import core.table_store as table_store + return table_store.make(f"{table_key}_table_workspace", st=session.runtime) + + +#: WAVE 27 item 2 (contract C2 / amendment A2) — the relation refresh is COALESCED and runs OFF +#: the request path. `{tenant: True}` while a pass is queued; the worker holds the lock. +_REL_LOCK = threading.Lock() +_REL_DIRTY = {} +_REL_RUNNING = {} + + +def _refresh_relations(session): + """Mark this tenant's Links/Rollups stale and refresh them AFTER the response, once. + + ⛔ WHY THIS IS NOT A DIRECT CALL ANY MORE, and the numbers are the argument. The owner's + item 2 was "adding a new record visually takes too long, I need to be able to spam it", and + this function was the largest single cost inside `POST /tables/{key}/rows`: + `engine.refresh_relations` DEEP-COPIES every table and every row in the tenant before it can + decide whether anything needs doing (`automation_engine.py:9383-9388`), and when something + does it commits with `flush="sync"` — a store round trip, i.e. an HF Dataset commit on the + default backend. Every added record paid a whole-tenant snapshot plus a synchronous commit + before the 201 came back. + + ⛔ AND BACKGROUNDING ALONE WOULD HAVE BEEN A WORSE BUG. Twenty rapid adds would queue twenty + whole-tenant snapshots, each one re-reading a store the previous one just wrote — the spam + the item asks us to support is exactly the load that would melt it. So this COALESCES: at + most one pass runs, and at most one is queued behind it. + + ⚠ THE DIRTY FLAG IS THE LOAD-BEARING PART, not the lock. A write landing WHILE a pass is in + flight must still get a pass afterwards, because the in-flight one snapshotted before that + write existed. Without the flag the LAST add in a burst is precisely the one whose rollups + never update — the failure nobody would notice until a total was quietly wrong. + + Eventual consistency is the accepted trade and was already the documented posture: the tick + repairs materialised cells regardless, and answering 503 here would invite the browser to + repeat a mutation that already succeeded. + """ + tenant = str(getattr(session, "tenant", "") or "") + rt = session.runtime + with _REL_LOCK: + _REL_DIRTY[tenant] = True + if _REL_RUNNING.get(tenant): + return # a worker is live; it will see the flag and loop + _REL_RUNNING[tenant] = True + + def _worker(): + import automation_engine as engine + try: + while True: + with _REL_LOCK: + if not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + return + _REL_DIRTY.pop(tenant, None) + try: + engine.refresh_relations(rt, log=lambda *_args: None) + except Exception as exc: # noqa: BLE001 + # The source write already landed and the response is already sent. Log and + # let the tick repair it; never retry in a tight loop. + print(f"[tables] relation refresh deferred: {type(exc).__name__}: {exc}") + finally: + # Belt for an unexpected raise on the bookkeeping itself: a tenant left marked + # RUNNING would never refresh again for the life of the process. + with _REL_LOCK: + if _REL_RUNNING.get(tenant) and not _REL_DIRTY.get(tenant): + _REL_RUNNING.pop(tenant, None) + + threading.Thread(target=_worker, daemon=True, + name=f"rel-refresh:{tenant or 'default'}").start() + + +def _defn_or_refuse(session, table_key, st=None): + """The per-table wall: 404 for a key that does not exist, 403 for one this session may not + open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but + not yours' is exactly what may_open is for. + + ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the + nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked + about the same table — it is simply not re-reading a 28.5 MB document to ask it. + """ + ut = _ut() + # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. + # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* + # This guard has FOURTEEN call sites and cost TWO whole-document deep copies at every one + # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions + # about ONE table. Lending here fixes every caller at once, including the eight routes the + # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · + # `POST /rows/import` · `POST|PATCH /fields` · `PATCH /rows/{pid}`). + # ⛔ WHY IT IS SAFE HERE AND WOULD NOT BE IN THE ROUTES: a lend is a PRE-WRITE snapshot. The + # guard runs before any mutation and returns only the DEFINITION, so the snapshot never + # survives to serve a read-back. Blanket-replacing `st=session.runtime` inside the routes + # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which + # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it + # replaced. The remaining in-route reads are deliberately untouched and booked instead. + st = st if st is not None else ut.lend(session.runtime) + defn = ut.get(table_key, st=st) + if not defn: + raise err(404, "unknown_table", "that database does not exist") + if not ut.may_open(table_key, session.uname, session.admin, st=st): + raise err(403, "forbidden", "that database belongs to another user") + return defn + + +def _records_or_refuse(session, table_key, st=None): + """The human record-write wall for a database the automation engine owns.""" + # One lend for BOTH questions this wall asks — the definition wall and the record-mode wall — + # so the two are answered from one read instead of two. Same reasoning as `_defn_or_refuse`. + st = st if st is not None else _ut().lend(session.runtime) + defn = _defn_or_refuse(session, table_key, st=st) + if not _ut().records_mutable(table_key, st=st): + raise err(403, "records_read_only", + "records in this automation-owned database are read-only — add Instagram " + "handles in a Profile database and let enrichment populate this database") + return defn + + +#: ⛔ THE PER-USER CANDIDATE WALL IS RETIRED (WAVE 27, DEBT D-72). **THE TENANT IS THE UNIT, NOT +#: THE USER** — one profile is ONE row, and everyone who may open the database sees all of it. +#: +#: WHY IT HAD TO GO, and it is not a preference: wave 26's R4 made the candidate identity +#: `(platform, handle)` and `_merge_candidates` stamps only the FIRST finder, later finders never +#: overwriting. Combined with a wall that then showed a non-admin only `created_by == me`, the +#: two were SILENT DATA LOSS — the second finder's row was merged away into the first finder's, +#: and the wall then hid the survivor from the person who just found it. They searched, they +#: paid, and the screen said nothing arrived. +#: +#: ⚠ AND THE PAIR WAS MUTUALLY MASKING, which is why it stayed green for a wave: the wall named +#: `ut_ig_candidates`, and a READ-ONLY census of all four tenant stores +#: (`ops/w26_candidate_census.py`) proved that table exists in NO tenant — since wave 25's R2 the +#: write target is whatever database the user points the Create-record action at. So the wall +#: governed a table nobody writes, and fixing EITHER half alone would have armed the other +#: ([[defects-that-mask-each-other]]). +#: +#: The register offered two exits and R4 already implied this one. Restoring per-user visibility +#: instead would have required R4's merge to stop crossing users — a bigger change, against the +#: ruling, to bring back a wall that never governed anything real. +#: +#: ⛔ DO NOT RE-ADD THIS BY INFERRING THE RULE FROM A `created_by` COLUMN. Every +#: automation-written table has one (a scraped row says `automation`), so inference would hide +#: every scraped row from every non-admin — the same disappearance defect, one table wide instead +#: of one table narrow. `created_by` survives as W26/R4's informational "Found by" stamp ONLY. + + +def _too_big(): + """`routes_odoo_tables.TooBigToMaterialise`, imported lazily — one name, two policies below.""" + import routes_odoo_tables + return routes_odoo_tables.TooBigToMaterialise + + +def _read_through_rows(table_key, field_keys, rt=None): + """The mirror's rows for one read-through grid, projected to this table's declared columns. + + ⭐⭐ W31-T45 / D-169 — `rt` IS THE SESSION'S TENANT RUNTIME AND IT IS PASSED THROUGH (D's ask, + `mailbox/D.md` D-2). `_defn_or_refuse` above answers *"may this SESSION open this DATABASE"*; + the guard `whole_pool` fires on `rt` answers a DIFFERENT question — *"is the DuckDB file this + process has open THIS TENANT's"* — and R2 gives GTM Lab connected tables in its own document, + which is exactly the shape that satisfies the first wall while failing the second. + `datastore.ro_con()` reads a process-global `DB_PATH` and one Space process serves every + tenant, so the two walls are not substitutes for each other. + + ⛔ ONE FETCH, TWO POLICIES — and the split is the whole of W31-T20. Both `scoped_pool` (which + owes a caller every row) and `scoped_pids` (which owes only the row SET) reach the mirror + through this function, so the pid set the cheap path answers with is IDENTICAL to the pool's + by construction rather than by a second query that agrees today. A pid-only `SELECT` would be + cheaper and would also be a SECOND statement of what a row of this table is + ([[one-question-two-normalizers]]) — the two callers differ in what they do with the REFUSAL, + never in how they ask. + + Raises `TooBigToMaterialise` when the population exceeds one window; the caller decides + whether that is a 409 or an unresolved pid scope. + """ + import routes_odoo_tables + + rows_src = [{k: v for k, v in r.items() if k in field_keys or k == "pid"} + for r in routes_odoo_tables.whole_pool(table_key, rt=rt)] + rows_src.sort(key=lambda r: r["pid"]) + return rows_src + + +#: ⭐⭐ W31-T20 / D-174 — R6's SECOND SENTENCE FOR A PID SCOPE THAT CANNOT BE RESOLVED. +#: +#: Wave 30 un-capped the ROW door and left the WORKSPACE envelope, so `GET /workspace?scope= +#: ut_odoo_gl_lines` answered `409 window_required` six times out of six on the live deploy and the +#: grid painted an ERROR PAGE with a Retry button — a whole shipped feature nobody could open. +#: The cause: an envelope needs no row, but it asked for every one of 963,783 of them to derive a +#: pid set it uses for exactly two things (cohort membership and a shared view's `memberPids`). +#: +#: ⛔ SO THE ENVELOPE STOPS ASKING, AND SAYS SO. An empty pid set is not "no rows" — it is +#: "membership is unresolved on this grid", which is a different claim and has to be made out +#: loud, on the wire, or it is the silent truncation R6 is actually about. Both consequences are +#: fail-closed: a stored cohort's members are reported MISSING by `aios_grid.workspace_wire` +#: rather than silently dropped, and any WRITE that names a pid is refused by `routes_grid`. +_PID_SCOPE_LIMIT = { + "subject": "pids", "effect": "unresolved", + "recommendation": "cohorts and shared-view membership are resolved per page on this grid; " + "filter and read it a window at a time (`/odoo-tables/{key}/rows`), where " + "every total is a SQL count over the whole table", +} + + +def scoped_pool(session: Session, table_key: str): + """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. + + `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a + caller that only needs "which rows of this database may this session touch" — record comments + — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second + idea of what a user table's pool is. + """ + defn = _defn_or_refuse(session, table_key) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + field_keys = {f["key"] for f in fields_base} + # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. + # + # This is the one function that turns "what is stored" into "what this session may see", which + # is exactly why the read-through arm belongs HERE and nowhere else: the rows route, the events + # route, the comments wall and the assembly all reach rows through it, so they all convert + # together or not at all. ⛔ Reading `defn["rows"]` for such a table would find `{}` and serve + # an EMPTY GRID — correct-looking, wrong, and silent. + # ⚠ The wall above has already run. This adds no scope of its own and takes none away. + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows_src = _read_through_rows(table_key, field_keys, rt=session.runtime) + except _too_big() as e: + # R6's second sentence: a limit that cannot be met is a SENTENCE, never a short grid. + # ⚠ THE ROWS PATH STILL REFUSES, AND THAT IS CORRECT (W31-T20 changed the ENVELOPE, not + # this): a caller that asked for every row of a 963,783-row grid cannot be served a + # short one. `scoped_pids` below takes the same refusal and answers a different + # question with it, because an envelope needs no row. + raise err(409, "window_required", str(e)) + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + # WAVE 27 / D-72: no per-row owner filter. The table-level wall above + # (`_defn_or_refuse` -> `may_open`) is the WHOLE wall — see the retirement note on + # PER_USER_TABLES' former home. A row this session can reach is a row the tenant owns. + r = {k: v for k, v in (row or {}).items() if k in field_keys} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + return frozenset(r["pid"] for r in rows_src), rows_src, fields_base, defn + + +@router.patch("/tables/{table_key}/shared/{pid}") +def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a cell into the TENANT-WIDE overlay — the product door `core/shared_overlay.py` has + been waiting for since it shipped (W29-T62, wave 30 T28). + + ⭐ WHY A SEPARATE STRATUM AT ALL, restated because it is the whole feature and it is not + "sharing would be nice": a user-created column and its values live PER USER, so a shared view + filtering on one names a column other accounts do not have — and an unknown column is an + INACTIVE condition in the tri-state engine, which IGNORES it and therefore WIDENS. The buy + list would silently show the whole catalogue to everyone but its author. A column whose value + is the same for every reader is the precondition for editing it at all. + + ⛔ THE WALL IS `_defn_or_refuse`, AND THE STRATUM IS NOT ONE. `shared_overlay` refuses no + reader and no writer by design; "may this session open this surface" is answered HERE, where + the session is. Do not push the question down there. + ⚠ A tenant-wide write is not a private one: every account that may open this database sees it. + That is the point, and it is why this door declares the column too — a value with no + definition is a cell nobody can find. + """ + body = body if isinstance(body, dict) else {} + key = str(body.get("field") or "").strip() + if not key: + raise err(400, "bad_request", "a field key is required") + _defn_or_refuse(session, table_key) + from core import shared_overlay + if not shared_overlay.is_shared(table_key, key, st=session.runtime): + shared_overlay.put_field(table_key, key, { + "key": key, "label": str(body.get("label") or key), "source": "overlay", + "type": str(body.get("type") or "text"), "shared": True, + "createdBy": session.uname}, st=session.runtime) + try: + # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is + # the API that says so. It delegates to the plural, so both stay reachable through the one + # caller; before this, the singular had no caller at all and `verify_reachability` LENS 2 + # named it (the same lens that found `drop_field` had no door either). + stored = {key: shared_overlay.put_cell(table_key, pid, key, body.get("value"), + st=session.runtime)} + except ValueError as e: + # A non-scalar RAISES in the stratum rather than being dropped; relay it as the answer. + raise err(400, "bad_value", str(e)) + return {"ok": True, "pid": pid, "cells": stored, + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +@router.delete("/tables/{table_key}/shared/fields/{field_key}") +def delete_shared_field(table_key: str, field_key: str, + session: Session = Depends(require_session)): + """Remove a TENANT-WIDE column and every value in it. + + ⛔ WHY THIS EXISTS AT ALL, said plainly: W30-T28 shipped the door that CREATES a shared column + and none that removes one, so a column anybody added was permanent for the whole tenant. The + reachability gate found it from the other end — `shared_overlay.drop_field` was complete, + correct, gated, and callable by nothing but its own gate ([[reachable-is-not-the-same-as-built]]). + + ⛔ AND THIS ONE IS CREATOR-OR-ADMIN, WHICH THE WRITE DOOR IS NOT. Writing a cell changes a + value; dropping the column deletes that value for EVERY account at once, so it is the + destructive-op wall this repo already uses for a database delete — not `editRole`, which + governs renaming and is not a value wall ([[schema-role-is-not-a-value-wall]]). + ⚠ `createdBy` is stamped by the write door above; a column stored before that stamp existed + is admin-only, which is the safe direction. + """ + _defn_or_refuse(session, table_key) + from core import shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(str(field_key)) + if not defn: + raise err(404, "unknown_field", "that column is not a shared column on this database") + owner = str(defn.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a tenant-wide column can be removed by its creator or an admin — this one " + f"was added by {owner or 'somebody else'}, and dropping it would delete the " + f"value for every account") + dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) + return {"ok": True, "dropped": bool(dropped), + "fields": list(shared_overlay.fields(table_key, st=session.runtime))} + + +def scoped_pids(session: Session, table_key: str, limits=None): + """`(pids, fields_base, defn)` — the SAME wall and the SAME row set as `scoped_pool`, without + building a row. + + ⭐⭐ WAVE 30 / W30-T30 — THIS IS WHY ONE HIDE-FIELDS CHECKBOX WAS EXPENSIVE. A view write + (`view_upsert`) reaches `grid_events_route`, which built a FULL assembly purely to validate + it: `scoped_pool` allocates a fresh dict per row and then sorts them — ~33k order rows, on + every toggle — and the six keys the events route actually reads from that assembly + (`fields`, `pids`, `measures`, `measure_sets`, `lists`, `views`) contain no row at all. + `rows_src` was computed and discarded. + + ⛔ THE PID SET IS IDENTICAL, NOT MERELY EQUIVALENT, and that is the whole safety argument: + `scoped_pool` derives its pids as `frozenset(r["pid"] for r in rows_src)` over exactly the + row ids that pass `str(rid).isdigit()`, which is this comprehension with a dict build in the + middle. The row WALL is unchanged — a narrower or wider set here would be a permission + change, and this is a performance change. + + ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs two whole-document + deep copies (`ut.get` then `may_open`), which is D-87 and W30-T31. This removes the row pass. + + ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a + list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a + read-through grid whose population exceeds one window). The pid set is then EMPTY, and every + consumer of an empty pid set is fail-closed — but "fail-closed and unannounced" is exactly the + silent limit R6's second sentence forbids, so a caller that renders an envelope or admits a + write is expected to carry the sentence through. Omitting the list means the caller accepts an + unannounced empty scope, which is only ever right for a caller that does not use the pids. + """ + defn = _defn_or_refuse(session, table_key) + fields_base = [dict(f) for f in (defn.get("fields") or [])] + # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the + # comprehension below would answer an EMPTY pid set — and the promise this function makes is + # that its set is IDENTICAL to `scoped_pool`'s, not merely cheaper. It reaches the mirror + # through the SAME fetch that function uses rather than growing a second idea of the row set; + # the saving W30-T30 bought stays on every materialised table, which is all of the big ones. + if not _ut().materialises(table_key, st=session.runtime, defn=defn): + try: + rows = _read_through_rows(table_key, {f["key"] for f in fields_base}, + rt=session.runtime) + return frozenset(r["pid"] for r in rows), fields_base, defn + except _too_big() as e: + # ⛔ THE REFUSAL BECOMES AN ANSWER HERE, WHICH IT MUST NOT ON THE ROWS PATH. Turning + # this into a 409 is what made both line grids unopenable: the envelope was refused + # over rows it never renders. The scope is empty and SAID to be empty. + if limits is not None: + limits.append({**_PID_SCOPE_LIMIT, "cause": str(e)}) + return frozenset(), fields_base, defn + except RuntimeError as e: + raise err(503, "store_not_ready", str(e)) + pids = frozenset(int(rid) for rid in (defn.get("rows") or {}) if str(rid).isdigit()) + return pids, fields_base, defn + + +def ut_write_ctx(session: Session, table_key: str): + """The g-dict a WRITE needs — same keys as `ut_assembly`, no rows. + + Returns the six keys `routes_grid.grid_events_route` reads, so the events route consumes this + or a full assembly interchangeably. `rows_src` is `[]` on purpose rather than absent: a caller + that starts needing rows should fail on an empty list it can see, not on a KeyError. + """ + import aios_grid + from core import grid_events + + limits = [] + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(), + admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, + scope_key=table_key, table=_ops(session, table_key)) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", + fields_base=fields_base) + return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, + "derived": aios_grid.cohort_cells(lists), + "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather + # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. + "limits": limits, "defn": defn} + + +def ut_assembly(session: Session, table_key: str, storage_key: str = "", + consume_corrections: bool = True, with_rows: bool = True): + """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so + `/workspace` and the events route consume any of the three interchangeably. + + Honest absence: `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is + customer-grain, so there is nothing to offer over user rows. + + ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. "For ANY database new/old": a user table gets + cohorts like every other database, out of its OWN bucket (`ut__cohorts`), holding its + own row ids. The wave-18 refusal was correct while there was one customer-keyed bucket and + wrong the moment the store learned about topics. + + ⭐⭐ W31-T20 / D-174 — `with_rows=False` BUILDS THE ENVELOPE AND NOT THE TABLE, and the + caller that wants it is `/workspace`, which renders no row at all (the grid fetches rows from + `/tables/{key}/rows` or `/odoo-tables/{key}/rows` beside it). Two things follow: + * the read-through line grains become OPENABLE — `scoped_pool` refused their envelope over + 963,783 rows nobody was going to look at, which is D-174 in one sentence; + * every materialised `ut_*` database stops allocating a dict per row and sorting them on a + route whose payload has no rows in it — `ut_odoo_orders` was rebuilding 32,826 of them per + database switch (owner item 7). + ⛔ NOTHING IS VALIDATED LESS. `scoped_pids` runs the SAME `_defn_or_refuse` wall and answers + the identical pid set; the flag removes work, never a check — the shape W30-T30 already proved + on the write door. + """ + import aios_grid + from core import grid_events + + limits = [] + if with_rows: + pids, rows_src, fields_base, defn = scoped_pool(session, table_key) + else: + pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) + rows_src = [] + + ops = _ops(session, table_key) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=frozenset(), + admin=session.admin, fallback_ws=None, seen_ids={}, + # R6b (D-16): the tenant handle rides every ctx this layer builds, not just the ones + # that happen to carry a scoped `table`. + st=session.runtime, + scope_key=table_key, table=ops) + ws = grid_events.table_workspace(ctx, allowed_pids=pids, + consume_corrections=consume_corrections) + workspace, fields, views, lists = aios_grid.workspace_wire( + ws, session.uname, set(pids), defs={}, scope_key=table_key, + storage_key=storage_key, fields_base=fields_base) + + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, + "fields": fields, "views": views, "lists": lists, + # R9: the Cohorts column's cells from this table's own lists. + "derived": aios_grid.cohort_cells(lists), + "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test + # for is a key a consumer forgets to test for, and this one carries a refusal. + "limits": limits, "defn": defn} + + +def ut_label(defn, key, meta=None): + """THE name of a user table, resolved ONCE (wave 20, item 6a). + + `nav_meta`'s rename wins, then the definition's own label, then the key. Every surface that + shows a database name reads through here, because the alternative is what wave 20 found: the + rail showed the renamed name (it reads `nav_meta`) while the automation editor's picker + showed the original (it reads the definition), and neither looked broken. + + ⚠ `set_label` now writes the DEFINITION too, so the two agree at the source. This resolver + stays because it makes every row already stored — renamed before that fix landed — read + correctly today, without a migration. + """ + return ((meta or {}).get(key, {}).get("name") + or (defn or {}).get("label") or key) + + +def nav_meta(session): + """The tenant's nav_meta bucket, read defensively. A store blip must not take a list down.""" + try: + got = session.runtime.get("nav_meta") + return got if isinstance(got, dict) else {} + except Exception: # noqa: BLE001 + return {} + + +@router.get("/tables") +def list_tables(session: Session = Depends(require_session)): + """This session's user tables — the list the '+ New database' surface renders. + + ⭐⭐ W31-T12 (contract C1, D-175's third instance) — ONE DOCUMENT READ, NOT `1 + 2N`. This + route was never ticketed and has the same shape `/nav` and `/automations` were fixed for: + `all_tables` once, then `may_open` per key (another whole-document deep copy each, 28.6 MB on + tenant #0 measured, 703 ms warm) and `records_mutable` per key on top of that — so a tenant + with ten databases paid twenty-one copies to list them. The wall is UNCHANGED and still asked + about every table; it is handed the document this function already holds. See + `user_tables.lend`'s own note for why inlining the predicate is the one fix that is not + available. + """ + ut = _ut() + meta = nav_meta(session) + out = [] + tables = ut.all_tables(st=session.runtime) + lent = ut.lend(session.runtime, **{ut.STORE_KEY: tables}) + for key, t in sorted(tables.items(), + key=lambda kv: (ut_label(kv[1], kv[0], meta) or "").lower()): + if not ut.may_open(key, session.uname, session.admin, st=lent): + continue + out.append({"key": key, "label": ut_label(t, key, meta), + "source": t.get("source") or "Blank", + "recordsMutable": ut.records_mutable(key, st=lent), + "createdBy": t.get("createdBy") or "", + "created": t.get("created") or "", + "fields": [dict(f) for f in (t.get("fields") or [])], + "rowCount": len(t.get("rows") or {})}) + return {"tables": out} + + +#: ⭐⭐ THE ROLLUP SOURCE OFFER — what makes the read-through rollup a FEATURE rather than a +#: capability. Owner 2026-08-09: *"Full field editor: pick topic → metric → window."* +#: +#: ⛔ THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since +#: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product +#: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client +#: could make one, so the whole read-through path was reachable by editing Python — the +#: [[artifact-with-no-importer]] shape, twice burned in this repo already. +#: +#: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and +#: `model/metrics/*.yml` already state which measures belong to which topic and which dims that +#: topic can group by; a second list here would be a second definition of the same fact, and the +#: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding +#: to a metric KEY instead of carrying SQL. +_ROLLUP_CACHE = {} + + +def _rollup_source_offer(): + """`{topics:[…], windows:[…]}` — every (topic, measure, dim) the engine can actually answer. + + ⛔ ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE + time refuses silently — the cells are simply left blank, hours later, on a column that looks + configured. Two exclusions do real work: + * a topic with NO dims cannot be grouped at all, so it can never key a parent row; + * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` — `agg: ratio`/`derived` whose + inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present: + *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only + ever error. + Each dim also declares HOW it keys — by an Odoo id or by its own value — because that is what + the user is matching their own column against, and `payment_state` (a value) and + `partner` (an id) are matched to very different columns. + """ + if _ROLLUP_CACHE.get("offer"): + return _ROLLUP_CACHE["offer"] + from harness import semantic as sem + from harness import windows as W + ut = _ut() + + topics, metrics = sem.topics(), sem.metrics() + by_topic = {} + for key, m in metrics.items(): + # A ratio/derived metric whose parts sit on another topic cannot be grouped — see above. + if m.get("agg") in ("ratio", "derived"): + continue + by_topic.setdefault(m["topic"], []).append( + {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd", + "description": m.get("description") or ""}) + + out = [] + for tkey, t in sorted(topics.items()): + dims = ((t.get("store") or {}).get("dims") or {}) + measures = by_topic.get(tkey) or [] + if not dims or not measures: + continue + out.append({ + "key": tkey, + "label": t.get("label") or tkey, + "grain": t.get("grain") or "", + "dims": [{"key": dkey, + "label": d.get("label") or dkey, + # `store_query` emits `_id` only when the dim carries a display name + # alongside the key; otherwise the value IS the key. `rollup_sql` handles + # both, and the editor says which so the user matches the right column. + "keyedBy": "id" if d.get("name_col") else "value"} + for dkey, d in dims.items()], + "measures": sorted(measures, key=lambda m: m["label"].lower()), + }) + + # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS` + # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds + # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the + # validator refuses would let the editor build a field the save door rejects. + windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} + for k in ut.ROLLUP_SOURCE_WINDOWS] + offer = {"topics": out, "windows": windows} + _ROLLUP_CACHE["offer"] = offer + return offer + + +@router.get("/tables/rollup-sources") +def rollup_sources(session: Session = Depends(require_session)): + """The topic → metric → dim → window offer the rollup field editor renders. + + ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI + matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below + this line it still resolves; added ABOVE it, this endpoint would silently start arriving as + `table_key='rollup-sources'` and 404 from the table wall. There is no such route today — + this is the cheap ordering that keeps it from mattering. + + ⛔ TENANT-SCOPED, AND IT WAS NOT WHEN FIRST WRITTEN. `sem.topics()` reads the GLOBAL model + files, so nurilab and gtmlab were served the full Odoo offer — they would have seen "Live + Odoo data" in the field editor and been able to build a column that can only ever be blank, + because there is no mirror behind it. Three comments (here, in `apiBridge` and on the + `rollupSourceOffer` prop) each asserted that a tenant with nothing connected receives `[]`, + and the mode switch's "render only when there is a choice" guard is built on that promise. + ⚠ `odoo_relational.is_royal` is the authority, reused rather than re-decided: it is already + what `refresh` consults to decide whether these tables may exist at all, and a second copy of + the rule would be a second answer the day a tenant gains a mirror. + """ + import odoo_relational + if not odoo_relational.is_royal(session.tenant): + return {"topics": [], "windows": []} + return _rollup_source_offer() + + +@router.post("/tables", status_code=201) +def create_table(body: dict = Body(default=None), + session: Session = Depends(require_session)): + ut = _ut() + body = body or {} + label = str(body.get("label") or "").strip() + if not label: + raise err(400, "bad_label", "give the database a name") + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable — nothing was created") + source = body.get("source") + try: + key = ut.create(label, session.uname, fields=body.get("fields"), + source=source, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", + "the tenant store refused the write — nothing was created") + if not key: + raise err(400, "refused", + f"could not create it — the name may be empty or this tenant already has " + f"{ut.MAX_TABLES} databases") + return {"key": key} + + +@router.patch("/tables/{table_key}") +def patch_table(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Rename a database — IN ITS DEFINITION (wave 20, item 6a). + + ⚠ THE RENAME DOOR IN THE NAV WRITES `nav_meta` AND MUST ALSO CALL THIS. `nav_meta` is the + nav's display layer; the definition is what the automation editor's database picker, the + schema drawer and every future reader see. A rename that lands in only one of them leaves a + picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an + amendment for whoever owns that door. + """ + defn = _defn_or_refuse(session, table_key) + ut = _ut() + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can rename it") + label = ut.set_label(table_key, (body or {}).get("label"), st=session.runtime) + if not label: + raise err(400, "bad_label", "give the database a name") + return {"key": table_key, "label": label} + + +@router.get("/tables/{table_key}/footprint") +def table_footprint(table_key: str, session: Session = Depends(require_session)): + """What dies with this database — the confirm dialog's disclosure (wave 21, item 6a / C3). + + Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories + without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled + like the delete itself: only someone who could delete may case the joint.""" + defn = _defn_or_refuse(session, table_key) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + s = session.runtime + views, fields = set(), len(defn.get("fields") or []) + try: + bucket = s.get(f"{table_key}_table_workspace") or {} + for _u, ws in bucket.items(): + if isinstance(ws, dict): + views |= set((ws.get("views") or {}).keys()) + fields += len(ws.get("fields") or {}) # per-user custom/measure strata + except Exception: + pass + import core.shares as shares + g = shares.grants("database", table_key, st=s) + auto = [] + try: + import automation_engine as engine + for aid, d in (engine.all_definitions(s) or {}).items(): + if (d.get("config") or {}).get("targetTable") == str(table_key): + auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) + except Exception: + pass + return {"rows": len(defn.get("rows") or {}), "fields": fields, "views": len(views), + "sharedUsers": len(g.get("entries") or []), + "automations": sorted(auto, key=lambda a: a["name"].lower())} + + +@router.delete("/tables/{table_key}") +def delete_table(table_key: str, session: Session = Depends(require_session)): + """CREATOR OR ADMIN — checked explicitly (wave 21, item 6a / C3). + + ⛔ The wave-20 docstring said "the same actors may_open admits" and that stopped being the + creator-or-admin set the day de5037f taught `may_open` to admit share GRANTEES: a view-role + grantee could reach this route and delete the database somebody shared with them. The wall + is now the definition's own `createdBy`, the same check the rename route always had. + + Deletion cleans the artifact families server-side (`user_tables.delete` lists them) and + DISABLES bound automations with a status note — never deletes them. The client's confirm + dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the + dialog is a product requirement, not a formality.""" + defn = _defn_or_refuse(session, table_key) + if not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can delete it") + try: + import automation_engine as engine + engine.disable_for_table(session.runtime, table_key) + except Exception: + pass + try: + _ut().delete(table_key, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + return {"ok": True} + + +#: ⭐ 2026-08-07 — tenants whose Instagram tables THIS PROCESS has already brought forward. +_IG_FORWARDED = set() + + +def _ig_forward(session): + """Bring this tenant's Instagram tables onto the current schema, at most once per process. + + ⛔ WHY A MIGRATION RUNS ON A READ AT ALL, when the module's own rule is that it rides the WRITE + path. `ut_ensure` calling it is right for a schema an automation is about to append to, and + useless for a change a PERSON is waiting to see: the owner's report was *"the first field is + still blank"*, and "re-save the automation and it will fix itself" is not an answer to that. + The write path stays exactly as it was — this is a second door to the same idempotent call, + not a replacement for it. + + ⚠ BOUNDED THREE WAYS, because a write on a read is otherwise how a grid gets slow: once per + tenant per process; `migrate_ig_tables` returns without a write when every table is already + current (the common case after the first read); and a failure is SWALLOWED — a migration must + never be the reason a database will not open. + + ⚠ THE TENANT IS MARKED BEFORE THE ATTEMPT, deliberately. A migration that raises must not be + retried on every subsequent read of every table for the life of the process — the write path is + still the backstop, so the cost of skipping is a delay, while the cost of retrying is a failing + store call on the hot path of a grid that is trying to render. + """ + tenant = str(getattr(session, "tenant", "") or "") + if tenant in _IG_FORWARDED: + return + _IG_FORWARDED.add(tenant) + try: + import automation_engine as engine + engine.migrate_ig_tables(session.runtime, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[tables] ig forward-migration skipped: {type(e).__name__}: {e}") + + + +#: Above this many characters a `json` cell is replaced by a stand-in in the LIST envelope. Sized +#: so an ordinary config document (a few hundred bytes) is untouched while a vendor response is +#: not — the shape this exists for is one already-paid provider payload per row. +JSON_LIST_MAX = 400 + + +def _thin_json(fields, merged, table_key=""): + """Replace oversized `json` cells with a stand-in for the LIST response. Pure; returns a copy. + + ⚠ THE STAND-IN IS ITSELF VALID JSON and carries the byte count, so the grid preview reads + `{...} 3 keys` rather than a broken brace, and a reader can see the column holds something + large rather than something empty. `_truncated` is what the viewer keys its fetch on. + + ⭐ THE STAND-IN CARRIES ITS OWN `_url`, which is what keeps this change small AND correct: the + viewer needs no table key, no record id and no new props threaded down through three + components to find the document — the route that removed the value says where it went. One + writer of that address instead of a server rule and a client rule that must agree forever. + """ + json_keys = [str(f.get("key")) for f in (fields or []) + if str(f.get("type") or "") == "json"] + if not json_keys: + return merged + out = {} + for pid, cells in (merged or {}).items(): + row = cells + for key in json_keys: + raw = cells.get(key) + if isinstance(raw, str) and len(raw) > JSON_LIST_MAX: + if row is cells: + row = dict(cells) + row[key] = json.dumps({ + "_truncated": True, "bytes": len(raw), + "_url": f"/api/v1/tables/{table_key}/rows/{pid}/fields/{key}"}) + out[pid] = row + return out + + +@router.get("/tables/{table_key}/rows") +def table_rows(table_key: str, session: Session = Depends(require_session)): + """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at, + identity}` — so the client's generic topic fetch consumes it with zero new parsing. + + ⚠ THE MERGE ORDER IS THE CONTRACT. `rows_from_pool` sources an overlay-typed field's cell + from the OVERLAY stratum only (that is what makes a custom column render standalone) — a + user table's base values live in its DEFINITION rows, so they are layered UNDER the user's + overlay edits here: base first, overlay wins. Without this every base cell reads empty + (found by this route's own gate check, not by luck).""" + import aios_grid + + _ig_forward(session) + g = ut_assembly(session, table_key) + # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge + # this line does not perform since the route was written (owner item 3, 2026-08-09: + # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). + # + # `merged` was built from `rows_src` ALONE — the DEFINITION rows. But `rows_from_pool` + # sources an overlay-typed field's cell from this dict, and a CUSTOM column on a `ut_*` table + # is overlay-typed by construction, so it looked up a key that could not be there and every + # such cell rendered blank. + # + # ⭐ THE WRITES WERE NEVER LOST — MEASURED. `ws['overlays']` holds + # `{"1": {"custom_geography_yf6vi": "Jakarta"}, ...}` for ten rows: the owner's swipes landed + # in the store exactly as they should. Only the READ-BACK dropped them, which is why the + # value survived the gesture, vanished on reload, and looked like "it reset itself" — and + # why `patch_row`'s `_took()` then reported a perfectly good write as `refused`. + # + # ⚠ OVERLAY WINS, base underneath — the order the docstring already specifies. A definition + # value must not shadow an edit the user has made on top of it. + # ⚠ AND IT IS THIS USER'S OWN OVERLAY (`table_workspace` is keyed by `ctx.uname`), so this + # widens what a caller can SEE by exactly their own edits and nothing else. + _overlays = (g.get("ws") or {}).get("overlays") or {} + merged = {} + for _r in g["rows_src"]: + _pid = str(_r["pid"]) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _ov = _overlays.get(_pid) + if isinstance(_ov, dict): + _cells.update(_ov) + merged[_pid] = _cells + # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE + # JSON DOCUMENTS DO NOT RIDE THE LIST. + # + # MEASURED on nurilab, and the numbers are the whole argument: `source_payload` is **95.6% + # to 98.5%** of every IG grid's bytes — `ut_ig_post_snapshots` shipped **11.6 MB of a 12.2 MB + # response**, `ut_ig_snapshots` 1.85 MB of 2.03 MB, the profile table 1.83 MB of 2.11 MB. The + # grid renders those cells as a SIXTY-CHARACTER preview (`display.jsonPreview`), so the whole + # vendor response crossed the wire, was parsed by the browser and held in memory purely so a + # clipped first line could be drawn. + # + # ⚠ IT IS NOT A CAP AND NOTHING IS LOST. The full document is served by + # `GET /tables/{key}/rows/{pid}/fields/{fkey}`, which the JSON viewer fetches when it opens — + # the one place a person actually reads it, for the one row they opened. The cell that rides + # the list is a VALID small document saying what it stands for, so the preview renders + # honestly instead of showing half a truncated brace. + # ⛔ ONLY `json` COLUMNS, and only over the threshold: a small document still travels whole, + # so a tenant using `json` for a short config sees no change at all. + rows = aios_grid.rows_from_pool( + g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) + # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you + # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this + # database says so here, with its cause and the recommendation, rather than waiting to be + # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the + # honest answer for a table nothing limits — never an absent key, which a client cannot tell + # apart from an older server. + _report = _ut().limit_report(table_key, st=session.runtime) + # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the + # only thing that ever set this key; the write door never stopped working and every client + # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a + # missing key has been silently switching the feature off. ⛔ ONE shared serialiser with the + # customer scope — `grid_events.docs_for` — never a twin here. + from core import grid_events as _ge + return {"fields": g["fields"], "rows": rows, "today": g["today"], + "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, + admin=session.admin, st=session.runtime), + "pulled_at": time.strftime("%Y-%m-%d %H:%M"), + "identity": {"pid": "pid"}, + "scope": {"table": table_key, "rowCount": len(rows)}, + "limits": [_report] if _report else [], + "recordsMutable": _ut().records_mutable(table_key, st=session.runtime)} + + +@router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") +def table_cell(table_key: str, pid: str, fkey: str, + session: Session = Depends(require_session)): + """ONE cell, whole — the other half of `_thin_json`. + + ⛔ WITHOUT THIS THE THINNING WOULD BE A CAP, and a cap on data somebody already paid a vendor + for is exactly what this product refuses everywhere else. The list ships a stand-in; the JSON + viewer opens this for the one row a person is actually reading. + + ⚠ SAME PERMISSION WALL AS THE LIST, reached the same way (`ut_assembly` resolves the session's + view of the table), so this cannot become a side door onto a table the caller may not open — + which is the failure a "just fetch the raw cell" helper invites. + ⚠ THE OVERLAY WINS, exactly as it does in `table_rows`: a user who has typed over a cell must + read back what they typed, not the definition value underneath it. + """ + g = ut_assembly(session, table_key) + field = next((f for f in (g.get("fields") or []) if str(f.get("key")) == str(fkey)), None) + if field is None: + raise err(404, "unknown field", f"{fkey!r} is not a column on this database") + row = next((r for r in g["rows_src"] if str(r.get("pid")) == str(pid)), None) + if row is None: + raise err(404, "unknown record", f"no record {pid!r} in this database") + overlay = ((g.get("ws") or {}).get("overlays") or {}).get(str(pid)) or {} + value = overlay.get(fkey, row.get(fkey)) + return {"table": table_key, "pid": str(pid), "field": str(fkey), + "value": "" if value is None else str(value)} + + +@router.post("/tables/{table_key}/rows", status_code=201) +def add_row(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Append a row — or RESTORE one under its old id (contract C-ADDROW / C-UNDO). + + ⚠ THE ANSWER IS ALWAYS THE ID THAT WAS STORED, never the one that was asked for. An undo + that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the + response rather than assume; the client re-anchors on what came back. + """ + _records_or_refuse(session, table_key) + ut = _ut() + values = (body or {}).get("values") or {} + if not isinstance(values, dict): + raise err(400, "bad_values", "values must be an object of {fieldKey: value}") + try: + rid = ut.add_row(table_key, values, session.uname, st=session.runtime, + rid=(body or {}).get("rid")) + except Exception: + raise err(503, "store_unavailable", "the row was not saved — the store refused") + if rid is None: + # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap + # sentence alone would misdirect — the reader would go and count rows. Ask the same + # validator the law used rather than re-deciding here (one rule, two voices). + pf = ut.profile_field(table_key, st=session.runtime) + if pf and pf["key"] in values: + _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) + if not ok: + raise err(400, "refused", + f"{str(values[pf['key']])[:80]!r} is not an Instagram profile — " + f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " + f"profile link (instagram.com/name)") + raise err(400, "refused", + f"row refused — the table may be at its {ut.MAX_ROWS}-row cap") + _refresh_relations(session) + return {"rid": rid, "pid": int(rid)} + + +@router.post("/tables/{table_key}/rows/import", status_code=201) +def import_rows(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. + + Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a + spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, + nothing is matched or overwritten, and the dialog says so before the button is pressed. + + ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate + the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in + an import is not a stray to be tidied away — it means the client offered a target it should + not have, and silently dropping it would leave the user looking for a column of values that + never arrived. The refusal names the column. + + ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every + profile cell validates, because a half-imported file is the worst outcome available: the user + cannot tell which rows landed without reconciling the spreadsheet by hand. + """ + _records_or_refuse(session, table_key) + ut = _ut() + rows_in = (body or {}).get("rows") + if not isinstance(rows_in, list) or not rows_in: + raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") + if any(not isinstance(r, dict) for r in rows_in): + raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") + defn = _defn_or_refuse(session, table_key) + by_key = {f["key"]: f for f in (defn.get("fields") or [])} + asked = {k for r in rows_in for k in r} + unknown = sorted(k for k in asked if k not in by_key) + if unknown: + raise err(400, "unknown_field", + f"this database has no column {unknown[0]!r}") + computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) + if computed: + label = by_key[computed[0]].get("label") or computed[0] + raise err(400, "computed_field", + f"{label!r} is worked out from other columns, so it cannot be imported into") + # ⛔ W29-T81 — THE TYPE WALL, AND IT LIVES HERE BECAUSE THE ONLY OTHER ONE IS IN THE BROWSER. + # `coerceClipboardValue` refuses "seventeen-ish" at an `int` column in the dialog; curl, a + # second client, or a future importer met no wall at all and the string landed verbatim in a + # typed column, where the grid then painted it as a fabricated `0`. Refused whole and BEFORE + # `add_rows`, matching this door's own atomicity: a half-imported file is the outcome every + # rule on this route exists to prevent. The sentence names the row and the column, because + # "invalid value" sends somebody hunting through 2,000 lines of spreadsheet. + for index, row in enumerate(rows_in): + for key, value in row.items(): + why = ut.cell_type_refusal(by_key[key], value) + if why: + raise err(400, "bad_value", f"row {index + 1}: {why} — nothing was imported") + try: + made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "nothing was imported — the store refused") + if made is None: + raise err(400, "refused", + f"nothing was imported — {len(rows_in)} rows would take this database past " + f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") + _refresh_relations(session) + return {"imported": len(made), "pids": [int(r) for r in made]} + + +# --------------------------------------------------------------------------------------------- +# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) +# --------------------------------------------------------------------------------------------- +# R2: a `ut_*` table's fields are the TABLE'S schema — everyone with access sees the same +# columns, the creator or an admin edits them, and a per-field `editRole` can open ONE column's +# definition to everyone without handing over the table. This supersedes wave 17's "fields are +# per-user" law for this path only; the connector scopes keep their own model. +# +# ⚠ WHY IT MATTERS BEYOND TIDINESS: a grid add-field on a `ut_` scope used to land in the +# per-user workspace overlay, which is why the automation editor's "Automation column" picker +# could not see a column the user had just created — it reads the DEFINITION. Same defect shape +# as the rename (item 6a): two places to look, and the surfaces disagreed silently. + +def _field_or_refuse(session, table_key, fkey=""): + """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule.""" + defn = _defn_or_refuse(session, table_key) + ut = _ut() + if not ut.is_user_table(table_key, st=session.runtime): + raise err(400, "not_a_user_table", + "only a user-created database has an editable schema — a connected source " + "owns its own columns") + if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, + st=session.runtime): + field = next((f for f in (defn.get("fields") or []) + if f.get("key") == str(fkey)), None) + if isinstance((field or {}).get("automation"), dict) \ + and field["automation"].get("preset") is True: + # ⚠ THIS BRANCH NARRATES `may_edit_field`, it does not re-decide (the `and not + # ut.may_edit_field` above is the wall). Said out loud because the sentence itself + # went stale on 2026-08-09: preset ROLLUPS became editable by owner ruling, so a + # blanket "pre-set fields are locked" would now be the server explaining a refusal + # it did not make — `preset_editable` is the one predicate that answers this. + raise err(403, "preset_field_locked", + "this is a pre-set column, so its name and type are fixed; you may sort, " + "filter or hide it, edit any Rollup column, and add your own columns") + raise err(403, "forbidden", "that column can only be changed by the database's creator " + "or an admin") + if not fkey and not (session.admin or defn.get("createdBy") == session.uname): + raise err(403, "forbidden", "only the database's creator or an admin can add a column") + return defn + + +@router.post("/tables/{table_key}/fields", status_code=201) +def add_field(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _field_or_refuse(session, table_key) + ut = _ut() + field = ut.add_field(table_key, body or {}, st=session.runtime) + if not field: + # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers + # None for every refusal, so this route said "check the name and type" to somebody whose + # name and type were fine and whose automation column named a flow that does not exist. + # A refusal that misdirects is worse than a bare 400: it sends the reader to look at the + # one thing that was never wrong. Checked HERE, in the route's own words, because the + # law itself stays enforced in `user_tables.flow_bound` — this narrates it, never + # re-implements it (a second copy of the rule is how two doors start disagreeing). + raise err(400, "refused", + _refusal_sentence(ut, session, body or {}, table_key=table_key)) + if field.get("type") == "link": + synced = ut.sync_reciprocal_link(table_key, field["key"], st=session.runtime) + field = synced.get("field") or field + # ⭐⭐ 2026-08-09 — `rollup` REFRESHES TOO, and the omission was invisible until this route + # became reachable for one. It was gated on `link` alone, while `patch_field` and + # `delete_field` next door refresh unconditionally — so a newly created Rollup got its first + # fold from `_store_resync_loop`, which sleeps 1800 s BEFORE its first pass (D-107's shape). + # The user would have created the column, watched a 201 come back, and read a blank cell for + # half an hour: *"the Rollup doesn't work"*, arriving through the door opened to fix it. + # ⚠ Still conditional rather than unconditional: a pass deep-copies every table and row in + # the tenant, and adding a text column has nothing to fold. The condition is now "is this + # field relational", which is the question that was always meant. + if field.get("type") in ("link", "rollup"): + _refresh_relations(session) + return {"field": field} + + +def _refusal_sentence(ut, session, body, table_key="", fkey=""): + """Why was this column refused? The specific reason when we can name one, the general list + otherwise — never a specific-sounding guess.""" + # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is + # what the contract asks for and what makes it actionable — "at most one" sends the reader + # hunting through a 40-column schema for a flag they cannot see from the header. + if isinstance((body or {}).get("profile"), dict): + # ⚠ THE TYPE IS NAMED FIRST, and the order is the point. Both refusals can be true at + # once (an `int` profile column on a table that already has a profile column), and the + # TYPE is the one that is wrong about what the caller just sent — unconditionally, no + # matter what else is on the table. Answering "you already have one" to somebody whose + # real mistake was the column type sends them to fix the wrong thing, which is the + # misdirection D-46 closed one door over. + if str((body or {}).get("type") or "text").strip().lower() != "text": + return ("a profile column is a flag on an ordinary TEXT column — it validates what " + "is typed into it, which it can only do for text") + existing = ut.profile_field(table_key, st=session.runtime) if table_key else None + if existing and existing.get("key") != str(fkey): + return (f"this database already has a profile column — " + f"{existing.get('label') or existing.get('key')!r}. A database has at most " + f"one, so the automation knows which handle to enrich; edit that column, or " + f"take the flag off it first") + bag = (body or {}).get("automation") + if isinstance(bag, dict): + flow = str(bag.get("flowId") or "").strip() + if not flow: + return ("an automation column has to name the automation that fills it — pick a " + "flow, or make this an ordinary column") + if not ut.flow_bound(bag, st=session.runtime): + return (f"this column names automation {flow!r}, which does not exist in this " + f"workspace — it may have been deleted; pick a flow that is still there") + kind = str((body or {}).get("type") or "").strip() + if kind and kind not in ut.UT_FIELD_TYPES: + return (f"{kind!r} is not a column type here (types: " + f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") + return (f"the column was refused — check the name and type, or the table may be at its " + f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") + + +@router.patch("/tables/{table_key}/fields/{fkey}") +def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Edit one column's definition, and MIGRATE its values when options are renamed. + + ⛔ A CHOICE RENAME IS AN EXPLICIT MAPPING, NEVER A DIFF (contract C-RENAME). `{renames: + [{from, to}]}` arrives alongside the new options list, because a diff cannot tell "renamed + Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and + every saved view that filtered on it. + """ + _field_or_refuse(session, table_key, fkey) + ut = _ut() + body = body or {} + migrated = None + renames = body.get("renames") + if renames: + try: + migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "the rename did not land — try again") + # The per-user workspace strata and any view filter naming the old value are the OTHER + # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an + # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a + # missing counterpart must not lose the half that DID land. + try: + import core.table_store as table_store + fn = getattr(table_store, "rename_choice_values", None) + if callable(fn): + fn(table_key, fkey, renames, st=session.runtime) + migrated = dict(migrated or {}, workspace=True) + except Exception: # noqa: BLE001 + migrated = dict(migrated or {}, workspace=False) + field = ut.patch_field(table_key, fkey, body, st=session.runtime) + if not field: + # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as + # adding one, so it must get the same sentence naming the column that already holds the + # flag — a patch that answered "check the name and type" would send the reader to the + # one thing that was never wrong (the D-46 lesson, one door over). + raise err(400, "refused", + _refusal_sentence(ut, session, body, table_key=table_key, fkey=fkey)) + synced = ut.sync_reciprocal_link(table_key, fkey, st=session.runtime) + field = synced.get("field") or field + _refresh_relations(session) + out = {"field": field} + if migrated is not None: + out["migrated"] = migrated + return out + + +@router.delete("/tables/{table_key}/fields/{fkey}") +def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): + _field_or_refuse(session, table_key, fkey) + if not _ut().delete_field(table_key, fkey, st=session.runtime): + raise err(400, "refused", + "that column could not be removed — a database must keep at least one") + _refresh_relations(session) + return {"deleted": fkey} + + +@router.delete("/tables/{table_key}/rows/{rid}") +def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): + # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. + # Owner, verbatim (2026-08-13): *"it still takes forever to delete a record from TT Profile."* + # A single DELETE was FIVE whole-document deep copies before the commit even began — three in + # the guard (`get` + `may_open` + `records_mutable`) and two more inside + # `core.user_tables.delete_row` (`is_user_table` + `records_mutable` again). MEASURED: 2 reads + # cost 45 ms on an 0.8 MB fixture, and tenant #0's `user_tables` document is **28.5 MB** + # (D-185), so the guard alone was seconds of copying to answer questions about one row. + # ⚠ THE LEND IS READ-ONLY AND THE WRITE STILL GOES THROUGH THE REAL RUNTIME — `_Lent` + # `__getattr__`-passes `update` straight to it, and `_drop` runs against the LIVE document + # under the store lock, so a lent snapshot can never be the thing written back. + # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually + # consistent delete is indistinguishable from one that did not work. This makes the guard + # cheap; it does not make the commit optimistic. + lent = _ut().lend(session.runtime) + _records_or_refuse(session, table_key, st=lent) + try: + ok = _ut().delete_row(table_key, rid, st=lent) + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + if not ok: + raise err(400, "refused", "rows can only be deleted from user-created databases") + _refresh_relations(session) + return {"ok": True} + + +@router.patch("/tables/{table_key}/rows/{pid}") +def patch_row(table_key: str, pid: int, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Cell edits — the products PATCH on the user-table ctx. Routed through + `core.grid_events.handle_one` so truncation and permission rules stay ONE implementation; + the accepted values are read BACK from the bucket, never echoed from the request.""" + from core import grid_events + + updates = dict(body or {}) + if not updates: + raise err(400, "empty_patch", "no fields to update") + _records_or_refuse(session, table_key) + g = ut_assembly(session, table_key, consume_corrections=False) + if pid not in g["pids"]: + raise err(403, "out_of_scope", "that row is not in this database") + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, hidden_keys=frozenset(), + st=session.runtime, # R6b (D-16) + scope_key=table_key, table=_ops(session, table_key)) + try: + grid_events.handle_one( + {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", + "pid": pid, "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", + "the tenant store is unavailable — your change was not saved") + # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. + # + # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH + # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. + # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's + # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began + # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` + # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring + # claiming otherwise is what makes the next reader look for a merge bug that is not here. + # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY + # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` + # asks whether THIS write landed, and this write goes to the definition. + stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) + .get(str(pid)) or {}) + accepted = {k: stored.get(k) for k in updates if k in stored} + + def _took(k): + """Did the cell TAKE this write? Normally that is "stored == asked". + + ⚠ A PROFILE COLUMN CANONICALISES, so "stored != asked" is its NORMAL success: `@Nurilab` + and `instagram.com/nurilab` both store `nurilab`. Reporting those as refused would tell + the client to roll back a write that landed. But it cannot simply be exempted either — + a junk handle leaves the OLD value sitting in `stored`, which would then read as + accepted. So the question asked is the exact one: **is what is stored the canonical form + of what was asked?** Anything else is a genuine refusal. + """ + if k not in accepted: + return False + want = str(updates[k]) + if stored.get(k) == want: + return True + pf = _ut().profile_field(table_key, st=session.runtime) + if pf and pf["key"] == k: + handle, ok = _ut().normalize_profile(want, pf["profile"].get("source")) + return bool(ok) and stored.get(k) == handle + return False + + refused = sorted(k for k in updates if not _took(k)) + out = {"pid": pid, "updates": accepted} + if refused: + out["refused"] = refused + # ⭐ R6: the cells the SERVER changed that the client never typed — the preset cells a + # profile blank cleared. Without this the grid keeps painting a stale follower count under + # an empty handle until something else forces a refetch, which is the NO-BLIP LAW's other + # half: the client may keep only what the server actually took, and must be TOLD what else + # moved. Derived by diffing this row against what was asked for, so it cannot drift from + # whatever the clear rule decides to touch. + also = {k: v for k, v in stored.items() if k not in updates and str(v or "") == ""} + cleared = sorted(k for k in also if k in _ut().PROFILE_PRESET_KEYS) + if cleared: + out["cleared"] = cleared + # ⭐⭐ R9's SECOND RE-ARM DOOR — the one call that makes `engine.clear_gone` live (wave 28, + # amendment A5; SESSION B built and gated the function and correctly declared it INERT until + # this line existed, citing [[flag-shipped-without-its-writer]]). + # + # ⛔ R9 makes a `not_found` handle a TOMBSTONE, not a 30-day backoff: nothing re-buys a dead + # account on a timer any more. Door 1 — correcting the handle — needs no wiring, because the + # verdict is keyed on `(platform, handle)` and a corrected handle simply is not the verdict we + # recorded. THIS is door 2: a human re-typing the SAME handle, which is how somebody says "try + # it again, the account is back". Without this call that person has no way back at all, and + # the failure costs nothing and raises nothing — so no spend-shaped test would ever find it. + # + # ⚠ GATED ON `updates`, NOT ON `accepted`: re-typing the identical value is the whole case this + # door exists for, and a no-op write can be filtered out of `accepted`. What matters is that a + # human touched the handle cell. + # ⚠ NOT a bare `except: pass`. A swallowed AttributeError here would be exactly the optional- + # prop silence this wiring exists to prevent — if the engine ever loses `clear_gone`, that must + # be readable in the log rather than degrade into "the re-arm quietly stopped working". + _pf = _ut().profile_field(table_key, st=session.runtime) + if _pf and _pf["key"] in updates: + try: + import automation_engine as _engine + _engine.clear_gone(session.runtime, table_key, stored.get(_pf["key"])) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] clear_gone failed: {type(e).__name__}: {e}") + _refresh_relations(session) + return out diff --git a/jobs/web_agent_job.py b/jobs/web_agent_job.py index 358deb37cc3c541a1e1dee8da0771a3b10e41b2f..43396a0e4907fa63b56988428df4ab5e41df2c6b 100644 --- a/jobs/web_agent_job.py +++ b/jobs/web_agent_job.py @@ -113,21 +113,6 @@ INSTALL_DEPS = os.environ.get("WEB_AGENT_INSTALL_DEPS", "1") != "0" INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0" MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload MAX_STEPS = 20 # matches the engine's MAX_ACTIONS — one flow, one job -#: ⛔ D-193 — HOW OLD A CHECKPOINT MAY BE AND STILL BE SERVED, in hours. **Deliberately SHORT.** -#: The key is content-addressed (`runId` + a hash of the steps), so a scheduled automation with a -#: stable run id hits the same key every time it runs; with no ceiling it would be served the same -#: stored page for as long as the file existed, which is a measurement of a web page presented with -#: today's date on it. Six hours makes a same-morning retry cheap and a next-day run honest. -#: ⚠ A CEILING, NOT A CACHE POLICY. The point of the checkpoint is that a RETRY of a job that -#: already paid for a fetch does not pay again; that window is minutes, not days. -try: - CHECKPOINT_MAX_AGE_H = float(os.environ.get("WEB_AGENT_CHECKPOINT_MAX_AGE_H", "6") or 6) -except (TypeError, ValueError): - CHECKPOINT_MAX_AGE_H = 6.0 -#: How many expired `runs/*.json` one job may delete, in ONE commit. ⛔ Bounded because the repo -#: already spends a commit per run against HF's 256-commits/hr budget, and a sweep that spent one -#: commit per FILE would be a denial of service on the store it is tidying. -CHECKPOINT_SWEEP_MAX = 200 def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None: @@ -177,8 +162,6 @@ def _checkpoint(blob: str, payload: dict) -> None: repo_id=CHECKPOINT_REPO, repo_type="dataset", commit_message=f"web-agent run {run_id}") say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}") - # ⚠ AFTER the write, never before: the sweep is housekeeping and the result is the job. - _sweep_expired() except Exception as exc: # noqa: BLE001 — a boundary # ⛔ CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can # fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not @@ -188,94 +171,8 @@ def _checkpoint(blob: str, payload: dict) -> None: f"the stdout frame still carries the result") -def _too_old(payload: dict) -> str: - """"" if this checkpoint may still be served, else the SENTENCE saying why it may not. - - ⛔ D-193 — A RESUME THAT IS A YEAR OLD IS NOT A RESUME, IT IS A STALE ANSWER SERVED AS FRESH. - The key is deliberately content-addressed (`runId` + a hash of the steps), so a scheduled - automation reusing a stable run id resumed the SAME STORED PAGE for as long as the file - existed. `finishedAt` was written by `_emit` from the first day and read by nothing. - ⚠ AN UNREADABLE OR ABSENT TIMESTAMP IS TREATED AS TOO OLD, not as fresh. A checkpoint written - before this field existed carries no date, and "no date" must not be the one input that gets - served forever — fail toward re-fetching, which costs a job, rather than toward a wrong answer, - which costs a number nobody can explain. - """ - stamp = str(payload.get("finishedAt") or "") - if not stamp: - return "it carries no finishedAt, so its age cannot be established" - try: - when = datetime.fromisoformat(stamp) - if when.tzinfo is None: - when = when.replace(tzinfo=timezone.utc) - except Exception: # noqa: BLE001 - return f"its finishedAt ({stamp[:40]}) is not a date this job can read" - hours = (datetime.now(timezone.utc) - when).total_seconds() / 3600.0 - if hours > CHECKPOINT_MAX_AGE_H: - return (f"it finished {hours:.1f} h ago and the ceiling is {CHECKPOINT_MAX_AGE_H} h " - f"(WEB_AGENT_CHECKPOINT_MAX_AGE_H)") - return "" - - -def _sweep_expired() -> None: - """Delete checkpoints past the ceiling, ONE COMMIT, at most `CHECKPOINT_SWEEP_MAX` of them. - - ⛔ D-193's SECOND CLAUSE, and it is a different problem from the first. Ageing out the READ - stops a stale answer being served; it does not stop the repo growing one file per run forever. - ⛔ ONE COMMIT FOR THE WHOLE SWEEP. `upload_file`/`delete_file` each spend a commit, and this - repo already spends one per run against HF's 256-per-hour budget — a per-file sweep would - exhaust the budget it exists to protect, and the failure would land on the next run's WRITE. - `create_commit` with a list of delete operations is one commit whatever the count. - ⚠ AGE IS READ FROM THE LISTING, not by downloading each file. `last_commit` is what the API - gives cheaply; downloading N files to read a timestamp inside them would make a tidy-up cost - more than the runs it is tidying. - ⚠ REPORT-DON'T-RAISE, like `_checkpoint`: this runs AFTER the result is in hand, and a - housekeeping failure must never destroy a result that has already been paid for. - """ - if not (CHECKPOINT_REPO and HF_TOKEN): - return - try: - from huggingface_hub import CommitOperationDelete, HfApi - api = HfApi(token=HF_TOKEN) - cutoff = datetime.now(timezone.utc).timestamp() - CHECKPOINT_MAX_AGE_H * 3600.0 - doomed = [] - for info in api.list_repo_tree(CHECKPOINT_REPO, repo_type="dataset", path_in_repo="runs", - expand=True, recursive=False): - path = getattr(info, "path", "") - when = getattr(getattr(info, "last_commit", None), "date", None) - if not path.endswith(".json") or when is None: - continue - if when.timestamp() < cutoff: - doomed.append(path) - if len(doomed) >= CHECKPOINT_SWEEP_MAX: - break - if not doomed: - return - api.create_commit(repo_id=CHECKPOINT_REPO, repo_type="dataset", - operations=[CommitOperationDelete(path_in_repo=p) for p in doomed], - commit_message=f"sweep {len(doomed)} checkpoints past " - f"{CHECKPOINT_MAX_AGE_H} h") - say(f"CHECKPOINT swept {len(doomed)} expired run(s) in one commit") - except Exception as exc: # noqa: BLE001 — a boundary - say(f"CHECKPOINT sweep skipped ({type(exc).__name__}: {str(exc)[:160]}) — " - f"the result is unaffected") - - def _resume(run_id: str): - """A prior run's SUCCESSFUL and RECENT result, if one exists. None otherwise. - - ⛔ D-199 — A FAILED RUN'S CHECKPOINT USED TO REPLAY FOREVER. `_fail` checkpoints too - (`_fail` → `_emit` → `_checkpoint`), so the stored envelope for a failure is exactly as - truthy as one for a success; the caller tested `if prior:` and nothing ever looked at `ok`. - Under a stable `runId` a failed fetch could then never be re-attempted — it replayed the - stored failure unchanged, with `resumedFromCheckpoint: True` on it. The defect was LATENT - only because no deployment set `WEB_AGENT_REPO`, i.e. it armed itself on the day somebody - switched checkpointing on, which is the day nobody would be reading this function. - ⚠ THE TEST BELONGS HERE, NOT AT THE CALL SITE. `_resume` is the one place that decides what a - resumable checkpoint IS; a caller-side `if prior and prior.get("ok")` would be a second - definition, and the next caller would be written from the first one. - ⚠ A REFUSED CHECKPOINT SAYS SO IN THE LOG. Silently re-running looks identical to never - having had a checkpoint, and those two need to be distinguishable at 3am. - """ + """A prior run's result, if this exact run already finished. None otherwise.""" if not (CHECKPOINT_REPO and HF_TOKEN and run_id): return None try: @@ -284,21 +181,9 @@ def _resume(run_id: str): filename=f"runs/{run_id}.json", token=HF_TOKEN, force_download=True) with open(path, encoding="utf-8") as fh: - prior = json.load(fh) + return json.load(fh) except Exception: # noqa: BLE001 return None # absent is the normal case - if not isinstance(prior, dict): - say(f"RESUME refused — runs/{run_id}.json is not an object; re-running") - return None - if not prior.get("ok"): - say(f"RESUME refused — the stored run FAILED " - f"({str(prior.get('error') or 'no reason recorded')[:120]}); re-running") - return None - stale = _too_old(prior) - if stale: - say(f"RESUME refused — {stale}; re-running") - return None - return prior # ── the plan ──────────────────────────────────────────────────────────────────────────────── diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 03a7d98d3137781d6895f5af1804d8dfd144fa8c..5728254c4029ef2e883da439dfd1f9757c83e5f0 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -1,2245 +1,2216 @@ -"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. - -This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the -standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit -component. The React bundle reads its data from `window.__AIOS_GRID__` (an object -`{fields, rows}`) that we inject into the page before the app's module script runs -— so there is NO backend and NO /api call in the Streamlit container; the browser only ever -sees the derived JSON we hand it. - -Usage (from any page): - import aios_grid - aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) - -Design notes: - * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and - `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and - sidesteps deploy_hf.py's import guard entirely. - * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to - deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy - dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py). - For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. - * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args - into the React app and returns guarded events for persistence in the tenant store. The old - injected-HTML path remains a read/local-write fallback when component assets are absent. -""" -import json -import re -from pathlib import Path - -_HERE = Path(__file__).resolve().parent - -# Where the inlined single-file build lives. FIRST match wins: -# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) -# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) -_EMBED_CANDIDATES = [ - _HERE / "aios_grid_embed.html", - _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", -] -_COMPONENT_CANDIDATES = [ - _HERE / "aios_grid_component", - _HERE.parent / "aios-web" / "web" / "dist-embed", -] -_DECLARED_COMPONENTS = {} - -# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS -# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is -# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. -# -# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so -# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the -# module preserves that exactly — no import, no guard interaction — while still single-sourcing -# the values (a build-time copy from another tree would reintroduce the drift we're removing). -# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run -# aios-web/verify_fields_contract.py. --- -_FIELDS_PATH = _HERE / "aios_grid_fields.json" - - -def _load_fields(): - if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " - "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." - ) - doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) - fields = doc.get("fields") if isinstance(doc, dict) else doc - if not isinstance(fields, list) or not fields: - raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") - return fields - - -FIELDS = _load_fields() - - -def product_fields(): - """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same - JSON). A separate accessor rather than a second module constant so the one file-read and the - one failure mode stay shared with `FIELDS`.""" - doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) - fields = (doc.get("product_data") or {}).get("fields") - if not isinstance(fields, list) or not fields: - raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.") - return fields - -# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — -# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. - - -def _round(v): - return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v - - -# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors -# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. -# select — a single-select with its own `options` (the "Status" a user wants to add; distinct -# from the Odoo-sourced `status` lifecycle, which is not user-defined) -# user — an assignee, whose choices come from the HOST's real user list, never from here -#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared -#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every -#: member of it (the `multi` grouping/cell contract the Cohorts column established). -#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · -#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level -#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's -#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). -#: ⭐ Wave-19 R7 / contract C5: `image` — a PICTURE on a record. The cell holds a string -#: REFERENCE, never bytes: a product `code`, an `ed:` editorial asset, or `rec:` for -#: something uploaded through the field (`POST /api/v1/assets/records`). Storing a reference is -#: what lets Royal's 1,142 existing SKU masters appear with nothing re-uploaded, and it keeps the -#: overlay stratum the size it is — base64 bytes in a JSON blob read on every render would be a -#: megabyte-per-row tax on the whole store. Editable by NATURE (deliberately NOT in -#: `READONLY_CUSTOM_TYPES`): the ref is what the upload endpoint hands back, and the client PATCHes -#: it through the ordinary overlay wall rather than the asset route writing cells behind it. -#: ⭐ WAVE 23 (C7) — `json` joined: a cell holding a whole DOCUMENT (an Instagram comment thread, -#: a webhook payload, a scraped blob) that opens in its own viewer instead of being flattened -#: into one unreadable line. It is EDITABLE by nature, like `image`: the value is still a plain -#: string on the wire, so it rides the ordinary overlay wall — what makes it a json field is that -#: `grid_events` REFUSES a write that does not parse (a column promising structure must not -#: silently hold something that isn't). -#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They ride here because -#: `UT_FIELD_TYPES` must stay a SUBSET of this set (gated in verify_api's W18-UT section) — the -#: surfaces that CREATE them are the user-table databases, where a relation between two tables is -#: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to -#: point at, so the column menu there simply never offers one. -CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", - "checkbox", "phone", "email", "url", "rating", "created_time", "formula", - "automation", "image", "json", "link", "rollup", "code"} - -#: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. -#: -#: R13 is explicit that this kind is "syntax-highlighted storage + language config, NO execution -#: engine". So a language is a RENDERING hint and nothing else: it selects a highlighter, it never -#: selects an interpreter, and no value here may ever grow a run path. The list is short on -#: purpose — every entry costs a highlighter the client actually has to implement, and an -#: unimplemented language would paint plain text under a label promising colour. -#: -#: `plain` is the default and the fallback, so it is never a second way to say nothing: it is the -#: honest answer for a snippet whose language the user has not chosen. -CODE_LANGUAGES = {"plain", "json", "sql", "python", "javascript", "typescript", - "html", "css", "markdown", "yaml", "xml", "shell"} - - -def _clean_code(raw): - """The `code` field's config bag -> `{'language': ...}`, or None. - - Deliberately OPTIONAL rather than required (the `automation` posture, not `link`'s): a code - column with no declared language is a legitimate state — it stores and highlights as plain - text — so refusing the FIELD over a missing bag would block the ordinary create path. An - unknown language falls back to `plain` rather than refusing, because the value is a rendering - hint: dropping the user's column to punish a typo in a highlighter name would be the - disproportionate half of the fail-closed rule. - - ⚠ `plain` RETURNS NONE, and that is what makes the control reversible rather than one-way. - Absent already means plain, so storing `{'language': 'plain'}` would be the default wearing a - second name (the `kanbanClamp` law). But the patch path resolves an OMITTED key to the - previous value — so if plain were merely omitted by the client, switching a column back from - SQL to Plain text would keep storing SQL and read as a control that does not save. Sending - the bag explicitly and having it evaluate to None here means: omit = keep, plain = clear. - """ - if not isinstance(raw, dict): - return None - lang = str(raw.get('language') or 'plain').strip().lower() - if lang not in CODE_LANGUAGES or lang == 'plain': - return None - return {'language': lang} -#: User-created types whose CELLS are read-only: their values are computed (formula — client -#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). -#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so -#: the client never offers an editor and the host never accepts a cell write for them. -READONLY_CUSTOM_TYPES = {"created_time", "formula"} -MAX_FIELD_OPTIONS = 50 -MAX_FORMULA_LEN = 500 -#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. -RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 - -#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that -#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's -#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata -#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a -#: measure field is READ-ONLY and its values are computed by the host per render. -MEASURE_FIELD_PREFIX = "measure_" -#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). -MEASURE_FIELD_TYPES = {"currency", "int", "pct"} - - -def _clean_options(raw): - """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. - Mirrors types.ts parseOptions — a choice list that means one thing in the picker and - another in the filter dropdown is a column with two vocabularies.""" - out, seen = [], set() - for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: - if not isinstance(v, (str, int, float)) or isinstance(v, bool): - continue - s = str(v).strip()[:120] - if not s or s.lower() in seen: - continue - seen.add(s.lower()) - out.append(s) - return out[:MAX_FIELD_OPTIONS] - - -def _clean_option_colors(raw, options): - """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary.""" - if not isinstance(raw, dict): - return {} - supplied = {} - for label, color in raw.items(): - if not isinstance(label, str) or not isinstance(color, str): - continue - clean = color.strip().upper() - if re.fullmatch(r"#[0-9A-F]{6}", clean): - supplied[label.strip().lower()] = clean - out = {} - for option in options or []: - color = supplied.get(str(option).strip().lower()) - if color: - out[str(option)] = color - return out - - -def _choice_appearance(raw, options): - """Validated select-family appearance. Absent colour toggle means legacy-on.""" - if not isinstance(raw, dict): - return {} - out = {} - if isinstance(raw.get("colorCodeOptions"), bool): - out["colorCodeOptions"] = raw["colorCodeOptions"] - colors = _clean_option_colors(raw.get("optionColors"), options) - if colors: - out["optionColors"] = colors - return out - - -def _clean_rating_max(raw): - """A rating's star count, bounded. Anything unusable is the default, not a refusal — the - field still holds its 1..max integers either way.""" - try: - return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) - except (TypeError, ValueError): - return RATING_MAX_DEFAULT - - -#: The field types a number-style display format may apply to. `formula` is here because its -#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. -_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} - - -def _clean_format(raw, ftype): - """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong - types return None (the property is simply absent). Rendering-only — a format can change how - a value reads, never what it is, which is why this needs no parity gate of its own.""" - if not isinstance(raw, dict): - return None - out = {} - if ftype in _NUMBER_FORMAT_TYPES: - if isinstance(raw.get("thousands"), bool): - out["thousands"] = raw["thousands"] - if raw.get("decimals") is not None: - try: - d = int(raw["decimals"]) - except (TypeError, ValueError): - d = None - if d is not None and 0 <= d <= 4: - out["decimals"] = d - if isinstance(raw.get("abbrev"), bool): - out["abbrev"] = raw["abbrev"] - elif ftype in ("date", "created_time"): - if isinstance(raw.get("time"), bool): - out["time"] = raw["time"] - if raw.get("tz") in ("local", "utc"): - out["tz"] = raw["tz"] - return out or None - - -def _clean_permissions(raw): - """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is - the host handler's business (creator/admin, enforced fail-closed there); this validates - only the shape, like every other property here.""" - if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): - return {"edit": raw["edit"]} - return None - - -#: ⛔ THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is -#: refused by a wall that is supposed to be structural (2026-08-03). -#: -#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client -#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` — CONCATENATE, TEXT, -#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so -#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace` -#: dropped the column on read. Nothing went red — a refused create looks like a quiet failure -#: and a dropped column looks like a column nobody made. -#: -#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now", -#: "OK")` and could not be created through the product UI at all. -#: -#: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar — that is the filter_sql-class -#: drift risk this file's docstring names. The charset says which characters may appear; the -#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote -#: character brings with it. -_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") -_FORMULA_REF = re.compile(r"\{([^{}]*)\}") - - -def _quotes_balanced(s): - """An even number of `"` — the structural half of string support. - - Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it - contributes TWO characters. So a well-formed expression always has an even count and an - unterminated string always has an odd one. What a balanced pair MEANS is the engine's - business, exactly as with parentheses. - """ - return s.count('"') % 2 == 0 - - -#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that -#: silently never runs; `source` names where the run's subject comes from. -AUTOMATION_KINDS = {"instagram_profile"} -AUTOMATION_SOURCES = {"record_url_field", "self"} -MAX_AUTOMATION_SETTINGS = 12 - - -def _clean_automation(raw, valid_keys=None, flow_ids=None): - """Validate an `automation` config bag. Returns the clean dict, or None when there is - nothing valid to store (the column then renders as unconfigured — never invented). - - ⛔ WAVE 22 (contract C8, owner item 5) — NO FIELD WITHOUT A FLOW. With `flow_ids` given - (the WRITE path: grid_events / user_tables pass the tenant's automation-definition ids), - the bag MUST carry a `flowId` naming one of them — absent or naming a deleted flow is - refused, the same fail-closed direction as a formula ref that names no field. With - `flow_ids=None` (the READ path, `fields_from_workspace`) the law is NOT applied: a column - stored before the law must keep projecting — enforcement at read time would vaporise it, - which is the `_clean_formula` write/read split exactly. - """ - if not isinstance(raw, dict): - return None - kind = str(raw.get('kind') or '').strip() - if kind not in AUTOMATION_KINDS: - return None - source = str(raw.get('source') or 'record_url_field').strip() - if source not in AUTOMATION_SOURCES: - source = 'record_url_field' - out = {'kind': kind, 'source': source} - flow = str(raw.get('flowId') or '').strip()[:40] - if flow_ids is not None and (not flow or flow not in flow_ids): - return None - if flow: - out['flowId'] = flow - url_field = str(raw.get('urlField') or '').strip()[:80] - # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time - if url_field and (valid_keys is None or url_field in valid_keys): - out['urlField'] = url_field - settings = {} - for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]: - if isinstance(v, bool) or isinstance(v, (int, float)): - settings[str(k)[:40]] = v - elif isinstance(v, str): - settings[str(k)[:40]] = v[:200] - if settings: - out['settings'] = settings - return out - - -def _clean_formula(raw, valid_keys=None): - """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). - - Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` - refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a - wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a - Python mirror of the grammar: a second engine is the filter_sql-class drift risk. - Structure IS checked — charset, length, balanced parens, BALANCED QUOTES, well-formed - non-empty `{refs}` — and at WRITE time (`valid_keys` given) every ref must name a field this - table has, fail closed. At READ time refs are left alone: a referenced field deleted later - must blank the CELLS, not vaporise the column. - - ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so - `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the - fail-closed direction: the alternative is teaching this function where strings begin and - end, which is the second grammar the paragraph above refuses to write. - """ - if not isinstance(raw, str): - return None - s = raw.strip() - if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): - return None - depth = 0 - for ch in s: - if ch == "(": - depth += 1 - elif ch == ")": - depth -= 1 - if depth < 0: - return None - if depth: - return None - if not _quotes_balanced(s): - return None - refs = _FORMULA_REF.findall(s) - leftover = _FORMULA_REF.sub("", s) - if "{" in leftover or "}" in leftover: # unbalanced / nested braces - return None - if any(not r.strip() for r in refs): # a `{}` ref names nothing - return None - if valid_keys is not None and any(r not in valid_keys for r in refs): - return None - return s - - -def _field_extras(saved, ftype): - """createdBy / permissions / format / scope — the validated passthrough the created strata - share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it - is carried so the client can gate its menus and the handler can enforce against it. - `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler - stamps it at create (cohort page only) and preserves it like createdBy; carried here so the - client can label the field, filtered OUT of other pages by fields_from_workspace.""" - out = {} - who = saved.get("createdBy") - if isinstance(who, str) and who.strip(): - out["createdBy"] = who.strip()[:80] - perms = _clean_permissions(saved.get("permissions")) - if perms: - out["permissions"] = perms - fmt = _clean_format(saved.get("format"), ftype) - if fmt: - out["format"] = fmt - if saved.get("scope") == "cohort": - out["scope"] = "cohort" - corrected_from = saved.get("labelCorrectedFrom") - correction_id = saved.get("labelCorrectionId") - if (isinstance(corrected_from, str) and corrected_from.strip() - and isinstance(correction_id, str) and correction_id.strip()): - out["labelCorrectedFrom"] = corrected_from.strip()[:120] - out["labelCorrectionId"] = correction_id.strip()[:180] - return out - - -#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). -#: -#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the -#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the -#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. -#: Putting it in the canonical contract would mean an always-present column that is empty for -#: everyone else, plus three consumers to keep in step for a value none of them can produce. -COHORT_COLUMN = 'cohorts' - - -def cohort_field(label='Locked views'): - """The derived membership column's descriptor. - - ⚠ WAVE 17 item 14 / C-STR — THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES - NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because - every stored view that shows or groups by this column names it by KEY, and every gate in - two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it - Cohort, but locked instead"), which is a change to what a reader sees — renaming the - identifiers would break saved views to change a word nobody reads. - - `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability - off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay - keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what - stops the column menu calling it a "source field" on that basis. - - ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another - column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains - "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse - than one that is absent. - """ - return { - 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', - 'default': False, 'filterable': False, 'derived': True, 'multi': True, - 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' - 'so grouping by this column puts a customer under EVERY view they are in — the ' - 'group counts therefore add up to more than the record count, which stays the ' - 'number of distinct customers. Read-only: membership changes only by adding or ' - 'removing customers on the locked view itself.', - } - - -def cohort_cells(cohorts, allowed_pids=None): - """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. - - Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, - never merged into the cached pool rows — those are shared across users, and stamping one - user's cohorts onto them would leak the membership to everybody else on the next render. - """ - cells = {} - for c in cohorts or []: - # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY - # cohort they are in, so it cannot also occur inside a name. Cohort names are free text - # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather - # than left to break the split silently — one group called "Q3 calls" and another called - # "west" would be two lists that do not exist. The cost is cosmetic and confined to the - # cell; the Cohort page still shows the name the user typed. - name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() - for pid in c.get('pids') or (): - if allowed_pids is not None and pid not in allowed_pids: - continue - cells.setdefault(pid, []).append(name) - return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} - - -def clean_measure_field(raw, offered): - """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. - - `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — - the same admission the condition builder uses, so a field can only name a measure this - caller could also filter by. Returns the canonical stored shape, or None (fail closed). - - `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: - the client keys editability off `source == 'overlay'` and the host accepts cell writes only - for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` - because the REPLACEMENT is the measure CONDITION with the same measure and window — the - governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. - """ - if not isinstance(raw, dict): - return None - key = str(raw.get("key") or "") - if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: - return None - spec = raw.get("measure") - if not isinstance(spec, dict): - return None - m = (offered or {}).get(spec.get("key")) - if not m: - return None # not admitted for this caller -> fail closed - window = _clean_window(spec.get("window")) - if window is None: - return None # a measure column with no period is not a column - mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" - return { - "key": key, - "label": str(raw.get("label") or m.get("label") or "Measure")[:120], - "type": mtype, - "source": "odoo", - "default": True, - "custom": True, - "derived": True, - "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, - "note": str(raw.get("note") or "")[:2000], - "measure": {"key": str(spec.get("key"))[:80], "window": window}, - } - - -def measure_fields_of(fields): - """The formula-measure columns among `fields` — the ones whose values the host must compute - per render (see `rows_from_pool`'s `derived`).""" - return [f for f in fields or [] if isinstance(f.get("measure"), dict)] - - -def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None): - """Overlay persisted notes/custom fields onto the immutable source-field contract. - - `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and - `default: False`, so it never displaces a column somebody already reads; the Fields menu is - where you turn it on. - - Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), - and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure - field was validated against the caller's measure OFFER when it was written - (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero - app-internal imports and cannot know the offer. A measure that has since become - unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value - time, never to an error. - - `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A - saved def carrying `scope` is emitted only when it matches — so a cohort-specific field - never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees - only unscoped (global) fields; base contract fields are never scoped. - - `fields_base` (wave 16 C-TOPIC) — the canonical contract to overlay onto. Absent = the - CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product - surface passes `product_fields()`. The workspace dict a caller hands in must already be the - matching table object's bucket — this function cannot tell a customer overlay from a - product one, which is exactly why the buckets are separate stores. - """ - saved = dict((workspace or {}).get("fields") or {}) - out = [] - base_fields = fields_base if fields_base is not None else FIELDS - base_keys = {field["key"] for field in base_fields} - for base in base_fields: - meta = saved.get(base["key"]) or {} - field = dict(base) - if isinstance(meta.get("note"), str): - field["note"] = meta["note"][:2000] - # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a - # number or date READS, per user. Rendering-only, so this is the whole acceptance. - fmt = _clean_format(meta.get("format"), base.get("type")) - if fmt: - field["format"] = fmt - # ⭐ W29-T83 — the saved COLUMN SUMMARY, the read half of the write door in - # `grid_events.field_upsert`. Without this the value round-trips into the store and is - # never served back, which looks exactly like a write that never happened - # ([[read-path-cannot-witness-write-path]]). Absent = whatever the contract declares. - if meta.get("agg") in FIELD_AGGS: - field["agg"] = meta["agg"] - # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved - # window/label override. Wave 6 deleted every preset member (the owner's - # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the - # measure_ path's twin for any future preset-carrying contract, and because deleting - # it would silently change what a re-added preset means. - if base.get("preset") and isinstance(base.get("measure"), dict): - saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} - window = _clean_window(saved_measure.get("window")) - if window is not None: - field["measure"] = {"key": base["measure"]["key"], "window": window} - if isinstance(meta.get("label"), str) and meta["label"].strip(): - field["label"] = meta["label"][:120] - out.append(field) - for key, field in saved.items(): - if key in base_keys or not isinstance(field, dict): - continue - if field.get("scope") and field.get("scope") != scope_key: - continue # a cohort-specific field on another page (wave-6 item 9) - if (str(key).startswith(MEASURE_FIELD_PREFIX) - and isinstance(field.get("measure"), dict)): - window = _clean_window(field["measure"].get("window")) - mkey = str(field["measure"].get("key") or "") - if window is None or not mkey: - continue - mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" - out.append({ - "key": str(key)[:80], - "label": str(field.get("label") or "Measure")[:120], - "type": mtype, - "source": "odoo", - "default": bool(field.get("default", True)), - "custom": True, - "derived": True, - "filterable": False, - "agg": "sum" if mtype in ("currency", "int") else None, - "note": str(field.get("note") or "")[:2000], - "measure": {"key": mkey[:80], "window": window}, - **_field_extras(field, mtype), - }) - continue - if not str(key).startswith("custom_"): - continue - ftype = field.get("type") - if ftype not in CUSTOM_FIELD_TYPES: - continue - if ftype in READONLY_CUSTOM_TYPES: - # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort - # column's mechanism (source 'odoo' + derived) so the client never offers an - # editor and the host's overlay-write guard excludes them by construction. - # FILTERABLE since wave 6 (owner item 6): their values live client-side - # (formula computes over the row, created_time renders `_created`), this - # table's counts are client-mode, and the windowed count path never sees - # these tables — so the client engine answers them soundly. - entry = { - "key": str(key)[:80], - "label": str(field.get("label") or "Untitled")[:120], - "type": ftype, - "source": "odoo", - "derived": True, - "filterable": True, - "default": bool(field.get("default", True)), - "custom": True, - "note": str(field.get("note") or "")[:2000], - **_field_extras(field, ftype), - } - if ftype == "formula": - formula = _clean_formula(field.get("formula")) - if formula is None: - continue # a formula field without a formula is nothing - entry["formula"] = formula - out.append(entry) - continue - if field.get("source") != "overlay": - continue - options = (_clean_options(field.get("options")) - if ftype in ("select", "multiselect") else []) - if ftype in ("select", "multiselect") and not options: - # A select with no surviving choices can never hold a value. Dropping the COLUMN - # would lose the user's data; degrading it to text keeps every stored value - # readable and lets them re-add choices. - ftype = "text" - out.append({ - "key": str(key)[:80], - "label": str(field.get("label") or "Untitled")[:120], - "type": ftype, - "source": "overlay", - "default": bool(field.get("default", True)), - "custom": True, - # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a - # picker offering Average against a projection that only passes Sum through is the - # silent half of this feature. - "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, - "note": str(field.get("note") or "")[:2000], - **({"options": options} if ftype in ("select", "multiselect") else {}), - **(_choice_appearance(field, options) - if ftype in ("select", "multiselect") else {}), - # comma-joined SET semantics (the Cohorts column's contract): the row belongs to - # every member, groups count it under each, the toolbar count stays distinct. - **({"multi": True} if ftype == "multiselect" else {}), - **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), - **({"automation": _clean_automation(field.get("automation"))} - if ftype == "automation" and _clean_automation(field.get("automation")) else {}), - # ⭐ WAVE-27 item 13 (R13) — the code column's language rides the wire, because the - # highlighter is chosen per column and the client cannot infer a language from a - # string. Absent = `plain`, which is what an unconfigured code column renders as. - **({"code": _clean_code(field.get("code"))} - if ftype == "code" and _clean_code(field.get("code")) else {}), - **_field_extras(field, ftype), - }) - if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): - # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. - # A user table's field keys are slugged from whatever its creator typed, so a column - # literally called "Cohorts" produces the key `cohorts` — and appending here unguarded - # would put TWO fields with one key on the wire. The client indexes fields by key, so the - # duplicate does not error: it silently paints one column's values under the other's - # header. The user's own column wins; the derived one steps aside rather than shadowing it. - out.append(cohort_field()) - return out - - -def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): - """Map customer_data.pool() dicts -> the API row shape the grid expects: - pid + each Odoo field (numeric fields rounded, text/date passed through) + the - persisted external overlay. Mirrors the standalone API payload contract. - - `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than - reads off the pool row — today just the cohort column. A separate argument from `overlays` - on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is - never written back would make the dict mean two things. - """ - fields = fields or FIELDS - odoo_fields = [field for field in fields if field["source"] == "odoo"] - overlay_fields = [field for field in fields if field["source"] == "overlay"] - derived_keys = [field["key"] for field in fields if field.get("derived")] - overlays = overlays or {} - derived = derived or {} - out = [] - for r in pool_rows: - pid = r.get("pid") - # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the - # `created_time` field type renders, regardless of that field's own key. Not a Field: - # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride - # the same way: the Map VIEW's data, nullable, deliberately not a column. - row = {"pid": pid, "_created": r.get("_created") or "", - "lat": r.get("lat"), "lon": r.get("lon")} - for field in odoo_fields: - k = field["key"] - if field.get("derived"): - continue # not on the pool row — filled from `derived` below - v = r.get(k) - row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) - saved = overlays.get(str(pid), {}) or {} - for field in overlay_fields: - row[field["key"]] = saved.get(field["key"], "") - got = derived.get(pid) or {} - for k in derived_keys: - # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a - # text column is the question somebody will ask of it. - row[k] = got.get(k, "") - out.append(row) - return out - - -# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- -# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are -# VALUE-FREE (they legitimately carry no value and must never be dropped for it). -FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', - 'gt', 'gte', 'lt', 'lte', 'between', 'within', - # Wave 2026-08-02 (C-OPS): RANK operators — evaluated as a SET pass over the - # sibling-filtered domain by the client engine (useVisibleRows). The validator - # accepts them like any op (structural, not semantic); filter_sql REFUSES to - # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg / - # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a - # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100, - # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a - # measure-carrying column they rank the field's own derived values. - 'topN', 'bottomN', 'inTopPct', 'inBottomPct', - 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'} -#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not -#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean -#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. -COHORT_FIELD = '__cohort__' -#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. -#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: -#: a set op reaching a column leaf would fall through the client engine's switch to "no -#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. -COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} -#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: -#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the -#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. -COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} -#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. -MAX_COHORT_IDS = 20 - - -def parse_cohort_ids(value): - """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. - - Comma-separated in one string because `FilterRule.value` is what all four layers persist and - round-trip, and a one-element list is byte-identical to what the single-cohort leaf already - stored — so every shipped view parses with no migration. Safe because a cohort id is built - from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. - """ - out = [] - for raw in ('' if value is None else str(value)).split(','): - cid = raw.strip()[:120] - if not cid or cid in out: - continue - out.append(cid) - if len(out) >= MAX_COHORT_IDS: - break - return out -# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays -# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. -MAX_FILTER_DEPTH = 3 -MAX_FILTER_NODES = 100 # total nodes across the whole tree -MAX_FILTER_SIBLINGS = 50 # per level - - -#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by -#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored -#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already -#: need a gate to prevent. This validates SHAPE only. -WINDOW_MAX_N = 3650 - - -def _clean_window(raw): - """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. - - Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by - `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this - way keeps the grid module reusable and keeps one definition of what "last quarter" means. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if not isinstance(kind, str) or not kind or len(kind) > 40: - return None - out = {'kind': kind} - if raw.get('n') is not None: - try: - out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) - except (TypeError, ValueError): - return None - for side in ('from', 'to'): - if raw.get(side) not in (None, ''): - out[side] = str(raw[side])[:32] - return out - - -def _clean_rhs(raw, valid_keys): - """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. - - SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` - with the measure keys, exactly as it does for the left side), and a measure rhs must carry a - window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if kind not in ('field', 'measure', 'stat'): - return None - if kind == 'stat': - # A STATISTIC carries no column: the population is the comparand. Shape only — which - # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one - # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. - stat = raw.get('stat') - if not isinstance(stat, str) or not stat or len(stat) > 24: - return None - return {'kind': 'stat', 'stat': stat} - col = raw.get('colId') - if col not in valid_keys: - return None - out = {'kind': kind, 'colId': col} - if kind == 'measure': - window = _clean_window(raw.get('window')) - if window is None: - return None # a measure comparand with no period is not a question - out['window'] = window - return out - - -#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 -#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown -#: display means, so it is never stored. -#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to -#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the -#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing -#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live -#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The -#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on -#: READ forever, only ever WRITE 'chart'. -#: ⭐ WAVE-27 item 8 (owner ruling R2, contract C3): 'swipe' — a DECK of the records whose bound -#: single-select is EMPTY, triaged one at a time by swiping left or right into two of that -#: field's options. Landed here FIRST and in the same change as the client registry, which is -#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an -#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view -#: that silently reverts to a grid on the next read. -#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has -#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never -#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — -#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door -#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at -#: anything. The mirror is one name, and it is the half nobody could see was missing because -#: BOTH sides were individually consistent. -#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out -#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law -#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by -#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered -#: one is red too, so the hold cannot outlive its reason the way wave 27's did). -DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', - 'timeseries', 'catalog', 'swipe', 'form'} - -#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at -#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read -#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a -#: legal mode and this function had no branch for the key, so every spec a client sent was dropped -#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. -#: -#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a -#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — -#: so one tenant setting its token to another tenant's value would silently receive that tenant's -#: submissions. The token is minted server-side and lives in a bucket no client write can reach; -#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. -FORM_ACCESS = ('public', 'emails') -MAX_FORM_FIELDS = 60 -MAX_FORM_EMAILS = 200 -MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 -MAX_FORM_EMAIL = 254 - -#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the -#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing -#: from here is a question the builder cannot ask yet; a type wrongly present is a public door -#: writing values the column cannot mean (an `image` with no upload channel, a `json` document -#: typed into a text box, a `link` naming a record id a stranger guessed). -#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: -#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an -#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks -#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather -#: than re-derived ([[one-evaluator-per-question]]). -#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares -#: the two files name-for-name. -FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', - 'checkbox', 'phone', 'email', 'url', 'rating') - -#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in -#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the -#: expensive failure, not an odd address that gets in. -_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') - - -def _clean_form(raw, valid_keys): - """One form spec, fail-closed. Returns None when nothing is configured. - - ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the - builder let somebody arrange these questions, and sorting them by column order would silently - rearrange a live form every time a column was added (`_public_form` states the same rule from - the serving end). - - ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A - swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck - that can never write. A form is a LIST of questions — losing the column behind question three - costs the asker question three, and taking the whole form away because one field was deleted - would be a far larger loss than the one that happened. - """ - if not isinstance(raw, dict): - return None - fields, seen = [], set() - for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: - if k in valid_keys and k not in seen: - seen.add(k) - fields.append(k) - out = {} - if fields: - out['fields'] = fields - # A required flag on a question the form no longer asks is not a rule, it is a trap: the - # submitter can never satisfy it and the sentence names a field they cannot see. - req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] - if req: - out['required'] = req - for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), - ('submitLabel', MAX_FORM_SUBMIT)): - text = str(raw.get(key) or '').strip()[:cap] - if text: - out[key] = text - # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only - # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which - # is why `emails` is what gets written rather than a `public: false`. - if raw.get('access') == 'emails': - out['access'] = 'emails' - emails = [] - for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: - e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] - if _FORM_EMAIL.match(e) and e not in emails: - emails.append(e) - # Kept even while `access` is public: a person toggling the door open to test it and back - # again must not lose the list of people they typed. It is never served publicly. - if emails: - out['emails'] = emails - return out or None - - -#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a -#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same -#: number rather than a second opinion about it: a longer string cannot name a real option, and -#: a SHORTER cap here would silently refuse a binding to a legal one. -MAX_SWIPE_OPTION = 120 - -#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the -#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N / -#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops, -#: so an accepted save reads back byte-identically on both engines. -TS_BUCKETS = {'week', 'month', 'quarter', 'year'} -TS_MAX_LAST_N = 120 -TS_MAX_FIELDS = 12 -MAX_CALENDAR_METRICS = 4 -_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') - -#: C6-CATALOG (wave 18) — the catalog view's vocabulary and caps. types.ts mirrors every name -#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads -#: back identically on both engines. The code budget is the order-sensitive one — see -#: `_clean_catalogs`. -MAX_CATALOGS = 12 -MAX_CATALOG_PAGES = 40 -MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER -CATALOG_PAPERS = {'letter', 'a4', 'tabloid'} -CATALOG_ORIENTATIONS = {'portrait', 'landscape'} -CATALOG_QUALITIES = {'web', 'print'} -CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'} -CATALOG_COLS = (2, 3, 4) -CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80 -CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60 -_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') - -#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts -#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES — the C-DISP byte-identical law. -TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') -TS_MAX_CUSTOM_ROWS = 12 -TS_MAX_STYLES = 200 -#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client -#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole -#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. -_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') - -#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown -#: mode is still rejected rather than accidentally aliased. -_LEGACY_MODES = {'dashboard': 'chart'} - -#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's -#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a -#: kind the client cannot draw is worse than one that does not exist yet. -#: Wave-16 C-CHARTCAP: + 'table' — the group-by aggregate table (by-rep / by-BU / -#: top-customers, the third Sales block shape). Client renderer: DashboardView's -#: GroupTableView over salesParity.tableFromSpec. -CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} -CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} - -#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is -#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is -#: the picker's order; membership tests read it as a tuple perfectly well. -#: -#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` -#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): -#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole -#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum -#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are -#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both -#: would have to be the intersection of what each can express. -#: -#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` -#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, -#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would -#: give the product two words for one operation on two menus a click apart. -#: -#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column -#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. -#: -#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. -#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, -#: and must not grow a second spelling of it. -#: -#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by -#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to -#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. -FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') -MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop -MAX_CHART_TITLE = 60 - -#: Wave-9 I11 (contract C2) — chart customisation, host-validated. -#: -#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: -#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) -#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored -#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — -#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. -CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} -CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} -MAX_AXIS_LABEL = 40 -#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. -CHART_W_RANGE = (1, 12) -CHART_H_RANGE = (120, 800) - - -def _clean_chart(raw, valid_keys): - """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. - - A chart's `y` is optional (absent = count of rows, which is what "how many customers per - state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and - silently keeping it would put an empty card on the dashboard with no way to tell why. - """ - if not isinstance(raw, dict): - return None - kind = raw.get('kind') - if kind not in CHART_KINDS: - return None - x = raw.get('x') - if x not in valid_keys: - return None # dead category ref -> the chart goes, not the board - cid = raw.get('id') - if not isinstance(cid, str) or not cid.strip(): - return None # the client owns chart ids; an unidentified card - # cannot be edited or removed, so it must not persist - out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, - 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} - if raw.get('y') in valid_keys: - out['y'] = raw['y'] - else: - # no measurable column -> the only honest aggregation left is "how many rows" - out['agg'] = 'count' - title = raw.get('title') - if isinstance(title, str) and title.strip(): - out['title'] = title.strip()[:MAX_CHART_TITLE] - - # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── - # `splitBy` is the field whose values become the SERIES. It is deliberately not called - # `colorBy`: that name already means two other things here (`config.colorBy` = row - # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. - if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: - out['splitBy'] = raw['splitBy'] - # Stacking is only a question once there are series to stack, and only for the two kinds - # that can express it. Anywhere else it is dropped rather than stored as a lie the client - # would have to re-decide. - if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: - out['stacked'] = True - if raw.get('palette') in CHART_PALETTES: - out['palette'] = raw['palette'] - - axis = raw.get('axis') - if isinstance(axis, dict): - # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: - # two y-scales on one frame can manufacture any correlation you like by rescaling, and - # the honest alternatives are two charts, small multiples, or indexing to a common base. - # Ruled explicitly in contract C2 against the "Tableau versatility" brief. - clean_axis = {} - for side in ('x', 'y'): - spec = axis.get(side) - if not isinstance(spec, dict): - continue - one = {} - lab = spec.get('label') - if isinstance(lab, str) and lab.strip(): - one['label'] = lab.strip()[:MAX_AXIS_LABEL] - if spec.get('format') in CHART_FORMATS: - one['format'] = spec['format'] - if one: - clean_axis[side] = one - if clean_axis: - out['axis'] = clean_axis - - size = raw.get('size') - if isinstance(size, dict): - one = {} - for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): - try: - one[key] = max(lo, min(hi, int(size[key]))) - except (KeyError, TypeError, ValueError): - pass # a partial size is fine: the client defaults the missing axis - if one: - out['size'] = one - - # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD — the - # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed: - # a category column has no time dimension, and a stored period on it would promise a - # trend the TS channel must refuse. `span` is meaningful only beside `bucket`. - if isinstance(out.get('y'), str) and out['y'].startswith('measure_'): - if raw.get('bucket') in TS_BUCKETS: - out['bucket'] = raw['bucket'] - sp = raw.get('span') - if isinstance(sp, dict): - n = sp.get('lastN') - if (isinstance(n, int) and not isinstance(n, bool) - and 1 <= n <= TS_MAX_LAST_N): - out['span'] = {'lastN': n} - # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something — - # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta - # line). Anything else is a stored claim the renderer would have to re-refuse. - # Mirrors the client's cleanCharts rule key for key. - if raw.get('compare') == 'prior_year' and ( - out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')): - out['compare'] = 'prior_year' - return out - - -def _clean_catalog_page(raw, budget): - """C6-CATALOG — one page of a catalog. Returns `(page | None, codes_spent)`. - - `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a - page and not across the catalog: a product legitimately appears on a gallery page and again - in its section listing, and de-duplicating globally would silently delete the second - appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of - its last pages — never a random scatter, and never a page (a page with no products is a - heading the user can still see and fix). - """ - if not isinstance(raw, dict): - return None, 0 - page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX] - kind = raw.get('kind') - if not page_id or kind not in CATALOG_PAGE_KINDS: - return None, 0 - out = {'id': page_id, 'kind': kind} - for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX), - ('imageCode', CATALOG_CODE_MAX)): - v = raw.get(key) - if isinstance(v, str) and v: - out[key] = v[:cap] - products = raw.get('products') - if isinstance(products, list) and budget > 0: - clean_p, seen_p = [], set() - for c in products: - if not isinstance(c, str) or not c: - continue - c = c[:CATALOG_CODE_MAX] - if c in seen_p: - continue - seen_p.add(c) - clean_p.append(c) - if len(clean_p) >= budget: - break - if clean_p: - out['products'] = clean_p - layout = raw.get('layout') - if isinstance(layout, dict): - clean_l = {} - cols = layout.get('cols') - if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS: - clean_l['cols'] = cols - # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by - # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all). - # So only the opt-OUT is storable for the first two and only the opt-IN for the third — - # a second spelling of a default is how a round trip starts churning. - if layout.get('showPack') is False: - clean_l['showPack'] = False - if layout.get('showColor') is False: - clean_l['showColor'] = False - if layout.get('showPrice') is True: - clean_l['showPrice'] = True - if clean_l: - out['layout'] = clean_l - return out, len(out.get('products') or ()) - - -def _clean_catalogs(raw, valid_keys): - """C6-CATALOG (wave 18) — `display.catalogs`, fail-closed. Returns a list or None. - - A catalog is a PRINT artifact, so the two structural keys that decide how it paginates - (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with - no size is not a smaller catalog, it is an unrenderable one. Everything else follows the - house rules — unknown keys dropped, per-entry drops never cost the neighbours, empty - sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and - an empty one are not two spellings of the same nothing. - - `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU / - pack / colour, and `product_data` carries no pack or colour of its own — the user binds - custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same - split `dateField`/`stackField` already run. - """ - if not isinstance(raw, list): - return None - out = [] - for c in raw: - if len(out) >= MAX_CATALOGS: - break - if not isinstance(c, dict): - continue - cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX] - name = c.get('name') - # An EMPTY name is legal (the user cleared the box and will type again) — an ABSENT one - # is a malformed record. The `tsRows` label rule, same reasoning. - if not cat_id or not isinstance(name, str): - continue - cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]} - cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter' - cat['orientation'] = (c['orientation'] - if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait') - if c.get('quality') in CATALOG_QUALITIES: - cat['quality'] = c['quality'] - brand = c.get('brand') - if isinstance(brand, dict): - clean_b = {} - for k in ('primary', 'accent'): - v = brand.get(k) - if isinstance(v, str) and _HEX6.match(v): - clean_b[k] = v - company = brand.get('company') - if isinstance(company, str) and company: - clean_b['company'] = company[:CATALOG_NAME_MAX] - # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside - # print CSS is exactly the tokens-not-values rule this contract carries. - logo = brand.get('logo') - if isinstance(logo, str) and logo: - clean_b['logo'] = logo[:CATALOG_CODE_MAX] - if clean_b: - cat['brand'] = clean_b - binds = c.get('fields') - if isinstance(binds, dict): - clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price') - if binds.get(k) in valid_keys} - if clean_bind: - cat['fields'] = clean_bind - pages, budget = [], MAX_CATALOG_CODES - raw_pages = c.get('pages') - if isinstance(raw_pages, list): - for p in raw_pages: - if len(pages) >= MAX_CATALOG_PAGES: - break - page, spent = _clean_catalog_page(p, budget) - if page is None: - continue - budget -= spent - pages.append(page) - # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog - # starts in, and dropping the key would make "new" and "corrupt" the same wire value. - cat['pages'] = pages - out.append(cat) - return out or None - - -def _clean_display(raw, valid_keys): - """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. - - `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode - must be a known non-grid mode (grid is the absent default, so storing it would be a second - way to say nothing); every field ref must name a field this table has (a ref to a deleted - field is DROPPED and the client falls back to its per-mode default); unknown keys are - dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family - stack, map a single-select to colour by and a numeric to size by — is the client's - business: it is the only layer that renders them, and a wrong-typed ref degrades to that - surface's default rather than to an error (the `_clean_window` split). - - Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and - dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — - never the whole array, because losing one column should not cost the user a dashboard they - spent time building. - - W33-T45 (contract C1, ruling R5) adds `published` + `publishAccess`. ⚠ THE HALF OF THE ROUND - TRIP THIS FUNCTION CANNOT ENFORCE: `customer-grid/types.ts::cleanDisplay` is a SECOND - normalizer, in the browser, which rebuilds the config key by key on every autosave and drops - anything it does not name. A key accepted here and unknown there dies on the next column - resize — silently, because the client then POSTs the stripped config and the host REPLACES - the stored one. "The host accepts it" is half a round trip; that file is the other half. - """ - if not isinstance(raw, dict): - return None - mode = raw.get('mode') - if mode not in DISPLAY_MODES or mode == 'grid': - return None - # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown - # mode is still rejected rather than accidentally aliased into a real one. Every already - # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. - mode = _LEGACY_MODES.get(mode, mode) - out = {'mode': mode} - for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): - if raw.get(ref) in valid_keys: - out[ref] = raw[ref] - # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── - # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped — the new - # standardized default — so storing True would be a second way to say nothing (the same - # rule that keeps mode:'grid' out of the store). - if raw.get('kanbanClamp') is False: - out['kanbanClamp'] = False - if raw.get('calendarMode') in ('records', 'summary'): - out['calendarMode'] = raw['calendarMode'] - metrics = raw.get('calendarMetrics') - if isinstance(metrics, list): - clean_m, seen_m = [], set() - for m in metrics[:MAX_CALENDAR_METRICS]: - # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the - # user the summary card they configured around it. - if not isinstance(m, dict): - continue - mid = str(m.get('id') or '')[:40] - if (not mid or mid in seen_m or m.get('field') not in valid_keys - or m.get('agg') not in CHART_AGGS): - continue - seen_m.add(mid) - clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']}) - if clean_m: - out['calendarMetrics'] = clean_m - if raw.get('tsBucket') in TS_BUCKETS: - out['tsBucket'] = raw['tsBucket'] - span = raw.get('tsSpan') - if isinstance(span, dict): - clean_span = {} - n = span.get('lastN') - if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N: - clean_span['lastN'] = n - else: - f, t = span.get('from'), span.get('to') - f = f if isinstance(f, str) and _ISO_DAY.match(f) else None - t = t if isinstance(t, str) and _ISO_DAY.match(t) else None - if f and t and f > t: - f, t = t, f - if f: - clean_span['from'] = f - if t: - clean_span['to'] = t - if clean_span: - out['tsSpan'] = clean_span - ts_fields = raw.get('tsFields') - if isinstance(ts_fields, list): - clean_f, seen_f = [], set() - for k in ts_fields[:TS_MAX_FIELDS]: - if k in valid_keys and k not in seen_f: - seen_f.add(k) - clean_f.append(k) - if clean_f: - out['tsFields'] = clean_f - # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ──────────────────────── - deltas = raw.get('tsDeltas') - if isinstance(deltas, list): - clean_d, seen_d = [], set() - for d in deltas: - if d in TS_DELTA_KINDS and d not in seen_d: - seen_d.add(d) - clean_d.append(d) - if clean_d: - out['tsDeltas'] = clean_d - # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the - # literal opt-IN (absent = off) — the kanbanClamp asymmetry, one per direction. - if raw.get('tsGridlines') is False: - out['tsGridlines'] = False - if raw.get('tsSparkline') is True: - out['tsSparkline'] = True - rows = raw.get('tsRows') - if isinstance(rows, list): - clean_r, seen_r = [], set() - for r in rows[:TS_MAX_CUSTOM_ROWS]: - if not isinstance(r, dict): - continue - rid = str(r.get('id') or '')[:40] - r_kind = r.get('kind') - if not rid or rid in seen_r or r_kind not in ('note', 'formula'): - continue - label = r.get('label') - if not isinstance(label, str): - continue # ABSENT label = malformed; an EMPTY one is a legal spacer row - # (GRID's dated asymmetry amendments, 2026-08-02) - one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} - if r_kind == 'formula': - expr = r.get('expr') - if (isinstance(expr, str) and expr.strip() - and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): - one['expr'] = expr - # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row - # would delete the user's label to punish their arithmetic (GRID's dated - # asymmetry amendment; the calendarMetrics per-entry-drop precedent). - seen_r.add(rid) - clean_r.append(one) - if clean_r: - out['tsRows'] = clean_r - styles = raw.get('tsStyles') - if isinstance(styles, dict): - clean_s = {} - for s_key, s_val in styles.items(): - if len(clean_s) >= TS_MAX_STYLES: - break # capped, not truncated silently: the gate names this - if not isinstance(s_key, str) or not s_key or len(s_key) > 96: - continue # key = rowId or "rowId:colKey" — the client's grammar - if not isinstance(s_val, dict): - continue - one = {} - if s_val.get('bold') is True: - one['bold'] = True - if s_val.get('line') is True: - one['line'] = True - if one: - clean_s[s_key] = one - if clean_s: - out['tsStyles'] = clean_s - charts = raw.get('charts') - if isinstance(charts, list): - clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] - # de-dupe by id: two cards sharing an id are one card as far as the client's keyed - # render is concerned, and the second would silently shadow the first - seen, uniq = set(), [] - for c in clean: - if c['id'] in seen: - continue - seen.add(c['id']) - uniq.append(c) - if uniq: - out['charts'] = uniq - # ── C6-CATALOG (wave 18) ───────────────────────────────────────────────────────────── - catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) - if catalogs: - out['catalogs'] = catalogs - # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── - # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that - # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. - # Those are LISTS of independent cards, so losing one entry costs the user one card. This is - # a single three-part BINDING: a swipe view holding a fieldKey with one option, or two - # options and no field, is not a degraded swipe view — it is a deck that can never write - # anything, rendered as though it were configured. Dropping the key entirely puts the view - # back in its honest unconfigured state, which is the one state the client has a UI for. - # - # ⚠ What this CANNOT check, deliberately, and why the client must: whether `fieldKey` names - # a SELECT, and whether the two options are still in that select's vocabulary. `valid_keys` - # is a key set, and the docstring above draws this exact line — "what each mode MEANS ... is - # the client's business". So SwipeView owns three losses this function is blind to (field - # deleted, field retyped away from select, option removed) and must SHOW each one rather - # than fall back to the first option, per the `viewModes.tsx` house rule. - swipe = raw.get('swipe') - if isinstance(swipe, dict): - f_key = swipe.get('fieldKey') - left, right = swipe.get('leftOption'), swipe.get('rightOption') - ok = (f_key in valid_keys - and isinstance(left, str) and isinstance(right, str)) - if ok: - left, right = left.strip()[:MAX_SWIPE_OPTION], right.strip()[:MAX_SWIPE_OPTION] - # Both non-empty, and DISTINCT: one option on both sides is a deck whose two - # gestures do the same thing, which is two spellings of one state (the - # `kanbanClamp` law) wearing a control that promises a choice. - if left and right and left.casefold() != right.casefold(): - out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} - # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. - form = _clean_form(raw.get('form'), valid_keys) - if form: - out['form'] = form - # ── ⭐⭐ W33-T45 / CONTRACT C1 / RULING R5 (owner item 8b): IS THIS INTERFACE PUBLISHED ──── - # - # ⛔ TWO KEYS LIVE HERE AND TWO DELIBERATELY DO NOT. The `published` flag and the sharer's - # `public | password` choice are DISPLAY state — the view says what it is, the client renders - # a badge from it, and it travels with the view like every other key in this dict. The SECRET - # TOKEN and the PASSPHRASE HASH do not: they live in the server-only bucket, exactly as - # `routes_forms.py`'s `TOKENS_KEY` holds the form token. `config.display` is echoed back to - # every user who can open the view, so a token in here is a token published to the audience - # the password was meant to exclude. `_clean_form` above carries the same rule and the same - # reason; this is the second door, not a new one. - # - # ⛔ AND THE COERCION IS FAIL-CLOSED, WHICH IS WHY THIS IS NOT A BARE ALLOWLIST. A plain - # allowlist drops an unrecognised `publishAccess` and KEEPS `published: True` — leaving a - # published view with no stated access, i.e. a third state neither the ruling nor the client - # has a meaning for, on the one key where guessing wrong publishes a tenant's data to the - # open internet. So: a published view ALWAYS carries an access, and anything that is not the - # literal `'public'` reads as `'password'`. The unpublished case stores nothing at all — - # absent means unpublished, and a `published: False` would be the second way to say nothing - # that `kanbanClamp` and `mode: 'grid'` are both here to forbid. - if raw.get('published') is True: - out['published'] = True - out['publishAccess'] = 'public' if raw.get('publishAccess') == 'public' else 'password' - return out - - -#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). -#: -#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each -#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. -#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort -#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated -#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of -#: what a view IS: keeping it out of the view config means duplicating or exporting a view does -#: not drag a folder reference along with it. One map, one home, both surfaces. -FOLDER_SURFACES = {"views", "cohorts"} -MAX_FOLDERS = 60 -MAX_FOLDER_NAME = 80 - -#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. -#: -#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already -#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, -#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the -#: client cannot render can never reach the store. -#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — -#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional -#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set -#: without the matching client geometry: an unknown shape falls back to the default folder mark, -#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. -FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} -#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the -#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. -FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} -FOLDER_ICON_DEFAULT_TONE = "grey" - - -#: Wave-9 I17 (contract C4) — who may EDIT a saved view. -#: -#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today -#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one -#: user's views are invisible to every other user and "collaborative" has nothing to act on -#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable -#: and fail-closed now, so that when shared views land the permission does not need a data -#: migration and no saved view is retro-restricted. It does NOT make anything shared, and -#: nothing in the app currently reads it to grant or deny cross-user access. -#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. -VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} -MAX_VIEW_USERS = 50 - - -def clean_view_permissions(raw, default, known_users=None): - """{edit, users?} — fail-closed on both halves. - - `default` is supplied by the CALLER because it splits by path, and that split is a - permission rule rather than a formatting one: absent on a view that already exists means a - pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a - silent takeaway), while absent on CREATE must be 'personal' (a new view must never be - anyone-can-edit purely by omission). - - `known_users` (when given) is the real account list: an unknown name is DROPPED, and an - 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. - """ - mode = (raw or {}).get('edit') if isinstance(raw, dict) else None - if mode not in VIEW_EDIT_MODES: - mode = default if default in VIEW_EDIT_MODES else 'personal' - if mode != 'users': - return {'edit': mode} - names, seen = [], set() - for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: - u = str(u or '').strip() - if not u or u.lower() in seen: - continue - if known_users is not None and u not in known_users: - continue # fail-closed: a name we cannot resolve grants nothing - seen.add(u.lower()) - names.append(u) - if not names: - return {'edit': 'personal'} # an empty grant is NOT "everyone" - return {'edit': 'users', 'users': names} - - -def clean_folder_icon(raw): - """{shape, tone} or None. Fail-closed on both halves, independently. - - A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather - than losing the icon entirely — losing a user's pick because one half was wrong is the kind - of silent data loss the folder events already avoid elsewhere. - """ - if not isinstance(raw, dict): - return None - shape = raw.get("shape") - if shape not in FOLDER_ICON_SHAPES: - return None - tone = raw.get("tone") - return {"shape": shape, - "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} - - -def clean_folders(raw): - """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" - out = {} - for surface in FOLDER_SURFACES: - items, seen = [], set() - for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: - if not isinstance(f, dict): - continue - fid = str(f.get("id") or "").strip()[:80] - name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] - if not fid or not name or fid in seen: - continue # an unidentified or unnamed folder cannot be shown or edited - seen.add(fid) - try: - order = int(f.get("order", len(items))) - except (TypeError, ValueError): - order = len(items) - row = {"id": fid, "name": name, "order": order} - icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark - if icon: - row["icon"] = icon - items.append(row) - items.sort(key=lambda x: x["order"]) - for i, f in enumerate(items): - f["order"] = i # re-index so `order` is always dense and total - if items: - out[surface] = items - return out - - -#: ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, raised by SESSION C as ASK C-16) — "FILED AT ROOT". -#: -#: ⛔ THE DEFECT IS THAT ROOT WAS REPRESENTED BY *ABSENCE*, AND ABSENCE CANNOT HOLD TWO FACTS. -#: "this arrived by grant and was never filed" and "the receiver deliberately dragged this OUT of -#: the Shared group" were the same stored state — nothing — so the client had to GUESS, and -#: `folders.ts::groupByFolder` guessed "Shared". That is why only folder→folder moves appeared to -#: work: **the root bucket was unreachable for a shared view by construction.** -#: -#: ⚠ A RESERVED FOLDER ID, NOT A NEW FIELD, deliberately. The placement map is `{itemId: folderId}` -#: and every reader on both sides already understands it; a parallel "filedAtRoot" set would be a -#: second source of truth for one question, and the two would disagree the first time one of them -#: was written without the other. This id names no folder BY DESIGN and is therefore exempt from -#: the folder-exists test below — it is the one value that means "no folder, on purpose". -#: ⚠ Spelled `ROOT_FOLDER_ID` on the client (`customer-grid/folders.ts`, C's file). Two spellings -#: of one constant is [[a-constant-two-features-share]]; `verify_folders`/`verify_api` assert they -#: agree rather than a comment asking nicely. -ROOT_PLACEMENT = "__root__" - - -def clean_item_folders(raw, folders, valid_ids): - """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. - - This is what makes a deleted folder's contents fall back to the root rather than vanish: - nothing stores "this item is in no folder", so an unresolvable placement simply disappears - and the item renders at the top level. Same for an item that was deleted elsewhere — its - stale placement can never resurrect it, because the sidebars render ITEMS and consult this - map, never the other way round. - - ⭐⭐ WAVE 32 — THE PARAGRAPH ABOVE STATES THE FEATURE AND THE BUG IN ONE SENTENCE, and it took - owner item 20 to notice they were the same mechanism. *"Nothing stores 'this item is in no - folder', so an unresolvable placement simply disappears"* is exactly right for a DELETED FOLDER - (its contents should fall to the root) and exactly wrong for a SHARED VIEW (falling back means - falling back INTO the Shared group, which is where it started). `ROOT_PLACEMENT` is the value - that survives this function so the second case can be said out loud. - """ - out = {} - for surface in FOLDER_SURFACES: - fids = {f["id"] for f in (folders or {}).get(surface, [])} - ok = {} - for item_id, fid in ((raw or {}).get(surface) or {}).items(): - if not isinstance(item_id, str) or not isinstance(fid, str): - continue - # ⛔ `fid == ROOT_PLACEMENT` FIRST, and it is NOT in `fids` — it names no folder, which - # is the whole point. Without this clause the value is written by `item_move` and - # scrubbed here on the way back out, so the mark would be stored and instantly lost: - # the two halves are ONE change and shipping either alone is worse than shipping - # neither ([[lost-write-looks-like-failed-read]]). - if item_id in (valid_ids or {}).get(surface, ()) and (fid == ROOT_PLACEMENT - or fid in fids): - ok[item_id[:120]] = fid - if ok: - out[surface] = ok - return out - - -def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): - """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). - - Returns a clean tree of leaf conditions ({colId, op, value, value2}) and - groups ({conj, children}). Module-agnostic on purpose: any module embedding - the grid validates its own view state through this one function. - - Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — - the same contract the rest of the view sanitiser follows, so one bad rule can - never cost a user their whole saved view. Depth, per-level width and total - node count are all capped: the tree is re-evaluated for every row on every - render, so an unbounded structure would be a persistent client-side DoS. - Empty groups are dropped (they carry no meaning once persisted). - - `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is - DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a - condition that can only match nothing, so `List is not [deleted]` would show an empty table - forever with no way to tell why. `None` means this host has no cohorts, and then every - cohort leaf is dropped: fail-closed, like every other unknown key. - """ - if budget is None: - budget = [MAX_FILTER_NODES] - out = [] - for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: - if budget[0] <= 0: - break - if not isinstance(node, dict): - continue - if isinstance(node.get('children'), list): # a condition GROUP - if depth >= MAX_FILTER_DEPTH: - continue # too deep -> drop - budget[0] -= 1 - children = clean_filter_tree(node['children'], valid_keys, - depth + 1, budget, cohort_ids) - if children: - out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', - 'children': children}) - continue - if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf - op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) - named = parse_cohort_ids(node.get('value')) - # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT - # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to - # `is none of [A]` would show every row in B under a count nobody would doubt. This - # is the same all-or-nothing the single-cohort leaf already had, extended to a set. - if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): - budget[0] -= 1 - out.append({'colId': COHORT_FIELD, 'op': op, - 'value': ','.join(named), 'value2': ''}) - continue - if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: - budget[0] -= 1 - # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the - # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client - # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently - # stop filtering and show every row instead of the zero-revenue ones. - val, val2 = node.get('value'), node.get('value2') - leaf = {'colId': node['colId'], 'op': node['op'], - 'value': ('' if val is None else str(val))[:500], - 'value2': ('' if val2 is None else str(val2))[:500]} - # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two - # extra members: a stable client-generated `id`, which is how the server's answer - # finds its way back to the condition that asked (positional matching silently - # re-associates every answer the moment a user deletes a condition), and the - # `window`. Emitted ONLY when the input has them — a column condition's cleaned - # shape is unchanged, so every persisted view deserialises byte-identically and - # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by - # exact structural comparison). - rid = node.get('id') - if rid not in (None, ''): - leaf['id'] = str(rid)[:64] - window = _clean_window(node.get('window')) - if window is not None: - leaf['window'] = window - # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted - # only when the input has them, so a plain column condition's cleaned shape is - # byte-identical to what it always was and `clean_filter_tree` stays idempotent - # (verify_filter_engine.py asserts that by exact structural comparison). Drop the - # carry-through and the next autosave silently strips a date condition back to a - # bare comparison against an empty value — i.e. back to INACTIVE. - date_window = _clean_window(node.get('dateWindow')) - if date_window is not None: - leaf['dateWindow'] = date_window - mode = node.get('dateMode') - # SHAPE only. An unrecognised mode survives here and is refused by - # `windows.resolve_anchor`, which returns None and makes the condition match - # NOTHING — the same split as `_clean_window`, and the reason this module can stay - # free of the date vocabulary it would otherwise have to keep in step. - if isinstance(mode, str) and 0 < len(mode) <= 40: - leaf['dateMode'] = mode - rhs = _clean_rhs(node.get('rhs'), valid_keys) - if rhs is not None: - leaf['rhs'] = rhs - out.append(leaf) - return out - - -def _default_view_config(fields): - # ⭐⭐ W30-T41's SERVER HALF (F's ask F-1, answered by D — this file is D's fence). - # - # ⛔ THE SECOND ARM USED TO BE `or field["source"] == "overlay"`, AND IT SWALLOWED THE FIRST - # ONE FOR EVERY CONNECTED COLUMN. `user_tables._clean_field` stamps `source: "overlay"` on - # every `ut_` field, so on an Odoo grid the arm was true for ALL of them and `default: False` - # meant nothing: `odoo_id`, `state`, `customer_link` and `partner_id` opened SHOWN however - # they were declared. The exception had become the rule ([[fallback-that-became-the-rule]]), - # and it is the same predicate `useGridColumns.isDefaultVisible` carried on the client. - # - # ⚠ AND THE TWO HALVES MUST MOVE TOGETHER, which is why this is not cosmetic. `CustomerGrid` - # compares the stored view against its own `defaultViewConfig` by JSON equality; with the - # client fixed (T41) and this left alone, the system view would differ from the client's - # default on every render — a view that looks permanently dirty and autosaves forever, which - # is the failure `verify_filter_engine`'s key-ORDER check exists to prevent, one level down. - # - # ⚠ A USER-CREATED COLUMN IS UNAFFECTED, and that is why the fix is a DELETION rather than a - # carve-out for the four Odoo keys: it carries no `default` key at all, so `is not False` - # keeps it visible. On the main Customer grid exactly one field moves — `notes`, which asks - # to be hidden in its own declaration and was being shown against it. - shown = [field["key"] for field in fields if field.get("default") is not False] - hidden = [field["key"] for field in fields if field["key"] not in shown] - return { - # `filters` is the ROOT of the filter tree: leaf conditions and/or nested - # condition groups ({conj, children}); `filterConj` joins the root level. - "filters": [], "filterConj": "and", - "sorts": [], "groupBy": None, "colorBy": None, - "rowHeightMode": "short", "order": shown + hidden, "visible": shown, - "widths": {}, "memberPids": [], - } - - -#: Wave 17 R1 / C-LOCKV — the `kind` a PROJECTED locked view wears. A cohort is not a separate -#: kind of object any more: it is a saved view whose rows are a hand-curated set. -LOCKED_VIEW_KIND = 'locked' - - -def locked_view_projection(entry, base_config): - """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV). - - ⛔ THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the - view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the - named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do - all the work with no second mechanism. Membership itself is NEVER copied in here — it stays - in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped - collection inside a client-writable `config` is deleted by the next autosave (see the - contract's reason 2). - - ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set: - these views are ordinary in every respect the owner asked for — reorder, folder, sort, - filter, change display mode. The lock mark in the rail is driven by `kind`. - """ - return { - 'id': entry['id'], - 'name': entry.get('name') or entry['id'], - 'kind': LOCKED_VIEW_KIND, - 'config': {**base_config, 'cohortLock': entry['id']}, - } - - -#: ⭐ WAVE-27 item 27 (owner ruling R8) — the IG "Overview" view's CURATED COLUMNS, in the -#: owner's own order: handle, followers, engagement, location, last enriched. -#: -#: Written as candidates rather than as a requirement. The template registry REFUSES a template -#: whose columns the target lacks (`view_templates.missing_columns`) because applying one writes -#: the user's own views and a filter on a missing column silently WIDENS. This view is INJECTED, -#: not applied, and it filters nothing — so the proportionate rule is the opposite one: take the -#: columns the table has, in this order, and skip the rest. An IG database that predates a -#: column simply shows the other four. -#: -#: ⚠ `location_guess` is SESSION B's item-16 column and may not exist yet. That is exactly why -#: this list is intersected rather than asserted: a hard requirement here would make the whole -#: view vanish (or the assembly refuse) on every tenant until B lands, and then appear by -#: surprise. `profile_url` closes the list as the click-through, which is what makes the view -#: usable rather than merely informative. -IG_OVERVIEW_COLUMNS = ('handle', 'full_name', 'followers', 'avg_engagement', - 'location_guess', 'enriched_at', 'profile_url') - -#: The id is PINNED, the `view_templates` discipline: re-assembling must update the same view -#: rather than mint "Overview 2". It also lets a user's own edits overlay it through the saved -#: -config loop below, exactly as a cohort projection does. -IG_OVERVIEW_ID = 'tpl_overview' - -#: How this function recognises an IG preset database WITHOUT importing the engine: two of the -#: profile preset columns is a stronger signal than any single one (a hand-made table could -#: plausibly own a column called `followers`; owning `followers` AND `avg_engagement` AND -#: `handle` is the preset set). `core/` must stay importable without the API layer, so this -#: mirrors `user_tables.PROFILE_PRESET_KEYS` the way that module mirrors the engine's. -_IG_SIGNATURE = ('handle', 'followers', 'avg_engagement') - - -def _ig_overview_view(fields, base): - """R8's curated Overview, or None when this table is not an Instagram one.""" - keys = {f['key'] for f in fields} - if not all(k in keys for k in _IG_SIGNATURE): - return None - visible = [k for k in IG_OVERVIEW_COLUMNS if k in keys] - return { - 'id': IG_OVERVIEW_ID, - 'name': 'Overview', - 'kind': 'system', - # NOT `locked`. The system view is locked because it is the identity of the table ("show - # me everything"); this one is a STARTING LAYOUT, and R8 calls it curated rather than - # fixed. A user who wants a sixth column should get one. - 'note': 'The five things worth seeing first on a creator. Sorted by reach.', - 'config': { - **dict(base), - 'visible': visible, - 'order': visible + [k for k in (f['key'] for f in fields) if k not in visible], - 'sorts': ([{'colId': 'followers', 'dir': 'desc'}] - if 'followers' in keys else []), - }, - } - - -def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None, - view_order=None): - """Convert legacy list formulas into the shared serializable SavedView contract. - - `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the - product surface). The ID stays "all-customers" on every topic — the client pins it - (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork - that contract for a label's sake. - - `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose - id IS the cohort id — so every stored reference to that id (a `cohortLock` on another view, - an `is part of` condition, a folder placement) keeps pointing at the same thing and no - rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the - `list:` views have always used, which is what gives a locked view its own sort, filter, - columns and display mode with no new storage.""" - base = _default_view_config(fields) - views = [{ - "id": "all-customers", "name": system_name, "kind": "system", - "locked": True, "config": dict(base), - }] - # ⭐ WAVE-27 item 27 (R8) — the IG Overview, ABOVE All records. - # - # ⚠ INJECTED, not seeded into the store, and that is what makes "existing IG databases gain - # it too" true with no migration and no write on a read path. It is the same mechanism the - # system view above has always used; the pinned id means a user's own edits overlay it - # through the saved-config loop below rather than forking a second view. - _overview = _ig_overview_view(fields, base) - if _overview: - views.insert(0, _overview) - op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", - "contains": "contains"} - for name, definition in (defs or {}).items(): - filters = [] - for rule in definition.get("rules") or []: - if rule.get("field") not in {field["key"] for field in fields}: - continue - filters.append({ - "colId": rule["field"], - "op": op_map.get(rule.get("op"), "eq"), - "value": str(rule.get("value") if rule.get("value") is not None else ""), - }) - sort = str(definition.get("sort") or "") - sorts = ([{"colId": sort.lstrip("-"), - "dir": "desc" if sort.startswith("-") else "asc"}] - if sort.lstrip("-") in {field["key"] for field in fields} else []) - views.append({ - "id": "list:" + str(name), - "name": str(name), - "kind": "list", - "note": str(definition.get("note") or ""), - "config": { - **base, "filters": filters, "sorts": sorts, - "memberPids": [int(pid) for pid in definition.get("members") or [] - if isinstance(pid, int) or str(pid).isdigit()], - }, - }) - # Wave 17 R1 — the cohorts, as ordinary views. Appended BEFORE the saved-config overlay - # below so a user's own edits to a locked view (its sort, its columns, its display mode) - # land on the projection instead of creating a second row with the same id. - for _entry in (locked_lists or []): - if isinstance(_entry, dict) and _entry.get('id'): - views.append(locked_view_projection(_entry, base)) - index = {view["id"]: i for i, view in enumerate(views)} - for view_id, saved in (saved_views or {}).items(): - if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): - continue - clean = dict(saved) - clean["id"] = str(view_id) - if view_id in index: - views[index[view_id]] = clean - else: - views.append(clean) - # ⛔ WAVE 17 R1 — RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected - # view with its saved record, and a saved record that omits `cohortLock` would hand back a - # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the - # client rebuilds `config` on every autosave (a column resize is enough), and the lock is - # identity here, not something the browser is the source of truth for. Read-side rather than - # write-side-only on purpose — this also repairs any record already written by another path. - _locked_ids = {e['id']: e for e in (locked_lists or []) - if isinstance(e, dict) and e.get('id')} - if _locked_ids: - for _v in views: - _entry = _locked_ids.get(_v.get('id')) - if not _entry: - continue - _v['kind'] = LOCKED_VIEW_KIND - _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']} - # One thing, one name: the cohort store owns it (the rename event routes there), so - # a stale `name` on the saved record can never fork into a second title. - _v['name'] = _entry.get('name') or _v['id'] - # ── ⭐ WAVE-27 item 5, contract C7: the PER-USER VIEW ORDER ─────────────────────────────── - # - # `view_order` is a list of view ids this user dragged into place. Applied LAST, over the - # finished list, so it reorders whatever the assembly produced without having to know how any - # of it got there (system, list:, cohort projection, saved, injected Overview). - # - # ⛔ THE SYSTEM VIEW STAYS AT INDEX 0 (C7), and it is re-pinned here rather than trusted to - # sort correctly: `all-customers` is the client's landing default and one of its - # UNDELETABLE_VIEW_IDS, so a stored order that happened to omit it — or list it third — - # would move the rail's home row. ⚠ R8's Overview is the ONE thing allowed above it, because - # the owner put it there; it is re-pinned with the system view so a drag cannot bury it - # either. Both are facts about the table rather than the user's arrangement of it. - # - # ⚠ UNKNOWN IDS APPEND IN SERVER ORDER (C7). A view created since this order was stored, or - # one shared to this user yesterday, must APPEAR — dropping it would make sharing look - # broken, and the failure would be invisible to whoever shared it. Ids in the stored order - # that no longer resolve are simply skipped. - if view_order: - _rank = {vid: i for i, vid in enumerate(view_order) if isinstance(vid, str)} - _pinned = [v for v in views if v.get('id') in (IG_OVERVIEW_ID, 'all-customers')] - _rest = [v for v in views if v.get('id') not in (IG_OVERVIEW_ID, 'all-customers')] - # A stable sort over a rank that DEFAULTS TO THE END keeps unranked views in their - # server order behind the ranked ones, rather than interleaving them by accident. - _rest.sort(key=lambda v: _rank.get(v.get('id'), len(_rank) + 1)) - views = _pinned + _rest - return views - - -def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, - fields_base=None, with_cohorts=True): - """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE - projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`). - - ⛔ WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no - `storageKey` — and the client validator (`fetchWorkspace`) requires one, so the standalone - shell silently discarded the whole workspace: saved views never rendered and `cohortMode` - never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline - projection into the route would have re-created the same drift one wave later; extracting it - means the wire can only be one thing. - - Returns `(workspace, fields, views, cohort_lists)` — the extra three because the host - interleaves further work (docs, derived cells, measure sets) that consumes them. - - HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`, - `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`). - - Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer, - byte-identical). - - ⭐ WAVE 19 / R9 — `with_cohorts` NO LONGER MEANS "customer only". Wave 16 set it False on the - product surface because cohorts were a single customer-keyed bucket, so resolving them against - product pids would have intersected two unrelated id spaces and printed a plausible, - meaningless member count. `modules.cohort` is scope-parameterized now: the lists come from - THIS topic's bucket (`cohort_mod.scoped(scope_key)`), so their ids are this topic's ids and - the intersection with `pool_pids` is the ordinary one. The flag survives as an honest OFF - switch for a surface that wants no membership channel at all — it is not a topic wall. - """ - import modules.cohort as cohort_mod - - cohort_lists = [] - if with_cohorts: - for cid, c in sorted(cohort_mod.scoped(scope_key).visible(uname, pool_pids).items(), - key=lambda kv: (kv[1].get('name') or '').lower()): - members = [p for p in (c.get('members') or []) if p in pool_pids] - entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members} - # Rule 8b: a member can drop out of the 24-month pool without the cohort being - # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule - # forbids. - missing = len(c.get('members') or []) - len(members) - if missing: - entry['missing'] = missing - cohort_lists.append(entry) - - fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key, - fields_base=fields_base) - views = views_from_defs(defs or {}, ws.get('views'), fields, - # Wave 21 (item 3, R6): the system view's name is TOPIC-DERIVED. A - # user database's default view used to read "All customers" — a - # compiled customer literal minted on every topic, one half of the - # owner's "my new database looks like RI's Customer table". The ID - # stays 'all-customers' everywhere (pinned client+server — the - # client's UNDELETABLE set and the view pin both name it). - system_name=("All products" if scope_key == 'product' - else "All records" - if str(scope_key or '').startswith('ut_') - else "All customers"), - locked_lists=cohort_lists, - # ⭐ WAVE-27 item 5 (C7) — this user's own rail arrangement, from - # their own stratum. Read here rather than sorted by the client so - # the ORDER a request answers with is the order that was stored: - # sorting client-side would make the rail settle after a paint on - # every load, and shared views would land in server order first. - view_order=ws.get('viewOrder')) - workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists} - # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy - # wins when present; this is the server's answer for a FRESH browser, which used to fall - # all the way to the system default view (and whatever display mode was stored on it). - if ws.get('activeViewId'): - workspace['activeViewId'] = str(ws['activeViewId']) - - # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be - # deleted by a path that knows nothing about folders, and the placement map must not outlive - # the thing it points at. - _folders = clean_folders(ws.get('folders')) - _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')} - # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT - # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so - # its folder has to be an ordinary view folder — but rewriting the stored map would be a - # one-shot write that has to be got right once, while this is a projection that is right - # every time it runs. New drags write to `views` anyway (the client only knows that - # surface), so `cohorts` drains on its own and never needs a second pass. - # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged: - # re-parenting somebody's list into a folder that merely shares an id is a worse outcome - # than the list appearing at the root, where it is visible and one drag from home. - _cf = list(_folders.get('cohorts') or []) - if _cf: - _vf = list(_folders.get('views') or []) - _taken = {f['id'] for f in _vf} - _order = len(_vf) - for _f in _cf: - if _f['id'] in _taken: - continue - _vf.append({**_f, 'order': _order}) - _order += 1 - _folders['views'] = _vf - _raw_item_folders = dict(ws.get('itemFolders') or {}) - if _raw_item_folders.get('cohorts'): - # Cohort placements now describe VIEWS (same ids — that is the point of preserving them). - # A placement already stored on the views surface WINS: it is the more recent act. - _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}), - **dict(_raw_item_folders.get('views') or {})} - _placed = clean_item_folders( - _raw_item_folders, _folders, - {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}}) - if _folders.get('views'): - workspace['folders'] = _folders['views'] - # ⛔ `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a - # wire that still described one would invite a second rendering of rows that are now views. - _vplaced = _placed.get('views') or {} - for _v in (views or []): - if isinstance(_v, dict) and _v.get('id') in _vplaced: - _v['folderId'] = _vplaced[_v['id']] - _cplaced = _placed.get('cohorts') or {} - for _c in cohort_lists: - if _c['id'] in _cplaced: - _c['folderId'] = _cplaced[_c['id']] - - # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order, - # re-validated at SERVE time exactly like folders — a field can be deleted by a path - # that knows nothing about this stratum, and a stale key must not outlive its field. - _rl = ws.get('recordLayout') - if isinstance(_rl, dict) and isinstance(_rl.get('order'), list): - _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')} - _order, _seen = [], set() - for _k in _rl['order'][:200]: - _k = str(_k or '') - if _k and _k in _fkeys and _k not in _seen: - _seen.add(_k) - _order.append(_k) - if _order: - workspace['recordLayout'] = {'order': _order} - - return workspace, fields, views, cohort_lists - - -def embed_html_path(): - """The first existing candidate path for the inlined single-file build, or None.""" - for p in _EMBED_CANDIDATES: - if p.is_file(): - return p - return None - - -def scope_counts(shown, matched, total): - """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). - - `matched` and `total` MUST come from their own queries over the whole scope. Never pass - `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would - report the window size as though it were the result size. - - Refuses the shapes that could only be a mistake, because a wrong count here is invisible on - screen (it looks like a smaller dataset, not like an error). - """ - shown, matched, total = int(shown), int(matched), int(total) - if matched > total: - raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " - f"more rows than the scope holds") - if shown > matched: - raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " - f"more rows than the filter matched") - return {"shown": shown, "matched": matched, "total": total, "windowed": True} - - - -# ⛔ EXIT-6 (2026-08-04): `build_html`, `component_dir`, `render` and `_DECLARED_COMPONENTS` WERE -# HERE, and they are gone with Streamlit. They were the EMBED HOST — the path that declared the -# prebuilt bundle as a `streamlit.components.v1` custom component (or injected the single-file -# HTML build as a fallback) so the React grid could be drawn inside a Streamlit page. -# -# THIS MODULE ITSELF SURVIVES, and that distinction is the whole point: `aios_grid.py` is imported -# at 11 sites across `aios-web/api/` plus `harness/semantic.py` — it owns the canonical field -# contract, the workspace wire and the count envelope. Only the ~95 lines that knew about a HOST -# went; the rest never did. Its one and only `import streamlit` lived inside `render`, lazily, and -# left with it. `api/verify_no_streamlit.py` now gates that nothing here re-imports it. -# -# Deleted with them: `aios_grid_embed.html` + `aios_grid_component/index.html` (a 2.1 MB prebuilt -# bundle), `build_embed.py` that produced them, and `deploy_hf.py`'s embed-staleness guard. The -# React app is now served directly by the FastAPI container — there is no twin to keep fresh, so -# the entire class of "the code shipped but the bundle did not" is retired rather than guarded. +"""aios_grid — embed the AIOS React/glide Airtable-style grid inside Streamlit. + +This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the +standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit +component. The React bundle reads its data from `window.__AIOS_GRID__` (an object +`{fields, rows}`) that we inject into the page before the app's module script runs +— so there is NO backend and NO /api call in the Streamlit container; the browser only ever +sees the derived JSON we hand it. + +Usage (from any page): + import aios_grid + aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS) + +Design notes: + * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and + `rows_from_pool` takes already-built rows — so this helper is tenant/module-agnostic and + sidesteps deploy_hf.py's import guard entirely. + * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to + deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy + dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py). + For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly. + * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args + into the React app and returns guarded events for persistence in the tenant store. The old + injected-HTML path remains a read/local-write fallback when component assets are absent. +""" +import json +import re +from pathlib import Path + +_HERE = Path(__file__).resolve().parent + +# Where the inlined single-file build lives. FIRST match wins: +# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space) +# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy) +_EMBED_CANDIDATES = [ + _HERE / "aios_grid_embed.html", + _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html", +] +_COMPONENT_CANDIDATES = [ + _HERE / "aios_grid_component", + _HERE.parent / "aios-web" / "web" / "dist-embed", +] +_DECLARED_COMPONENTS = {} + +# --- the FIELD CONTRACT — loaded from the CANONICAL source `aios_grid_fields.json` in THIS +# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is +# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo. +# +# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so +# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the +# module preserves that exactly — no import, no guard interaction — while still single-sourcing +# the values (a build-time copy from another tree would reintroduce the drift we're removing). +# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run +# aios-web/verify_fields_contract.py. --- +_FIELDS_PATH = _HERE / "aios_grid_fields.json" + + +def _load_fields(): + if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single " + "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)." + ) + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = doc.get("fields") if isinstance(doc, dict) else doc + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.") + return fields + + +FIELDS = _load_fields() + + +def product_fields(): + """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same + JSON). A separate accessor rather than a second module constant so the one file-read and the + one failure mode stay shared with `FIELDS`.""" + doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8")) + fields = (doc.get("product_data") or {}).get("fields") + if not isinstance(fields, list) or not fields: + raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.") + return fields + +# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded — +# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte. + + +def _round(v): + return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + +# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors +# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. +# select — a single-select with its own `options` (the "Status" a user wants to add; distinct +# from the Odoo-sourced `status` lifecycle, which is not user-defined) +# user — an assignee, whose choices come from the HOST's real user list, never from here +#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select — declared +#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every +#: member of it (the `multi` grouping/cell contract the Cohorts column established). +#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') · +#: `phone` / `email` / `url` (text-family with per-type rendering) · `rating` (top-level +#: `max`, SVG stars client-side — never emoji) · `created_time` (READ-ONLY, renders the row's +#: `_created`) · `formula` (READ-ONLY, client-computed from the row's other cells). +#: ⭐ Wave-19 R7 / contract C5: `image` — a PICTURE on a record. The cell holds a string +#: REFERENCE, never bytes: a product `code`, an `ed:` editorial asset, or `rec:` for +#: something uploaded through the field (`POST /api/v1/assets/records`). Storing a reference is +#: what lets Royal's 1,142 existing SKU masters appear with nothing re-uploaded, and it keeps the +#: overlay stratum the size it is — base64 bytes in a JSON blob read on every render would be a +#: megabyte-per-row tax on the whole store. Editable by NATURE (deliberately NOT in +#: `READONLY_CUSTOM_TYPES`): the ref is what the upload endpoint hands back, and the client PATCHes +#: it through the ordinary overlay wall rather than the asset route writing cells behind it. +#: ⭐ WAVE 23 (C7) — `json` joined: a cell holding a whole DOCUMENT (an Instagram comment thread, +#: a webhook payload, a scraped blob) that opens in its own viewer instead of being flattened +#: into one unreadable line. It is EDITABLE by nature, like `image`: the value is still a plain +#: string on the wire, so it rides the ordinary overlay wall — what makes it a json field is that +#: `grid_events` REFUSES a write that does not parse (a column promising structure must not +#: silently hold something that isn't). +#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They ride here because +#: `UT_FIELD_TYPES` must stay a SUBSET of this set (gated in verify_api's W18-UT section) — the +#: surfaces that CREATE them are the user-table databases, where a relation between two tables is +#: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to +#: point at, so the column menu there simply never offers one. +CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", + "checkbox", "phone", "email", "url", "rating", "created_time", "formula", + "automation", "image", "json", "link", "rollup", "code"} + +#: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. +#: +#: R13 is explicit that this kind is "syntax-highlighted storage + language config, NO execution +#: engine". So a language is a RENDERING hint and nothing else: it selects a highlighter, it never +#: selects an interpreter, and no value here may ever grow a run path. The list is short on +#: purpose — every entry costs a highlighter the client actually has to implement, and an +#: unimplemented language would paint plain text under a label promising colour. +#: +#: `plain` is the default and the fallback, so it is never a second way to say nothing: it is the +#: honest answer for a snippet whose language the user has not chosen. +CODE_LANGUAGES = {"plain", "json", "sql", "python", "javascript", "typescript", + "html", "css", "markdown", "yaml", "xml", "shell"} + + +def _clean_code(raw): + """The `code` field's config bag -> `{'language': ...}`, or None. + + Deliberately OPTIONAL rather than required (the `automation` posture, not `link`'s): a code + column with no declared language is a legitimate state — it stores and highlights as plain + text — so refusing the FIELD over a missing bag would block the ordinary create path. An + unknown language falls back to `plain` rather than refusing, because the value is a rendering + hint: dropping the user's column to punish a typo in a highlighter name would be the + disproportionate half of the fail-closed rule. + + ⚠ `plain` RETURNS NONE, and that is what makes the control reversible rather than one-way. + Absent already means plain, so storing `{'language': 'plain'}` would be the default wearing a + second name (the `kanbanClamp` law). But the patch path resolves an OMITTED key to the + previous value — so if plain were merely omitted by the client, switching a column back from + SQL to Plain text would keep storing SQL and read as a control that does not save. Sending + the bag explicitly and having it evaluate to None here means: omit = keep, plain = clear. + """ + if not isinstance(raw, dict): + return None + lang = str(raw.get('language') or 'plain').strip().lower() + if lang not in CODE_LANGUAGES or lang == 'plain': + return None + return {'language': lang} +#: User-created types whose CELLS are read-only: their values are computed (formula — client +#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`). +#: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so +#: the client never offers an editor and the host never accepts a cell write for them. +READONLY_CUSTOM_TYPES = {"created_time", "formula"} +MAX_FIELD_OPTIONS = 50 +MAX_FORMULA_LEN = 500 +#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. +RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 + +#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that +#: IS a measure over a window — `Sales · the last 90 days` as a column. Mirrors the client's +#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata +#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a +#: measure field is READ-ONLY and its values are computed by the host per render. +MEASURE_FIELD_PREFIX = "measure_" +#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range). +MEASURE_FIELD_TYPES = {"currency", "int", "pct"} + + +def _clean_options(raw): + """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. + Mirrors types.ts parseOptions — a choice list that means one thing in the picker and + another in the filter dropdown is a column with two vocabularies.""" + out, seen = [], set() + for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: + if not isinstance(v, (str, int, float)) or isinstance(v, bool): + continue + s = str(v).strip()[:120] + if not s or s.lower() in seen: + continue + seen.add(s.lower()) + out.append(s) + return out[:MAX_FIELD_OPTIONS] + + +def _clean_option_colors(raw, options): + """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary.""" + if not isinstance(raw, dict): + return {} + supplied = {} + for label, color in raw.items(): + if not isinstance(label, str) or not isinstance(color, str): + continue + clean = color.strip().upper() + if re.fullmatch(r"#[0-9A-F]{6}", clean): + supplied[label.strip().lower()] = clean + out = {} + for option in options or []: + color = supplied.get(str(option).strip().lower()) + if color: + out[str(option)] = color + return out + + +def _choice_appearance(raw, options): + """Validated select-family appearance. Absent colour toggle means legacy-on.""" + if not isinstance(raw, dict): + return {} + out = {} + if isinstance(raw.get("colorCodeOptions"), bool): + out["colorCodeOptions"] = raw["colorCodeOptions"] + colors = _clean_option_colors(raw.get("optionColors"), options) + if colors: + out["optionColors"] = colors + return out + + +def _clean_rating_max(raw): + """A rating's star count, bounded. Anything unusable is the default, not a refusal — the + field still holds its 1..max integers either way.""" + try: + return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX)) + except (TypeError, ValueError): + return RATING_MAX_DEFAULT + + +#: The field types a number-style display format may apply to. `formula` is here because its +#: RESULT is a number the client renders; `pct` already renders in points and takes decimals. +_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"} + + +def _clean_format(raw, ftype): + """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong + types return None (the property is simply absent). Rendering-only — a format can change how + a value reads, never what it is, which is why this needs no parity gate of its own.""" + if not isinstance(raw, dict): + return None + out = {} + if ftype in _NUMBER_FORMAT_TYPES: + if isinstance(raw.get("thousands"), bool): + out["thousands"] = raw["thousands"] + if raw.get("decimals") is not None: + try: + d = int(raw["decimals"]) + except (TypeError, ValueError): + d = None + if d is not None and 0 <= d <= 4: + out["decimals"] = d + if isinstance(raw.get("abbrev"), bool): + out["abbrev"] = raw["abbrev"] + elif ftype in ("date", "created_time"): + if isinstance(raw.get("time"), bool): + out["time"] = raw["time"] + if raw.get("tz") in ("local", "utc"): + out["tz"] = raw["tz"] + return out or None + + +def _clean_permissions(raw): + """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is + the host handler's business (creator/admin, enforced fail-closed there); this validates + only the shape, like every other property here.""" + if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"): + return {"edit": raw["edit"]} + return None + + +#: ⛔ THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is +#: refused by a wall that is supposed to be structural (2026-08-03). +#: +#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client +#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` — CONCATENATE, TEXT, +#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so +#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace` +#: dropped the column on read. Nothing went red — a refused create looks like a quiet failure +#: and a dropped column looks like a column nobody made. +#: +#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now", +#: "OK")` and could not be created through the product UI at all. +#: +#: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar — that is the filter_sql-class +#: drift risk this file's docstring names. The charset says which characters may appear; the +#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote +#: character brings with it. +_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") +_FORMULA_REF = re.compile(r"\{([^{}]*)\}") + + +def _quotes_balanced(s): + """An even number of `"` — the structural half of string support. + + Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it + contributes TWO characters. So a well-formed expression always has an even count and an + unterminated string always has an odd one. What a balanced pair MEANS is the engine's + business, exactly as with parentheses. + """ + return s.count('"') % 2 == 0 + + +#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that +#: silently never runs; `source` names where the run's subject comes from. +AUTOMATION_KINDS = {"instagram_profile"} +AUTOMATION_SOURCES = {"record_url_field", "self"} +MAX_AUTOMATION_SETTINGS = 12 + + +def _clean_automation(raw, valid_keys=None, flow_ids=None): + """Validate an `automation` config bag. Returns the clean dict, or None when there is + nothing valid to store (the column then renders as unconfigured — never invented). + + ⛔ WAVE 22 (contract C8, owner item 5) — NO FIELD WITHOUT A FLOW. With `flow_ids` given + (the WRITE path: grid_events / user_tables pass the tenant's automation-definition ids), + the bag MUST carry a `flowId` naming one of them — absent or naming a deleted flow is + refused, the same fail-closed direction as a formula ref that names no field. With + `flow_ids=None` (the READ path, `fields_from_workspace`) the law is NOT applied: a column + stored before the law must keep projecting — enforcement at read time would vaporise it, + which is the `_clean_formula` write/read split exactly. + """ + if not isinstance(raw, dict): + return None + kind = str(raw.get('kind') or '').strip() + if kind not in AUTOMATION_KINDS: + return None + source = str(raw.get('source') or 'record_url_field').strip() + if source not in AUTOMATION_SOURCES: + source = 'record_url_field' + out = {'kind': kind, 'source': source} + flow = str(raw.get('flowId') or '').strip()[:40] + if flow_ids is not None and (not flow or flow not in flow_ids): + return None + if flow: + out['flowId'] = flow + url_field = str(raw.get('urlField') or '').strip()[:80] + # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time + if url_field and (valid_keys is None or url_field in valid_keys): + out['urlField'] = url_field + settings = {} + for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]: + if isinstance(v, bool) or isinstance(v, (int, float)): + settings[str(k)[:40]] = v + elif isinstance(v, str): + settings[str(k)[:40]] = v[:200] + if settings: + out['settings'] = settings + return out + + +def _clean_formula(raw, valid_keys=None): + """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9). + + Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}` + refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell — never a + wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a + Python mirror of the grammar: a second engine is the filter_sql-class drift risk. + Structure IS checked — charset, length, balanced parens, BALANCED QUOTES, well-formed + non-empty `{refs}` — and at WRITE time (`valid_keys` given) every ref must name a field this + table has, fail closed. At READ time refs are left alone: a referenced field deleted later + must blank the CELLS, not vaporise the column. + + ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so + `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the + fail-closed direction: the alternative is teaching this function where strings begin and + end, which is the second grammar the paragraph above refuses to write. + """ + if not isinstance(raw, str): + return None + s = raw.strip() + if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s): + return None + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return None + if depth: + return None + if not _quotes_balanced(s): + return None + refs = _FORMULA_REF.findall(s) + leftover = _FORMULA_REF.sub("", s) + if "{" in leftover or "}" in leftover: # unbalanced / nested braces + return None + if any(not r.strip() for r in refs): # a `{}` ref names nothing + return None + if valid_keys is not None and any(r not in valid_keys for r in refs): + return None + return s + + +def _field_extras(saved, ftype): + """createdBy / permissions / format / scope — the validated passthrough the created strata + share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it + is carried so the client can gate its menus and the handler can enforce against it. + `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific — the handler + stamps it at create (cohort page only) and preserves it like createdBy; carried here so the + client can label the field, filtered OUT of other pages by fields_from_workspace.""" + out = {} + who = saved.get("createdBy") + if isinstance(who, str) and who.strip(): + out["createdBy"] = who.strip()[:80] + perms = _clean_permissions(saved.get("permissions")) + if perms: + out["permissions"] = perms + fmt = _clean_format(saved.get("format"), ftype) + if fmt: + out["format"] = fmt + if saved.get("scope") == "cohort": + out["scope"] = "cohort" + corrected_from = saved.get("labelCorrectedFrom") + correction_id = saved.get("labelCorrectionId") + if (isinstance(corrected_from, str) and corrected_from.strip() + and isinstance(correction_id, str) and correction_id.strip()): + out["labelCorrectedFrom"] = corrected_from.strip()[:120] + out["labelCorrectionId"] = correction_id.strip()[:180] + return out + + +#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27). +#: +#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the +#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the +#: caller has cohorts — the same condition under which the `__cohort__` FILTER field is offered. +#: Putting it in the canonical contract would mean an always-present column that is empty for +#: everyone else, plus three consumers to keep in step for a value none of them can produce. +COHORT_COLUMN = 'cohorts' + + +def cohort_field(label='Locked views'): + """The derived membership column's descriptor. + + ⚠ WAVE 17 item 14 / C-STR — THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES + NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because + every stored view that shows or groups by this column names it by KEY, and every gate in + two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it + Cohort, but locked instead"), which is a change to what a reader sees — renaming the + identifiers would break saved views to change a word nobody reads. + + `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability + off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay + keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what + stops the column menu calling it a "source field" on that basis. + + ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another + column. Text ops over a joined string would ALMOST work and disagree at the edges — `contains + "VIP"` also matches a cohort called "VIP club" — and a filter that is nearly right is worse + than one that is absent. + """ + return { + 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo', + 'default': False, 'filterable': False, 'derived': True, 'multi': True, + 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, ' + 'so grouping by this column puts a customer under EVERY view they are in — the ' + 'group counts therefore add up to more than the record count, which stays the ' + 'number of distinct customers. Read-only: membership changes only by adding or ' + 'removing customers on the locked view itself.', + } + + +def cohort_cells(cohorts, allowed_pids=None): + """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`. + + Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`, + never merged into the cached pool rows — those are shared across users, and stamping one + user's cohorts onto them would leak the membership to everybody else on the next render. + """ + cells = {} + for c in cohorts or []: + # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY + # cohort they are in, so it cannot also occur inside a name. Cohort names are free text + # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather + # than left to break the split silently — one group called "Q3 calls" and another called + # "west" would be two lists that do not exist. The cost is cosmetic and confined to the + # cell; the Cohort page still shows the name the user typed. + name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip() + for pid in c.get('pids') or (): + if allowed_pids is not None and pid not in allowed_pids: + continue + cells.setdefault(pid, []).append(name) + return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()} + + +def clean_measure_field(raw, offered): + """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER. + + `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` — + the same admission the condition builder uses, so a field can only name a measure this + caller could also filter by. Returns the canonical stored shape, or None (fail closed). + + `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused: + the client keys editability off `source == 'overlay'` and the host accepts cell writes only + for overlay keys, so a measure column cannot be typed into at either end. `filterable:False` + because the REPLACEMENT is the measure CONDITION with the same measure and window — the + governed, gate-proved path (CG-8/CG-12) — not text ops over a derived cell. + """ + if not isinstance(raw, dict): + return None + key = str(raw.get("key") or "") + if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80: + return None + spec = raw.get("measure") + if not isinstance(spec, dict): + return None + m = (offered or {}).get(spec.get("key")) + if not m: + return None # not admitted for this caller -> fail closed + window = _clean_window(spec.get("window")) + if window is None: + return None # a measure column with no period is not a column + mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency" + return { + "key": key, + "label": str(raw.get("label") or m.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": True, + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(raw.get("note") or "")[:2000], + "measure": {"key": str(spec.get("key"))[:80], "window": window}, + } + + +def measure_fields_of(fields): + """The formula-measure columns among `fields` — the ones whose values the host must compute + per render (see `rows_from_pool`'s `derived`).""" + return [f for f in fields or [] if isinstance(f.get("measure"), dict)] + + +def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None): + """Overlay persisted notes/custom fields onto the immutable source-field contract. + + `cohorts=True` appends the derived cohort column — see `cohort_field`. Appended LAST and + `default: False`, so it never displaces a column somebody already reads; the Fields menu is + where you turn it on. + + Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable), + and `measure_` formula-measure fields (read-only, host-computed — owner item 7). A measure + field was validated against the caller's measure OFFER when it was written + (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero + app-internal imports and cannot know the offer. A measure that has since become + unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value + time, never to an error. + + `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A + saved def carrying `scope` is emitted only when it matches — so a cohort-specific field + never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees + only unscoped (global) fields; base contract fields are never scoped. + + `fields_base` (wave 16 C-TOPIC) — the canonical contract to overlay onto. Absent = the + CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product + surface passes `product_fields()`. The workspace dict a caller hands in must already be the + matching table object's bucket — this function cannot tell a customer overlay from a + product one, which is exactly why the buckets are separate stores. + """ + saved = dict((workspace or {}).get("fields") or {}) + out = [] + base_fields = fields_base if fields_base is not None else FIELDS + base_keys = {field["key"] for field in base_fields} + for base in base_fields: + meta = saved.get(base["key"]) or {} + field = dict(base) + if isinstance(meta.get("note"), str): + field["note"] = meta["note"][:2000] + # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) — how a + # number or date READS, per user. Rendering-only, so this is the whole acceptance. + fmt = _clean_format(meta.get("format"), base.get("type")) + if fmt: + field["format"] = fmt + # ⭐ W29-T83 — the saved COLUMN SUMMARY, the read half of the write door in + # `grid_events.field_upsert`. Without this the value round-trips into the store and is + # never served back, which looks exactly like a write that never happened + # ([[read-path-cannot-witness-write-path]]). Absent = whatever the contract declares. + if meta.get("agg") in FIELD_AGGS: + field["agg"] = meta["agg"] + # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved + # window/label override. Wave 6 deleted every preset member (the owner's + # no-buildable-presets rule) so this branch is currently MEMBERLESS — kept as the + # measure_ path's twin for any future preset-carrying contract, and because deleting + # it would silently change what a re-added preset means. + if base.get("preset") and isinstance(base.get("measure"), dict): + saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {} + window = _clean_window(saved_measure.get("window")) + if window is not None: + field["measure"] = {"key": base["measure"]["key"], "window": window} + if isinstance(meta.get("label"), str) and meta["label"].strip(): + field["label"] = meta["label"][:120] + out.append(field) + for key, field in saved.items(): + if key in base_keys or not isinstance(field, dict): + continue + if field.get("scope") and field.get("scope") != scope_key: + continue # a cohort-specific field on another page (wave-6 item 9) + if (str(key).startswith(MEASURE_FIELD_PREFIX) + and isinstance(field.get("measure"), dict)): + window = _clean_window(field["measure"].get("window")) + mkey = str(field["measure"].get("key") or "") + if window is None or not mkey: + continue + mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Measure")[:120], + "type": mtype, + "source": "odoo", + "default": bool(field.get("default", True)), + "custom": True, + "derived": True, + "filterable": False, + "agg": "sum" if mtype in ("currency", "int") else None, + "note": str(field.get("note") or "")[:2000], + "measure": {"key": mkey[:80], "window": window}, + **_field_extras(field, mtype), + }) + continue + if not str(key).startswith("custom_"): + continue + ftype = field.get("type") + if ftype not in CUSTOM_FIELD_TYPES: + continue + if ftype in READONLY_CUSTOM_TYPES: + # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort + # column's mechanism (source 'odoo' + derived) so the client never offers an + # editor and the host's overlay-write guard excludes them by construction. + # FILTERABLE since wave 6 (owner item 6): their values live client-side + # (formula computes over the row, created_time renders `_created`), this + # table's counts are client-mode, and the windowed count path never sees + # these tables — so the client engine answers them soundly. + entry = { + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "odoo", + "derived": True, + "filterable": True, + "default": bool(field.get("default", True)), + "custom": True, + "note": str(field.get("note") or "")[:2000], + **_field_extras(field, ftype), + } + if ftype == "formula": + formula = _clean_formula(field.get("formula")) + if formula is None: + continue # a formula field without a formula is nothing + entry["formula"] = formula + out.append(entry) + continue + if field.get("source") != "overlay": + continue + options = (_clean_options(field.get("options")) + if ftype in ("select", "multiselect") else []) + if ftype in ("select", "multiselect") and not options: + # A select with no surviving choices can never hold a value. Dropping the COLUMN + # would lose the user's data; degrading it to text keeps every stored value + # readable and lets them re-add choices. + ftype = "text" + out.append({ + "key": str(key)[:80], + "label": str(field.get("label") or "Untitled")[:120], + "type": ftype, + "source": "overlay", + "default": bool(field.get("default", True)), + "custom": True, + # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a + # picker offering Average against a projection that only passes Sum through is the + # silent half of this feature. + "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, + "note": str(field.get("note") or "")[:2000], + **({"options": options} if ftype in ("select", "multiselect") else {}), + **(_choice_appearance(field, options) + if ftype in ("select", "multiselect") else {}), + # comma-joined SET semantics (the Cohorts column's contract): the row belongs to + # every member, groups count it under each, the toolbar count stays distinct. + **({"multi": True} if ftype == "multiselect" else {}), + **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}), + **({"automation": _clean_automation(field.get("automation"))} + if ftype == "automation" and _clean_automation(field.get("automation")) else {}), + # ⭐ WAVE-27 item 13 (R13) — the code column's language rides the wire, because the + # highlighter is chosen per column and the client cannot infer a language from a + # string. Absent = `plain`, which is what an unconfigured code column renders as. + **({"code": _clean_code(field.get("code"))} + if ftype == "code" and _clean_code(field.get("code")) else {}), + **_field_extras(field, ftype), + }) + if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): + # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. + # A user table's field keys are slugged from whatever its creator typed, so a column + # literally called "Cohorts" produces the key `cohorts` — and appending here unguarded + # would put TWO fields with one key on the wire. The client indexes fields by key, so the + # duplicate does not error: it silently paints one column's values under the other's + # header. The user's own column wins; the derived one steps aside rather than shadowing it. + out.append(cohort_field()) + return out + + +def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): + """Map customer_data.pool() dicts -> the API row shape the grid expects: + pid + each Odoo field (numeric fields rounded, text/date passed through) + the + persisted external overlay. Mirrors the standalone API payload contract. + + `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than + reads off the pool row — today just the cohort column. A separate argument from `overlays` + on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is + never written back would make the dict mean two things. + """ + fields = fields or FIELDS + odoo_fields = [field for field in fields if field["source"] == "odoo"] + overlay_fields = [field for field in fields if field["source"] == "overlay"] + derived_keys = [field["key"] for field in fields if field.get("derived")] + overlays = overlays or {} + derived = derived or {} + out = [] + for r in pool_rows: + pid = r.get("pid") + # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the + # `created_time` field type renders, regardless of that field's own key. Not a Field: + # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride + # the same way: the Map VIEW's data, nullable, deliberately not a column. + row = {"pid": pid, "_created": r.get("_created") or "", + "lat": r.get("lat"), "lon": r.get("lon")} + for field in odoo_fields: + k = field["key"] + if field.get("derived"): + continue # not on the pool row — filled from `derived` below + v = r.get(k) + row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) + saved = overlays.get(str(pid), {}) or {} + for field in overlay_fields: + row[field["key"]] = saved.get(field["key"], "") + got = derived.get(pid) or {} + for k in derived_keys: + # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a + # text column is the question somebody will ask of it. + row[k] = got.get(k, "") + out.append(row) + return out + + +# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) --------------- +# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are +# VALUE-FREE (they legitimately carry no value and must never be dropped for it). +FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty', + 'gt', 'gte', 'lt', 'lte', 'between', 'within', + # Wave 2026-08-02 (C-OPS): RANK operators — evaluated as a SET pass over the + # sibling-filtered domain by the client engine (useVisibleRows). The validator + # accepts them like any op (structural, not semantic); filter_sql REFUSES to + # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg / + # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a + # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100, + # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a + # measure-carrying column they rank the field's own derived values. + 'topN', 'bottomN', 'inTopPct', 'inBottomPct', + 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'} +#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not +#: a Field and never will be — a cohort is a hand-curated SET, so making it a column would mean +#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD. +COHORT_FIELD = '__cohort__' +#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts. +#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS — see types.ts COHORT_OPS: +#: a set op reaching a column leaf would fall through the client engine's switch to "no +#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. +COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} +#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: +#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the +#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. +COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'} +#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS. +MAX_COHORT_IDS = 20 + + +def parse_cohort_ids(value): + """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`. + + Comma-separated in one string because `FilterRule.value` is what all four layers persist and + round-trip, and a one-element list is byte-identical to what the single-cohort leaf already + stored — so every shipped view parses with no migration. Safe because a cohort id is built + from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one. + """ + out = [] + for raw in ('' if value is None else str(value)).split(','): + cid = raw.strip()[:120] + if not cid or cid in out: + continue + out.append(cid) + if len(out) >= MAX_COHORT_IDS: + break + return out +# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays +# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this. +MAX_FILTER_DEPTH = 3 +MAX_FILTER_NODES = 100 # total nodes across the whole tree +MAX_FILTER_SIBLINGS = 50 # per level + + +#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by +#: design, so it does NOT know the window VOCABULARY — `harness/windows.py` owns that, mirrored +#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already +#: need a gate to prevent. This validates SHAPE only. +WINDOW_MAX_N = 3650 + + +def _clean_window(raw): + """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window. + + Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by + `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this + way keeps the grid module reusable and keeps one definition of what "last quarter" means. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if not isinstance(kind, str) or not kind or len(kind) > 40: + return None + out = {'kind': kind} + if raw.get('n') is not None: + try: + out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N)) + except (TypeError, ValueError): + return None + for side in ('from', 'to'): + if raw.get(side) not in (None, ''): + out[side] = str(raw[side])[:32] + return out + + +def _clean_rhs(raw, valid_keys): + """CG-9 — validate `{kind, colId, window?}`, the "compare against another attribute" side. + + SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys` + with the measure keys, exactly as it does for the left side), and a measure rhs must carry a + window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in ('field', 'measure', 'stat'): + return None + if kind == 'stat': + # A STATISTIC carries no column: the population is the comparand. Shape only — which + # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one + # is REFUSED there rather than guessed at, exactly like an unrecognised window kind. + stat = raw.get('stat') + if not isinstance(stat, str) or not stat or len(stat) > 24: + return None + return {'kind': 'stat', 'stat': stat} + col = raw.get('colId') + if col not in valid_keys: + return None + out = {'kind': kind, 'colId': col} + if kind == 'measure': + window = _clean_window(raw.get('window')) + if window is None: + return None # a measure comparand with no period is not a question + out['window'] = window + return out + + +#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8 +#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown +#: display means, so it is never stored. +#: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to +#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the +#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing +#: 'dashboard' here would silently downgrade every already-saved chart view to grid — and live +#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The +#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on +#: READ forever, only ever WRITE 'chart'. +#: ⭐ WAVE-27 item 8 (owner ruling R2, contract C3): 'swipe' — a DECK of the records whose bound +#: single-select is EMPTY, triaged one at a time by swiping left or right into two of that +#: field's options. Landed here FIRST and in the same change as the client registry, which is +#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an +#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view +#: that silently reverts to a grid on the next read. +#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has +#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never +#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — +#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door +#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at +#: anything. The mirror is one name, and it is the half nobody could see was missing because +#: BOTH sides were individually consistent. +#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out +#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law +#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by +#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered +#: one is red too, so the hold cannot outlive its reason the way wave 27's did). +DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', + 'timeseries', 'catalog', 'swipe', 'form'} + +#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at +#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read +#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a +#: legal mode and this function had no branch for the key, so every spec a client sent was dropped +#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. +#: +#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a +#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — +#: so one tenant setting its token to another tenant's value would silently receive that tenant's +#: submissions. The token is minted server-side and lives in a bucket no client write can reach; +#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. +FORM_ACCESS = ('public', 'emails') +MAX_FORM_FIELDS = 60 +MAX_FORM_EMAILS = 200 +MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 +MAX_FORM_EMAIL = 254 + +#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the +#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing +#: from here is a question the builder cannot ask yet; a type wrongly present is a public door +#: writing values the column cannot mean (an `image` with no upload channel, a `json` document +#: typed into a text box, a `link` naming a record id a stranger guessed). +#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: +#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an +#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks +#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather +#: than re-derived ([[one-evaluator-per-question]]). +#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares +#: the two files name-for-name. +FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', + 'checkbox', 'phone', 'email', 'url', 'rating') + +#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in +#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the +#: expensive failure, not an odd address that gets in. +_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') + + +def _clean_form(raw, valid_keys): + """One form spec, fail-closed. Returns None when nothing is configured. + + ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the + builder let somebody arrange these questions, and sorting them by column order would silently + rearrange a live form every time a column was added (`_public_form` states the same rule from + the serving end). + + ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A + swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck + that can never write. A form is a LIST of questions — losing the column behind question three + costs the asker question three, and taking the whole form away because one field was deleted + would be a far larger loss than the one that happened. + """ + if not isinstance(raw, dict): + return None + fields, seen = [], set() + for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: + if k in valid_keys and k not in seen: + seen.add(k) + fields.append(k) + out = {} + if fields: + out['fields'] = fields + # A required flag on a question the form no longer asks is not a rule, it is a trap: the + # submitter can never satisfy it and the sentence names a field they cannot see. + req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] + if req: + out['required'] = req + for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), + ('submitLabel', MAX_FORM_SUBMIT)): + text = str(raw.get(key) or '').strip()[:cap] + if text: + out[key] = text + # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only + # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which + # is why `emails` is what gets written rather than a `public: false`. + if raw.get('access') == 'emails': + out['access'] = 'emails' + emails = [] + for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: + e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] + if _FORM_EMAIL.match(e) and e not in emails: + emails.append(e) + # Kept even while `access` is public: a person toggling the door open to test it and back + # again must not lose the list of people they typed. It is never served publicly. + if emails: + out['emails'] = emails + return out or None + + +#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a +#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same +#: number rather than a second opinion about it: a longer string cannot name a real option, and +#: a SHORTER cap here would silently refuse a binding to a legal one. +MAX_SWIPE_OPTION = 120 + +#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the +#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N / +#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops, +#: so an accepted save reads back byte-identically on both engines. +TS_BUCKETS = {'week', 'month', 'quarter', 'year'} +TS_MAX_LAST_N = 120 +TS_MAX_FIELDS = 12 +MAX_CALENDAR_METRICS = 4 +_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$') + +#: C6-CATALOG (wave 18) — the catalog view's vocabulary and caps. types.ts mirrors every name +#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads +#: back identically on both engines. The code budget is the order-sensitive one — see +#: `_clean_catalogs`. +MAX_CATALOGS = 12 +MAX_CATALOG_PAGES = 40 +MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER +CATALOG_PAPERS = {'letter', 'a4', 'tabloid'} +CATALOG_ORIENTATIONS = {'portrait', 'landscape'} +CATALOG_QUALITIES = {'web', 'print'} +CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'} +CATALOG_COLS = (2, 3, 4) +CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80 +CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60 +_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') + +#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts +#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES — the C-DISP byte-identical law. +TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') +TS_MAX_CUSTOM_ROWS = 12 +TS_MAX_STYLES = 200 +#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client +#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole +#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. +_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') + +#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown +#: mode is still rejected rather than accidentally aliased. +_LEGACY_MODES = {'dashboard': 'chart'} + +#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's +#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a +#: kind the client cannot draw is worse than one that does not exist yet. +#: Wave-16 C-CHARTCAP: + 'table' — the group-by aggregate table (by-rep / by-BU / +#: top-customers, the third Sales block shape). Client renderer: DashboardView's +#: GroupTableView over salesParity.tableFromSpec. +CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} +CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} + +#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is +#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is +#: the picker's order; membership tests read it as a tuple perfectly well. +#: +#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` +#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): +#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole +#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum +#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are +#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both +#: would have to be the intersection of what each can express. +#: +#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` +#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, +#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would +#: give the product two words for one operation on two menus a click apart. +#: +#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column +#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. +#: +#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. +#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, +#: and must not grow a second spelling of it. +#: +#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by +#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to +#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. +FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') +MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop +MAX_CHART_TITLE = 60 + +#: Wave-9 I11 (contract C2) — chart customisation, host-validated. +#: +#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex: +#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity) +#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored +#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable — +#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying. +CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'} +CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'} +MAX_AXIS_LABEL = 40 +#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px. +CHART_W_RANGE = (1, 12) +CHART_H_RANGE = (120, 800) + + +def _clean_chart(raw, valid_keys): + """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn. + + A chart's `y` is optional (absent = count of rows, which is what "how many customers per + state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and + silently keeping it would put an empty card on the dashboard with no way to tell why. + """ + if not isinstance(raw, dict): + return None + kind = raw.get('kind') + if kind not in CHART_KINDS: + return None + x = raw.get('x') + if x not in valid_keys: + return None # dead category ref -> the chart goes, not the board + cid = raw.get('id') + if not isinstance(cid, str) or not cid.strip(): + return None # the client owns chart ids; an unidentified card + # cannot be edited or removed, so it must not persist + out = {'id': cid.strip()[:64], 'kind': kind, 'x': x, + 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'} + if raw.get('y') in valid_keys: + out['y'] = raw['y'] + else: + # no measurable column -> the only honest aggregation left is "how many rows" + out['agg'] = 'count' + title = raw.get('title') + if isinstance(title, str) and title.strip(): + out['title'] = title.strip()[:MAX_CHART_TITLE] + + # ── wave-9 I11 (contract C2): customisation ──────────────────────────────────────────── + # `splitBy` is the field whose values become the SERIES. It is deliberately not called + # `colorBy`: that name already means two other things here (`config.colorBy` = row + # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable. + if raw.get('splitBy') in valid_keys and raw['splitBy'] != x: + out['splitBy'] = raw['splitBy'] + # Stacking is only a question once there are series to stack, and only for the two kinds + # that can express it. Anywhere else it is dropped rather than stored as a lie the client + # would have to re-decide. + if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True: + out['stacked'] = True + if raw.get('palette') in CHART_PALETTES: + out['palette'] = raw['palette'] + + axis = raw.get('axis') + if isinstance(axis, dict): + # ⛔ ONE y-scale, always. There is no second-axis key here and there must never be: + # two y-scales on one frame can manufacture any correlation you like by rescaling, and + # the honest alternatives are two charts, small multiples, or indexing to a common base. + # Ruled explicitly in contract C2 against the "Tableau versatility" brief. + clean_axis = {} + for side in ('x', 'y'): + spec = axis.get(side) + if not isinstance(spec, dict): + continue + one = {} + lab = spec.get('label') + if isinstance(lab, str) and lab.strip(): + one['label'] = lab.strip()[:MAX_AXIS_LABEL] + if spec.get('format') in CHART_FORMATS: + one['format'] = spec['format'] + if one: + clean_axis[side] = one + if clean_axis: + out['axis'] = clean_axis + + size = raw.get('size') + if isinstance(size, dict): + one = {} + for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)): + try: + one[key] = max(lo, min(hi, int(size[key]))) + except (KeyError, TypeError, ValueError): + pass # a partial size is fine: the client defaults the missing axis + if one: + out['size'] = one + + # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD — the + # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed: + # a category column has no time dimension, and a stored period on it would promise a + # trend the TS channel must refuse. `span` is meaningful only beside `bucket`. + if isinstance(out.get('y'), str) and out['y'].startswith('measure_'): + if raw.get('bucket') in TS_BUCKETS: + out['bucket'] = raw['bucket'] + sp = raw.get('span') + if isinstance(sp, dict): + n = sp.get('lastN') + if (isinstance(n, int) and not isinstance(n, bool) + and 1 <= n <= TS_MAX_LAST_N): + out['span'] = {'lastN': n} + # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something — + # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta + # line). Anything else is a stored claim the renderer would have to re-refuse. + # Mirrors the client's cleanCharts rule key for key. + if raw.get('compare') == 'prior_year' and ( + out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')): + out['compare'] = 'prior_year' + return out + + +def _clean_catalog_page(raw, budget): + """C6-CATALOG — one page of a catalog. Returns `(page | None, codes_spent)`. + + `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a + page and not across the catalog: a product legitimately appears on a gallery page and again + in its section listing, and de-duplicating globally would silently delete the second + appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of + its last pages — never a random scatter, and never a page (a page with no products is a + heading the user can still see and fix). + """ + if not isinstance(raw, dict): + return None, 0 + page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX] + kind = raw.get('kind') + if not page_id or kind not in CATALOG_PAGE_KINDS: + return None, 0 + out = {'id': page_id, 'kind': kind} + for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX), + ('imageCode', CATALOG_CODE_MAX)): + v = raw.get(key) + if isinstance(v, str) and v: + out[key] = v[:cap] + products = raw.get('products') + if isinstance(products, list) and budget > 0: + clean_p, seen_p = [], set() + for c in products: + if not isinstance(c, str) or not c: + continue + c = c[:CATALOG_CODE_MAX] + if c in seen_p: + continue + seen_p.add(c) + clean_p.append(c) + if len(clean_p) >= budget: + break + if clean_p: + out['products'] = clean_p + layout = raw.get('layout') + if isinstance(layout, dict): + clean_l = {} + cols = layout.get('cols') + if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS: + clean_l['cols'] = cols + # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by + # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all). + # So only the opt-OUT is storable for the first two and only the opt-IN for the third — + # a second spelling of a default is how a round trip starts churning. + if layout.get('showPack') is False: + clean_l['showPack'] = False + if layout.get('showColor') is False: + clean_l['showColor'] = False + if layout.get('showPrice') is True: + clean_l['showPrice'] = True + if clean_l: + out['layout'] = clean_l + return out, len(out.get('products') or ()) + + +def _clean_catalogs(raw, valid_keys): + """C6-CATALOG (wave 18) — `display.catalogs`, fail-closed. Returns a list or None. + + A catalog is a PRINT artifact, so the two structural keys that decide how it paginates + (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with + no size is not a smaller catalog, it is an unrenderable one. Everything else follows the + house rules — unknown keys dropped, per-entry drops never cost the neighbours, empty + sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and + an empty one are not two spellings of the same nothing. + + `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU / + pack / colour, and `product_data` carries no pack or colour of its own — the user binds + custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same + split `dateField`/`stackField` already run. + """ + if not isinstance(raw, list): + return None + out = [] + for c in raw: + if len(out) >= MAX_CATALOGS: + break + if not isinstance(c, dict): + continue + cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX] + name = c.get('name') + # An EMPTY name is legal (the user cleared the box and will type again) — an ABSENT one + # is a malformed record. The `tsRows` label rule, same reasoning. + if not cat_id or not isinstance(name, str): + continue + cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]} + cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter' + cat['orientation'] = (c['orientation'] + if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait') + if c.get('quality') in CATALOG_QUALITIES: + cat['quality'] = c['quality'] + brand = c.get('brand') + if isinstance(brand, dict): + clean_b = {} + for k in ('primary', 'accent'): + v = brand.get(k) + if isinstance(v, str) and _HEX6.match(v): + clean_b[k] = v + company = brand.get('company') + if isinstance(company, str) and company: + clean_b['company'] = company[:CATALOG_NAME_MAX] + # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside + # print CSS is exactly the tokens-not-values rule this contract carries. + logo = brand.get('logo') + if isinstance(logo, str) and logo: + clean_b['logo'] = logo[:CATALOG_CODE_MAX] + if clean_b: + cat['brand'] = clean_b + binds = c.get('fields') + if isinstance(binds, dict): + clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price') + if binds.get(k) in valid_keys} + if clean_bind: + cat['fields'] = clean_bind + pages, budget = [], MAX_CATALOG_CODES + raw_pages = c.get('pages') + if isinstance(raw_pages, list): + for p in raw_pages: + if len(pages) >= MAX_CATALOG_PAGES: + break + page, spent = _clean_catalog_page(p, budget) + if page is None: + continue + budget -= spent + pages.append(page) + # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog + # starts in, and dropping the key would make "new" and "corrupt" the same wire value. + cat['pages'] = pages + out.append(cat) + return out or None + + +def _clean_display(raw, valid_keys): + """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed. + + `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` — mode + must be a known non-grid mode (grid is the absent default, so storing it would be a second + way to say nothing); every field ref must name a field this table has (a ref to a deleted + field is DROPPED and the client falls back to its per-mode default); unknown keys are + dropped. What each mode MEANS — calendar wants a date-family field, kanban a select-family + stack, map a single-select to colour by and a numeric to size by — is the client's + business: it is the only layer that renders them, and a wrong-typed ref degrades to that + surface's default rather than to an error (the `_clean_window` split). + + Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and + dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — + never the whole array, because losing one column should not cost the user a dashboard they + spent time building. + """ + if not isinstance(raw, dict): + return None + mode = raw.get('mode') + if mode not in DISPLAY_MODES or mode == 'grid': + return None + # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown + # mode is still rejected rather than accidentally aliased into a real one. Every already + # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. + mode = _LEGACY_MODES.get(mode, mode) + out = {'mode': mode} + for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── + # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped — the new + # standardized default — so storing True would be a second way to say nothing (the same + # rule that keeps mode:'grid' out of the store). + if raw.get('kanbanClamp') is False: + out['kanbanClamp'] = False + if raw.get('calendarMode') in ('records', 'summary'): + out['calendarMode'] = raw['calendarMode'] + metrics = raw.get('calendarMetrics') + if isinstance(metrics, list): + clean_m, seen_m = [], set() + for m in metrics[:MAX_CALENDAR_METRICS]: + # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the + # user the summary card they configured around it. + if not isinstance(m, dict): + continue + mid = str(m.get('id') or '')[:40] + if (not mid or mid in seen_m or m.get('field') not in valid_keys + or m.get('agg') not in CHART_AGGS): + continue + seen_m.add(mid) + clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']}) + if clean_m: + out['calendarMetrics'] = clean_m + if raw.get('tsBucket') in TS_BUCKETS: + out['tsBucket'] = raw['tsBucket'] + span = raw.get('tsSpan') + if isinstance(span, dict): + clean_span = {} + n = span.get('lastN') + if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N: + clean_span['lastN'] = n + else: + f, t = span.get('from'), span.get('to') + f = f if isinstance(f, str) and _ISO_DAY.match(f) else None + t = t if isinstance(t, str) and _ISO_DAY.match(t) else None + if f and t and f > t: + f, t = t, f + if f: + clean_span['from'] = f + if t: + clean_span['to'] = t + if clean_span: + out['tsSpan'] = clean_span + ts_fields = raw.get('tsFields') + if isinstance(ts_fields, list): + clean_f, seen_f = [], set() + for k in ts_fields[:TS_MAX_FIELDS]: + if k in valid_keys and k not in seen_f: + seen_f.add(k) + clean_f.append(k) + if clean_f: + out['tsFields'] = clean_f + # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ──────────────────────── + deltas = raw.get('tsDeltas') + if isinstance(deltas, list): + clean_d, seen_d = [], set() + for d in deltas: + if d in TS_DELTA_KINDS and d not in seen_d: + seen_d.add(d) + clean_d.append(d) + if clean_d: + out['tsDeltas'] = clean_d + # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the + # literal opt-IN (absent = off) — the kanbanClamp asymmetry, one per direction. + if raw.get('tsGridlines') is False: + out['tsGridlines'] = False + if raw.get('tsSparkline') is True: + out['tsSparkline'] = True + rows = raw.get('tsRows') + if isinstance(rows, list): + clean_r, seen_r = [], set() + for r in rows[:TS_MAX_CUSTOM_ROWS]: + if not isinstance(r, dict): + continue + rid = str(r.get('id') or '')[:40] + r_kind = r.get('kind') + if not rid or rid in seen_r or r_kind not in ('note', 'formula'): + continue + label = r.get('label') + if not isinstance(label, str): + continue # ABSENT label = malformed; an EMPTY one is a legal spacer row + # (GRID's dated asymmetry amendments, 2026-08-02) + one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} + if r_kind == 'formula': + expr = r.get('expr') + if (isinstance(expr, str) and expr.strip() + and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): + one['expr'] = expr + # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row + # would delete the user's label to punish their arithmetic (GRID's dated + # asymmetry amendment; the calendarMetrics per-entry-drop precedent). + seen_r.add(rid) + clean_r.append(one) + if clean_r: + out['tsRows'] = clean_r + styles = raw.get('tsStyles') + if isinstance(styles, dict): + clean_s = {} + for s_key, s_val in styles.items(): + if len(clean_s) >= TS_MAX_STYLES: + break # capped, not truncated silently: the gate names this + if not isinstance(s_key, str) or not s_key or len(s_key) > 96: + continue # key = rowId or "rowId:colKey" — the client's grammar + if not isinstance(s_val, dict): + continue + one = {} + if s_val.get('bold') is True: + one['bold'] = True + if s_val.get('line') is True: + one['line'] = True + if one: + clean_s[s_key] = one + if clean_s: + out['tsStyles'] = clean_s + charts = raw.get('charts') + if isinstance(charts, list): + clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c] + # de-dupe by id: two cards sharing an id are one card as far as the client's keyed + # render is concerned, and the second would silently shadow the first + seen, uniq = set(), [] + for c in clean: + if c['id'] in seen: + continue + seen.add(c['id']) + uniq.append(c) + if uniq: + out['charts'] = uniq + # ── C6-CATALOG (wave 18) ───────────────────────────────────────────────────────────── + catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) + if catalogs: + out['catalogs'] = catalogs + # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── + # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that + # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. + # Those are LISTS of independent cards, so losing one entry costs the user one card. This is + # a single three-part BINDING: a swipe view holding a fieldKey with one option, or two + # options and no field, is not a degraded swipe view — it is a deck that can never write + # anything, rendered as though it were configured. Dropping the key entirely puts the view + # back in its honest unconfigured state, which is the one state the client has a UI for. + # + # ⚠ What this CANNOT check, deliberately, and why the client must: whether `fieldKey` names + # a SELECT, and whether the two options are still in that select's vocabulary. `valid_keys` + # is a key set, and the docstring above draws this exact line — "what each mode MEANS ... is + # the client's business". So SwipeView owns three losses this function is blind to (field + # deleted, field retyped away from select, option removed) and must SHOW each one rather + # than fall back to the first option, per the `viewModes.tsx` house rule. + swipe = raw.get('swipe') + if isinstance(swipe, dict): + f_key = swipe.get('fieldKey') + left, right = swipe.get('leftOption'), swipe.get('rightOption') + ok = (f_key in valid_keys + and isinstance(left, str) and isinstance(right, str)) + if ok: + left, right = left.strip()[:MAX_SWIPE_OPTION], right.strip()[:MAX_SWIPE_OPTION] + # Both non-empty, and DISTINCT: one option on both sides is a deck whose two + # gestures do the same thing, which is two spellings of one state (the + # `kanbanClamp` law) wearing a control that promises a choice. + if left and right and left.casefold() != right.casefold(): + out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} + # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. + form = _clean_form(raw.get('form'), valid_keys) + if form: + out['form'] = form + return out + + +#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4). +#: +#: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each +#: view or cohort — a deliberate amendment to C4's first wording, recorded in the split doc. +#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort +#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated +#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of +#: what a view IS: keeping it out of the view config means duplicating or exporting a view does +#: not drag a folder reference along with it. One map, one home, both surfaces. +FOLDER_SURFACES = {"views", "cohorts"} +MAX_FOLDERS = 60 +MAX_FOLDER_NAME = 80 + +#: Wave-9 I15 (contract C5) — a user-chosen folder icon, as {shape, tone}. +#: +#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already +#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token, +#: not a colour — so a browser cannot post a hex and defeat a tenant restyle, and a shape the +#: client cannot render can never reach the store. +#: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors — +#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional +#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set +#: without the matching client geometry: an unknown shape falls back to the default folder mark, +#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder. +FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"} +#: Tones are the C1 pastels — FILLS ONLY, never text (the standing palette rule). 'grey' is the +#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed. +FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"} +FOLDER_ICON_DEFAULT_TONE = "grey" + + +#: Wave-9 I17 (contract C4) — who may EDIT a saved view. +#: +#: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today +#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one +#: user's views are invisible to every other user and "collaborative" has nothing to act on +#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable +#: and fail-closed now, so that when shared views land the permission does not need a data +#: migration and no saved view is retro-restricted. It does NOT make anything shared, and +#: nothing in the app currently reads it to grant or deny cross-user access. +#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md. +VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'} +MAX_VIEW_USERS = 50 + + +def clean_view_permissions(raw, default, known_users=None): + """{edit, users?} — fail-closed on both halves. + + `default` is supplied by the CALLER because it splits by path, and that split is a + permission rule rather than a formatting one: absent on a view that already exists means a + pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a + silent takeaway), while absent on CREATE must be 'personal' (a new view must never be + anyone-can-edit purely by omission). + + `known_users` (when given) is the real account list: an unknown name is DROPPED, and an + 'users' grant left with nobody in it collapses to 'personal' rather than to everyone. + """ + mode = (raw or {}).get('edit') if isinstance(raw, dict) else None + if mode not in VIEW_EDIT_MODES: + mode = default if default in VIEW_EDIT_MODES else 'personal' + if mode != 'users': + return {'edit': mode} + names, seen = [], set() + for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]: + u = str(u or '').strip() + if not u or u.lower() in seen: + continue + if known_users is not None and u not in known_users: + continue # fail-closed: a name we cannot resolve grants nothing + seen.add(u.lower()) + names.append(u) + if not names: + return {'edit': 'personal'} # an empty grant is NOT "everyone" + return {'edit': 'users', 'users': names} + + +def clean_folder_icon(raw): + """{shape, tone} or None. Fail-closed on both halves, independently. + + A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather + than losing the icon entirely — losing a user's pick because one half was wrong is the kind + of silent data loss the folder events already avoid elsewhere. + """ + if not isinstance(raw, dict): + return None + shape = raw.get("shape") + if shape not in FOLDER_ICON_SHAPES: + return None + tone = raw.get("tone") + return {"shape": shape, + "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE} + + +def clean_folders(raw): + """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}.""" + out = {} + for surface in FOLDER_SURFACES: + items, seen = [], set() + for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]: + if not isinstance(f, dict): + continue + fid = str(f.get("id") or "").strip()[:80] + name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME] + if not fid or not name or fid in seen: + continue # an unidentified or unnamed folder cannot be shown or edited + seen.add(fid) + try: + order = int(f.get("order", len(items))) + except (TypeError, ValueError): + order = len(items) + row = {"id": fid, "name": name, "order": order} + icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark + if icon: + row["icon"] = icon + items.append(row) + items.sort(key=lambda x: x["order"]) + for i, f in enumerate(items): + f["order"] = i # re-index so `order` is always dense and total + if items: + out[surface] = items + return out + + +#: ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, raised by SESSION C as ASK C-16) — "FILED AT ROOT". +#: +#: ⛔ THE DEFECT IS THAT ROOT WAS REPRESENTED BY *ABSENCE*, AND ABSENCE CANNOT HOLD TWO FACTS. +#: "this arrived by grant and was never filed" and "the receiver deliberately dragged this OUT of +#: the Shared group" were the same stored state — nothing — so the client had to GUESS, and +#: `folders.ts::groupByFolder` guessed "Shared". That is why only folder→folder moves appeared to +#: work: **the root bucket was unreachable for a shared view by construction.** +#: +#: ⚠ A RESERVED FOLDER ID, NOT A NEW FIELD, deliberately. The placement map is `{itemId: folderId}` +#: and every reader on both sides already understands it; a parallel "filedAtRoot" set would be a +#: second source of truth for one question, and the two would disagree the first time one of them +#: was written without the other. This id names no folder BY DESIGN and is therefore exempt from +#: the folder-exists test below — it is the one value that means "no folder, on purpose". +#: ⚠ Spelled `ROOT_FOLDER_ID` on the client (`customer-grid/folders.ts`, C's file). Two spellings +#: of one constant is [[a-constant-two-features-share]]; `verify_folders`/`verify_api` assert they +#: agree rather than a comment asking nicely. +ROOT_PLACEMENT = "__root__" + + +def clean_item_folders(raw, folders, valid_ids): + """{surface: {itemId: folderId}} — dropping any placement whose ITEM or FOLDER is gone. + + This is what makes a deleted folder's contents fall back to the root rather than vanish: + nothing stores "this item is in no folder", so an unresolvable placement simply disappears + and the item renders at the top level. Same for an item that was deleted elsewhere — its + stale placement can never resurrect it, because the sidebars render ITEMS and consult this + map, never the other way round. + + ⭐⭐ WAVE 32 — THE PARAGRAPH ABOVE STATES THE FEATURE AND THE BUG IN ONE SENTENCE, and it took + owner item 20 to notice they were the same mechanism. *"Nothing stores 'this item is in no + folder', so an unresolvable placement simply disappears"* is exactly right for a DELETED FOLDER + (its contents should fall to the root) and exactly wrong for a SHARED VIEW (falling back means + falling back INTO the Shared group, which is where it started). `ROOT_PLACEMENT` is the value + that survives this function so the second case can be said out loud. + """ + out = {} + for surface in FOLDER_SURFACES: + fids = {f["id"] for f in (folders or {}).get(surface, [])} + ok = {} + for item_id, fid in ((raw or {}).get(surface) or {}).items(): + if not isinstance(item_id, str) or not isinstance(fid, str): + continue + # ⛔ `fid == ROOT_PLACEMENT` FIRST, and it is NOT in `fids` — it names no folder, which + # is the whole point. Without this clause the value is written by `item_move` and + # scrubbed here on the way back out, so the mark would be stored and instantly lost: + # the two halves are ONE change and shipping either alone is worse than shipping + # neither ([[lost-write-looks-like-failed-read]]). + if item_id in (valid_ids or {}).get(surface, ()) and (fid == ROOT_PLACEMENT + or fid in fids): + ok[item_id[:120]] = fid + if ok: + out[surface] = ok + return out + + +def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): + """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). + + Returns a clean tree of leaf conditions ({colId, op, value, value2}) and + groups ({conj, children}). Module-agnostic on purpose: any module embedding + the grid validates its own view state through this one function. + + Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised — + the same contract the rest of the view sanitiser follows, so one bad rule can + never cost a user their whole saved view. Depth, per-level width and total + node count are all capped: the tree is re-evaluated for every row on every + render, so an unbounded structure would be a persistent client-side DoS. + Empty groups are dropped (they carry no meaning once persisted). + + `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is + DROPPED here rather than left for the engine — a deleted cohort would otherwise leave a + condition that can only match nothing, so `List is not [deleted]` would show an empty table + forever with no way to tell why. `None` means this host has no cohorts, and then every + cohort leaf is dropped: fail-closed, like every other unknown key. + """ + if budget is None: + budget = [MAX_FILTER_NODES] + out = [] + for node in list(raw or [])[:MAX_FILTER_SIBLINGS]: + if budget[0] <= 0: + break + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): # a condition GROUP + if depth >= MAX_FILTER_DEPTH: + continue # too deep -> drop + budget[0] -= 1 + children = clean_filter_tree(node['children'], valid_keys, + depth + 1, budget, cohort_ids) + if children: + out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', + 'children': children}) + continue + if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf + op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op')) + named = parse_cohort_ids(node.get('value')) + # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT + # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to + # `is none of [A]` would show every row in B under a count nobody would doubt. This + # is the same all-or-nothing the single-cohort leaf already had, extended to a set. + if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): + budget[0] -= 1 + out.append({'colId': COHORT_FIELD, 'op': op, + 'value': ','.join(named), 'value2': ''}) + continue + if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: + budget[0] -= 1 + # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the + # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client + # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently + # stop filtering and show every row instead of the zero-revenue ones. + val, val2 = node.get('value'), node.get('value2') + leaf = {'colId': node['colId'], 'op': node['op'], + 'value': ('' if val is None else str(val))[:500], + 'value2': ('' if val2 is None else str(val2))[:500]} + # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two + # extra members: a stable client-generated `id`, which is how the server's answer + # finds its way back to the condition that asked (positional matching silently + # re-associates every answer the moment a user deletes a condition), and the + # `window`. Emitted ONLY when the input has them — a column condition's cleaned + # shape is unchanged, so every persisted view deserialises byte-identically and + # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by + # exact structural comparison). + rid = node.get('id') + if rid not in (None, ''): + leaf['id'] = str(rid)[:64] + window = _clean_window(node.get('window')) + if window is not None: + leaf['window'] = window + # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted + # only when the input has them, so a plain column condition's cleaned shape is + # byte-identical to what it always was and `clean_filter_tree` stays idempotent + # (verify_filter_engine.py asserts that by exact structural comparison). Drop the + # carry-through and the next autosave silently strips a date condition back to a + # bare comparison against an empty value — i.e. back to INACTIVE. + date_window = _clean_window(node.get('dateWindow')) + if date_window is not None: + leaf['dateWindow'] = date_window + mode = node.get('dateMode') + # SHAPE only. An unrecognised mode survives here and is refused by + # `windows.resolve_anchor`, which returns None and makes the condition match + # NOTHING — the same split as `_clean_window`, and the reason this module can stay + # free of the date vocabulary it would otherwise have to keep in step. + if isinstance(mode, str) and 0 < len(mode) <= 40: + leaf['dateMode'] = mode + rhs = _clean_rhs(node.get('rhs'), valid_keys) + if rhs is not None: + leaf['rhs'] = rhs + out.append(leaf) + return out + + +def _default_view_config(fields): + # ⭐⭐ W30-T41's SERVER HALF (F's ask F-1, answered by D — this file is D's fence). + # + # ⛔ THE SECOND ARM USED TO BE `or field["source"] == "overlay"`, AND IT SWALLOWED THE FIRST + # ONE FOR EVERY CONNECTED COLUMN. `user_tables._clean_field` stamps `source: "overlay"` on + # every `ut_` field, so on an Odoo grid the arm was true for ALL of them and `default: False` + # meant nothing: `odoo_id`, `state`, `customer_link` and `partner_id` opened SHOWN however + # they were declared. The exception had become the rule ([[fallback-that-became-the-rule]]), + # and it is the same predicate `useGridColumns.isDefaultVisible` carried on the client. + # + # ⚠ AND THE TWO HALVES MUST MOVE TOGETHER, which is why this is not cosmetic. `CustomerGrid` + # compares the stored view against its own `defaultViewConfig` by JSON equality; with the + # client fixed (T41) and this left alone, the system view would differ from the client's + # default on every render — a view that looks permanently dirty and autosaves forever, which + # is the failure `verify_filter_engine`'s key-ORDER check exists to prevent, one level down. + # + # ⚠ A USER-CREATED COLUMN IS UNAFFECTED, and that is why the fix is a DELETION rather than a + # carve-out for the four Odoo keys: it carries no `default` key at all, so `is not False` + # keeps it visible. On the main Customer grid exactly one field moves — `notes`, which asks + # to be hidden in its own declaration and was being shown against it. + shown = [field["key"] for field in fields if field.get("default") is not False] + hidden = [field["key"] for field in fields if field["key"] not in shown] + return { + # `filters` is the ROOT of the filter tree: leaf conditions and/or nested + # condition groups ({conj, children}); `filterConj` joins the root level. + "filters": [], "filterConj": "and", + "sorts": [], "groupBy": None, "colorBy": None, + "rowHeightMode": "short", "order": shown + hidden, "visible": shown, + "widths": {}, "memberPids": [], + } + + +#: Wave 17 R1 / C-LOCKV — the `kind` a PROJECTED locked view wears. A cohort is not a separate +#: kind of object any more: it is a saved view whose rows are a hand-curated set. +LOCKED_VIEW_KIND = 'locked' + + +def locked_view_projection(entry, base_config): + """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV). + + ⛔ THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the + view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the + named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do + all the work with no second mechanism. Membership itself is NEVER copied in here — it stays + in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped + collection inside a client-writable `config` is deleted by the next autosave (see the + contract's reason 2). + + ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set: + these views are ordinary in every respect the owner asked for — reorder, folder, sort, + filter, change display mode. The lock mark in the rail is driven by `kind`. + """ + return { + 'id': entry['id'], + 'name': entry.get('name') or entry['id'], + 'kind': LOCKED_VIEW_KIND, + 'config': {**base_config, 'cohortLock': entry['id']}, + } + + +#: ⭐ WAVE-27 item 27 (owner ruling R8) — the IG "Overview" view's CURATED COLUMNS, in the +#: owner's own order: handle, followers, engagement, location, last enriched. +#: +#: Written as candidates rather than as a requirement. The template registry REFUSES a template +#: whose columns the target lacks (`view_templates.missing_columns`) because applying one writes +#: the user's own views and a filter on a missing column silently WIDENS. This view is INJECTED, +#: not applied, and it filters nothing — so the proportionate rule is the opposite one: take the +#: columns the table has, in this order, and skip the rest. An IG database that predates a +#: column simply shows the other four. +#: +#: ⚠ `location_guess` is SESSION B's item-16 column and may not exist yet. That is exactly why +#: this list is intersected rather than asserted: a hard requirement here would make the whole +#: view vanish (or the assembly refuse) on every tenant until B lands, and then appear by +#: surprise. `profile_url` closes the list as the click-through, which is what makes the view +#: usable rather than merely informative. +IG_OVERVIEW_COLUMNS = ('handle', 'full_name', 'followers', 'avg_engagement', + 'location_guess', 'enriched_at', 'profile_url') + +#: The id is PINNED, the `view_templates` discipline: re-assembling must update the same view +#: rather than mint "Overview 2". It also lets a user's own edits overlay it through the saved +#: -config loop below, exactly as a cohort projection does. +IG_OVERVIEW_ID = 'tpl_overview' + +#: How this function recognises an IG preset database WITHOUT importing the engine: two of the +#: profile preset columns is a stronger signal than any single one (a hand-made table could +#: plausibly own a column called `followers`; owning `followers` AND `avg_engagement` AND +#: `handle` is the preset set). `core/` must stay importable without the API layer, so this +#: mirrors `user_tables.PROFILE_PRESET_KEYS` the way that module mirrors the engine's. +_IG_SIGNATURE = ('handle', 'followers', 'avg_engagement') + + +def _ig_overview_view(fields, base): + """R8's curated Overview, or None when this table is not an Instagram one.""" + keys = {f['key'] for f in fields} + if not all(k in keys for k in _IG_SIGNATURE): + return None + visible = [k for k in IG_OVERVIEW_COLUMNS if k in keys] + return { + 'id': IG_OVERVIEW_ID, + 'name': 'Overview', + 'kind': 'system', + # NOT `locked`. The system view is locked because it is the identity of the table ("show + # me everything"); this one is a STARTING LAYOUT, and R8 calls it curated rather than + # fixed. A user who wants a sixth column should get one. + 'note': 'The five things worth seeing first on a creator. Sorted by reach.', + 'config': { + **dict(base), + 'visible': visible, + 'order': visible + [k for k in (f['key'] for f in fields) if k not in visible], + 'sorts': ([{'colId': 'followers', 'dir': 'desc'}] + if 'followers' in keys else []), + }, + } + + +def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None, + view_order=None): + """Convert legacy list formulas into the shared serializable SavedView contract. + + `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the + product surface). The ID stays "all-customers" on every topic — the client pins it + (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork + that contract for a label's sake. + + `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose + id IS the cohort id — so every stored reference to that id (a `cohortLock` on another view, + an `is part of` condition, a folder placement) keeps pointing at the same thing and no + rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the + `list:` views have always used, which is what gives a locked view its own sort, filter, + columns and display mode with no new storage.""" + base = _default_view_config(fields) + views = [{ + "id": "all-customers", "name": system_name, "kind": "system", + "locked": True, "config": dict(base), + }] + # ⭐ WAVE-27 item 27 (R8) — the IG Overview, ABOVE All records. + # + # ⚠ INJECTED, not seeded into the store, and that is what makes "existing IG databases gain + # it too" true with no migration and no write on a read path. It is the same mechanism the + # system view above has always used; the pinned id means a user's own edits overlay it + # through the saved-config loop below rather than forking a second view. + _overview = _ig_overview_view(fields, base) + if _overview: + views.insert(0, _overview) + op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq", + "contains": "contains"} + for name, definition in (defs or {}).items(): + filters = [] + for rule in definition.get("rules") or []: + if rule.get("field") not in {field["key"] for field in fields}: + continue + filters.append({ + "colId": rule["field"], + "op": op_map.get(rule.get("op"), "eq"), + "value": str(rule.get("value") if rule.get("value") is not None else ""), + }) + sort = str(definition.get("sort") or "") + sorts = ([{"colId": sort.lstrip("-"), + "dir": "desc" if sort.startswith("-") else "asc"}] + if sort.lstrip("-") in {field["key"] for field in fields} else []) + views.append({ + "id": "list:" + str(name), + "name": str(name), + "kind": "list", + "note": str(definition.get("note") or ""), + "config": { + **base, "filters": filters, "sorts": sorts, + "memberPids": [int(pid) for pid in definition.get("members") or [] + if isinstance(pid, int) or str(pid).isdigit()], + }, + }) + # Wave 17 R1 — the cohorts, as ordinary views. Appended BEFORE the saved-config overlay + # below so a user's own edits to a locked view (its sort, its columns, its display mode) + # land on the projection instead of creating a second row with the same id. + for _entry in (locked_lists or []): + if isinstance(_entry, dict) and _entry.get('id'): + views.append(locked_view_projection(_entry, base)) + index = {view["id"]: i for i, view in enumerate(views)} + for view_id, saved in (saved_views or {}).items(): + if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict): + continue + clean = dict(saved) + clean["id"] = str(view_id) + if view_id in index: + views[index[view_id]] = clean + else: + views.append(clean) + # ⛔ WAVE 17 R1 — RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected + # view with its saved record, and a saved record that omits `cohortLock` would hand back a + # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the + # client rebuilds `config` on every autosave (a column resize is enough), and the lock is + # identity here, not something the browser is the source of truth for. Read-side rather than + # write-side-only on purpose — this also repairs any record already written by another path. + _locked_ids = {e['id']: e for e in (locked_lists or []) + if isinstance(e, dict) and e.get('id')} + if _locked_ids: + for _v in views: + _entry = _locked_ids.get(_v.get('id')) + if not _entry: + continue + _v['kind'] = LOCKED_VIEW_KIND + _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']} + # One thing, one name: the cohort store owns it (the rename event routes there), so + # a stale `name` on the saved record can never fork into a second title. + _v['name'] = _entry.get('name') or _v['id'] + # ── ⭐ WAVE-27 item 5, contract C7: the PER-USER VIEW ORDER ─────────────────────────────── + # + # `view_order` is a list of view ids this user dragged into place. Applied LAST, over the + # finished list, so it reorders whatever the assembly produced without having to know how any + # of it got there (system, list:, cohort projection, saved, injected Overview). + # + # ⛔ THE SYSTEM VIEW STAYS AT INDEX 0 (C7), and it is re-pinned here rather than trusted to + # sort correctly: `all-customers` is the client's landing default and one of its + # UNDELETABLE_VIEW_IDS, so a stored order that happened to omit it — or list it third — + # would move the rail's home row. ⚠ R8's Overview is the ONE thing allowed above it, because + # the owner put it there; it is re-pinned with the system view so a drag cannot bury it + # either. Both are facts about the table rather than the user's arrangement of it. + # + # ⚠ UNKNOWN IDS APPEND IN SERVER ORDER (C7). A view created since this order was stored, or + # one shared to this user yesterday, must APPEAR — dropping it would make sharing look + # broken, and the failure would be invisible to whoever shared it. Ids in the stored order + # that no longer resolve are simply skipped. + if view_order: + _rank = {vid: i for i, vid in enumerate(view_order) if isinstance(vid, str)} + _pinned = [v for v in views if v.get('id') in (IG_OVERVIEW_ID, 'all-customers')] + _rest = [v for v in views if v.get('id') not in (IG_OVERVIEW_ID, 'all-customers')] + # A stable sort over a rank that DEFAULTS TO THE END keeps unranked views in their + # server order behind the ranked ones, rather than interleaving them by accident. + _rest.sort(key=lambda v: _rank.get(v.get('id'), len(_rank) + 1)) + views = _pinned + _rest + return views + + +def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None, + fields_base=None, with_cohorts=True): + """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE + projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`). + + ⛔ WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no + `storageKey` — and the client validator (`fetchWorkspace`) requires one, so the standalone + shell silently discarded the whole workspace: saved views never rendered and `cohortMode` + never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline + projection into the route would have re-created the same drift one wave later; extracting it + means the wire can only be one thing. + + Returns `(workspace, fields, views, cohort_lists)` — the extra three because the host + interleaves further work (docs, derived cells, measure sets) that consumes them. + + HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`, + `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`). + + Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer, + byte-identical). + + ⭐ WAVE 19 / R9 — `with_cohorts` NO LONGER MEANS "customer only". Wave 16 set it False on the + product surface because cohorts were a single customer-keyed bucket, so resolving them against + product pids would have intersected two unrelated id spaces and printed a plausible, + meaningless member count. `modules.cohort` is scope-parameterized now: the lists come from + THIS topic's bucket (`cohort_mod.scoped(scope_key)`), so their ids are this topic's ids and + the intersection with `pool_pids` is the ordinary one. The flag survives as an honest OFF + switch for a surface that wants no membership channel at all — it is not a topic wall. + """ + import modules.cohort as cohort_mod + + cohort_lists = [] + if with_cohorts: + for cid, c in sorted(cohort_mod.scoped(scope_key).visible(uname, pool_pids).items(), + key=lambda kv: (kv[1].get('name') or '').lower()): + members = [p for p in (c.get('members') or []) if p in pool_pids] + entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members} + # Rule 8b: a member can drop out of the 24-month pool without the cohort being + # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule + # forbids. + missing = len(c.get('members') or []) - len(members) + if missing: + entry['missing'] = missing + cohort_lists.append(entry) + + fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key, + fields_base=fields_base) + views = views_from_defs(defs or {}, ws.get('views'), fields, + # Wave 21 (item 3, R6): the system view's name is TOPIC-DERIVED. A + # user database's default view used to read "All customers" — a + # compiled customer literal minted on every topic, one half of the + # owner's "my new database looks like RI's Customer table". The ID + # stays 'all-customers' everywhere (pinned client+server — the + # client's UNDELETABLE set and the view pin both name it). + system_name=("All products" if scope_key == 'product' + else "All records" + if str(scope_key or '').startswith('ut_') + else "All customers"), + locked_lists=cohort_lists, + # ⭐ WAVE-27 item 5 (C7) — this user's own rail arrangement, from + # their own stratum. Read here rather than sorted by the client so + # the ORDER a request answers with is the order that was stored: + # sorting client-side would make the rail settle after a paint on + # every load, and shared views would land in server order first. + view_order=ws.get('viewOrder')) + workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists} + # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy + # wins when present; this is the server's answer for a FRESH browser, which used to fall + # all the way to the system default view (and whatever display mode was stored on it). + if ws.get('activeViewId'): + workspace['activeViewId'] = str(ws['activeViewId']) + + # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be + # deleted by a path that knows nothing about folders, and the placement map must not outlive + # the thing it points at. + _folders = clean_folders(ws.get('folders')) + _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')} + # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT + # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so + # its folder has to be an ordinary view folder — but rewriting the stored map would be a + # one-shot write that has to be got right once, while this is a projection that is right + # every time it runs. New drags write to `views` anyway (the client only knows that + # surface), so `cohorts` drains on its own and never needs a second pass. + # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged: + # re-parenting somebody's list into a folder that merely shares an id is a worse outcome + # than the list appearing at the root, where it is visible and one drag from home. + _cf = list(_folders.get('cohorts') or []) + if _cf: + _vf = list(_folders.get('views') or []) + _taken = {f['id'] for f in _vf} + _order = len(_vf) + for _f in _cf: + if _f['id'] in _taken: + continue + _vf.append({**_f, 'order': _order}) + _order += 1 + _folders['views'] = _vf + _raw_item_folders = dict(ws.get('itemFolders') or {}) + if _raw_item_folders.get('cohorts'): + # Cohort placements now describe VIEWS (same ids — that is the point of preserving them). + # A placement already stored on the views surface WINS: it is the more recent act. + _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}), + **dict(_raw_item_folders.get('views') or {})} + _placed = clean_item_folders( + _raw_item_folders, _folders, + {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}}) + if _folders.get('views'): + workspace['folders'] = _folders['views'] + # ⛔ `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a + # wire that still described one would invite a second rendering of rows that are now views. + _vplaced = _placed.get('views') or {} + for _v in (views or []): + if isinstance(_v, dict) and _v.get('id') in _vplaced: + _v['folderId'] = _vplaced[_v['id']] + _cplaced = _placed.get('cohorts') or {} + for _c in cohort_lists: + if _c['id'] in _cplaced: + _c['folderId'] = _cplaced[_c['id']] + + # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order, + # re-validated at SERVE time exactly like folders — a field can be deleted by a path + # that knows nothing about this stratum, and a stale key must not outlive its field. + _rl = ws.get('recordLayout') + if isinstance(_rl, dict) and isinstance(_rl.get('order'), list): + _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')} + _order, _seen = [], set() + for _k in _rl['order'][:200]: + _k = str(_k or '') + if _k and _k in _fkeys and _k not in _seen: + _seen.add(_k) + _order.append(_k) + if _order: + workspace['recordLayout'] = {'order': _order} + + return workspace, fields, views, cohort_lists + + +def embed_html_path(): + """The first existing candidate path for the inlined single-file build, or None.""" + for p in _EMBED_CANDIDATES: + if p.is_file(): + return p + return None + + +def scope_counts(shown, matched, total): + """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2). + + `matched` and `total` MUST come from their own queries over the whole scope. Never pass + `len(rows)` as `matched` — that is the silent [:N] this exists to prevent: the page would + report the window size as though it were the result size. + + Refuses the shapes that could only be a mistake, because a wrong count here is invisible on + screen (it looks like a smaller dataset, not like an error). + """ + shown, matched, total = int(shown), int(matched), int(total) + if matched > total: + raise ValueError(f"matched ({matched}) exceeds total ({total}) — a filter cannot match " + f"more rows than the scope holds") + if shown > matched: + raise ValueError(f"shown ({shown}) exceeds matched ({matched}) — the window cannot hold " + f"more rows than the filter matched") + return {"shown": shown, "matched": matched, "total": total, "windowed": True} + + + +# ⛔ EXIT-6 (2026-08-04): `build_html`, `component_dir`, `render` and `_DECLARED_COMPONENTS` WERE +# HERE, and they are gone with Streamlit. They were the EMBED HOST — the path that declared the +# prebuilt bundle as a `streamlit.components.v1` custom component (or injected the single-file +# HTML build as a fallback) so the React grid could be drawn inside a Streamlit page. +# +# THIS MODULE ITSELF SURVIVES, and that distinction is the whole point: `aios_grid.py` is imported +# at 11 sites across `aios-web/api/` plus `harness/semantic.py` — it owns the canonical field +# contract, the workspace wire and the count envelope. Only the ~95 lines that knew about a HOST +# went; the rest never did. Its one and only `import streamlit` lived inside `render`, lazily, and +# left with it. `api/verify_no_streamlit.py` now gates that nothing here re-imports it. +# +# Deleted with them: `aios_grid_embed.html` + `aios_grid_component/index.html` (a 2.1 MB prebuilt +# bundle), `build_embed.py` that produced them, and `deploy_hf.py`'s embed-staleness guard. The +# React app is now served directly by the FastAPI container — there is no twin to keep fresh, so +# the entire class of "the code shipped but the bundle did not" is retired rather than guarded. diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index ff4c5f6504e02c921b0f565d62714da3c0a6f0c8..d52218949811a431b8d60c2cae68fe1b0d12cb35 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -1,474 +1,464 @@ -{ - "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", - "fields": [ - { - "key": "customer", - "label": "Customer", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." - }, - { - "key": "partner_id", - "label": "Odoo ID", - "type": "int", - "source": "odoo", - "derived": true, - "default": false, - "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." - }, - { - "key": "odoo_status", - "label": "Odoo record", - "type": "status", - "source": "odoo", - "default": false, - "options": ["Active", "Archived"], - "description": "Whether this customer still exists in Odoo. Archived means deleted there." - }, - { - "key": "agent", - "label": "Agent", - "type": "text", - "source": "odoo", - "default": true, - "description": "The sales agent who owns this account." - }, - { - "key": "dba", - "label": "DBA", - "type": "select", - "source": "odoo", - "default": false, - "options": [ - "Fisch", - "Royal", - "Both" - ], - "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." - }, - { - "key": "salesperson", - "label": "Salesperson", - "type": "text", - "source": "odoo", - "default": false, - "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." - }, - { - "key": "street", - "label": "Street", - "type": "text", - "source": "odoo", - "default": false, - "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." - }, - { - "key": "street2", - "label": "Street 2", - "type": "text", - "source": "odoo", - "default": false, - "description": "Second address line (suite, unit, floor) on the customer's Odoo address." - }, - { - "key": "city", - "label": "City", - "type": "text", - "source": "odoo", - "default": true, - "description": "City on the customer's Odoo address." - }, - { - "key": "state", - "label": "State", - "type": "text", - "source": "odoo", - "default": true, - "description": "State or province on the customer's Odoo address." - }, - { - "key": "country", - "label": "Country", - "type": "text", - "source": "odoo", - "default": false, - "description": "Country on the customer's Odoo address." - }, - { - "key": "zip", - "label": "ZIP", - "type": "text", - "source": "odoo", - "default": false, - "description": "Postal code on the customer's Odoo address." - }, - { - "key": "customer_since", - "label": "Customer since", - "type": "date", - "source": "odoo", - "default": false, - "description": "When this customer was first set up in Odoo." - }, - { - "key": "tags", - "label": "Tags", - "type": "text", - "source": "odoo", - "default": false, - "description": "Odoo labels on this customer, comma-separated." - }, - { - "key": "pricelist", - "label": "Price list", - "type": "text", - "source": "odoo", - "default": false, - "description": "The price list this customer buys on." - }, - { - "key": "payment_terms", - "label": "Payment terms", - "type": "text", - "source": "odoo", - "default": false, - "description": "Payment terms on this customer's account — Net 30, for example." - }, - { - "key": "last_order", - "label": "Last order", - "type": "date", - "source": "odoo", - "default": true, - "description": "Date of the most recent confirmed order." - }, - { - "key": "overdue_days", - "label": "Overdue days", - "type": "int", - "source": "odoo", - "default": true, - "description": "How many days late this customer is running against their own usual ordering rhythm." - }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, - { - "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", - "key": "ar_open", - "label": "AR current $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." - }, - { - "key": "ar_overdue", - "label": "AR overdue $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money past due — same basis as the Collections page." - }, - { - "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", - "key": "ar_outstanding", - "label": "AR outstanding $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." - }, - { - "key": "ar_exposure", - "label": "Credit exposure $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." - }, - { - "key": "ar_aged_1_30", - "label": "1-30 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_31_60", - "label": "31-60 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_61_90", - "label": "61-90 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_90_plus", - "label": "90+ days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "days_to_pay", - "label": "Days to pay", - "type": "int", - "source": "odoo", - "default": false, - "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." - }, - { - "key": "top_category", - "label": "Top category", - "type": "text", - "source": "odoo", - "default": false, - "description": "The category this customer spent the most on in the last 12 months." - }, - { - "key": "top_category_pct", - "label": "Top category %", - "type": "pct", - "source": "odoo", - "default": false, - "description": "Share of last-12-months spend that went to the top category." - }, - { - "key": "sku_count", - "label": "SKUs bought", - "type": "int", - "source": "odoo", - "default": false, - "description": "Distinct products bought in the last 12 months." - }, - { - "key": "top_sku", - "label": "Top SKU", - "type": "text", - "source": "odoo", - "default": false, - "description": "The product this customer spent the most on in the last 12 months." - }, - { - "key": "days_since", - "label": "Days since order", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days since the last confirmed order." - }, - { - "key": "typical_gap_days", - "label": "Typical gap days", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days this customer usually goes between orders, from their own history." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "description": "Your notes on this customer. Saved in this app only, visible only to you." - } - ], - "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.", - "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).", - "product_data": { - "identity": "pid", - "business_key": "code", - "fields": [ - { - "key": "code", - "label": "SKU", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." - }, - { - "key": "product", - "label": "Product", - "type": "text", - "source": "odoo", - "default": true, - "description": "Product name as it appears in Odoo." - }, - { - "key": "category", - "label": "Category", - "type": "select", - "source": "odoo", - "default": true, - "description": "Product category; '(uncategorized)' when Odoo carries none." - }, - { - "key": "supplier", - "label": "Supplier", - "type": "text", - "source": "overlay", - "default": true, - "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.", - "shared": true - }, - { - "key": "origin_country", - "label": "Country", - "type": "text", - "source": "overlay", - "default": false, - "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.", - "shared": true - }, - { - "key": "lead_days", - "label": "Lead time (days)", - "type": "int", - "source": "overlay", - "default": true, - "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.", - "shared": true - }, - { - "key": "first_cost", - "label": "First cost", - "type": "currency", - "source": "overlay", - "default": false, - "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.", - "shared": true - }, - { - "key": "price_fisch", - "label": "Fisch price", - "type": "currency", - "source": "odoo", - "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "price_royal_1", - "label": "Royal 1 price", - "type": "currency", - "source": "odoo", - "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "price_royal_2", - "label": "Royal 2 price", - "type": "currency", - "source": "odoo", - "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." - }, - { - "key": "rev_ytd", - "label": "Revenue YTD", - "type": "currency", - "source": "odoo", - "default": true, - "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is." - }, - { - "key": "rev_ly", - "label": "Revenue LY", - "type": "currency", - "source": "odoo", - "description": "Same period last year — seasonal wholesale compares like for like." - }, - { - "key": "yoy_pct", - "label": "YoY %", - "type": "pct", - "source": "odoo", - "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." - }, - { - "key": "qty_ytd", - "label": "Units YTD", - "type": "int", - "source": "odoo", - "description": "Units sold year to date." - }, - { - "key": "orders_ytd", - "label": "Orders YTD", - "type": "int", - "source": "odoo", - "description": "Distinct orders containing this SKU, year to date." - }, - { - "key": "on_hand", - "label": "On hand", - "type": "int", - "source": "odoo", - "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." - }, - { - "key": "unit_cost", - "label": "Unit cost", - "type": "currency", - "source": "odoo", - "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "inv_value", - "label": "Stock value", - "type": "currency", - "source": "odoo", - "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "qty_ltm", - "label": "Units LTM", - "type": "int", - "source": "odoo", - "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "dos", - "label": "Days of supply", - "type": "int", - "source": "odoo", - "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller." - }, - { - "key": "cover_gap_d", - "label": "Cover gap (days)", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands." - }, - { - "key": "stock_bucket", - "label": "Stock status", - "type": "select", - "source": "odoo", - "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller." - } - ] - } -} +{ + "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "dba", + "label": "DBA", + "type": "select", + "source": "odoo", + "default": false, + "options": [ + "Fisch", + "Royal", + "Both" + ], + "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "street", + "label": "Street", + "type": "text", + "source": "odoo", + "default": false, + "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." + }, + { + "key": "street2", + "label": "Street 2", + "type": "text", + "source": "odoo", + "default": false, + "description": "Second address line (suite, unit, floor) on the customer's Odoo address." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", + "key": "ar_open", + "label": "AR current $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_outstanding", + "label": "AR outstanding $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "ar_aged_1_30", + "label": "1-30 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_31_60", + "label": "31-60 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_61_90", + "label": "61-90 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_90_plus", + "label": "90+ days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.", + "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).", + "product_data": { + "identity": "pid", + "business_key": "code", + "fields": [ + { + "key": "code", + "label": "SKU", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." + }, + { + "key": "product", + "label": "Product", + "type": "text", + "source": "odoo", + "default": true, + "description": "Product name as it appears in Odoo." + }, + { + "key": "category", + "label": "Category", + "type": "select", + "source": "odoo", + "default": true, + "description": "Product category; '(uncategorized)' when Odoo carries none." + }, + { + "key": "supplier", + "label": "Supplier", + "type": "text", + "source": "overlay", + "default": true, + "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.", + "shared": true + }, + { + "key": "origin_country", + "label": "Country", + "type": "text", + "source": "overlay", + "default": false, + "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.", + "shared": true + }, + { + "key": "lead_days", + "label": "Lead time (days)", + "type": "int", + "source": "overlay", + "default": true, + "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.", + "shared": true + }, + { + "key": "first_cost", + "label": "First cost", + "type": "currency", + "source": "overlay", + "default": false, + "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.", + "shared": true + }, + { + "key": "price_fisch", + "label": "Fisch price", + "type": "currency", + "source": "odoo", + "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "price_royal_1", + "label": "Royal 1 price", + "type": "currency", + "source": "odoo", + "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "price_royal_2", + "label": "Royal 2 price", + "type": "currency", + "source": "odoo", + "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." + }, + { + "key": "rev_ytd", + "label": "Revenue YTD", + "type": "currency", + "source": "odoo", + "default": true, + "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is." + }, + { + "key": "rev_ly", + "label": "Revenue LY", + "type": "currency", + "source": "odoo", + "description": "Same period last year — seasonal wholesale compares like for like." + }, + { + "key": "yoy_pct", + "label": "YoY %", + "type": "pct", + "source": "odoo", + "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." + }, + { + "key": "qty_ytd", + "label": "Units YTD", + "type": "int", + "source": "odoo", + "description": "Units sold year to date." + }, + { + "key": "orders_ytd", + "label": "Orders YTD", + "type": "int", + "source": "odoo", + "description": "Distinct orders containing this SKU, year to date." + }, + { + "key": "on_hand", + "label": "On hand", + "type": "int", + "source": "odoo", + "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." + }, + { + "key": "unit_cost", + "label": "Unit cost", + "type": "currency", + "source": "odoo", + "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "inv_value", + "label": "Stock value", + "type": "currency", + "source": "odoo", + "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "qty_ltm", + "label": "Units LTM", + "type": "int", + "source": "odoo", + "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "dos", + "label": "Days of supply", + "type": "int", + "source": "odoo", + "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "cover_gap_d", + "label": "Cover gap (days)", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands." + }, + { + "key": "stock_bucket", + "label": "Stock status", + "type": "select", + "source": "odoo", + "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller." + } + ] + } +} diff --git a/platform/core/alerts.py b/platform/core/alerts.py index b8cd80e417fa849ce11a35f653f4f1a199104152..c7b9ac070259e1351acb2b1c3ef2d2be065f8423 100644 --- a/platform/core/alerts.py +++ b/platform/core/alerts.py @@ -209,7 +209,7 @@ def _queue(rec, pids, labels, st=None): _st(st).update(NOTIFICATIONS_KEY, _apply, flush='async') -def notify(owner, label, *, topic='automation', key='', row_id='', detail='', actor='', st=None): +def notify(owner, label, *, topic='automation', key='', row_id='', detail='', st=None): """⭐ WAVE 23 (C6) — put ONE notification in a named user's inbox, for a producer that is not a view-alert evaluation. @@ -222,24 +222,6 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', ac The inbox shape is UNCHANGED, deliberately: `AlertsPane` reads one list, and a second row shape would mean a client that must branch on where a notification came from. `alertId` carries the producer's key so a click-through can route (W23-W5). - - ⭐⭐ W33-T28 (`ASK C-14`) — `actor` IS THE ONLY THING THAT KNOWS WHO. Owner item: *"the Inbox - reads like email"*, and a verifier reading the carried wave-32 surface found the SENDER position - occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you", i.e. a - CATEGORY where mail puts a who — with no sender field anywhere on the wire, in the model or in - the markup. An alert and an automation have no person behind them and are honestly named by - their machine; **a SHARE has a real person, and only the PRODUCER knows their name.** By the time - the read path sees the row that fact is gone, so it has to be carried at write time. - - ⛔ ADDITIVE, AND THAT IS THE WHOLE DESIGN. No existing caller changes, no stored row changes - shape, and every reader already tolerates the key's absence — so an inbox written before this - line reads exactly as it did. The alternative C rejected is worth recording because it is the - tempting one: recovering the name by REGEXING the body prose (`" shared this with you"`). - A sender parsed out of a sentence breaks the first time the sentence is reworded, and it breaks - SILENTLY, in the header — [[one-question-two-normalizers]] with the second normaliser being - English. - ⚠ Blank is a legitimate value and means *"no person did this"*, NOT "unknown". The client falls - back BY KIND rather than to an empty From column, so an automation never borrows a human's name. """ owner = str(owner or '').strip() if not owner or not label: @@ -248,8 +230,7 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', ac item = {'id': f"{key or topic}:{row_id or 'n'}:{at}", 'alertId': str(key or ''), 'viewId': '', 'topic': str(topic), 'rowId': str(row_id or ''), 'label': str(detail or label), - 'alertLabel': str(label), 'at': at, 'read': False, - 'actor': str(actor or '')} + 'alertLabel': str(label), 'at': at, 'read': False} def _apply(data): box = list(data.get(owner) or []) diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 0eea69bf7d4a477836c49314aae8dd395744b3c5..9a384301f9c0c65de31d5a9ad8e6c9f61e78cdea 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -1,2147 +1,2125 @@ -"""core/grid_events.py — THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30). - -This is the grid's event handler: the code that takes an UNTRUSTED event object from the -browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` / -`_cl_handle_one` and moved here as a PURE MOVE — no behavior change inside a move (MA principle -5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the -Streamlit couplings the contract names: - - st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state) - st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc) - st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast) - st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback) - is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default) - -> ctx.admin - -There is NO `import streamlit` in this file and there must never be one — `verify_no_streamlit.py` -enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring. -(That gate was `verify_seam.py` until EXIT-6 retired it with `app.py`; the block, its negative -control and the AST walk moved over intact.) - -ONE ADAPTER, ONE HANDLER — it was TWO until 2026-08-04: - * ⛔ the Streamlit adapter is GONE. `app.py::_cl_handle_grid_event` built an EventCtx whose - `seen_ids` and `fallback_ws` were session-state-backed dicts and applied the Result to session - state. EXIT-6 deleted it. The host-neutrality this file was written for is what let that - happen without touching a line of the handler — which is the argument for the design, recorded - at the moment it paid out. - * the HTTP adapter — `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None - fallback with the store unavailable is a **503**, never a silent in-memory workspace: an - API request has no durable session dict to fall back to, so pretending to persist would - lose the write with a 200 on it. - -⚠ LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE §1's -one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer -table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The -contracted path for the strangler is `core/grid_events.py` (both adapters import it from here), -so the import stays and the violation is RECORDED rather than hidden. It is the one file in -`core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move -down, and this note is the marker for that work. -""" -import datetime as dt -import json as _json_ev # wave 23 (C7): the json field's parse wall -from dataclasses import dataclass, field -from typing import MutableMapping, Optional - -import core.store as store -import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates - -#: ⭐ WAVE-27 item 5 (C7) — how many view ids one rail arrangement may name. Generous on purpose: -#: it bounds a hostile payload, not the user (the rail holds saved views + cohort projections + -#: shared views, and a busy Customer grid already carries dozens). An order longer than this is -#: not a person dragging rows. -MAX_VIEW_ORDER = 500 -import core.user_tables as _ut_limits # wave 23 (C7): MAX_JSON_CELL — one ceiling, one definer -import core.users as users -import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log) -import modules.cohort as cohort_mod -import modules.customer_data as cl_mod - -#: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte -#: count — the client checks the same number first, but a client check only spares the user a -#: pointless upload; this is what bounds what the tenant's dataset can be made to hold. -#: MOVED from app.py:1410–1412 with the handler (app.py re-imports these during the strangler). -_DOCS_KEY = 'customer_docs' -_DOC_MAX_BYTES = 5 * 1024 * 1024 -_DOC_MAX_PER_ROW = 50 - -#: The props of a field def the CLIENT actually draws — the wave-6 no-blip comparison set. -#: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders -#: nothing from its own copy of those, and a permissions refusal is answered explicitly. -_FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'colorCodeOptions', - 'optionColors', 'max', 'formula', 'agg', 'scope', 'format', 'measure') - - -class StoreUnavailable(RuntimeError): - """The store is down AND the caller offered no durable fallback (HTTP adapter → 503). - - The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so - its store-down path degrades to the same captioned in-memory workspace it always had. An - API caller has no such dict, and the honest answer to "persist this" with nowhere to - persist it is an error — not a 200 over a write that evaporates. - """ - - -@dataclass -class EventResult: - """What the HANDLER needs to tell its adapter — the three things it cannot do itself. - - `rerender` is the old return value with its old meaning: "this page must re-render from - server state", NOT "handled" (see `handle_one`'s docstring — the distinction is the - difference between a snappy grid and a sluggish one). - """ - rerender: bool = False - doc: Optional[dict] = None # doc_fetch's answer — the bridge has no response channel - toast: Optional[str] = None # add_to_list's confirmation - - -@dataclass -class EventCtx: - """Everything the handler is allowed to know, and nothing about how it is being served. - - ⚠ `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and - every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids) - is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity — - never anything the browser said about itself. - - `seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every - emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP - passes the caller's per-session mapping. - - `fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username - (exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes - None, and None + store down raises StoreUnavailable (→ 503). - - `out` is the sink for the two parked answers, so `handle_one` can keep returning a bare - bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's - field list, 2026-07-30 — defaulted; recorded in the S1 mailbox.) - """ - uname: str - allowed_pids: frozenset - fields: list - measure_keys: frozenset = frozenset() - resolved_ids: frozenset = frozenset() - cohort_ids: frozenset = frozenset() - measure_offer: tuple = () - visible_views: tuple = () - admin: bool = False - #: ⛔ WAVE 15 C-PERM — the fields this caller's PERMANENT PERMISSIONS hide, transitive - #: closure included (`core.perm_scope.hidden_keys`). A SECURITY parameter in the same class - #: as `allowed_pids`, and here for the same reason: the read paths strip a hidden field from - #: both wires, so a caller cannot SEE it — but without this they could still WRITE it by - #: naming the key, because `overlay_keys` only asks whether the column exists. Empty means - #: "nothing hidden", which is every legacy record and therefore every caller today. - hidden_keys: frozenset = frozenset() - scope_key: str = 'customer' - #: Wave 16 C-TOPIC — the TABLE OPS this ctx reads and writes (a `core.table_store.make` - #: object). None = the CUSTOMER table (`modules.customer_data.TABLE_OPS`), which keeps - #: every existing caller byte-identical. The PRODUCT surface passes - #: `modules.product_data.TABLE_OPS` so its views/fields/overlays land in the product - #: bucket — never the customer one, whose pids are a different id space entirely. - table: Optional[object] = None - #: ⭐ WAVE 25 (R6b, closes DEBT D-16) — THE TENANT STORE HANDLE, and it is a RESIDENCY - #: parameter in the same class as `allowed_pids` is a security one. Everything this handler - #: could not reach through `ctx.table` used to go through the MODULE-LEVEL `core.store`, - #: i.e. tenant #0's dataset repo: the documents family, `_store_of(ctx).available()`, and — through - #: `_tops`' fallback — the whole customer and product table workspace. Owner, R6b: - #: *"Tenant 0 should not intertwine anywhere."* - #: - #: None keeps the module default, which is byte-identical for tenant #0 (empty prefix, - #: shared repo) and is what every non-HTTP caller still gets. The HTTP adapters pass - #: `session.runtime`, so a Nurilab write resolves against Nurilab. - st: Optional[object] = None - seen_ids: Optional[MutableMapping] = None - fallback_ws: Optional[MutableMapping] = None - out: EventResult = field(default_factory=EventResult) - - def __post_init__(self): - # A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is - # what makes re-delivery idempotent, and its absence must not be the thing that decides - # whether an event is processed at all. - if self.seen_ids is None: - self.seen_ids = {} - - -def _store_of(ctx): - """THE store handle this ctx reads and writes through (wave 25, R6b / D-16). ONE resolver, - so "whose repo does this land in" has exactly one answer per event. - - Order, and each step is deliberate: the ctx's own tenant handle · else the handle the TABLE - OPS object was built with (`table_store.make(..., st=…)` keeps it on `.st`, which is how the - `ut_` path has been tenant-correct since wave 18) · else the module default. - - ⛔ THE MODULE DEFAULT IS TENANT #0, AND THAT IS THE BUG D-16 NAMES — not a safe base case. - It is kept because it is byte-identical for tenant #0 (empty prefix, shared repo) and every - non-HTTP caller still relies on it; what changed is that the HTTP adapters now pass a real - handle, so the fallback is no longer what serves a Nurilab request. `verify_scopes` proves - that with a negative control: force this to fall through and the residency legs go RED. - """ - st = getattr(ctx, 'st', None) - if st is not None: - return st - tbl = getattr(ctx, 'table', None) - inner = getattr(tbl, 'st', None) if tbl is not None else None - return inner if inner is not None else store - - -def _blobs(ctx): - """The BYTE store for this ctx — `_store_of`'s sibling, split out because it is NOT yet - equivalent and pretending otherwise would hide a live residency hole. - - ⚠ `TenantRuntime` exposes `get/put/update/exists/available` and **no byte methods** - (`harness/runtime.py`), so a tenant with its OWN repo has nowhere tenant-scoped to put a - document's BYTES. The metadata half of D-16 is fixed here; the byte half needs - `upload_bytes`/`download_bytes`/`delete_path` on the runtime, which is SESSION A's file — - asked for in the mailbox and booked as debt rather than papered over with a fallback that - reads as intentional. The moment those methods exist this resolver picks them up with no - other change, which is why it is written as a capability test and not as a version check. - """ - st = _store_of(ctx) - return st if hasattr(st, 'upload_bytes') else store - - - -def _ag_overview_id(): - """`aios_grid.IG_OVERVIEW_ID`, resolved defensively. - - Function-local import, this module's convention for the layer above it. Falls back to the - literal rather than raising: a store blip must not turn "refuse to delete" into "allow", and - an unguarded delete here is the one that reads as success and undoes itself. - """ - try: - import aios_grid as _ago - return _ago.IG_OVERVIEW_ID - except Exception: - return 'tpl_overview' - -def _tops(ctx): - """The table-ops object this ctx operates on — the customer table unless the caller chose - another topic. ONE resolution point so 'which bucket does this write land in' has exactly - one answer per event. - - ⛔ WAVE 25 (R6b) — THIS FALLBACK IS STILL TENANT #0's, DELIBERATELY, AND IT IS THE ONE PIECE - OF THE RESIDENCY FIX THAT DID NOT LAND. `cl_mod.TABLE_OPS` is a module-level singleton built - with no store handle, so the customer and product table workspaces (views, fields, overlays, - folders) resolve to tenant #0 for every caller. Scoping it here was written, measured and - then REVERTED, because it would have been worse than the bug: - - the WRITE door (`routes_grid._ctx`) is this session's file and now carries a handle; - the READ door is `routes_customers._ctx_for` / `routes_products` — files in NO session's - fence this wave — and they carry none. - - Moving only the write side means a prefix tenant saves a view into `t//…` and reads - back from the unprefixed bucket forever: their own view vanishes the moment they save it. - A latent split is worse than a stated residency error, and read and write must move in ONE - change. Booked as an ask + debt with that exact shape. Tenant #0 is unaffected either way - (empty namespace, default repo), which is why nothing today would have shown it. - - ⛔ Untouched by this wave regardless: the EMIT GUARD is D-40, deferred by ruling A-15. - """ - if ctx.table is not None: - return ctx.table - return cl_mod.TABLE_OPS - - -def _cohorts(ctx): - """The COHORT store this ctx operates on — `_tops`'s sibling, and here for the same reason. - - ⭐ WAVE 19 / R9: a cohort belongs to its database. Every cohort verb below used to name - `cohort_mod` directly, so a list built on the Product surface landed in `customer_cohorts` - beside somebody's real customer lists — with PRODUCT pids in it (CRC32 hashes of SKU codes), - which the customer page would then try to resolve against partner ids. Wave 16 answered that - by REFUSING cohorts on other topics (`with_cohorts=False`); R9 says scope-parameterize - instead. `modules.cohort.scoped` returns the module itself for the customer topic, so the - customer path is byte-unchanged. - """ - return cohort_mod.scoped(ctx.scope_key) - - -#: The keys a stored document entry may put on the wire. ⛔ `path` IS NOT ONE OF THEM — it is the -#: byte address inside the tenant's repo, of no use to a browser and a needless disclosure of the -#: store's layout. `canDelete` is not stored either: it is a fact about the READER, computed per -#: request below, so one person's answer can never be cached into another's. -_DOC_WIRE_KEYS = ('id', 'name', 'mime', 'size', 'by', 'ts') - - -def docs_for(pids, scope_key='customer', uname='', admin=False, st=None): - """`{"": [CustomerDoc, ...]}` for a SCOPED row set — the producer `payload.docs` lost. - - ⭐⭐ WAVE 30 / CONTRACT C4 (D-138). The write door has been live and UNREACHABLE since EXIT-6: - `doc_add`/`doc_fetch`/`doc_delete` all still work, and the comment inside that handler says - exactly why nobody noticed — *"the React shell cannot reach it yet (`payload.docs` is set only - by app.py)"*. `app.py` was deleted, and the producer went with it. Every client half survived: - `RecordDetail`, `SwipeView` and `Documents` are complete, and all six `onDoc*` handlers in - `CustomerGrid.tsx` are written as `payload?.docs ? … : undefined`, so an ABSENT `docs` key is - what has been switching the whole feature off. An empty dict is truthy and turns it on with an - honest empty state. - ⛔ THIS IS THE SIXTH INSTANCE OF THE REACHABILITY CLASS and the one `verify_reachability.py` - cannot see — a payload key assembled through helpers and dict merges is not a text-matchable - literal ([[reachable-is-not-the-same-as-built]], [[artifact-with-no-importer]]). - - ⛔ ONE READER, NOT A MATCHING PAIR. `routes_tables` (ut_*) and `routes_customers` (the customer - scope) both call THIS — they do not each serialise. Two surfaces answering one question - separately is how the same defect got reintroduced in the opposite direction inside a single - commit ([[one-question-two-normalizers]]). - - ⚠ `pids` IS REQUIRED and is the row set the CALLER has already scoped — `shared_overlay.cells`' - argument, for the same reason: there is no "every document in the tenant" call to reach for. - A record with no documents gets `[]`, never a missing key, so a client can tell "none" from - "not served". - """ - wanted = {str(int(p)) for p in (pids or [])} - if not wanted: - return {} - scope = str(scope_key or 'customer').strip().lower() - key = _DOCS_KEY if scope in cohort_mod.LEGACY_SCOPES else f'{scope}_docs' - try: - stored = (st if st is not None else store).get(key) or {} - except Exception: # noqa: BLE001 - stored = {} # a display read degrades; it never breaks the grid - out = {} - for pid in wanted: - row = stored.get(pid) - docs = [] - for d in (row if isinstance(row, list) else []): - if not isinstance(d, dict) or not d.get('id'): - continue - wire = {k: d.get(k) for k in _DOC_WIRE_KEYS} - # The delete wall, mirrored onto the wire so the button is absent rather than - # offered-and-refused. `doc_delete` enforces the same test server-side; this is the - # DISPLAY of that rule, never a substitute for it. - wire['canDelete'] = bool(admin or (d.get('by') and d.get('by') == uname)) - docs.append(wire) - out[pid] = docs - return out - - -def _docs_key(ctx): - """The DOCUMENTS bucket this ctx operates on — `_cohorts`'s sibling (wave 19, item 12's - silent third case). - - ⛔ `_DOCS_KEY` STAYS the module constant with its shipped value: app.py re-imports it for the - Streamlit docs payload and `verify_seam.py` asserts the two resolve to the same string. This - resolves the CUSTOMER topic to exactly that constant and gives every other database its own - bucket, so the pid keys inside one bucket all come from one id space. - """ - scope = str(getattr(ctx, 'scope_key', '') or 'customer').strip().lower() - if scope in cohort_mod.LEGACY_SCOPES: - return _DOCS_KEY - return f'{scope}_docs' - - -# ------------------------------------------------------------------ the workspace family (X1 amd.) -def table_workspace(ctx, allowed_pids=None, consume_corrections=True): - """Durable Airtable-style view/schema/overlay state, with an honest session fallback. - - Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED - with them. Own views win a same-id collision — a personal view is the user's own object and - must never be shadowed by somebody else's share. - - ⚠ `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries - `memberPids`, which the writer's scope validated on the way IN — but the READER may have a - narrower scope. Handing a Fisch-scoped user a view whose member list was built by a - full-access user would leak the other BU's customers straight through the strict-isolation - rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the - result reaches a renderer or an event handler; omitting it drops memberPids entirely rather - than trusting the stored list (fail-closed: a smaller view, never a wider one). - - MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live - `is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call - site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids` - stays an EXPLICIT argument because the folder leg deliberately omits it. - """ - uname = ctx.uname - if _store_of(ctx).available(): - ws = _tops(ctx).workspace( - uname, consume_corrections=bool(consume_corrections)) - try: - shared = _tops(ctx).shared_views(uname, bool(ctx.admin)) - except Exception as _se: - _tel.error('sharedviews:read', _se) - shared = {} - # ⭐ WAVE 21 (item 9, C1): the R10 registry's named-user grants, PROJECTED at last. - # Wave 20 wrote view grants and no reader ever consulted them — "Share view…" recorded - # a row and changed nothing on the receiver's screen. - try: - granted = _granted_views(ctx, uname) - except Exception as _ge: - _tel.error('sharedviews:granted', _ge) - granted = {} - if shared or granted: - merged = dict(granted) - merged.update(shared) # the everyone-bucket outranks a grant - merged.update(ws.get('views') or {}) # own views win the id collision - ws['views'] = {vid: scope_view(v, uname, allowed_pids) - for vid, v in merged.items()} - return ws - # Store down. Streamlit degrades to its captioned session workspace exactly as before; an - # API caller has no durable dict and must be told, not quietly served memory. - if ctx.fallback_ws is None: - raise StoreUnavailable('table workspace unavailable: no store and no session fallback') - return ctx.fallback_ws.setdefault( - uname, {'views': {}, 'fields': {}, 'overlays': {}}) - - -def _granted_views(ctx, uname): - """{view_id: STAMPED view} for every view GRANTED to `uname` by name (wave 21, item 9, C1). - - The R10 registry stores bare ids, tenant-scoped through the SAME store handle this topic's - bucket uses; the record itself is fetched out of the OWNER's personal stratum of THIS - bucket. An id granted on another topic simply finds nothing here and contributes nothing - (fail-closed). Each projected view is stamped `shared` / `sharedRole` / `owner` so the - client can synthesise the "Shared with me" group and disable edit affordances for - role=view — the stamps live on the PROJECTION only, never written back to the store. - `role_for` is asked with is_admin=False deliberately: the stamp reports the GRANT (the - registry listed only explicit entries), not the caller's rank. - """ - import core.shares as shares - tops = _tops(ctx) - st = getattr(tops, 'st', None) - ids = (shares.shared_with(uname, kind='view', st=st) or {}).get('view') or [] - out = {} - for vid in ids: - found = tops.find_view(vid) if hasattr(tops, 'find_view') else None - if not found: - continue - owner, view = found - if owner == uname: - continue - role = shares.role_for('view', vid, uname, st=st) - if role not in ('view', 'edit'): - continue - v = dict(view) - v['shared'] = True - v['sharedRole'] = role - v['owner'] = owner - out[str(vid)] = v - - # ⭐ D-37 (2026-08-05) — THE FOLDER LEG. W20's R10 said "a folder share = its views ride - # along"; W21 wired the view kind only, so a granted folder was accepted, stored and listed - # under "Shared with me" while changing NOTHING on the receiver's screen. Two waves of a - # control that reported success and did nothing — the same defect class the view leg above - # was written to close, surviving in its sibling. - # - # ⚠ A DIRECT VIEW GRANT WINS over the folder it happens to sit in, and the ordering above is - # what implements that: the direct pass has already filled `out`, so the folder pass skips - # ids it finds there. The specific grant is the more deliberate statement of intent, and it - # may carry a DIFFERENT role — a folder shared read-only must not downgrade a view somebody - # was explicitly given edit on, nor upgrade one they were not. - try: - folder_ids = (shares.shared_with(uname, kind='folder', st=st) or {}).get('folder') or [] - except Exception: - folder_ids = [] - for fid in folder_ids: - found = tops.find_folder(fid) if hasattr(tops, 'find_folder') else None - if not found: - continue # a folder on another topic contributes nothing here - owner, folder, inside = found - if owner == uname: - continue - role = shares.role_for('folder', fid, uname, st=st) - if role not in ('view', 'edit'): - continue - for vid, view in inside.items(): - if str(vid) in out: - continue - v = dict(view) - v['shared'] = True - v['sharedRole'] = role - v['owner'] = owner - # The folder's NAME rides so the client can group these together later; today it - # renders in the same "Shared with me" group the view leg feeds. Stamps live on the - # PROJECTION only and are never written back, exactly as above. - v['sharedFolder'] = str(folder.get('name') or '')[:80] - out[str(vid)] = v - return out - - -def scope_view(view, uname, allowed_pids): - """Re-scope a view to the READER. A no-op for a view the reader created. - - Two things travel inside a shared view that are SCOPE, not presentation: - 1. `memberPids` — a customer list. Re-filtered against the reader's permitted pool. - 2. cohort conditions — a cohort id is a permission handle owned by ONE user. For a reader - who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable - leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) — i.e. it would show MORE rows, - not fewer. So the whole filter tree is dropped for a foreign reader rather than - evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a - silently widened one is a leak. - """ - if not isinstance(view, dict) or view.get('createdBy') == uname: - return view - out = dict(view) - cfg = dict(out.get('config') or {}) - mp = cfg.get('memberPids') - if isinstance(mp, list): - cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())] - if allowed_pids is not None else []) - if tree_has_cohort(cfg.get('filters')): - cfg['filters'] = [] - cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending - out['config'] = cfg - return out - - -def tree_has_cohort(node): - """True when a filter tree references the Cohorts column anywhere (any depth).""" - import aios_grid as _agc - col = _agc.COHORT_COLUMN - if isinstance(node, list): - return any(tree_has_cohort(n) for n in node) - if not isinstance(node, dict): - return False - if node.get('colId') == col or node.get('field') == col: - return True - return tree_has_cohort(node.get('conditions') or node.get('children')) - - -def measure_leaves(nodes): - """Every leaf of a filter tree that carries a date WINDOW — i.e. every measure condition. - - Keyed off the window rather than the column name on purpose: a window is the one thing no - column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure. - """ - out = [] - for node in nodes or []: - if not isinstance(node, dict): - continue - if isinstance(node.get('children'), list): - out.extend(measure_leaves(node['children'])) - elif node.get('window') is not None: - out.append(node) - return out - - -# ------------------------------------------------------------------ private handler helpers -def _field_echo_equal(accepted, raw): - """Does the ACCEPTED def match what the client already renders? (wave-6 item 3) - - The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no - re-render — the client is the ORIGIN of the change, its optimistic copy is already right, - and the rerun it would buy costs a full payload re-serialisation plus an iframe remount - (the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms - ('' / [] / {} / absent all render identically): any normalisation a validator applied is a - real difference the client must be SHOWN. Fail-closed — unsure means not equal means rerun. - """ - empty = ('', [], {}) - for k in _FIELD_ECHO_PROPS: - a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None - if k == 'scope': - # 'global' and absent are the SAME rendering (an unscoped field) — the event may - # say 'global' explicitly while the stored def says it by omission. - a = None if a == 'global' else a - b = None if b == 'global' else b - if (None if a in empty else a) != (None if b in empty else b): - return False - return True - - -def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120): - """Return a durable, case-insensitively unique field/view display name.""" - limit = max(1, int(max_len)) - - def _clean(value): - return ' '.join(str(value or '').split()) - - base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip() - taken = {_clean(value).casefold() for value in existing if _clean(value)} - if base.casefold() not in taken: - return base - index = 2 - while True: - suffix = f' {index}' - stem = base[:max(0, limit - len(suffix))].rstrip() - candidate = f'{stem}{suffix}' if stem else str(index)[-limit:] - if candidate.casefold() not in taken: - return candidate - index += 1 - - -def _known_usernames(): - """Real account names, for C4's fail-closed `users` grant. None = "could not resolve". - - None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry - error would silently collapse a user's stored 'specific users' grant to 'personal' on the - very next autosave — destroying a setting because a read blipped. The grant is inert today - anyway (views are per-user; nothing reads it to allow cross-user access), so the honest - trade is: never invent access, never destroy a stored intent either. - """ - try: - reg = set(users.registry() or {}) - except Exception as _ue: - _tel.error('viewperms:users', _ue) - return None - # ⚠ EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising - # when the user store cannot be read (a store blip logs `store:get:users` and hands back an - # empty dict) — and a working app ALWAYS has at least the bootstrap admin. Treating {} as a - # real answer would filter every name out of a stored grant and collapse it to 'personal' - # on the next autosave. Caught by testing this in BARE mode with no store token. - return reg or None - - -# ------------------------------------------------------------------ THE SEAM -def handle_events(events, ctx): - """The event LOG entry point → one EventResult. The `[-24:]` window lives here. - - ⚠ THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component - has exactly one value slot, and two quick emits share it: creating a field emits - `field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later — with the - server still inside the first rerun, the second write clobbered the slot before any run - read it, and the field creation VANISHED with no error anywhere (measured live; the same - race sat latent under overlay-field creation, masked by localStorage). The client now - sends its recent window of events; each is processed exactly once, keyed by its id. The - single-event shape is still accepted — it is what an older client in a stale tab sends. - - Accepts everything the component value can be: the log dict, a bare list of events (what - the HTTP body's `events` array unwraps to), or one legacy single-event object. - - `EventResult.rerender` is True when ANY processed event demands a re-render from server - state — the same OR-fold `_cl_handle_grid_event` did. - """ - if isinstance(events, dict) and isinstance(events.get('events'), list): - seq = events['events'][-24:] - elif isinstance(events, list): - seq = events[-24:] - else: - seq = [events] - rerun = False - for one in seq: - rerun = handle_one(one, ctx) or rerun - ctx.out.rerender = rerun - return ctx.out - - -def handle_one(event, ctx): - """Validate ONE untrusted component event. - - Odoo-source fields are never accepted as cell updates. View and schema objects are - user-scoped; row patches additionally require the pid to be in the currently permitted - Customer List pool. - - RETURN VALUE = "this page must re-render from server state", NOT "handled". - The distinction is the difference between a snappy grid and a sluggish one. - Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the - script once simply to deliver the component value, and the client is the ORIGIN - of the change, so its own state is correct before the server hears about it. An - extra st.rerun() therefore buys nothing and costs a second full page run. - Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO - page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row - payload — and the handler on the second run is a guaranteed no-op, because the - event-id dedupe two lines below has already consumed that id. - - THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) — True ONLY when the next render - genuinely differs from what the client already draws: - field_upsert — False when the accepted def ECHOES the client's copy - (_field_echo_equal); True on any validator mutation, any - permissions refusal, and ALWAYS when a measure window changed - (values must be re-resolved server-side). - overlay_patch — False when every requested update was accepted verbatim; - True the moment anything was refused or truncated. - field_delete — False: the client removed the column optimistically and the - store scrub is invisible to the next render. - field_duplicate — True only when host-held VALUES must flow back (custom_ cell - copies, measure_ recomputes); formula/created_time clones - compute client-side. - view_delete — False (the deleted view simply stops being served). - view_upsert — unchanged: False, except an unresolved measure condition. - cohort_* / add_to_list — True: the lists payload is server truth, and these are - not the hot path. - - The id dedupe is a SET (not a last-id slot): the event log resends the whole recent - window on every emit, so every id must stay consumed, not just the latest one. - """ - # The ctx fields the moved body reads as plain locals — bound once here so every branch - # below is textually the app.py body (which is what makes this move reviewable). - uname = ctx.uname - allowed_pids = ctx.allowed_pids - fields = ctx.fields - measure_keys = ctx.measure_keys - resolved_ids = ctx.resolved_ids - cohort_ids = ctx.cohort_ids - measure_offer = ctx.measure_offer - visible_views = ctx.visible_views - admin = ctx.admin - scope_key = ctx.scope_key - - if not isinstance(event, dict): - return False - event_id = str(event.get('id') or '')[:180] - seen = ctx.seen_ids - if not event_id or event_id in seen: - return False - seen[event_id] = True - if len(seen) > 400: # a session-lifetime set, kept bounded - for stale in list(seen)[:200]: - seen.pop(stale, None) - kind = event.get('type') - field_by_key = {f['key']: f for f in fields} - valid_keys = set(field_by_key) - overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'} - # Event validation is not a render payload. It must not consume a one-shot label - # correction before the browser has had a chance to receive it. - ws = table_workspace( - ctx, allowed_pids=allowed_pids, consume_corrections=False) - - def _session_ready(): - ws.setdefault('views', {}) - ws.setdefault('fields', {}) - ws.setdefault('overlays', {}) - - if kind == 'view_select': - # Owner item 3 (2026-07-31): remember WHERE THE USER IS so a fresh browser resumes - # there instead of the system default view. Presentation state only — the read side - # re-validates the id against what the caller may see, so nothing here needs an - # authorisation wall beyond the length cap. Never a re-render: the client is already - # on that view. - vid = str(event.get('viewId') or '').strip()[:120] - if vid and _store_of(ctx).available(): - try: - _tops(ctx).save_active_view(uname, vid) - except Exception as _ae: - _tel.error('grid:view-select', _ae) - return False - - if kind == 'record_layout': - # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. Presentation - # state for one surface; grid column order is untouched. Unknown keys are pruned - # against the LIVE field set, dupes collapse, and the wire re-validates at serve - # time — so a client racing a field delete degrades to default order, never an error. - raw_order = event.get('order') - if not isinstance(raw_order, list): - return False - order, seen = [], set() - for k in raw_order[:200]: - k = str(k or '').strip()[:80] - if k and k in valid_keys and k not in seen: - seen.add(k) - order.append(k) - if _store_of(ctx).available(): - _tops(ctx).save_record_layout(uname, order) - else: - _session_ready() - if order: - ws['recordLayout'] = {'order': order} - else: - ws.pop('recordLayout', None) - # Autosave hot path, like view_select: the client already renders the order it asked - # for, and the wire echoes it back on the next workspace read. - return False - - if kind == 'view_upsert': - raw = event.get('view') - if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict): - return False - view_id = str(raw.get('id') or '').strip()[:120] - name = str(raw.get('name') or '').strip()[:120] - requested_name = name - if not view_id or not name: - return False - cfg = raw['config'] - # The filter TREE (leaf conditions and/or nested condition groups) is - # validated by the module-agnostic grid contract helper — imported lazily - # here for the same reason the page does it: a missing embed build must - # never break page import. - import aios_grid as _ag - # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered, - # shown, grouped or coloured by, because it is not a column of this table. Hence the - # widened key set here and the plain `valid_keys` everywhere below. - # `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything - # else is dropped here. Leaving it in would persist a condition the engine can only - # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table - # forever with nothing on screen explaining why. - filters = _ag.clean_filter_tree(cfg.get('filters'), - valid_keys | set(measure_keys), - cohort_ids=set(cohort_ids)) - sorts = [] - for rule in list(cfg.get('sorts') or [])[:20]: - if (isinstance(rule, dict) and rule.get('colId') in valid_keys - and rule.get('dir') in ('asc', 'desc')): - sorts.append({'colId': rule['colId'], 'dir': rule['dir']}) - order = list(dict.fromkeys( - key for key in list(cfg.get('order') or []) if key in valid_keys)) - order += [field['key'] for field in fields if field['key'] not in order] - visible = list(dict.fromkeys( - key for key in list(cfg.get('visible') or []) if key in valid_keys)) - widths = {} - raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {} - for key, value in raw_widths.items(): - if key in valid_keys: - try: - widths[key] = max(60, min(600, round(float(value)))) - except (TypeError, ValueError): - pass - # C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort. - # `cohort_ids` is the same permission list the cohort filter leaf uses — a lock naming - # a cohort this session cannot see is DROPPED at write, exactly like a cohort rule. - # (Readers of a SHARED locked view resolve the id against their OWN visible sets; an - # unresolvable lock matches NOTHING client-side — the cohort-leaf law, never a widen.) - cohort_lock = str(cfg.get('cohortLock') or '').strip()[:120] - # ⛔ WAVE 17 R1 (C-LOCKV guard a) — A PROJECTED LOCKED VIEW RE-STAMPS ITS OWN LOCK. - # A locked view IS its set: its id and its lock are the same fact. The client rebuilds - # `config` from scratch on every autosave (a column resize is enough), so taking the - # lock from the browser would mean one omitted key turns "the 40 accounts we agreed to - # call" into the whole book, under the same name, with nothing going red. The read side - # re-stamps too (aios_grid.views_from_defs) — this keeps the STORE from accumulating the - # wrong record in the first place. - _is_locked_view = view_id in set(cohort_ids) - if _is_locked_view: - cohort_lock = view_id - config = { - 'filters': filters, - # conjunction joining the ROOT-level conditions; anything but an - # explicit 'or' means 'and' (legacy views had an implicit AND). - 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and', - **({'cohortLock': cohort_lock} - if cohort_lock and cohort_lock in set(cohort_ids) else {}), - # ⭐⭐ WAVE 32 · R5 / CONTRACT C4 — "Mark important" (raised by SESSION C as ASK C-12). - # - # ⛔ WITHOUT THIS LINE THE MARK IS A LIE THAT SURVIVES UNTIL RELOAD. This dict is - # REBUILT FROM AN ALLOWLIST on every autosave — a column resize is enough — so a key - # the client sends and this literal does not name is silently dropped. The badge would - # paint, the write would answer 200, and the mark would be gone on the next read: the - # exact shape T24's done-when says "persists across a reload" to catch, and - # [[a-migration-that-runs-on-the-next-write]] on a write path. - # - # ⚠ UNCONDITIONAL, NOT SPREAD-CONDITIONAL LIKE `cohortLock` ABOVE, and the asymmetry is - # the whole point: R5's mark must be REMOVABLE. A key written only when present means - # unmarking sends `important: false`, the allowlist drops it, and the stored `true` - # survives — a mark you can set and never clear. - # ⚠ `is True`, not truthy: the client sends a real boolean, and a string `"false"` must - # not become a mark. - # ⚠ NO PERMISSION GATE, deliberately. R5 makes this a personal legibility mark, not a - # lock; `_may_administer` already decides who may write this view at all, and adding a - # second wall here would be the two-normalizers shape this wave is already fixing - # elsewhere. - 'important': cfg.get('important') is True, - 'sorts': sorts, - 'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None, - 'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None, - 'rowHeightMode': (cfg.get('rowHeightMode') - if cfg.get('rowHeightMode') in ('short', 'medium', 'tall') - else 'short'), - 'order': order, - 'visible': visible, - 'widths': widths, - 'memberPids': [pid for pid in list(cfg.get('memberPids') or []) - if isinstance(pid, int) and pid in allowed_pids], - } - # Wave-6 item 10: a non-grid display mode rides the view config, validated - # structurally (mode whitelist, refs must be this table's fields). - display = _ag._clean_display(cfg.get('display'), valid_keys) - if display: - config['display'] = display - # Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling - # sideways. Clamped 1..8 — 0 would unfreeze the identity column the whole - # surface pivots on; absent = the legacy single frozen column. - if cfg.get('frozenCount') is not None: - try: - config['frozenCount'] = max(1, min(8, int(cfg['frozenCount']))) - except (TypeError, ValueError): - pass - # ── wave-9 C3 (lock) + C4 (permissions) ──────────────────────────────────────────── - # The PRIOR stored view decides both: whether this is a create or an update (C4's - # default splits on exactly that) and who is allowed to change the lock (C3). - prior = (ws.get('views') or {}).get(view_id) or {} - # ⚠ THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the - # caller may not SEE reads as absent — which would look like a CREATE and let anybody - # overwrite somebody else's shared view by guessing its id. Ask the store for the raw - # record and refuse before deciding anything else. Fail-closed on both branches. - _raw_shared = _tops(ctx).shared_view(view_id) if _store_of(ctx).available() else None - if _raw_shared is not None and not prior: - return False # exists, not visible to this caller: refuse - if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin): - return False # visible but not editable by this caller - is_new = not prior - # C4: the creator is HOST-STAMPED and never read from the browser — the same rule as - # the field-level createdBy. An existing view keeps whoever made it. - creator = prior.get('createdBy') or uname - # Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who - # could rewrite them could grant themselves sole ownership of someone else's view, or - # quietly widen a users-scoped view to everyone — privilege escalation by edit. - if not is_new and not _tstore._may_administer(prior, uname, admin): - raw = dict(raw) - raw['permissions'] = prior.get('permissions') - raw['locked'] = prior.get('locked') - # ── I12/C3 BELT (2026-07-31): a LOCKED view's DISPLAY MODE is frozen HOST-SIDE too. - # The client's switcher already refuses (setDisplayMode), and its comment has claimed - # "the host refuses too" since wave 9 — this makes that claim true for direct API - # callers. ⚠ MIRRORS types.ts `isModeFrozen` EXACTLY: SYSTEM views and the undeletable - # list:/cohort: projections are EXEMPT — their `locked` is legacy "undeletable", not - # mode-frozen, and freezing them would trap the default view in whatever mode it last - # saved (the owner's stuck-in-Map report is that trap, client-made). An upsert that - # UNLOCKS in the same write (authorised above) may change display freely. - _mode_frozen = (prior.get('locked') and raw.get('locked') is not False - and prior.get('kind') != 'system' - and view_id != 'all-customers') - if _mode_frozen: - if prior.get('config', {}).get('display'): - config['display'] = dict(prior['config']['display']) - else: - config.pop('display', None) - # ⚠ A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9 - # integration pass, before deploy: `all-customers` is every user's DEFAULT view and it - # carries their own column widths, visible fields and row height. C4's read-default - # ('collaborative' when permissions are absent, so nobody's saved view is - # retro-restricted) is right for a view somebody AUTHORED — but applied to the system - # view it routed `all-customers` into the shared bucket, and the first user to resize a - # column would have published their layout to everyone. Verified in the store: - # shared_table_views() was returning ['all-customers']. - # System views are pinned personal, so they stay in each user's own workspace exactly as - # before this wave. `kind` is host-validated two lines below; read it here from `raw` and - # treat the stored kind as authoritative for an existing view. - _kind = (prior.get('kind') if not is_new - else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') - else 'custom')) - if _kind == 'system' or view_id == 'all-customers': - perms = {'edit': 'personal'} - else: - perms = _ag.clean_view_permissions( - raw.get('permissions'), - default='personal' if is_new else 'collaborative', - known_users=_known_usernames()) - # C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever - # is already stored. Enforced here rather than hidden in the client. This composes with - # C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze - # the display MODE. - # ⚠ Inert in practice TODAY — a user can only ever write their own per-user workspace, - # so `uname == creator` always holds. Correct plumbing for shared views, not a live gate. - may_lock = admin or uname == creator - locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked')) - # C3: the LOCK's actual job — a locked view's display mode is frozen, so a locked Kanban - # stays Kanban. This half IS live per-user and is the part the owner asked for. Filters, - # sorts and columns stay editable by design. - if _mode_frozen: - was = (prior.get('config') or {}).get('display') - now = config.get('display') - if (was or {}).get('mode') != (now or {}).get('mode'): - if was: - config['display'] = was - else: - config.pop('display', None) - # WAVE 17 R1 (C-LOCKV guard b) — ONE THING, ONE NAME. Renaming a locked view renames - # the SET it is, in the store that owns it; the view record must not grow a second title - # that the read side would then have to arbitrate. Routed rather than duplicated. - if _is_locked_view and _store_of(ctx).available(): - _sets = _cohorts(ctx) # R9: the SET lives in this topic's bucket - _cur = (_sets.all_for(uname).get(view_id) or {}).get('name') - if name and name != _cur: - try: - _sets.rename(uname, view_id, name) - except ValueError as _e: - _tel.error('cohort:rename-via-view', _e) - view = {'id': view_id, 'name': name, - # ⚠ DERIVED FROM THE ID, never taken from the browser: `kind` is what paints the - # lock mark, so a client claiming it for an ordinary view would be claiming a - # guarantee the engine is not making about that view's rows. - 'kind': _ag.LOCKED_VIEW_KIND if _is_locked_view - else raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') - else 'custom', - 'locked': locked, - 'createdBy': creator, - 'permissions': perms, - 'note': str(raw.get('note') or '')[:2000], - 'config': config} - # System/list projections are not all persisted in this table store, but custom views - # may not impersonate their visible names. Persisted view names are allocated - # tenant-globally by TableStore inside the write transaction. - _reserved_view_names = [ - value.get('name') for value in visible_views - if (isinstance(value, dict) and value.get('id') != view_id - and value.get('kind') in ('system', 'list')) - ] - # Wave 21 (item 3, R6): the reserved SYSTEM name is PER-TOPIC — a ut_ database's system - # view is "All records", and reserving the customer literal there uniquified a user's - # honest "All customers" view name against a string that never appears on that topic. - _scope = str(getattr(ctx, 'scope_key', '') or 'customer').strip().lower() - _sys_name = ('All products' if _scope == 'product' - else 'All records' if _scope.startswith('ut_') else 'All customers') - if view_id != 'all-customers': - _reserved_view_names.append(_sys_name) - # ⭐ WAVE 21 (item 9, C1) — THE FOREIGN-VIEW WALL. A granted view is projected into the - # receiver's rail, so their autosave can now emit an upsert carrying an id they do not - # own. Silently saving that into the RECEIVER's stratum would FORK the view (own views - # win the merge, so the receiver sees their private copy forever and calls sharing - # broken). Role `edit` (or admin) writes THROUGH to the owner's record — config only, - # never identity or reach: `createdBy`/`permissions` are pinned from the stored record. - # Role `view` (or no grant) refuses with a toast instead of forking. - # - # ⛔⛔ WAVE 33 · T13 (owner item 3) — A PINNED PER-USER ID IS NOT A FOREIGN VIEW, AND - # TREATING IT AS ONE IS A CROSS-USER WRITE THAT REPORTS SUCCESS. - # - # `find_view` scans EVERY personal stratum and returns the FIRST match. `all-customers` - # and `tpl_overview` are PINNED ids that every user of a table has their own copy of — - # so the moment ANY colleague autosaves one (a column resize is enough), the wall reads - # "somebody else owns this id" for everybody else. MEASURED, on this exact path, after - # one colleague resized a column on the Overview: - # · a NON-ADMIN marking it is REFUSED, with a toast blaming sharing — a view nobody - # ever shared, on their own database; - # · an ADMIN marking it answers 200 and writes the mark into the OTHER USER'S - # stratum. The admin's own workspace still serves the un-marked view, so the mark - # "does not work"; the colleague silently gains a mark they never made. - # That is item 3's second mechanism, and it is a residency bug wearing a sharing bug's - # error message. - # - # ⚠ THIS FUNCTION ALREADY STATES THE RULE THE WALL BREAKS, seventy lines up: "A SYSTEM - # VIEW IS PER-USER STATE AND MUST NEVER BE SHARED", enforced there by pinning - # `perms = {'edit': 'personal'}`. A per-user view can therefore never BE a grant, which - # is the only thing this wall exists to protect — so exempting it narrows nothing. - _pinned_per_user = (_kind == 'system' or view_id in ('all-customers', _ag_overview_id())) - _save_as = uname - if _store_of(ctx).available() and not _pinned_per_user: - _foreign = _tops(ctx).find_view(view_id) - if _foreign and _foreign[0] != uname: - _o_uname, _stored = _foreign - if not admin: - import core.shares as _shares - if _shares.role_for('view', view_id, uname, - st=getattr(_tops(ctx), 'st', None)) != 'edit': - ctx.out.toast = ('That view is shared with you read-only — ask its ' - 'owner for edit access.') - return False - view['createdBy'] = _stored.get('createdBy', _o_uname) - if _stored.get('permissions') is not None: - view['permissions'] = _stored.get('permissions') - else: - view.pop('permissions', None) - _save_as = _o_uname - if _store_of(ctx).available(): - view = (_tops(ctx).save_view( - _save_as, view, reserved_names=_reserved_view_names, is_admin=admin) or view) - else: - _session_ready() - _view_names = list(_reserved_view_names) - _view_names.extend( - value.get('name') - for candidate_id, value in ws['views'].items() - if candidate_id != view_id and isinstance(value, dict) - ) - view['name'] = _unique_name(view.get('name'), _view_names) - ws['views'][view_id] = view - name_changed = view.get('name') != requested_name - # PERSISTED, but deliberately NO rerun — see the return-value contract in - # this function's docstring. This is the hot path: every filter, sort, - # group, colour, column-resize and column-toggle autosaves through here. - # - # CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered - # server-side, so if this view carries one whose answer was NOT in the payload we just - # shipped, the client is showing a pending marker with nothing coming. Ask for exactly - # one more run — and only then, so the common case (a column filter) still costs zero. - # If the top-of-run read of the component's own value already covered it, this is False - # and nothing extra happens; if that read is ever stale, this is what stops the - # condition hanging unresolved. - # - # ⚠ COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev): - # picking a measure FIELD mints a valueless rule whose autosave used to land here as - # "unresolved" and trigger a rerun that could resolve NOTHING — `rule_complete` refuses - # half-asked questions — so the only effect was a remount that closed the builder - # popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being - # typed stays client-side PENDING by design; the run that matters fires when its VALUE - # arrives and the question becomes answerable. - from harness import measure_filter as _mfc - unresolved = {str(r.get('id') or '') for r in measure_leaves(filters) - if _mfc.rule_complete(r)} - set(resolved_ids) - return name_changed or bool(unresolved - {''}) - - if kind == 'view_delete': - view_id = str(event.get('viewId') or '')[:120] - # ⭐ WAVE-27 item 27 (R8) — `IG_OVERVIEW_ID` joins the system view here, and the comment - # eight lines down already explains why in the LOCKED-view case: dropping the saved - # config leaves the PROJECTION, which rebuilds the row on the very next render. Overview - # is injected on every read of an Instagram database, so a delete would look like it - # worked and the view would be back after a reload — which is precisely the - # reads-as-success failure the client's own `UNDELETABLE_VIEW_IDS` note describes. - if not view_id or view_id in ('all-customers', _ag_overview_id()): - return False - # WAVE 17 R1 (C-LOCKV guard c) — deleting a LOCKED view deletes the set it is. Without - # this the store would drop the saved config and the projection would rebuild the view - # on the very next render: the row would come back, minus the user's sort and columns, - # which reads as a corruption rather than as a refusal. Own sets only (`all_for`), the - # same wall `cohort_delete` rides. - if _store_of(ctx).available() and view_id in _cohorts(ctx).all_for(uname): - _cohorts(ctx).delete(uname, view_id) - _tops(ctx).delete_view(uname, view_id) - # True, unlike an ordinary view_delete: the lists payload IS server truth and the - # client cannot know the membership channel changed. - return True - # Wave-9 I17: deleting a SHARED view is narrower than editing it — creator or admin - # only. A collaborator can change what a shared view shows; they cannot destroy - # everybody else's copy of it, because there is only one copy. - if _store_of(ctx).available(): - _raw_shared = _tops(ctx).shared_view(view_id) - if _raw_shared is not None and not _tstore._may_administer( - _raw_shared, uname, admin): - return False - # ⭐ WAVE 21 (item 9, C1) — the foreign wall, delete half. Contract: a grant of - # `edit` may delete (there is only one copy and the owner handed over its content); - # `view` or no grant refuses. The dead grant record is emptied afterwards so the - # receiver's "Shared with me" heals instead of listing a ghost id forever. - _foreign = _tops(ctx).find_view(view_id) - if _foreign and _foreign[0] != uname: - import core.shares as _shares - _sst = getattr(_tops(ctx), 'st', None) - if not admin and _shares.role_for('view', view_id, uname, - st=_sst) != 'edit': - ctx.out.toast = 'That view is shared with you read-only.' - return False - _tops(ctx).delete_view(_foreign[0], view_id) - try: - _shares.set_grants('view', view_id, [], st=_sst) - except Exception: - pass - return False - _tops(ctx).delete_view(uname, view_id) - else: - _session_ready() - ws['views'].pop(view_id, None) - # Wave-6 item 3: the client removed the view optimistically; a deleted view simply - # stops being served, so the next render differs in nothing the client still shows. - return False - - if kind == 'field_upsert': - raw = event.get('field') - if not isinstance(raw, dict): - return False - key = str(raw.get('key') or '')[:80] - import aios_grid as _ag2 - if key.startswith(_ag2.MEASURE_FIELD_PREFIX): - # A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure - # offer — the same admission the condition builder uses — so a field can only name a - # measure this user could also filter by. Fail closed on anything else. - field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer}) - if field is None: - return False - fmt = _ag2._clean_format(raw.get('format'), field.get('type')) - if fmt: - field['format'] = fmt - elif key in field_by_key and not key.startswith('custom_'): - base = field_by_key[key] - field = {**base, 'note': str(raw.get('note') or '')[:2000]} - # ⭐ W29-T83 — THE COLUMN SUMMARY ON A SOURCE-BACKED COLUMN. This branch rebuilds the - # accepted def from the CONTRACT and adds only what a user may override, and `agg` was - # not in that list — so Average on Customer's `Overdue days` was accepted with a 200 - # and dropped here, before it ever reached the store. Validated against the shared - # vocabulary rather than a literal (see the custom branch below for what a literal - # costs). An ABSENT or unknown `agg` clears it: the client sends the whole field on - # every edit, so a missing key is a user who unset the summary, and `{**base}` has - # already re-applied whatever the contract itself declares. - _agg = raw.get('agg') - if _agg in _ag2.FIELD_AGGS: - field['agg'] = _agg - else: - field.pop('agg', None) - # Wave-5 item 10: a DISPLAY format on any base field (presets included) — how a - # number/date reads, per user. Rendering-only; validated fail-closed per type. - fmt = _ag2._clean_format(raw.get('format'), base.get('type')) - if fmt: - field['format'] = fmt - else: - field.pop('format', None) # cleared -> back to the type's default render - # PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the - # auto-renamed label) of revenue_ytd / revenue_ly / orders_24m — nothing else. The - # whitelist IS the base contract: only a field that shipped with preset+measure can - # take a window here, and the measure KEY is pinned to the base's (fail closed). - if (base.get('preset') and isinstance(base.get('measure'), dict) - and isinstance(raw.get('measure'), dict)): - window = _ag2._clean_window(raw['measure'].get('window')) - if window is not None: - field['measure'] = {'key': base['measure']['key'], 'window': window} - if isinstance(raw.get('label'), str) and raw['label'].strip(): - field['label'] = str(raw['label'])[:120] - elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES: - ftype = raw['type'] - # ⭐⭐ 2026-08-09 — THE RELATIONAL PAIR HAS NO HOME IN A PER-USER OVERLAY, and this - # door used to accept one silently. It builds the stored field from a fixed key list - # and adds only the bags it knows (`options`, `max`, `formula`, `automation`, `code`) - # — so a `link`/`rollup` arriving here was stored WITHOUT its bag, in one user's - # stratum, where `compute_relation_cells` (which walks `user_tables` definitions) - # never sees it. Measured on nurilab as `custom_video_views_90j26`: a Rollup column - # that rendered, could not be filled by anything, and reported no error. - # ⛔ REFUSED RATHER THAN REPAIRED. These two belong to the table's SHARED schema - # (`POST /tables/{key}/fields`), which is where the client now sends them; storing a - # second, private, non-computing copy here would be a second definition of the same - # column. A refusal repaints, so a stale client says something rather than nothing. - if ftype in ('link', 'rollup'): - return False - readonly = ftype in _ag2.READONLY_CUSTOM_TYPES - # The editable family must SAY it is overlay (the client's own contract); the - # read-only pair (formula / created_time) is stored under the cohort column's - # mechanism instead, so no cell write can ever be accepted for it. - if not readonly and raw.get('source') != 'overlay': - return False - field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120], - 'type': ftype, 'source': 'odoo' if readonly else 'overlay', - 'default': True, 'custom': True, - 'note': str(raw.get('note') or '')[:2000]} - if readonly: - field['derived'] = True - # FILTERABLE since wave 6 (owner item 6): formula/created_time values live - # client-side, and this table's counts are client-mode — the client engine - # answers conditions on them soundly. Mirrors fields_from_workspace. - field['filterable'] = True - # ⛔ W29-T83 — THE VOCABULARY, NOT THE LITERAL `'sum'`. This is the write gatekeeper - # for an overlay column's summary and it accepted exactly one name, while the READ - # projection (`aios_grid.py:679`) passes all six — so a user who chose Median got a - # 200, a menu that said Median, and no summary after the next reload. The same shape - # as the `agg:"sum"` defect this repo already shipped, where one door kept it and the - # other dropped it ([[read-path-cannot-witness-write-path]]). - _agg = raw.get('agg') - if _agg in _ag2.FIELD_AGGS and not readonly: - field['agg'] = _agg - # A `select`/`multiselect` carries its own vocabulary; a `user` does NOT — its - # choices are the tenant's accounts, resolved fresh on every render, so persisting - # a snapshot here would go stale the moment somebody joins or leaves. - if ftype in ('select', 'multiselect'): - field['options'] = _ag2._clean_options(raw.get('options')) - if not field['options']: - return False # a select with no choices can never hold a value - field.update(_ag2._choice_appearance(raw, field['options'])) - if ftype == 'rating': - field['max'] = _ag2._clean_rating_max(raw.get('max')) - if ftype == 'formula': - # WRITE-time is where refs must exist (fail closed); read-time leaves them - # alone so a later-deleted ref blanks the CELLS, not the column. - formula = _ag2._clean_formula(raw.get('formula'), valid_keys) - if formula is None: - return False - field['formula'] = formula - if ftype == 'automation': - # C5-AUTOFIELD (wave 18): persist the validated config bag. No `return False` - # when absent — an unconfigured automation column is a legitimate intermediate - # state (create the column, then configure it in the gear). - # ⛔ C8 (wave 22): a CONFIGURED bag must name an existing flow — the write door - # reads the tenant's definition ids and `_clean_automation` refuses a bag - # without one (or naming a deleted one). Refused = the whole field write is - # refused, loudly (repaint), because storing the field while dropping its bag - # would be a column that silently never runs. - if isinstance(raw.get('automation'), dict): - try: - _flow_ids = frozenset((ctx.table.st if ctx.table is not None - else _tops(ctx).st).get('automations') or {}) - except Exception: - _flow_ids = frozenset() - auto = _ag2._clean_automation(raw.get('automation'), valid_keys, - flow_ids=_flow_ids) - if auto is None: - return False - field['automation'] = auto - # ⭐ WAVE-27 item 13 (R13): the code column's language. OPTIONAL, like the - # `automation` bag above and unlike `formula` — an unconfigured code column is a - # legitimate intermediate state (create it, pick the language in the gear), and it - # stores and renders as plain text meanwhile. `_clean_code` never refuses over an - # unknown language; it falls back to `plain`, because the value chooses a - # HIGHLIGHTER, and dropping a user's column to punish a typo in one would be the - # disproportionate half of fail-closed. - if ftype == 'code' and isinstance(raw.get('code'), dict): - code_bag = _ag2._clean_code(raw.get('code')) - if code_bag: - field['code'] = code_bag - fmt = _ag2._clean_format(raw.get('format'), ftype) - if fmt: - field['format'] = fmt - else: - return False - # Wave-5 item 1 — createdBy + permissions on the CREATED strata (custom_/measure_). - # `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the - # client never supplies it, so a browser cannot claim someone else's field. A - # permissions CHANGE is accepted only from the creator or an admin; a legacy field - # with no createdBy is admin-only (fail closed, both directions). The table workspace - # is per-user today, so this wall becomes load-bearing when sharing arrives — but the - # admin path and the legacy path are enforceable (and NC'd) right now. - needs_values = False - refused_perms = False - if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX): - prior = ws.get('fields', {}).get(key) - prior = prior if isinstance(prior, dict) else None - if prior is None: - field['createdBy'] = uname - # Wave-6 item 9: scope is chosen at CREATE, on the cohort page only — - # 'cohort' marks the field cohort-specific; 'global' or absent stays - # unscoped, so an old bundle that sends no scope keeps today's behavior. - if scope_key == 'cohort' and raw.get('scope') == 'cohort': - field['scope'] = 'cohort' - else: - if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip(): - field['createdBy'] = prior['createdBy'] - # Scope is PRESERVED like createdBy: the client copy never moves an - # existing field between scopes. - if prior.get('scope') == 'cohort': - field['scope'] = 'cohort' - want = _ag2._clean_permissions(raw.get('permissions')) - have = _ag2._clean_permissions((prior or {}).get('permissions')) - owner = field.get('createdBy') - if want is not None and (admin or (owner and owner == uname)): - field['permissions'] = want - else: - if have is not None: - field['permissions'] = have - # Asked for a permissions change and did not get it: the client's - # optimistic copy is a lie that must be repainted (wave-6 item 3). - refused_perms = want is not None and want != have - # A NEW measure column, or an existing one whose window moved: the VALUES - # are host-computed, so the next render genuinely differs. - if key.startswith(_ag2.MEASURE_FIELD_PREFIX): - needs_values = prior is None or prior.get('measure') != field.get('measure') - # Canonical names are reserved outside the durable workspace. Persisted field names are - # allocated by TableStore from the transaction's current state, not this stale snapshot. - _workspace_field_keys = set((ws.get('fields') or {})) - _reserved_field_names = [ - value.get('label') for candidate_key, value in field_by_key.items() - if (candidate_key != key and candidate_key not in _workspace_field_keys - and isinstance(value, dict)) - ] - if _store_of(ctx).available(): - field = (_tops(ctx).save_field( - uname, field, reserved_names=_reserved_field_names, - correction_id=event_id) or field) - else: - _session_ready() - _field_names = list(_reserved_field_names) - _field_names.extend( - value.get('label') - for candidate_key, value in ws['fields'].items() - if candidate_key != key and isinstance(value, dict) - ) - requested_label = ' '.join( - str(field.get('label') or 'Untitled').split())[:120].rstrip() - field.pop('labelCorrectedFrom', None) - field.pop('labelCorrectionId', None) - field['label'] = _unique_name(requested_label, _field_names) - if field['label'] != requested_label and event_id: - field['labelCorrectedFrom'] = requested_label - field['labelCorrectionId'] = str(event_id)[:180] - # ⚠ W29-T83 — THE SAME PREDICATE the store uses, called rather than restated. This - # fallback carried its own shorter copy (note only), so a format or measure override - # survived one path and not the other, and `agg` survived neither. - if _tstore.source_override_is_empty(field): - ws['fields'].pop(key, None) - else: - ws['fields'][key] = field - if needs_values or refused_perms: - return True - # THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client - # already renders needs no rerun — see _field_echo_equal and the docstring's law. - return not _field_echo_equal(field, raw) - - if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'): - # Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 — - # replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed: - # only the caller's OWN cohorts (all_for), and member pids are intersected with the - # caller's permitted pool — the same guard add_to_list rides. - if not _store_of(ctx).available(): - return False - sets = _cohorts(ctx) # R9: this DATABASE's lists, never another's - cohort_id = str(event.get('cohortId') or '').strip()[:120] - if not cohort_id or cohort_id not in sets.all_for(uname): - return False - try: - if kind == 'cohort_rename': - name = str(event.get('name') or '').strip()[:120] - if not name: - return False - sets.rename(uname, cohort_id, name) - elif kind == 'cohort_delete': - sets.delete(uname, cohort_id) - else: - raw_pids = event.get('pids') - if not isinstance(raw_pids, list): - return False - pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids] - if not pids: - return False - if kind == 'cohort_add': - sets.add_members(uname, cohort_id, pids) - else: - sets.remove_members(uname, cohort_id, pids) - except ValueError as e: - _tel.error('cohort:panel-edit', e) - return False - # True: the lists payload (names, membership, counts) must reflect this on the next - # render — the panel is showing server truth, not local optimism. - return True - - if kind in ('doc_add', 'doc_delete', 'doc_fetch'): - # DOCUMENTS on a row (owner item 12, contract C5). Metadata in the store key - # `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a - # JSON blob — see core.store.upload_bytes for why that distinction is load-bearing. - # - # ⭐ WAVE 19 / R9 — item 1's SILENT THIRD CASE, and the one nobody reported because the - # React shell cannot reach it yet (`payload.docs` is set only by app.py). Both homes are - # keyed by pid and both were topic-blind: the metadata bucket AND the byte path - # `docs//…`. Two topics' pid spaces overlap by construction (a CRC32 hash lands - # anywhere an Odoo partner id can), so a product's file could occupy the path of a - # customer's — one upload silently shadowing another tenant-visible document. Per-topic - # bucket AND per-topic prefix; the customer topic keeps both of its shipped names. - import base64 as _b64c - import re as _re_mod - if not _store_of(ctx).available(): - return False - docs_key = _docs_key(ctx) - docs_dir = 'docs' if docs_key == _DOCS_KEY else f'docs/{docs_key[:-5]}' - pid = event.get('pid') - doc_id = str(event.get('docId') or '').strip()[:80] - # The pid wall is the SAME one every other row-scoped event rides: a caller may only - # touch rows inside their own permitted pool. Without it, documents would be the - # one surface where a scoped agent could read another book's rows. - if not isinstance(pid, int) or pid not in allowed_pids or not doc_id: - return False - # ⭐ WAVE 25 (R6b / D-16) — THE DOCUMENTS FAMILY WAS THE LAST MODULE-GLOBAL READER HERE. - # Every line below said `store.…`, so an R2 tenant's document metadata (and its bytes) - # landed in tenant #0's repo. Wave 19 made this family per-TOPIC; per-TENANT is the axis - # it was still missing, and the two failures look identical from the app. - _st_docs = _store_of(ctx) - meta_all = _st_docs.get(docs_key) or {} - row = list((meta_all.get(str(pid)) or [])) - - if kind == 'doc_add': - raw_b64 = event.get('data_b64') - name = str(event.get('name') or 'document')[:200] - if not isinstance(raw_b64, str) or not raw_b64: - return False - try: - blob = _b64c.b64decode(raw_b64, validate=True) - except Exception: - return False # not base64 -> not a file we will store - # Re-check the DECODED size host-side. The client checks it too, but a client check - # is a courtesy: this is the number that bounds what the tenant's repo can be made - # to hold by anyone who can reach the endpoint. - if not blob or len(blob) > _DOC_MAX_BYTES: - return False - if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row): - return False - safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file' - path = f'{docs_dir}/{pid}/{doc_id}_{safe}' - try: - _blobs(ctx).upload_bytes( - path, blob, message=f'doc {doc_id} for {scope_key} {pid}') - except Exception as _ue: - _tel.error('docs:upload', _ue) - return False - row.append({'id': doc_id, 'name': name, - 'mime': str(event.get('mime') or 'application/octet-stream')[:120], - 'size': len(blob), 'by': uname, - 'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), - 'path': path}) - elif kind == 'doc_delete': - hit = next((d for d in row if d.get('id') == doc_id), None) - if not hit: - return False - # THE WALL: the uploader or an admin. Enforced here regardless of what the client - # chose to render — `canDelete` in the payload only hides a button. - if hit.get('by') != uname and not admin: - return False - _blobs(ctx).delete_path(hit.get('path') or '') - row = [d for d in row if d.get('id') != doc_id] - else: # doc_fetch - hit = next((d for d in row if d.get('id') == doc_id), None) - if not hit: - return False - blob = _blobs(ctx).download_bytes(hit.get('path') or '') - if blob is None: - return False - # The bridge is a fire-and-forget event LOG with no response channel, so the answer - # is PARKED on the Result and rides the next render's payload (C5). - ctx.out.doc = { - 'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document', - 'mime': hit.get('mime') or 'application/octet-stream', - 'data_b64': _b64c.b64encode(blob).decode('ascii')} - return True # re-render to deliver it - - def _write(cur): - cur[str(pid)] = row - return cur - # ⚠ `docs_key`, NOT `_DOCS_KEY`. The read above and the byte path were both made - # per-topic and this line was not — so a product document's BYTES landed correctly under - # `docs/product/…` while its METADATA went into `customer_docs` beside the customer rows, - # keyed by a CRC32 hash. Found by `verify_scopes.py`'s own leg on its first run, which is - # the argument for asserting the ABSENCE from the legacy bucket and not just the presence - # in the new one: every "it saved" check here was green. - _st_docs.update(docs_key, _write, flush='async') - return True - - if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate', - 'item_move', 'folder_reorder'): - # FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five - # events touch ONE home — the table workspace's `folders` + `itemFolders` — so a cohort - # can be filed without the cohort STORE learning about folders at all. - import aios_grid as _agf - if not _store_of(ctx).available(): - return False - surface = str(event.get('surface') or '') - if surface not in _agf.FOLDER_SURFACES: - return False - # The ids this user may actually file: their own views, their own cohorts. Fail-closed — - # an item_move naming somebody else's cohort is dropped by clean_item_folders anyway, - # but refusing here means we never write it in the first place. - def _own_ids(sfc, cur_ws): - if sfc == 'views': - return set((cur_ws.get('views') or {})) - return set(_cohorts(ctx).all_for(uname)) - - def _mutate(cur_ws): - folders = _agf.clean_folders(cur_ws.get('folders')) - placed = dict(cur_ws.get('itemFolders') or {}) - lst = list(folders.get(surface) or []) - # ⚠ the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the - # built contract; the split doc's first draft said `id`). Converged here rather - # than asking the client to re-emit — see the C4 reconciliation note in the doc. - fid = str(event.get('folderId') or '').strip()[:80] - if kind == 'folder_create': - name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] - if not fid or not name or len(lst) >= _agf.MAX_FOLDERS: - return None - if any(f['id'] == fid for f in lst): - return None - row = {'id': fid, 'name': name, 'order': len(lst)} - icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5) - if icon: - row['icon'] = icon - lst.append(row) - elif kind == 'folder_rename': - name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] - hit = next((f for f in lst if f['id'] == fid), None) - if not hit or not name: - return None - hit['name'] = name - # wave-9 I15 (C5): the rename pane is also where the icon is changed. An event - # that carries NO icon key leaves the existing one alone (a rename must not - # silently strip a chosen icon); an explicit null clears it back to default. - if 'icon' in event: - icon = _agf.clean_folder_icon(event.get('icon')) - if icon: - hit['icon'] = icon - else: - hit.pop('icon', None) - elif kind == 'folder_delete': - if not any(f['id'] == fid for f in lst): - return None - lst = [f for f in lst if f['id'] != fid] - # CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that: - # an item with no placement renders at the top level. - placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items() - if v != fid} - elif kind == 'folder_reorder': - # ⭐ WAVE 20 (owner item 19, contract C-FOLDER-REORDER) — the user's own folder - # ORDER in the rail. Views could already be dragged between folders; the folders - # themselves could not be dragged past each other. - # - # THE WIRE CARRIES THE FULL ORDER, NEVER A DELTA, and that is the contract's - # doing: a partial list cannot say where an UNNAMED folder went, so a "moved B - # after C" message would leave every other folder's position to be re-derived by - # two sides that can disagree. The client sends the list it is looking at. - # - # ⚠ IDS THIS USER DOES NOT OWN ARE IGNORED, NOT REJECTED. A stale tab can emit an - # order containing a folder that has since been deleted; dropping the whole event - # would make the rail un-reorderable until reload, while dropping the unknown id - # reorders exactly what still exists. Anything the payload omits keeps its - # relative position AFTER the named ones — an order that silently deleted a - # folder it merely failed to mention would be a data loss dressed as a sort. - order = [str(i).strip()[:80] for i in (event.get('order') or [])] - if not order: - return None - known = {f['id']: f for f in lst} - seen, ranked = set(), [] - for fid_ in order: - if fid_ in known and fid_ not in seen: - seen.add(fid_) - ranked.append(known[fid_]) - ranked.extend(f for f in lst if f['id'] not in seen) - if len(ranked) != len(lst): # cannot happen; refuse rather than truncate - return None - for i, f in enumerate(ranked): - f['order'] = i - lst = ranked - elif kind == 'folder_duplicate': - new_id = str(event.get('newId') or '').strip()[:80] - src = next((f for f in lst if f['id'] == fid), None) - if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS: - return None - if any(f['id'] == new_id for f in lst): - return None - lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME], - 'order': len(lst)}) - # ⚠ The folder's CONTENTS are duplicated by the client emitting the ordinary - # per-item duplicate events it already owns (view_upsert with a fresh id, the - # cohort copy path) and then item_move-ing them here. Copying views/cohorts - # inside this handler would be a SECOND implementation of "duplicate an item", - # and the two would drift the first time either one changed. - else: # item_move - item_id = str(event.get('itemId') or '').strip()[:120] - target = event.get('folderId') - if not item_id or item_id not in _own_ids(surface, cur_ws): - return None - cur = dict(placed.get(surface) or {}) - if target is None: - # ⚠ "UNFILE" — the client says nothing about where it went. Popping is right - # here and always was: no placement = render at the top level, and that is the - # state a brand-new item is already in. - cur.pop(item_id, None) # back to the root - elif str(target) == _agf.ROOT_PLACEMENT: - # ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, ASK C-16) — "FILED AT ROOT", STORED. - # - # ⛔ THE THIRD CASE, AND ITS ABSENCE IS THE WHOLE DEFECT. Popping (above) and - # this branch produce the SAME rendering for an ordinary view, which is why - # nobody missed it for nine waves — but they are different FACTS, and for a - # SHARED view the difference is the entire feature: "never filed" sends it to - # the Shared group, "the receiver dragged it out" must not. Absence cannot hold - # two facts, so the second one gets a value. - # - # ⛔ AND IT IS DELIBERATELY EXEMPT FROM THE FOLDER-EXISTS TEST BELOW. This id - # names no folder by design; running it through `any(f['id'] == tid ...)` would - # refuse every root filing, which is the same silent no-op this ticket exists - # to remove. `aios_grid.clean_item_folders` carries the matching exemption on - # the READ side — the two are one change (see its note). - cur[item_id] = _agf.ROOT_PLACEMENT - else: - tid = str(target)[:80] - if not any(f['id'] == tid for f in lst): - return None - cur[item_id] = tid - placed[surface] = cur - folders[surface] = lst - clean = _agf.clean_folders(folders) - return clean, _agf.clean_item_folders( - placed, clean, - {sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES}) - - # ⚠ NO allowed_pids ON PURPOSE — this is app.py's bare `_cl_table_workspace(uname)` - # call at the old :6268, preserved exactly. Omitting the pool drops memberPids from - # any foreign shared view (fail-closed), which is harmless here because the folder - # legs read only view IDS and the folder maps. Passing the pool would be a behavior - # change inside a pure move. - _res = _mutate(table_workspace(ctx, consume_corrections=False)) - if _res is None: - return False # refused: nothing is written - _tops(ctx).save_folders(uname, _res[0], _res[1]) - # True: the sidebars render from server state (which folder holds what), not from local - # optimism — the same contract the cohort panel's membership edits ride. - return True - - if kind == 'view_reorder': - # ⭐ WAVE-27 item 5 (contract C7) — the user drags views up and down the rail. - # - # THE WIRE CARRIES THE FULL ORDER, NEVER A DELTA — the `folder_reorder` contract above, - # verbatim, and for its reason: a partial list cannot say where an unnamed view went. - # - # ⚠ NO OWNERSHIP FILTER, and that is the difference from `item_move`. Filing a view into - # a folder writes a placement keyed by a view this user must own; ARRANGING the rail is - # about the reader's own screen, and the rail legitimately contains views this user does - # not own — the system view, cohort projections, R8's injected Overview, and anything - # SHARED with them. Refusing ids they do not own would make exactly those un-draggable, - # which is the half of sharing people notice. - # - # ⚠ Unresolvable ids are harmless by construction rather than by filtering here: the READ - # side (`aios_grid.views_from_defs`) ranks by this list and appends anything unranked, so - # an id naming a deleted view simply never matches. Validating against today's view set - # at WRITE time would also drop an id whose view is merely not visible in this request. - raw_order = event.get('order') - if not isinstance(raw_order, list): - return False # a non-list order is malformed, not an empty arrangement - order, seen_v = [], set() - for vid in raw_order[:MAX_VIEW_ORDER]: - vid = str(vid or '').strip()[:120] - if vid and vid not in seen_v: - seen_v.add(vid) - order.append(vid) - # ⚠ THE SESSION FALLBACK IS `record_layout`'s, COPIED DELIBERATELY, and it is not just a - # testability convenience. Both keys are per-user PRESENTATION state for one surface, and - # both should keep working while the store is briefly unavailable — a rail that refuses - # to reorder because a write failed is a frozen control with no explanation. It also - # makes the round trip reachable from `verify_grid_events`, which is headless and where - # `store.available()` is False: the store-backed branch of every OTHER stratum writer is - # covered only by the live pass, and this one did not have to be. - if _store_of(ctx).available(): - _tops(ctx).save_view_order(uname, order) - else: - _session_ready() - if order: - ws['viewOrder'] = order - else: - ws.pop('viewOrder', None) # absent = server order, never a stored [] - # True: the rail renders from server state, like the folder sidebars it sits beside. - return True - - if kind == 'field_delete': - # Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS - # the permission: only the created strata (`custom_` overlay fields, `measure_` formula - # columns) are deletable — a base field's key fails the prefix test no matter what the - # browser claims, so the table's contract cannot be deleted from the client. The store - # scrubs the field's stored cell values with it (delete_table_field); views naming the - # key self-heal on their next autosave, the rule every stale colId already rides. - key = str(event.get('key') or '')[:80] - import aios_grid as _ag3 - if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)): - return False - if _store_of(ctx).available(): - _tops(ctx).delete_field(uname, key) - else: - _session_ready() - ws['fields'].pop(key, None) - for _row in ws['overlays'].values(): - if isinstance(_row, dict): - _row.pop(key, None) - # Wave-6 item 3: the client dropped the column optimistically; the store scrub is - # invisible to the next render (the values leave WITH the column). - return False - - if kind == 'choice_rename': - # ⭐ WAVE 20 (owner item 15, contract C-RENAME) — RENAMING AN OPTION CARRIES ITS VALUES. - # - # Owner: *"if I change the option from 'Aaron' to 'Aron' the existing field that has the - # Selected 'Aaron' doesn't change to 'Aron'."* Correct, and it is a data-model fact rather - # than an oversight: a select cell stores the option's LABEL, so renaming the option in - # the field definition orphans every cell holding the old string. They do not render as - # the new name; they render as a value that is no longer an option. - # - # ⛔ THE WIRE CARRIES AN EXPLICIT {from, to} MAPPING, NEVER A DIFF OF THE CHOICE LISTS. - # Diffing is what makes this unsafe: rename A→B while also DELETING C and adding D, and a - # differ sees two removals and two additions with no way to know which pairs up. It would - # cheerfully rewrite every C cell to D. The client knows which row of its editor the user - # typed in; that intent travels, and nothing here has to guess it. - # - # THREE PLACES HOLD THE OLD STRING, and missing any one is its own visible bug: - # 1. the CELLS (`overlays`) — the owner's complaint; - # 2. the FIELD's own `choices` list — else the picker still offers the old name; - # 3. every VIEW that FILTERS or COLOURS BY that value — a saved view filtering - # `Agent is "Aaron"` silently matches NOTHING the moment the value moves, which is - # the failure that looks like data loss ([[aios-shared-views]]). - key = str(event.get('key') or '')[:80] - pairs = [] - for r in (event.get('renames') or ()): - if not isinstance(r, dict): - continue - a, b = str(r.get('from') or '')[:200], str(r.get('to') or '')[:200] - if a and b and a != b: - pairs.append((a, b)) - field = field_by_key.get(key) - if not key or not pairs or not isinstance(field, dict): - return False - if str(field.get('type') or '') not in ('select', 'multiselect', 'status'): - return False - # Only a field this caller may edit. A rename is a write to the field CONTRACT, so it - # rides the same wall `field_upsert` does rather than a looser one of its own. - if key in (ctx.hidden_keys or frozenset()): - return False - mapping = dict(pairs) - - def _rename_in(ws_): - fields_ = ws_.setdefault('fields', {}) - fld = fields_.get(key) - if isinstance(fld, dict) and isinstance(fld.get('choices'), list): - seen, out = set(), [] - for c in fld['choices']: - if isinstance(c, dict): - nm = mapping.get(str(c.get('name') or ''), None) - if nm is not None: - c = {**c, 'name': nm} - ident = str(c.get('name') or '') - else: - c = mapping.get(str(c), str(c)) - ident = str(c) - # A rename ONTO an existing option MERGES rather than duplicating it: the - # user typed a name that already exists, and two identical options is a - # picker bug. Their cells merge too, which is what the value rewrite below - # does anyway. - if ident in seen: - continue - seen.add(ident) - out.append(c) - fld['choices'] = out - fields_[key] = fld - for row in (ws_.get('overlays') or {}).values(): - if not isinstance(row, dict) or key not in row: - continue - val = row[key] - if isinstance(val, list): # multiselect - merged, out2 = set(), [] - for v in val: - nv = mapping.get(str(v), str(v)) - if nv not in merged: - merged.add(nv) - out2.append(nv) - row[key] = out2 - elif isinstance(val, str) and val in mapping: - row[key] = mapping[val] - # Views: filter rule values + colour-by keys naming the old option. - for view in (ws_.get('views') or {}).values(): - if not isinstance(view, dict): - continue - cfg = view.get('config') if isinstance(view.get('config'), dict) else view - - def _walk(nodes): - for n in nodes or (): - if not isinstance(n, dict): - continue - if isinstance(n.get('children'), list): - _walk(n['children']) - continue - if str(n.get('colId') or '') != key: - continue - v = n.get('value') - if isinstance(v, str) and v in mapping: - n['value'] = mapping[v] - elif isinstance(v, list): - n['value'] = [mapping.get(str(x), x) for x in v] - - _walk(cfg.get('filters') or []) - # ⚠ `colorBy` NEEDS NO REWRITE, and the reason is worth stating because the - # opposite looks obvious: it stores a COLUMN KEY (a string, validated against - # `valid_keys` at `view_upsert`), not a {value: colour} map. Per-value colours - # ride the CHOICE OBJECTS themselves, which this function renames in place — - # `{**c, 'name': nm}` keeps every other key on the choice, colour included — so - # a renamed option keeps its swatch for free. An earlier version of this handler - # tried to rewrite `colorBy['rules']` and crashed the whole event pipeline with - # `TypeError: unhashable type: 'dict'` on the FIRST view it touched. - return ws_ - - if _store_of(ctx).available(): - _tops(ctx).rename_choice_values(uname, _rename_in) - else: - _session_ready() - _rename_in(ws) - # True = the host must re-read: the client dropped its optimistic copy of the CHOICES, - # but the rewritten cells and view filters only exist server-side until the next payload. - return True - - if kind == 'field_duplicate': - # Wave-5 item 1. The DESTINATION key is client-generated (the established pattern — - # the client must know it synchronously to place the clone right of the source), and - # both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate, - # because duplicating the source-of-truth into an editable copy would need a - # value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for - # `custom_` sources (a measure clone recomputes; a formula clone re-evaluates). - # The duplicator becomes the clone's creator — it is a NEW field they made. - src_key = str(event.get('sourceKey') or '')[:80] - new_key = str(event.get('key') or '')[:80] - import aios_grid as _ag5 - prefix = ('custom_' if src_key.startswith('custom_') - else _ag5.MEASURE_FIELD_PREFIX - if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None) - if (prefix is None or not new_key.startswith(prefix) or new_key == src_key - or new_key in field_by_key): - return False - src = ws.get('fields', {}).get(src_key) - if not isinstance(src, dict): - return False - clone = dict(src) - clone['key'] = new_key - label = str(event.get('label') or '').strip()[:120] - requested_label = label or (str(src.get('label') or 'Untitled')[:110] + ' copy') - _workspace_field_keys = set((ws.get('fields') or {})) - _reserved_field_names = [ - value.get('label') for candidate_key, value in field_by_key.items() - if (candidate_key != new_key and candidate_key not in _workspace_field_keys - and isinstance(value, dict)) - ] - clone['label'] = requested_label - clone['createdBy'] = uname - # Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort - # page the duplicate surface may explicitly place it instead (same create-time rule - # as field_upsert — a page that is not the cohort page cannot mint a cohort field). - if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'): - if event['scope'] == 'cohort': - clone['scope'] = 'cohort' - else: - clone.pop('scope', None) - if _store_of(ctx).available(): - clone = (_tops(ctx).duplicate_field( - uname, src_key, new_key, clone, - reserved_names=_reserved_field_names, - correction_id=event_id) or clone) - else: - _session_ready() - _field_names = list(_reserved_field_names) - _field_names.extend( - value.get('label') - for candidate_key, value in ws['fields'].items() - if candidate_key != new_key and isinstance(value, dict) - ) - clone.pop('labelCorrectedFrom', None) - clone.pop('labelCorrectionId', None) - clone['label'] = _unique_name(requested_label, _field_names) - if clone['label'] != requested_label and event_id: - clone['labelCorrectedFrom'] = requested_label - clone['labelCorrectionId'] = str(event_id)[:180] - ws['fields'][new_key] = clone - if src_key.startswith('custom_'): - for _row in ws['overlays'].values(): - if isinstance(_row, dict) and src_key in _row: - _row[new_key] = _row[src_key] - # Wave-6 item 3: True only when host-held VALUES must flow back — copied overlay - # cells (custom_ sources) or host-computed measure values. formula/created_time - # clones compute client-side, so the optimistic insert is already complete. - label_changed = clone['label'] != requested_label - return label_changed or src.get('type') not in ('formula', 'created_time') - - if kind == 'add_to_list': - # "Add to list" (owner item 6): a view's current matches become fixed cohort members. - # - # ⚠ `pids` arrives from the BROWSER. It is intersected with `allowed_pids` — the pool this - # user may actually see — before anything is stored. The client computing the set is a - # correctness decision (it is the only engine that already knows the answer, and asking a - # second engine the same question is the drift verify_filter_engine.py exists to prevent); - # it is NOT a trust decision. - raw_pids = event.get('pids') - if not isinstance(raw_pids, list): - return False - pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids] - if not pids: - return False - cohort_id = str(event.get('cohortId') or '').strip()[:120] - name = str(event.get('name') or '').strip()[:120] - if not _store_of(ctx).available(): - return False - # ⭐ WAVE 19 / R9 — THE BUG THIS RULING NAMES. This branch is where a list is BORN, and it - # named `cohort_mod` directly: a list created from the Product surface was written into - # `customer_cohorts` with CRC32 product pids in it, then rendered on the Customer page as - # a list of customers nobody recognised. The set now belongs to the database it was - # created on, and `ctx.scope_key` is the only thing that decides which. - sets = _cohorts(ctx) - try: - if not cohort_id: - if not name: - return False - cohort_id = sets.create(uname, name, pids) - else: - if cohort_id not in sets.all_for(uname): - return False # not this user's cohort — fail closed, do not create - sets.add_members(uname, cohort_id, pids) - except ValueError as e: - _tel.error('cohort:add_to_list', e) - return False - # True: the Cohort page's list and counts must reflect this, and the toast below only - # renders on a fresh run. - # - # ⚠ The NOUN is the TOPIC's, not the literal "customer" this line carried since wave 2. - # "Added 12 customers" over a set of SKUs is the surface stating, in words, that the list - # went somewhere it did not — the user-visible half of the same leak. - _noun = sets.noun - ctx.out.toast = ( - f"Added {len(pids):,} {_noun}{'' if len(pids) == 1 else 's'} to " - f"{name or 'the list'}.") - return True - - if kind == 'overlay_patch': - pid = event.get('pid') - if not isinstance(pid, int) or pid not in allowed_pids: - return False - updates = {} - refused = False - stage_moved = False - row_events = [] # C3 (wave 22): admitted writes, emitted AFTER they apply - raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {} - for key, value in raw_updates.items(): - if key not in overlay_keys or not isinstance(value, (str, int, float)): - refused = True - continue - # C-PERM: a field this caller's permissions HIDE cannot be written, only refused. - # `overlay_keys` asks whether the column exists, not whether this person may touch - # it — so without this line a restricted user who knows a hidden key could PATCH a - # value they are not allowed to read back. Refused LOUDLY (repaint) so the client - # reverts rather than showing a value the store never took. - if key in (ctx.hidden_keys or frozenset()): - refused = True - continue - # Wave-5 item 1: `permissions.edit` is enforced HERE, per key — the client hiding - # its editor is courtesy, this is the wall. Fail-closed: a restricted field with - # no readable createdBy admits only admins. - perms = (field_by_key.get(key) or {}).get('permissions') or {} - edit = perms.get('edit') - if edit == 'admins' and not admin: - refused = True - continue - if edit == 'creator' and not ( - admin or (field_by_key.get(key) or {}).get('createdBy') == uname): - refused = True - continue - # C5-AUTOFIELD (wave 18): an automation cell is MACHINE-WRITTEN. The client marks - # the type read-only (READONLY_CELL_TYPES) but that is courtesy — this is the wall. - # Without it the UI hides the editor while the API still takes a PATCH, which is - # exactly the shape a "read-only" type must not have. - if (field_by_key.get(key) or {}).get('type') == 'automation': - refused = True - continue - # C2 (wave 22): a STAGE FIELD is engine-written EXCEPT at a review gate, where a - # human may advance a card to a member of the review stage's `next` and nothing - # else. The law rides the field definition (`automation.humanMoves` / `arrive`, - # stamped by the engine that derives the stages) because this module must not - # import the engine — the engine states the law, this door enforces it - # mechanically. An ADMITTED move writes THROUGH to the definition row, the shared - # truth the engine and every viewer read: parked in this user's overlay stratum it - # would be a board position only its author could see. Everything else about the - # field is refused — the same wall the automation-cell type gets, extended, not - # replaced. - if str(ctx.scope_key or '').startswith('ut_'): - import core.user_tables as _ut_mod - _tdef = _ut_mod.get(ctx.scope_key, st=ctx.table.st) if ctx.table is not None \ - else None - _fdef = next((f for f in ((_tdef or {}).get('fields') or []) - if f.get('key') == key), None) - # C7 (wave 22): a METRIC cell is engine-computed from the master series — a - # human write would be an invented measurement, refused the way an automation - # cell is (the wall, not the courtesy). - # ⭐ 2026-08-07 — EXTENDED TO THE RELATIONAL PAIR, and read through the field - # layer's own predicate rather than re-tested here. A `rollup` cell is an - # aggregate the server computes; a DERIVED `link` cell is a join the server - # resolves. Both would be overwritten by the next refresh anyway, so accepting - # the write would not merely be wrong — it would look like it worked and then - # silently revert, which is worse than a refusal. - # ⚠ An ORDINARY (user-picked) link is deliberately NOT refused: `is_computed_cell` - # answers per FIELD, not per type, because one kind is read-only in one mode and - # editable in the other. - if _ut_mod.is_computed_cell(_fdef): - refused = True - continue - _auto = (_fdef or {}).get('automation') - # An ordinary Link is a shared database relationship. Validate its selected - # target ids and write it through to the definition row; a personal overlay - # would be invisible to reciprocal links and Rollups. - if (_fdef or {}).get('type') == 'link': - _linked = _ut_mod.patch_link_cell( - ctx.scope_key, pid, key, value, st=ctx.table.st) - if _linked is None: - refused = True - continue - if _linked != str(value): - refused = True - row_events.append((key, _linked)) - continue - # ⭐ WAVE 25 (C3 + amendment C3-A1, owner rulings R7/R6) — THE PROFILE CELL. - # Two things happen here and neither can live anywhere else: - # - # 1. IT WRITES THROUGH TO THE DEFINITION ROW, the stage field's mechanism two - # branches up, for the identical reason. MEASURED before it was written: an - # ordinary `ut_*` cell edit lands in the TYPIST'S OWN overlay stratum, and - # the automation engine reads `t['rows']` — so a handle somebody typed - # would be invisible to the enrich action and to every other viewer of the - # same database. The flag would be decorative. - # 2. BLANKING IT CLEARS THAT ROW'S PRESET CELLS IN THE SAME WRITE (R6) — - # `patch_profile_cell` does both inside ONE store update, because a blank - # that commits without its clear leaves a row with no handle and a live - # follower count, i.e. stale numbers attributed to nobody. - # - # ⛔ NOT VIA THE EMIT GUARD (HARD RULE 5 / D-40). Nothing here emits; this is - # `user_tables`' own write door, reached from the branch that already knows the - # scope is `ut_`. ⚠ `editRole` is deliberately NOT consulted: it governs the - # SCHEMA, and C3 is explicit that a profile field's VALUES stay fully editable — - # that is the whole point of item 5a. - if isinstance((_fdef or {}).get('profile'), dict): - _pw = _ut_mod.patch_profile_cell( - ctx.scope_key, pid, key, value, st=ctx.table.st) - if _pw is None: - # Not a handle we can enrich. Refused LOUDLY so the client repaints to - # the stored truth instead of showing a value the store never took. - refused = True - continue - # A normalisation (`@Name` -> `name`, a pasted URL -> its handle) is a real - # difference from what the client is drawing, so it must repaint. An - # untouched value echoes and stays on the no-blip path. - if _pw['handle'] != str(value): - refused = True - if _pw['cleared']: - stage_moved = True # cells the client never typed changed: re-read - row_events.append((key, _pw['handle'])) - continue - # ⭐ OWNER ITEM 4 (2026-08-06) — A COLUMN AN AUTOMATION FILLS IS NOT A COLUMN A - # PERSON TYPES IN. Owner, verbatim: *"I should not be able to edit any of the - # Pre-set fields on instagram automation, we talked about this extensively."* - # - # ⛔ IT HAD NEVER BEEN ENFORCED ANYWHERE. `editRole: 'admins'` rides on every - # preset column and reads exactly like this wall, which is why it was believed to - # be one — but it governs the SCHEMA (`user_tables.py:535`): who may rename or - # retype the column. The VALUES stayed open to anyone with grid access, and the - # owner typed into one to prove it. The metric branch above is the same rule for - # the same reason, written for a different column kind; this is its sibling. - # - # ⚠ REACHED ONLY AFTER stage AND profile have `continue`d, and the order is the - # rule rather than an accident: both of those columns carry the identical tag and - # both are cells a human is SUPPOSED to drive — the stage because moving a card - # IS the decision, the profile because it is where enrichment gets its subject. - # Refusing on the tag alone would freeze every kanban. - # - # `flowId` and not merely `automation`: a user-configured `automation` COLUMN - # carries its own gear bag (kind/source/urlField/settings) and no flowId, and it - # is not this ruling's subject. - if isinstance(_auto, dict) and (_auto.get('flowId') or _auto.get('preset')): - refused = True - continue - # ⭐ WAVE 23 (C7) — the json wall, BEFORE the generic 10 000-char truncation, which - # would otherwise cut a document in half and store the halves as valid text. A json - # column promises its readers a parseable document; a write that is not one is - # REFUSED with the cell unchanged rather than silently kept as a string nothing can - # open. Blank stays legal — an empty cell is "no document", not a malformed one. - if (field_by_key.get(key) or {}).get('type') == 'json': - raw_j = value if isinstance(value, str) else _json_ev.dumps(value) - if not str(raw_j).strip(): - updates[key] = '' - continue - if len(str(raw_j)) > _ut_limits.MAX_JSON_CELL: - refused = True - continue - try: - _json_ev.loads(raw_j) - except (ValueError, TypeError): - refused = True - continue - updates[key] = str(raw_j) - continue - clean = str(value)[:10000] - if isinstance(value, str) and clean != value: - # Truncated: the stored truth is shorter than the client's copy. (An - # int/float arriving as its string form renders identically — not a refusal.) - refused = True - definition = field_by_key.get(key) or {} - if definition.get('type') in ('select', 'multiselect') and clean: - options = [str(v) for v in definition.get('options') or []] - canonical = {v.strip().lower(): v for v in options} - if definition.get('type') == 'select': - normalized = canonical.get(clean.strip().lower()) - if normalized is None: - refused = True - continue - else: - normalized_parts = [] - seen = set() - invalid = False - for part in clean.split(','): - match = canonical.get(part.strip().lower()) - if match is None: - invalid = True - break - if match.lower() not in seen: - seen.add(match.lower()) - normalized_parts.append(match) - if invalid: - refused = True - continue - normalized = ','.join(normalized_parts) - if normalized != clean: - refused = True - clean = normalized - updates[key] = clean - if updates: - if str(ctx.scope_key or '').startswith('ut_') and ctx.table is not None: - # User-table cells are collaborative record data. Views/layout remain personal, - # but record values live in the definition rows so Links, Rollups, automations, - # and every viewer observe one truth. - import core.user_tables as _ut_values - _ut_values.patch_cells(ctx.scope_key, pid, updates, st=ctx.table.st) - elif _store_of(ctx).available(): - _tops(ctx).patch_overlay(uname, pid, updates) - else: - _session_ready() - ws['overlays'].setdefault(str(pid), {}).update(updates) - # C3 (wave 22): the admitted writes become ROW EVENTS — emitted AFTER they applied, on - # the ut scopes the event triggers watch this wave, through the seam in user_tables so - # this module never imports the engine. Human door only, which IS the loop law. - if str(ctx.scope_key or '').startswith('ut_') and ctx.table is not None: - import core.user_tables as _ut_ev - for _k, _v in list(updates.items()) + row_events: - _ut_ev.emit_row_event({'type': 'event_field', 'table': str(ctx.scope_key), - 'rowId': str(pid), 'field': _k, 'after': _v, - 'st': ctx.table.st, 'user': uname}) - # Wave-6 item 3: the client's cell already shows the typed value — rerun ONLY when - # some of what it shows was refused or clamped and must be repainted to the truth. - # A stage move reruns too: its `arrive` side-effects (the tracked flag) changed cells - # the client never typed. - return refused or stage_moved - - return False +"""core/grid_events.py — THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30). + +This is the grid's event handler: the code that takes an UNTRUSTED event object from the +browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` / +`_cl_handle_one` and moved here as a PURE MOVE — no behavior change inside a move (MA principle +5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the +Streamlit couplings the contract names: + + st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state) + st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc) + st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast) + st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback) + is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default) + -> ctx.admin + +There is NO `import streamlit` in this file and there must never be one — `verify_no_streamlit.py` +enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring. +(That gate was `verify_seam.py` until EXIT-6 retired it with `app.py`; the block, its negative +control and the AST walk moved over intact.) + +ONE ADAPTER, ONE HANDLER — it was TWO until 2026-08-04: + * ⛔ the Streamlit adapter is GONE. `app.py::_cl_handle_grid_event` built an EventCtx whose + `seen_ids` and `fallback_ws` were session-state-backed dicts and applied the Result to session + state. EXIT-6 deleted it. The host-neutrality this file was written for is what let that + happen without touching a line of the handler — which is the argument for the design, recorded + at the moment it paid out. + * the HTTP adapter — `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None + fallback with the store unavailable is a **503**, never a silent in-memory workspace: an + API request has no durable session dict to fall back to, so pretending to persist would + lose the write with a 200 on it. + +⚠ LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE §1's +one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer +table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The +contracted path for the strangler is `core/grid_events.py` (both adapters import it from here), +so the import stays and the violation is RECORDED rather than hidden. It is the one file in +`core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move +down, and this note is the marker for that work. +""" +import datetime as dt +import json as _json_ev # wave 23 (C7): the json field's parse wall +from dataclasses import dataclass, field +from typing import MutableMapping, Optional + +import core.store as store +import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates + +#: ⭐ WAVE-27 item 5 (C7) — how many view ids one rail arrangement may name. Generous on purpose: +#: it bounds a hostile payload, not the user (the rail holds saved views + cohort projections + +#: shared views, and a busy Customer grid already carries dozens). An order longer than this is +#: not a person dragging rows. +MAX_VIEW_ORDER = 500 +import core.user_tables as _ut_limits # wave 23 (C7): MAX_JSON_CELL — one ceiling, one definer +import core.users as users +import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log) +import modules.cohort as cohort_mod +import modules.customer_data as cl_mod + +#: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte +#: count — the client checks the same number first, but a client check only spares the user a +#: pointless upload; this is what bounds what the tenant's dataset can be made to hold. +#: MOVED from app.py:1410–1412 with the handler (app.py re-imports these during the strangler). +_DOCS_KEY = 'customer_docs' +_DOC_MAX_BYTES = 5 * 1024 * 1024 +_DOC_MAX_PER_ROW = 50 + +#: The props of a field def the CLIENT actually draws — the wave-6 no-blip comparison set. +#: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders +#: nothing from its own copy of those, and a permissions refusal is answered explicitly. +_FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'colorCodeOptions', + 'optionColors', 'max', 'formula', 'agg', 'scope', 'format', 'measure') + + +class StoreUnavailable(RuntimeError): + """The store is down AND the caller offered no durable fallback (HTTP adapter → 503). + + The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so + its store-down path degrades to the same captioned in-memory workspace it always had. An + API caller has no such dict, and the honest answer to "persist this" with nowhere to + persist it is an error — not a 200 over a write that evaporates. + """ + + +@dataclass +class EventResult: + """What the HANDLER needs to tell its adapter — the three things it cannot do itself. + + `rerender` is the old return value with its old meaning: "this page must re-render from + server state", NOT "handled" (see `handle_one`'s docstring — the distinction is the + difference between a snappy grid and a sluggish one). + """ + rerender: bool = False + doc: Optional[dict] = None # doc_fetch's answer — the bridge has no response channel + toast: Optional[str] = None # add_to_list's confirmation + + +@dataclass +class EventCtx: + """Everything the handler is allowed to know, and nothing about how it is being served. + + ⚠ `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and + every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids) + is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity — + never anything the browser said about itself. + + `seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every + emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP + passes the caller's per-session mapping. + + `fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username + (exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes + None, and None + store down raises StoreUnavailable (→ 503). + + `out` is the sink for the two parked answers, so `handle_one` can keep returning a bare + bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's + field list, 2026-07-30 — defaulted; recorded in the S1 mailbox.) + """ + uname: str + allowed_pids: frozenset + fields: list + measure_keys: frozenset = frozenset() + resolved_ids: frozenset = frozenset() + cohort_ids: frozenset = frozenset() + measure_offer: tuple = () + visible_views: tuple = () + admin: bool = False + #: ⛔ WAVE 15 C-PERM — the fields this caller's PERMANENT PERMISSIONS hide, transitive + #: closure included (`core.perm_scope.hidden_keys`). A SECURITY parameter in the same class + #: as `allowed_pids`, and here for the same reason: the read paths strip a hidden field from + #: both wires, so a caller cannot SEE it — but without this they could still WRITE it by + #: naming the key, because `overlay_keys` only asks whether the column exists. Empty means + #: "nothing hidden", which is every legacy record and therefore every caller today. + hidden_keys: frozenset = frozenset() + scope_key: str = 'customer' + #: Wave 16 C-TOPIC — the TABLE OPS this ctx reads and writes (a `core.table_store.make` + #: object). None = the CUSTOMER table (`modules.customer_data.TABLE_OPS`), which keeps + #: every existing caller byte-identical. The PRODUCT surface passes + #: `modules.product_data.TABLE_OPS` so its views/fields/overlays land in the product + #: bucket — never the customer one, whose pids are a different id space entirely. + table: Optional[object] = None + #: ⭐ WAVE 25 (R6b, closes DEBT D-16) — THE TENANT STORE HANDLE, and it is a RESIDENCY + #: parameter in the same class as `allowed_pids` is a security one. Everything this handler + #: could not reach through `ctx.table` used to go through the MODULE-LEVEL `core.store`, + #: i.e. tenant #0's dataset repo: the documents family, `_store_of(ctx).available()`, and — through + #: `_tops`' fallback — the whole customer and product table workspace. Owner, R6b: + #: *"Tenant 0 should not intertwine anywhere."* + #: + #: None keeps the module default, which is byte-identical for tenant #0 (empty prefix, + #: shared repo) and is what every non-HTTP caller still gets. The HTTP adapters pass + #: `session.runtime`, so a Nurilab write resolves against Nurilab. + st: Optional[object] = None + seen_ids: Optional[MutableMapping] = None + fallback_ws: Optional[MutableMapping] = None + out: EventResult = field(default_factory=EventResult) + + def __post_init__(self): + # A caller that forgets `seen_ids` gets per-call dedup rather than a crash — the SET is + # what makes re-delivery idempotent, and its absence must not be the thing that decides + # whether an event is processed at all. + if self.seen_ids is None: + self.seen_ids = {} + + +def _store_of(ctx): + """THE store handle this ctx reads and writes through (wave 25, R6b / D-16). ONE resolver, + so "whose repo does this land in" has exactly one answer per event. + + Order, and each step is deliberate: the ctx's own tenant handle · else the handle the TABLE + OPS object was built with (`table_store.make(..., st=…)` keeps it on `.st`, which is how the + `ut_` path has been tenant-correct since wave 18) · else the module default. + + ⛔ THE MODULE DEFAULT IS TENANT #0, AND THAT IS THE BUG D-16 NAMES — not a safe base case. + It is kept because it is byte-identical for tenant #0 (empty prefix, shared repo) and every + non-HTTP caller still relies on it; what changed is that the HTTP adapters now pass a real + handle, so the fallback is no longer what serves a Nurilab request. `verify_scopes` proves + that with a negative control: force this to fall through and the residency legs go RED. + """ + st = getattr(ctx, 'st', None) + if st is not None: + return st + tbl = getattr(ctx, 'table', None) + inner = getattr(tbl, 'st', None) if tbl is not None else None + return inner if inner is not None else store + + +def _blobs(ctx): + """The BYTE store for this ctx — `_store_of`'s sibling, split out because it is NOT yet + equivalent and pretending otherwise would hide a live residency hole. + + ⚠ `TenantRuntime` exposes `get/put/update/exists/available` and **no byte methods** + (`harness/runtime.py`), so a tenant with its OWN repo has nowhere tenant-scoped to put a + document's BYTES. The metadata half of D-16 is fixed here; the byte half needs + `upload_bytes`/`download_bytes`/`delete_path` on the runtime, which is SESSION A's file — + asked for in the mailbox and booked as debt rather than papered over with a fallback that + reads as intentional. The moment those methods exist this resolver picks them up with no + other change, which is why it is written as a capability test and not as a version check. + """ + st = _store_of(ctx) + return st if hasattr(st, 'upload_bytes') else store + + + +def _ag_overview_id(): + """`aios_grid.IG_OVERVIEW_ID`, resolved defensively. + + Function-local import, this module's convention for the layer above it. Falls back to the + literal rather than raising: a store blip must not turn "refuse to delete" into "allow", and + an unguarded delete here is the one that reads as success and undoes itself. + """ + try: + import aios_grid as _ago + return _ago.IG_OVERVIEW_ID + except Exception: + return 'tpl_overview' + +def _tops(ctx): + """The table-ops object this ctx operates on — the customer table unless the caller chose + another topic. ONE resolution point so 'which bucket does this write land in' has exactly + one answer per event. + + ⛔ WAVE 25 (R6b) — THIS FALLBACK IS STILL TENANT #0's, DELIBERATELY, AND IT IS THE ONE PIECE + OF THE RESIDENCY FIX THAT DID NOT LAND. `cl_mod.TABLE_OPS` is a module-level singleton built + with no store handle, so the customer and product table workspaces (views, fields, overlays, + folders) resolve to tenant #0 for every caller. Scoping it here was written, measured and + then REVERTED, because it would have been worse than the bug: + + the WRITE door (`routes_grid._ctx`) is this session's file and now carries a handle; + the READ door is `routes_customers._ctx_for` / `routes_products` — files in NO session's + fence this wave — and they carry none. + + Moving only the write side means a prefix tenant saves a view into `t//…` and reads + back from the unprefixed bucket forever: their own view vanishes the moment they save it. + A latent split is worse than a stated residency error, and read and write must move in ONE + change. Booked as an ask + debt with that exact shape. Tenant #0 is unaffected either way + (empty namespace, default repo), which is why nothing today would have shown it. + + ⛔ Untouched by this wave regardless: the EMIT GUARD is D-40, deferred by ruling A-15. + """ + if ctx.table is not None: + return ctx.table + return cl_mod.TABLE_OPS + + +def _cohorts(ctx): + """The COHORT store this ctx operates on — `_tops`'s sibling, and here for the same reason. + + ⭐ WAVE 19 / R9: a cohort belongs to its database. Every cohort verb below used to name + `cohort_mod` directly, so a list built on the Product surface landed in `customer_cohorts` + beside somebody's real customer lists — with PRODUCT pids in it (CRC32 hashes of SKU codes), + which the customer page would then try to resolve against partner ids. Wave 16 answered that + by REFUSING cohorts on other topics (`with_cohorts=False`); R9 says scope-parameterize + instead. `modules.cohort.scoped` returns the module itself for the customer topic, so the + customer path is byte-unchanged. + """ + return cohort_mod.scoped(ctx.scope_key) + + +#: The keys a stored document entry may put on the wire. ⛔ `path` IS NOT ONE OF THEM — it is the +#: byte address inside the tenant's repo, of no use to a browser and a needless disclosure of the +#: store's layout. `canDelete` is not stored either: it is a fact about the READER, computed per +#: request below, so one person's answer can never be cached into another's. +_DOC_WIRE_KEYS = ('id', 'name', 'mime', 'size', 'by', 'ts') + + +def docs_for(pids, scope_key='customer', uname='', admin=False, st=None): + """`{"": [CustomerDoc, ...]}` for a SCOPED row set — the producer `payload.docs` lost. + + ⭐⭐ WAVE 30 / CONTRACT C4 (D-138). The write door has been live and UNREACHABLE since EXIT-6: + `doc_add`/`doc_fetch`/`doc_delete` all still work, and the comment inside that handler says + exactly why nobody noticed — *"the React shell cannot reach it yet (`payload.docs` is set only + by app.py)"*. `app.py` was deleted, and the producer went with it. Every client half survived: + `RecordDetail`, `SwipeView` and `Documents` are complete, and all six `onDoc*` handlers in + `CustomerGrid.tsx` are written as `payload?.docs ? … : undefined`, so an ABSENT `docs` key is + what has been switching the whole feature off. An empty dict is truthy and turns it on with an + honest empty state. + ⛔ THIS IS THE SIXTH INSTANCE OF THE REACHABILITY CLASS and the one `verify_reachability.py` + cannot see — a payload key assembled through helpers and dict merges is not a text-matchable + literal ([[reachable-is-not-the-same-as-built]], [[artifact-with-no-importer]]). + + ⛔ ONE READER, NOT A MATCHING PAIR. `routes_tables` (ut_*) and `routes_customers` (the customer + scope) both call THIS — they do not each serialise. Two surfaces answering one question + separately is how the same defect got reintroduced in the opposite direction inside a single + commit ([[one-question-two-normalizers]]). + + ⚠ `pids` IS REQUIRED and is the row set the CALLER has already scoped — `shared_overlay.cells`' + argument, for the same reason: there is no "every document in the tenant" call to reach for. + A record with no documents gets `[]`, never a missing key, so a client can tell "none" from + "not served". + """ + wanted = {str(int(p)) for p in (pids or [])} + if not wanted: + return {} + scope = str(scope_key or 'customer').strip().lower() + key = _DOCS_KEY if scope in cohort_mod.LEGACY_SCOPES else f'{scope}_docs' + try: + stored = (st if st is not None else store).get(key) or {} + except Exception: # noqa: BLE001 + stored = {} # a display read degrades; it never breaks the grid + out = {} + for pid in wanted: + row = stored.get(pid) + docs = [] + for d in (row if isinstance(row, list) else []): + if not isinstance(d, dict) or not d.get('id'): + continue + wire = {k: d.get(k) for k in _DOC_WIRE_KEYS} + # The delete wall, mirrored onto the wire so the button is absent rather than + # offered-and-refused. `doc_delete` enforces the same test server-side; this is the + # DISPLAY of that rule, never a substitute for it. + wire['canDelete'] = bool(admin or (d.get('by') and d.get('by') == uname)) + docs.append(wire) + out[pid] = docs + return out + + +def _docs_key(ctx): + """The DOCUMENTS bucket this ctx operates on — `_cohorts`'s sibling (wave 19, item 12's + silent third case). + + ⛔ `_DOCS_KEY` STAYS the module constant with its shipped value: app.py re-imports it for the + Streamlit docs payload and `verify_seam.py` asserts the two resolve to the same string. This + resolves the CUSTOMER topic to exactly that constant and gives every other database its own + bucket, so the pid keys inside one bucket all come from one id space. + """ + scope = str(getattr(ctx, 'scope_key', '') or 'customer').strip().lower() + if scope in cohort_mod.LEGACY_SCOPES: + return _DOCS_KEY + return f'{scope}_docs' + + +# ------------------------------------------------------------------ the workspace family (X1 amd.) +def table_workspace(ctx, allowed_pids=None, consume_corrections=True): + """Durable Airtable-style view/schema/overlay state, with an honest session fallback. + + Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED + with them. Own views win a same-id collision — a personal view is the user's own object and + must never be shadowed by somebody else's share. + + ⚠ `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries + `memberPids`, which the writer's scope validated on the way IN — but the READER may have a + narrower scope. Handing a Fisch-scoped user a view whose member list was built by a + full-access user would leak the other BU's customers straight through the strict-isolation + rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the + result reaches a renderer or an event handler; omitting it drops memberPids entirely rather + than trusting the stored list (fail-closed: a smaller view, never a wider one). + + MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live + `is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call + site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids` + stays an EXPLICIT argument because the folder leg deliberately omits it. + """ + uname = ctx.uname + if _store_of(ctx).available(): + ws = _tops(ctx).workspace( + uname, consume_corrections=bool(consume_corrections)) + try: + shared = _tops(ctx).shared_views(uname, bool(ctx.admin)) + except Exception as _se: + _tel.error('sharedviews:read', _se) + shared = {} + # ⭐ WAVE 21 (item 9, C1): the R10 registry's named-user grants, PROJECTED at last. + # Wave 20 wrote view grants and no reader ever consulted them — "Share view…" recorded + # a row and changed nothing on the receiver's screen. + try: + granted = _granted_views(ctx, uname) + except Exception as _ge: + _tel.error('sharedviews:granted', _ge) + granted = {} + if shared or granted: + merged = dict(granted) + merged.update(shared) # the everyone-bucket outranks a grant + merged.update(ws.get('views') or {}) # own views win the id collision + ws['views'] = {vid: scope_view(v, uname, allowed_pids) + for vid, v in merged.items()} + return ws + # Store down. Streamlit degrades to its captioned session workspace exactly as before; an + # API caller has no durable dict and must be told, not quietly served memory. + if ctx.fallback_ws is None: + raise StoreUnavailable('table workspace unavailable: no store and no session fallback') + return ctx.fallback_ws.setdefault( + uname, {'views': {}, 'fields': {}, 'overlays': {}}) + + +def _granted_views(ctx, uname): + """{view_id: STAMPED view} for every view GRANTED to `uname` by name (wave 21, item 9, C1). + + The R10 registry stores bare ids, tenant-scoped through the SAME store handle this topic's + bucket uses; the record itself is fetched out of the OWNER's personal stratum of THIS + bucket. An id granted on another topic simply finds nothing here and contributes nothing + (fail-closed). Each projected view is stamped `shared` / `sharedRole` / `owner` so the + client can synthesise the "Shared with me" group and disable edit affordances for + role=view — the stamps live on the PROJECTION only, never written back to the store. + `role_for` is asked with is_admin=False deliberately: the stamp reports the GRANT (the + registry listed only explicit entries), not the caller's rank. + """ + import core.shares as shares + tops = _tops(ctx) + st = getattr(tops, 'st', None) + ids = (shares.shared_with(uname, kind='view', st=st) or {}).get('view') or [] + out = {} + for vid in ids: + found = tops.find_view(vid) if hasattr(tops, 'find_view') else None + if not found: + continue + owner, view = found + if owner == uname: + continue + role = shares.role_for('view', vid, uname, st=st) + if role not in ('view', 'edit'): + continue + v = dict(view) + v['shared'] = True + v['sharedRole'] = role + v['owner'] = owner + out[str(vid)] = v + + # ⭐ D-37 (2026-08-05) — THE FOLDER LEG. W20's R10 said "a folder share = its views ride + # along"; W21 wired the view kind only, so a granted folder was accepted, stored and listed + # under "Shared with me" while changing NOTHING on the receiver's screen. Two waves of a + # control that reported success and did nothing — the same defect class the view leg above + # was written to close, surviving in its sibling. + # + # ⚠ A DIRECT VIEW GRANT WINS over the folder it happens to sit in, and the ordering above is + # what implements that: the direct pass has already filled `out`, so the folder pass skips + # ids it finds there. The specific grant is the more deliberate statement of intent, and it + # may carry a DIFFERENT role — a folder shared read-only must not downgrade a view somebody + # was explicitly given edit on, nor upgrade one they were not. + try: + folder_ids = (shares.shared_with(uname, kind='folder', st=st) or {}).get('folder') or [] + except Exception: + folder_ids = [] + for fid in folder_ids: + found = tops.find_folder(fid) if hasattr(tops, 'find_folder') else None + if not found: + continue # a folder on another topic contributes nothing here + owner, folder, inside = found + if owner == uname: + continue + role = shares.role_for('folder', fid, uname, st=st) + if role not in ('view', 'edit'): + continue + for vid, view in inside.items(): + if str(vid) in out: + continue + v = dict(view) + v['shared'] = True + v['sharedRole'] = role + v['owner'] = owner + # The folder's NAME rides so the client can group these together later; today it + # renders in the same "Shared with me" group the view leg feeds. Stamps live on the + # PROJECTION only and are never written back, exactly as above. + v['sharedFolder'] = str(folder.get('name') or '')[:80] + out[str(vid)] = v + return out + + +def scope_view(view, uname, allowed_pids): + """Re-scope a view to the READER. A no-op for a view the reader created. + + Two things travel inside a shared view that are SCOPE, not presentation: + 1. `memberPids` — a customer list. Re-filtered against the reader's permitted pool. + 2. cohort conditions — a cohort id is a permission handle owned by ONE user. For a reader + who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable + leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) — i.e. it would show MORE rows, + not fewer. So the whole filter tree is dropped for a foreign reader rather than + evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a + silently widened one is a leak. + """ + if not isinstance(view, dict) or view.get('createdBy') == uname: + return view + out = dict(view) + cfg = dict(out.get('config') or {}) + mp = cfg.get('memberPids') + if isinstance(mp, list): + cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())] + if allowed_pids is not None else []) + if tree_has_cohort(cfg.get('filters')): + cfg['filters'] = [] + cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending + out['config'] = cfg + return out + + +def tree_has_cohort(node): + """True when a filter tree references the Cohorts column anywhere (any depth).""" + import aios_grid as _agc + col = _agc.COHORT_COLUMN + if isinstance(node, list): + return any(tree_has_cohort(n) for n in node) + if not isinstance(node, dict): + return False + if node.get('colId') == col or node.get('field') == col: + return True + return tree_has_cohort(node.get('conditions') or node.get('children')) + + +def measure_leaves(nodes): + """Every leaf of a filter tree that carries a date WINDOW — i.e. every measure condition. + + Keyed off the window rather than the column name on purpose: a window is the one thing no + column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure. + """ + out = [] + for node in nodes or []: + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + out.extend(measure_leaves(node['children'])) + elif node.get('window') is not None: + out.append(node) + return out + + +# ------------------------------------------------------------------ private handler helpers +def _field_echo_equal(accepted, raw): + """Does the ACCEPTED def match what the client already renders? (wave-6 item 3) + + The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no + re-render — the client is the ORIGIN of the change, its optimistic copy is already right, + and the rerun it would buy costs a full payload re-serialisation plus an iframe remount + (the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms + ('' / [] / {} / absent all render identically): any normalisation a validator applied is a + real difference the client must be SHOWN. Fail-closed — unsure means not equal means rerun. + """ + empty = ('', [], {}) + for k in _FIELD_ECHO_PROPS: + a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None + if k == 'scope': + # 'global' and absent are the SAME rendering (an unscoped field) — the event may + # say 'global' explicitly while the stored def says it by omission. + a = None if a == 'global' else a + b = None if b == 'global' else b + if (None if a in empty else a) != (None if b in empty else b): + return False + return True + + +def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120): + """Return a durable, case-insensitively unique field/view display name.""" + limit = max(1, int(max_len)) + + def _clean(value): + return ' '.join(str(value or '').split()) + + base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip() + taken = {_clean(value).casefold() for value in existing if _clean(value)} + if base.casefold() not in taken: + return base + index = 2 + while True: + suffix = f' {index}' + stem = base[:max(0, limit - len(suffix))].rstrip() + candidate = f'{stem}{suffix}' if stem else str(index)[-limit:] + if candidate.casefold() not in taken: + return candidate + index += 1 + + +def _known_usernames(): + """Real account names, for C4's fail-closed `users` grant. None = "could not resolve". + + None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry + error would silently collapse a user's stored 'specific users' grant to 'personal' on the + very next autosave — destroying a setting because a read blipped. The grant is inert today + anyway (views are per-user; nothing reads it to allow cross-user access), so the honest + trade is: never invent access, never destroy a stored intent either. + """ + try: + reg = set(users.registry() or {}) + except Exception as _ue: + _tel.error('viewperms:users', _ue) + return None + # ⚠ EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising + # when the user store cannot be read (a store blip logs `store:get:users` and hands back an + # empty dict) — and a working app ALWAYS has at least the bootstrap admin. Treating {} as a + # real answer would filter every name out of a stored grant and collapse it to 'personal' + # on the next autosave. Caught by testing this in BARE mode with no store token. + return reg or None + + +# ------------------------------------------------------------------ THE SEAM +def handle_events(events, ctx): + """The event LOG entry point → one EventResult. The `[-24:]` window lives here. + + ⚠ THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component + has exactly one value slot, and two quick emits share it: creating a field emits + `field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later — with the + server still inside the first rerun, the second write clobbered the slot before any run + read it, and the field creation VANISHED with no error anywhere (measured live; the same + race sat latent under overlay-field creation, masked by localStorage). The client now + sends its recent window of events; each is processed exactly once, keyed by its id. The + single-event shape is still accepted — it is what an older client in a stale tab sends. + + Accepts everything the component value can be: the log dict, a bare list of events (what + the HTTP body's `events` array unwraps to), or one legacy single-event object. + + `EventResult.rerender` is True when ANY processed event demands a re-render from server + state — the same OR-fold `_cl_handle_grid_event` did. + """ + if isinstance(events, dict) and isinstance(events.get('events'), list): + seq = events['events'][-24:] + elif isinstance(events, list): + seq = events[-24:] + else: + seq = [events] + rerun = False + for one in seq: + rerun = handle_one(one, ctx) or rerun + ctx.out.rerender = rerun + return ctx.out + + +def handle_one(event, ctx): + """Validate ONE untrusted component event. + + Odoo-source fields are never accepted as cell updates. View and schema objects are + user-scoped; row patches additionally require the pid to be in the currently permitted + Customer List pool. + + RETURN VALUE = "this page must re-render from server state", NOT "handled". + The distinction is the difference between a snappy grid and a sluggish one. + Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the + script once simply to deliver the component value, and the client is the ORIGIN + of the change, so its own state is correct before the server hears about it. An + extra st.rerun() therefore buys nothing and costs a second full page run. + Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO + page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row + payload — and the handler on the second run is a guaranteed no-op, because the + event-id dedupe two lines below has already consumed that id. + + THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) — True ONLY when the next render + genuinely differs from what the client already draws: + field_upsert — False when the accepted def ECHOES the client's copy + (_field_echo_equal); True on any validator mutation, any + permissions refusal, and ALWAYS when a measure window changed + (values must be re-resolved server-side). + overlay_patch — False when every requested update was accepted verbatim; + True the moment anything was refused or truncated. + field_delete — False: the client removed the column optimistically and the + store scrub is invisible to the next render. + field_duplicate — True only when host-held VALUES must flow back (custom_ cell + copies, measure_ recomputes); formula/created_time clones + compute client-side. + view_delete — False (the deleted view simply stops being served). + view_upsert — unchanged: False, except an unresolved measure condition. + cohort_* / add_to_list — True: the lists payload is server truth, and these are + not the hot path. + + The id dedupe is a SET (not a last-id slot): the event log resends the whole recent + window on every emit, so every id must stay consumed, not just the latest one. + """ + # The ctx fields the moved body reads as plain locals — bound once here so every branch + # below is textually the app.py body (which is what makes this move reviewable). + uname = ctx.uname + allowed_pids = ctx.allowed_pids + fields = ctx.fields + measure_keys = ctx.measure_keys + resolved_ids = ctx.resolved_ids + cohort_ids = ctx.cohort_ids + measure_offer = ctx.measure_offer + visible_views = ctx.visible_views + admin = ctx.admin + scope_key = ctx.scope_key + + if not isinstance(event, dict): + return False + event_id = str(event.get('id') or '')[:180] + seen = ctx.seen_ids + if not event_id or event_id in seen: + return False + seen[event_id] = True + if len(seen) > 400: # a session-lifetime set, kept bounded + for stale in list(seen)[:200]: + seen.pop(stale, None) + kind = event.get('type') + field_by_key = {f['key']: f for f in fields} + valid_keys = set(field_by_key) + overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'} + # Event validation is not a render payload. It must not consume a one-shot label + # correction before the browser has had a chance to receive it. + ws = table_workspace( + ctx, allowed_pids=allowed_pids, consume_corrections=False) + + def _session_ready(): + ws.setdefault('views', {}) + ws.setdefault('fields', {}) + ws.setdefault('overlays', {}) + + if kind == 'view_select': + # Owner item 3 (2026-07-31): remember WHERE THE USER IS so a fresh browser resumes + # there instead of the system default view. Presentation state only — the read side + # re-validates the id against what the caller may see, so nothing here needs an + # authorisation wall beyond the length cap. Never a re-render: the client is already + # on that view. + vid = str(event.get('viewId') or '').strip()[:120] + if vid and _store_of(ctx).available(): + try: + _tops(ctx).save_active_view(uname, vid) + except Exception as _ae: + _tel.error('grid:view-select', _ae) + return False + + if kind == 'record_layout': + # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. Presentation + # state for one surface; grid column order is untouched. Unknown keys are pruned + # against the LIVE field set, dupes collapse, and the wire re-validates at serve + # time — so a client racing a field delete degrades to default order, never an error. + raw_order = event.get('order') + if not isinstance(raw_order, list): + return False + order, seen = [], set() + for k in raw_order[:200]: + k = str(k or '').strip()[:80] + if k and k in valid_keys and k not in seen: + seen.add(k) + order.append(k) + if _store_of(ctx).available(): + _tops(ctx).save_record_layout(uname, order) + else: + _session_ready() + if order: + ws['recordLayout'] = {'order': order} + else: + ws.pop('recordLayout', None) + # Autosave hot path, like view_select: the client already renders the order it asked + # for, and the wire echoes it back on the next workspace read. + return False + + if kind == 'view_upsert': + raw = event.get('view') + if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict): + return False + view_id = str(raw.get('id') or '').strip()[:120] + name = str(raw.get('name') or '').strip()[:120] + requested_name = name + if not view_id or not name: + return False + cfg = raw['config'] + # The filter TREE (leaf conditions and/or nested condition groups) is + # validated by the module-agnostic grid contract helper — imported lazily + # here for the same reason the page does it: a missing embed build must + # never break page import. + import aios_grid as _ag + # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered, + # shown, grouped or coloured by, because it is not a column of this table. Hence the + # widened key set here and the plain `valid_keys` everywhere below. + # `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything + # else is dropped here. Leaving it in would persist a condition the engine can only + # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table + # forever with nothing on screen explaining why. + filters = _ag.clean_filter_tree(cfg.get('filters'), + valid_keys | set(measure_keys), + cohort_ids=set(cohort_ids)) + sorts = [] + for rule in list(cfg.get('sorts') or [])[:20]: + if (isinstance(rule, dict) and rule.get('colId') in valid_keys + and rule.get('dir') in ('asc', 'desc')): + sorts.append({'colId': rule['colId'], 'dir': rule['dir']}) + order = list(dict.fromkeys( + key for key in list(cfg.get('order') or []) if key in valid_keys)) + order += [field['key'] for field in fields if field['key'] not in order] + visible = list(dict.fromkeys( + key for key in list(cfg.get('visible') or []) if key in valid_keys)) + widths = {} + raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {} + for key, value in raw_widths.items(): + if key in valid_keys: + try: + widths[key] = max(60, min(600, round(float(value)))) + except (TypeError, ValueError): + pass + # C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort. + # `cohort_ids` is the same permission list the cohort filter leaf uses — a lock naming + # a cohort this session cannot see is DROPPED at write, exactly like a cohort rule. + # (Readers of a SHARED locked view resolve the id against their OWN visible sets; an + # unresolvable lock matches NOTHING client-side — the cohort-leaf law, never a widen.) + cohort_lock = str(cfg.get('cohortLock') or '').strip()[:120] + # ⛔ WAVE 17 R1 (C-LOCKV guard a) — A PROJECTED LOCKED VIEW RE-STAMPS ITS OWN LOCK. + # A locked view IS its set: its id and its lock are the same fact. The client rebuilds + # `config` from scratch on every autosave (a column resize is enough), so taking the + # lock from the browser would mean one omitted key turns "the 40 accounts we agreed to + # call" into the whole book, under the same name, with nothing going red. The read side + # re-stamps too (aios_grid.views_from_defs) — this keeps the STORE from accumulating the + # wrong record in the first place. + _is_locked_view = view_id in set(cohort_ids) + if _is_locked_view: + cohort_lock = view_id + config = { + 'filters': filters, + # conjunction joining the ROOT-level conditions; anything but an + # explicit 'or' means 'and' (legacy views had an implicit AND). + 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and', + **({'cohortLock': cohort_lock} + if cohort_lock and cohort_lock in set(cohort_ids) else {}), + # ⭐⭐ WAVE 32 · R5 / CONTRACT C4 — "Mark important" (raised by SESSION C as ASK C-12). + # + # ⛔ WITHOUT THIS LINE THE MARK IS A LIE THAT SURVIVES UNTIL RELOAD. This dict is + # REBUILT FROM AN ALLOWLIST on every autosave — a column resize is enough — so a key + # the client sends and this literal does not name is silently dropped. The badge would + # paint, the write would answer 200, and the mark would be gone on the next read: the + # exact shape T24's done-when says "persists across a reload" to catch, and + # [[a-migration-that-runs-on-the-next-write]] on a write path. + # + # ⚠ UNCONDITIONAL, NOT SPREAD-CONDITIONAL LIKE `cohortLock` ABOVE, and the asymmetry is + # the whole point: R5's mark must be REMOVABLE. A key written only when present means + # unmarking sends `important: false`, the allowlist drops it, and the stored `true` + # survives — a mark you can set and never clear. + # ⚠ `is True`, not truthy: the client sends a real boolean, and a string `"false"` must + # not become a mark. + # ⚠ NO PERMISSION GATE, deliberately. R5 makes this a personal legibility mark, not a + # lock; `_may_administer` already decides who may write this view at all, and adding a + # second wall here would be the two-normalizers shape this wave is already fixing + # elsewhere. + 'important': cfg.get('important') is True, + 'sorts': sorts, + 'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None, + 'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None, + 'rowHeightMode': (cfg.get('rowHeightMode') + if cfg.get('rowHeightMode') in ('short', 'medium', 'tall') + else 'short'), + 'order': order, + 'visible': visible, + 'widths': widths, + 'memberPids': [pid for pid in list(cfg.get('memberPids') or []) + if isinstance(pid, int) and pid in allowed_pids], + } + # Wave-6 item 10: a non-grid display mode rides the view config, validated + # structurally (mode whitelist, refs must be this table's fields). + display = _ag._clean_display(cfg.get('display'), valid_keys) + if display: + config['display'] = display + # Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling + # sideways. Clamped 1..8 — 0 would unfreeze the identity column the whole + # surface pivots on; absent = the legacy single frozen column. + if cfg.get('frozenCount') is not None: + try: + config['frozenCount'] = max(1, min(8, int(cfg['frozenCount']))) + except (TypeError, ValueError): + pass + # ── wave-9 C3 (lock) + C4 (permissions) ──────────────────────────────────────────── + # The PRIOR stored view decides both: whether this is a create or an update (C4's + # default splits on exactly that) and who is allowed to change the lock (C3). + prior = (ws.get('views') or {}).get(view_id) or {} + # ⚠ THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the + # caller may not SEE reads as absent — which would look like a CREATE and let anybody + # overwrite somebody else's shared view by guessing its id. Ask the store for the raw + # record and refuse before deciding anything else. Fail-closed on both branches. + _raw_shared = _tops(ctx).shared_view(view_id) if _store_of(ctx).available() else None + if _raw_shared is not None and not prior: + return False # exists, not visible to this caller: refuse + if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin): + return False # visible but not editable by this caller + is_new = not prior + # C4: the creator is HOST-STAMPED and never read from the browser — the same rule as + # the field-level createdBy. An existing view keeps whoever made it. + creator = prior.get('createdBy') or uname + # Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who + # could rewrite them could grant themselves sole ownership of someone else's view, or + # quietly widen a users-scoped view to everyone — privilege escalation by edit. + if not is_new and not _tstore._may_administer(prior, uname, admin): + raw = dict(raw) + raw['permissions'] = prior.get('permissions') + raw['locked'] = prior.get('locked') + # ── I12/C3 BELT (2026-07-31): a LOCKED view's DISPLAY MODE is frozen HOST-SIDE too. + # The client's switcher already refuses (setDisplayMode), and its comment has claimed + # "the host refuses too" since wave 9 — this makes that claim true for direct API + # callers. ⚠ MIRRORS types.ts `isModeFrozen` EXACTLY: SYSTEM views and the undeletable + # list:/cohort: projections are EXEMPT — their `locked` is legacy "undeletable", not + # mode-frozen, and freezing them would trap the default view in whatever mode it last + # saved (the owner's stuck-in-Map report is that trap, client-made). An upsert that + # UNLOCKS in the same write (authorised above) may change display freely. + _mode_frozen = (prior.get('locked') and raw.get('locked') is not False + and prior.get('kind') != 'system' + and view_id != 'all-customers') + if _mode_frozen: + if prior.get('config', {}).get('display'): + config['display'] = dict(prior['config']['display']) + else: + config.pop('display', None) + # ⚠ A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9 + # integration pass, before deploy: `all-customers` is every user's DEFAULT view and it + # carries their own column widths, visible fields and row height. C4's read-default + # ('collaborative' when permissions are absent, so nobody's saved view is + # retro-restricted) is right for a view somebody AUTHORED — but applied to the system + # view it routed `all-customers` into the shared bucket, and the first user to resize a + # column would have published their layout to everyone. Verified in the store: + # shared_table_views() was returning ['all-customers']. + # System views are pinned personal, so they stay in each user's own workspace exactly as + # before this wave. `kind` is host-validated two lines below; read it here from `raw` and + # treat the stored kind as authoritative for an existing view. + _kind = (prior.get('kind') if not is_new + else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') + else 'custom')) + if _kind == 'system' or view_id == 'all-customers': + perms = {'edit': 'personal'} + else: + perms = _ag.clean_view_permissions( + raw.get('permissions'), + default='personal' if is_new else 'collaborative', + known_users=_known_usernames()) + # C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever + # is already stored. Enforced here rather than hidden in the client. This composes with + # C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze + # the display MODE. + # ⚠ Inert in practice TODAY — a user can only ever write their own per-user workspace, + # so `uname == creator` always holds. Correct plumbing for shared views, not a live gate. + may_lock = admin or uname == creator + locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked')) + # C3: the LOCK's actual job — a locked view's display mode is frozen, so a locked Kanban + # stays Kanban. This half IS live per-user and is the part the owner asked for. Filters, + # sorts and columns stay editable by design. + if _mode_frozen: + was = (prior.get('config') or {}).get('display') + now = config.get('display') + if (was or {}).get('mode') != (now or {}).get('mode'): + if was: + config['display'] = was + else: + config.pop('display', None) + # WAVE 17 R1 (C-LOCKV guard b) — ONE THING, ONE NAME. Renaming a locked view renames + # the SET it is, in the store that owns it; the view record must not grow a second title + # that the read side would then have to arbitrate. Routed rather than duplicated. + if _is_locked_view and _store_of(ctx).available(): + _sets = _cohorts(ctx) # R9: the SET lives in this topic's bucket + _cur = (_sets.all_for(uname).get(view_id) or {}).get('name') + if name and name != _cur: + try: + _sets.rename(uname, view_id, name) + except ValueError as _e: + _tel.error('cohort:rename-via-view', _e) + view = {'id': view_id, 'name': name, + # ⚠ DERIVED FROM THE ID, never taken from the browser: `kind` is what paints the + # lock mark, so a client claiming it for an ordinary view would be claiming a + # guarantee the engine is not making about that view's rows. + 'kind': _ag.LOCKED_VIEW_KIND if _is_locked_view + else raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom') + else 'custom', + 'locked': locked, + 'createdBy': creator, + 'permissions': perms, + 'note': str(raw.get('note') or '')[:2000], + 'config': config} + # System/list projections are not all persisted in this table store, but custom views + # may not impersonate their visible names. Persisted view names are allocated + # tenant-globally by TableStore inside the write transaction. + _reserved_view_names = [ + value.get('name') for value in visible_views + if (isinstance(value, dict) and value.get('id') != view_id + and value.get('kind') in ('system', 'list')) + ] + # Wave 21 (item 3, R6): the reserved SYSTEM name is PER-TOPIC — a ut_ database's system + # view is "All records", and reserving the customer literal there uniquified a user's + # honest "All customers" view name against a string that never appears on that topic. + _scope = str(getattr(ctx, 'scope_key', '') or 'customer').strip().lower() + _sys_name = ('All products' if _scope == 'product' + else 'All records' if _scope.startswith('ut_') else 'All customers') + if view_id != 'all-customers': + _reserved_view_names.append(_sys_name) + # ⭐ WAVE 21 (item 9, C1) — THE FOREIGN-VIEW WALL. A granted view is projected into the + # receiver's rail, so their autosave can now emit an upsert carrying an id they do not + # own. Silently saving that into the RECEIVER's stratum would FORK the view (own views + # win the merge, so the receiver sees their private copy forever and calls sharing + # broken). Role `edit` (or admin) writes THROUGH to the owner's record — config only, + # never identity or reach: `createdBy`/`permissions` are pinned from the stored record. + # Role `view` (or no grant) refuses with a toast instead of forking. + _save_as = uname + if _store_of(ctx).available(): + _foreign = _tops(ctx).find_view(view_id) + if _foreign and _foreign[0] != uname: + _o_uname, _stored = _foreign + if not admin: + import core.shares as _shares + if _shares.role_for('view', view_id, uname, + st=getattr(_tops(ctx), 'st', None)) != 'edit': + ctx.out.toast = ('That view is shared with you read-only — ask its ' + 'owner for edit access.') + return False + view['createdBy'] = _stored.get('createdBy', _o_uname) + if _stored.get('permissions') is not None: + view['permissions'] = _stored.get('permissions') + else: + view.pop('permissions', None) + _save_as = _o_uname + if _store_of(ctx).available(): + view = (_tops(ctx).save_view( + _save_as, view, reserved_names=_reserved_view_names, is_admin=admin) or view) + else: + _session_ready() + _view_names = list(_reserved_view_names) + _view_names.extend( + value.get('name') + for candidate_id, value in ws['views'].items() + if candidate_id != view_id and isinstance(value, dict) + ) + view['name'] = _unique_name(view.get('name'), _view_names) + ws['views'][view_id] = view + name_changed = view.get('name') != requested_name + # PERSISTED, but deliberately NO rerun — see the return-value contract in + # this function's docstring. This is the hot path: every filter, sort, + # group, colour, column-resize and column-toggle autosaves through here. + # + # CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered + # server-side, so if this view carries one whose answer was NOT in the payload we just + # shipped, the client is showing a pending marker with nothing coming. Ask for exactly + # one more run — and only then, so the common case (a column filter) still costs zero. + # If the top-of-run read of the component's own value already covered it, this is False + # and nothing extra happens; if that read is ever stale, this is what stops the + # condition hanging unresolved. + # + # ⚠ COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev): + # picking a measure FIELD mints a valueless rule whose autosave used to land here as + # "unresolved" and trigger a rerun that could resolve NOTHING — `rule_complete` refuses + # half-asked questions — so the only effect was a remount that closed the builder + # popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being + # typed stays client-side PENDING by design; the run that matters fires when its VALUE + # arrives and the question becomes answerable. + from harness import measure_filter as _mfc + unresolved = {str(r.get('id') or '') for r in measure_leaves(filters) + if _mfc.rule_complete(r)} - set(resolved_ids) + return name_changed or bool(unresolved - {''}) + + if kind == 'view_delete': + view_id = str(event.get('viewId') or '')[:120] + # ⭐ WAVE-27 item 27 (R8) — `IG_OVERVIEW_ID` joins the system view here, and the comment + # eight lines down already explains why in the LOCKED-view case: dropping the saved + # config leaves the PROJECTION, which rebuilds the row on the very next render. Overview + # is injected on every read of an Instagram database, so a delete would look like it + # worked and the view would be back after a reload — which is precisely the + # reads-as-success failure the client's own `UNDELETABLE_VIEW_IDS` note describes. + if not view_id or view_id in ('all-customers', _ag_overview_id()): + return False + # WAVE 17 R1 (C-LOCKV guard c) — deleting a LOCKED view deletes the set it is. Without + # this the store would drop the saved config and the projection would rebuild the view + # on the very next render: the row would come back, minus the user's sort and columns, + # which reads as a corruption rather than as a refusal. Own sets only (`all_for`), the + # same wall `cohort_delete` rides. + if _store_of(ctx).available() and view_id in _cohorts(ctx).all_for(uname): + _cohorts(ctx).delete(uname, view_id) + _tops(ctx).delete_view(uname, view_id) + # True, unlike an ordinary view_delete: the lists payload IS server truth and the + # client cannot know the membership channel changed. + return True + # Wave-9 I17: deleting a SHARED view is narrower than editing it — creator or admin + # only. A collaborator can change what a shared view shows; they cannot destroy + # everybody else's copy of it, because there is only one copy. + if _store_of(ctx).available(): + _raw_shared = _tops(ctx).shared_view(view_id) + if _raw_shared is not None and not _tstore._may_administer( + _raw_shared, uname, admin): + return False + # ⭐ WAVE 21 (item 9, C1) — the foreign wall, delete half. Contract: a grant of + # `edit` may delete (there is only one copy and the owner handed over its content); + # `view` or no grant refuses. The dead grant record is emptied afterwards so the + # receiver's "Shared with me" heals instead of listing a ghost id forever. + _foreign = _tops(ctx).find_view(view_id) + if _foreign and _foreign[0] != uname: + import core.shares as _shares + _sst = getattr(_tops(ctx), 'st', None) + if not admin and _shares.role_for('view', view_id, uname, + st=_sst) != 'edit': + ctx.out.toast = 'That view is shared with you read-only.' + return False + _tops(ctx).delete_view(_foreign[0], view_id) + try: + _shares.set_grants('view', view_id, [], st=_sst) + except Exception: + pass + return False + _tops(ctx).delete_view(uname, view_id) + else: + _session_ready() + ws['views'].pop(view_id, None) + # Wave-6 item 3: the client removed the view optimistically; a deleted view simply + # stops being served, so the next render differs in nothing the client still shows. + return False + + if kind == 'field_upsert': + raw = event.get('field') + if not isinstance(raw, dict): + return False + key = str(raw.get('key') or '')[:80] + import aios_grid as _ag2 + if key.startswith(_ag2.MEASURE_FIELD_PREFIX): + # A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure + # offer — the same admission the condition builder uses — so a field can only name a + # measure this user could also filter by. Fail closed on anything else. + field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer}) + if field is None: + return False + fmt = _ag2._clean_format(raw.get('format'), field.get('type')) + if fmt: + field['format'] = fmt + elif key in field_by_key and not key.startswith('custom_'): + base = field_by_key[key] + field = {**base, 'note': str(raw.get('note') or '')[:2000]} + # ⭐ W29-T83 — THE COLUMN SUMMARY ON A SOURCE-BACKED COLUMN. This branch rebuilds the + # accepted def from the CONTRACT and adds only what a user may override, and `agg` was + # not in that list — so Average on Customer's `Overdue days` was accepted with a 200 + # and dropped here, before it ever reached the store. Validated against the shared + # vocabulary rather than a literal (see the custom branch below for what a literal + # costs). An ABSENT or unknown `agg` clears it: the client sends the whole field on + # every edit, so a missing key is a user who unset the summary, and `{**base}` has + # already re-applied whatever the contract itself declares. + _agg = raw.get('agg') + if _agg in _ag2.FIELD_AGGS: + field['agg'] = _agg + else: + field.pop('agg', None) + # Wave-5 item 10: a DISPLAY format on any base field (presets included) — how a + # number/date reads, per user. Rendering-only; validated fail-closed per type. + fmt = _ag2._clean_format(raw.get('format'), base.get('type')) + if fmt: + field['format'] = fmt + else: + field.pop('format', None) # cleared -> back to the type's default render + # PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the + # auto-renamed label) of revenue_ytd / revenue_ly / orders_24m — nothing else. The + # whitelist IS the base contract: only a field that shipped with preset+measure can + # take a window here, and the measure KEY is pinned to the base's (fail closed). + if (base.get('preset') and isinstance(base.get('measure'), dict) + and isinstance(raw.get('measure'), dict)): + window = _ag2._clean_window(raw['measure'].get('window')) + if window is not None: + field['measure'] = {'key': base['measure']['key'], 'window': window} + if isinstance(raw.get('label'), str) and raw['label'].strip(): + field['label'] = str(raw['label'])[:120] + elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES: + ftype = raw['type'] + # ⭐⭐ 2026-08-09 — THE RELATIONAL PAIR HAS NO HOME IN A PER-USER OVERLAY, and this + # door used to accept one silently. It builds the stored field from a fixed key list + # and adds only the bags it knows (`options`, `max`, `formula`, `automation`, `code`) + # — so a `link`/`rollup` arriving here was stored WITHOUT its bag, in one user's + # stratum, where `compute_relation_cells` (which walks `user_tables` definitions) + # never sees it. Measured on nurilab as `custom_video_views_90j26`: a Rollup column + # that rendered, could not be filled by anything, and reported no error. + # ⛔ REFUSED RATHER THAN REPAIRED. These two belong to the table's SHARED schema + # (`POST /tables/{key}/fields`), which is where the client now sends them; storing a + # second, private, non-computing copy here would be a second definition of the same + # column. A refusal repaints, so a stale client says something rather than nothing. + if ftype in ('link', 'rollup'): + return False + readonly = ftype in _ag2.READONLY_CUSTOM_TYPES + # The editable family must SAY it is overlay (the client's own contract); the + # read-only pair (formula / created_time) is stored under the cohort column's + # mechanism instead, so no cell write can ever be accepted for it. + if not readonly and raw.get('source') != 'overlay': + return False + field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120], + 'type': ftype, 'source': 'odoo' if readonly else 'overlay', + 'default': True, 'custom': True, + 'note': str(raw.get('note') or '')[:2000]} + if readonly: + field['derived'] = True + # FILTERABLE since wave 6 (owner item 6): formula/created_time values live + # client-side, and this table's counts are client-mode — the client engine + # answers conditions on them soundly. Mirrors fields_from_workspace. + field['filterable'] = True + # ⛔ W29-T83 — THE VOCABULARY, NOT THE LITERAL `'sum'`. This is the write gatekeeper + # for an overlay column's summary and it accepted exactly one name, while the READ + # projection (`aios_grid.py:679`) passes all six — so a user who chose Median got a + # 200, a menu that said Median, and no summary after the next reload. The same shape + # as the `agg:"sum"` defect this repo already shipped, where one door kept it and the + # other dropped it ([[read-path-cannot-witness-write-path]]). + _agg = raw.get('agg') + if _agg in _ag2.FIELD_AGGS and not readonly: + field['agg'] = _agg + # A `select`/`multiselect` carries its own vocabulary; a `user` does NOT — its + # choices are the tenant's accounts, resolved fresh on every render, so persisting + # a snapshot here would go stale the moment somebody joins or leaves. + if ftype in ('select', 'multiselect'): + field['options'] = _ag2._clean_options(raw.get('options')) + if not field['options']: + return False # a select with no choices can never hold a value + field.update(_ag2._choice_appearance(raw, field['options'])) + if ftype == 'rating': + field['max'] = _ag2._clean_rating_max(raw.get('max')) + if ftype == 'formula': + # WRITE-time is where refs must exist (fail closed); read-time leaves them + # alone so a later-deleted ref blanks the CELLS, not the column. + formula = _ag2._clean_formula(raw.get('formula'), valid_keys) + if formula is None: + return False + field['formula'] = formula + if ftype == 'automation': + # C5-AUTOFIELD (wave 18): persist the validated config bag. No `return False` + # when absent — an unconfigured automation column is a legitimate intermediate + # state (create the column, then configure it in the gear). + # ⛔ C8 (wave 22): a CONFIGURED bag must name an existing flow — the write door + # reads the tenant's definition ids and `_clean_automation` refuses a bag + # without one (or naming a deleted one). Refused = the whole field write is + # refused, loudly (repaint), because storing the field while dropping its bag + # would be a column that silently never runs. + if isinstance(raw.get('automation'), dict): + try: + _flow_ids = frozenset((ctx.table.st if ctx.table is not None + else _tops(ctx).st).get('automations') or {}) + except Exception: + _flow_ids = frozenset() + auto = _ag2._clean_automation(raw.get('automation'), valid_keys, + flow_ids=_flow_ids) + if auto is None: + return False + field['automation'] = auto + # ⭐ WAVE-27 item 13 (R13): the code column's language. OPTIONAL, like the + # `automation` bag above and unlike `formula` — an unconfigured code column is a + # legitimate intermediate state (create it, pick the language in the gear), and it + # stores and renders as plain text meanwhile. `_clean_code` never refuses over an + # unknown language; it falls back to `plain`, because the value chooses a + # HIGHLIGHTER, and dropping a user's column to punish a typo in one would be the + # disproportionate half of fail-closed. + if ftype == 'code' and isinstance(raw.get('code'), dict): + code_bag = _ag2._clean_code(raw.get('code')) + if code_bag: + field['code'] = code_bag + fmt = _ag2._clean_format(raw.get('format'), ftype) + if fmt: + field['format'] = fmt + else: + return False + # Wave-5 item 1 — createdBy + permissions on the CREATED strata (custom_/measure_). + # `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the + # client never supplies it, so a browser cannot claim someone else's field. A + # permissions CHANGE is accepted only from the creator or an admin; a legacy field + # with no createdBy is admin-only (fail closed, both directions). The table workspace + # is per-user today, so this wall becomes load-bearing when sharing arrives — but the + # admin path and the legacy path are enforceable (and NC'd) right now. + needs_values = False + refused_perms = False + if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX): + prior = ws.get('fields', {}).get(key) + prior = prior if isinstance(prior, dict) else None + if prior is None: + field['createdBy'] = uname + # Wave-6 item 9: scope is chosen at CREATE, on the cohort page only — + # 'cohort' marks the field cohort-specific; 'global' or absent stays + # unscoped, so an old bundle that sends no scope keeps today's behavior. + if scope_key == 'cohort' and raw.get('scope') == 'cohort': + field['scope'] = 'cohort' + else: + if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip(): + field['createdBy'] = prior['createdBy'] + # Scope is PRESERVED like createdBy: the client copy never moves an + # existing field between scopes. + if prior.get('scope') == 'cohort': + field['scope'] = 'cohort' + want = _ag2._clean_permissions(raw.get('permissions')) + have = _ag2._clean_permissions((prior or {}).get('permissions')) + owner = field.get('createdBy') + if want is not None and (admin or (owner and owner == uname)): + field['permissions'] = want + else: + if have is not None: + field['permissions'] = have + # Asked for a permissions change and did not get it: the client's + # optimistic copy is a lie that must be repainted (wave-6 item 3). + refused_perms = want is not None and want != have + # A NEW measure column, or an existing one whose window moved: the VALUES + # are host-computed, so the next render genuinely differs. + if key.startswith(_ag2.MEASURE_FIELD_PREFIX): + needs_values = prior is None or prior.get('measure') != field.get('measure') + # Canonical names are reserved outside the durable workspace. Persisted field names are + # allocated by TableStore from the transaction's current state, not this stale snapshot. + _workspace_field_keys = set((ws.get('fields') or {})) + _reserved_field_names = [ + value.get('label') for candidate_key, value in field_by_key.items() + if (candidate_key != key and candidate_key not in _workspace_field_keys + and isinstance(value, dict)) + ] + if _store_of(ctx).available(): + field = (_tops(ctx).save_field( + uname, field, reserved_names=_reserved_field_names, + correction_id=event_id) or field) + else: + _session_ready() + _field_names = list(_reserved_field_names) + _field_names.extend( + value.get('label') + for candidate_key, value in ws['fields'].items() + if candidate_key != key and isinstance(value, dict) + ) + requested_label = ' '.join( + str(field.get('label') or 'Untitled').split())[:120].rstrip() + field.pop('labelCorrectedFrom', None) + field.pop('labelCorrectionId', None) + field['label'] = _unique_name(requested_label, _field_names) + if field['label'] != requested_label and event_id: + field['labelCorrectedFrom'] = requested_label + field['labelCorrectionId'] = str(event_id)[:180] + # ⚠ W29-T83 — THE SAME PREDICATE the store uses, called rather than restated. This + # fallback carried its own shorter copy (note only), so a format or measure override + # survived one path and not the other, and `agg` survived neither. + if _tstore.source_override_is_empty(field): + ws['fields'].pop(key, None) + else: + ws['fields'][key] = field + if needs_values or refused_perms: + return True + # THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client + # already renders needs no rerun — see _field_echo_equal and the docstring's law. + return not _field_echo_equal(field, raw) + + if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'): + # Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 — + # replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed: + # only the caller's OWN cohorts (all_for), and member pids are intersected with the + # caller's permitted pool — the same guard add_to_list rides. + if not _store_of(ctx).available(): + return False + sets = _cohorts(ctx) # R9: this DATABASE's lists, never another's + cohort_id = str(event.get('cohortId') or '').strip()[:120] + if not cohort_id or cohort_id not in sets.all_for(uname): + return False + try: + if kind == 'cohort_rename': + name = str(event.get('name') or '').strip()[:120] + if not name: + return False + sets.rename(uname, cohort_id, name) + elif kind == 'cohort_delete': + sets.delete(uname, cohort_id) + else: + raw_pids = event.get('pids') + if not isinstance(raw_pids, list): + return False + pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids] + if not pids: + return False + if kind == 'cohort_add': + sets.add_members(uname, cohort_id, pids) + else: + sets.remove_members(uname, cohort_id, pids) + except ValueError as e: + _tel.error('cohort:panel-edit', e) + return False + # True: the lists payload (names, membership, counts) must reflect this on the next + # render — the panel is showing server truth, not local optimism. + return True + + if kind in ('doc_add', 'doc_delete', 'doc_fetch'): + # DOCUMENTS on a row (owner item 12, contract C5). Metadata in the store key + # `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a + # JSON blob — see core.store.upload_bytes for why that distinction is load-bearing. + # + # ⭐ WAVE 19 / R9 — item 1's SILENT THIRD CASE, and the one nobody reported because the + # React shell cannot reach it yet (`payload.docs` is set only by app.py). Both homes are + # keyed by pid and both were topic-blind: the metadata bucket AND the byte path + # `docs//…`. Two topics' pid spaces overlap by construction (a CRC32 hash lands + # anywhere an Odoo partner id can), so a product's file could occupy the path of a + # customer's — one upload silently shadowing another tenant-visible document. Per-topic + # bucket AND per-topic prefix; the customer topic keeps both of its shipped names. + import base64 as _b64c + import re as _re_mod + if not _store_of(ctx).available(): + return False + docs_key = _docs_key(ctx) + docs_dir = 'docs' if docs_key == _DOCS_KEY else f'docs/{docs_key[:-5]}' + pid = event.get('pid') + doc_id = str(event.get('docId') or '').strip()[:80] + # The pid wall is the SAME one every other row-scoped event rides: a caller may only + # touch rows inside their own permitted pool. Without it, documents would be the + # one surface where a scoped agent could read another book's rows. + if not isinstance(pid, int) or pid not in allowed_pids or not doc_id: + return False + # ⭐ WAVE 25 (R6b / D-16) — THE DOCUMENTS FAMILY WAS THE LAST MODULE-GLOBAL READER HERE. + # Every line below said `store.…`, so an R2 tenant's document metadata (and its bytes) + # landed in tenant #0's repo. Wave 19 made this family per-TOPIC; per-TENANT is the axis + # it was still missing, and the two failures look identical from the app. + _st_docs = _store_of(ctx) + meta_all = _st_docs.get(docs_key) or {} + row = list((meta_all.get(str(pid)) or [])) + + if kind == 'doc_add': + raw_b64 = event.get('data_b64') + name = str(event.get('name') or 'document')[:200] + if not isinstance(raw_b64, str) or not raw_b64: + return False + try: + blob = _b64c.b64decode(raw_b64, validate=True) + except Exception: + return False # not base64 -> not a file we will store + # Re-check the DECODED size host-side. The client checks it too, but a client check + # is a courtesy: this is the number that bounds what the tenant's repo can be made + # to hold by anyone who can reach the endpoint. + if not blob or len(blob) > _DOC_MAX_BYTES: + return False + if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row): + return False + safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file' + path = f'{docs_dir}/{pid}/{doc_id}_{safe}' + try: + _blobs(ctx).upload_bytes( + path, blob, message=f'doc {doc_id} for {scope_key} {pid}') + except Exception as _ue: + _tel.error('docs:upload', _ue) + return False + row.append({'id': doc_id, 'name': name, + 'mime': str(event.get('mime') or 'application/octet-stream')[:120], + 'size': len(blob), 'by': uname, + 'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), + 'path': path}) + elif kind == 'doc_delete': + hit = next((d for d in row if d.get('id') == doc_id), None) + if not hit: + return False + # THE WALL: the uploader or an admin. Enforced here regardless of what the client + # chose to render — `canDelete` in the payload only hides a button. + if hit.get('by') != uname and not admin: + return False + _blobs(ctx).delete_path(hit.get('path') or '') + row = [d for d in row if d.get('id') != doc_id] + else: # doc_fetch + hit = next((d for d in row if d.get('id') == doc_id), None) + if not hit: + return False + blob = _blobs(ctx).download_bytes(hit.get('path') or '') + if blob is None: + return False + # The bridge is a fire-and-forget event LOG with no response channel, so the answer + # is PARKED on the Result and rides the next render's payload (C5). + ctx.out.doc = { + 'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document', + 'mime': hit.get('mime') or 'application/octet-stream', + 'data_b64': _b64c.b64encode(blob).decode('ascii')} + return True # re-render to deliver it + + def _write(cur): + cur[str(pid)] = row + return cur + # ⚠ `docs_key`, NOT `_DOCS_KEY`. The read above and the byte path were both made + # per-topic and this line was not — so a product document's BYTES landed correctly under + # `docs/product/…` while its METADATA went into `customer_docs` beside the customer rows, + # keyed by a CRC32 hash. Found by `verify_scopes.py`'s own leg on its first run, which is + # the argument for asserting the ABSENCE from the legacy bucket and not just the presence + # in the new one: every "it saved" check here was green. + _st_docs.update(docs_key, _write, flush='async') + return True + + if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate', + 'item_move', 'folder_reorder'): + # FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five + # events touch ONE home — the table workspace's `folders` + `itemFolders` — so a cohort + # can be filed without the cohort STORE learning about folders at all. + import aios_grid as _agf + if not _store_of(ctx).available(): + return False + surface = str(event.get('surface') or '') + if surface not in _agf.FOLDER_SURFACES: + return False + # The ids this user may actually file: their own views, their own cohorts. Fail-closed — + # an item_move naming somebody else's cohort is dropped by clean_item_folders anyway, + # but refusing here means we never write it in the first place. + def _own_ids(sfc, cur_ws): + if sfc == 'views': + return set((cur_ws.get('views') or {})) + return set(_cohorts(ctx).all_for(uname)) + + def _mutate(cur_ws): + folders = _agf.clean_folders(cur_ws.get('folders')) + placed = dict(cur_ws.get('itemFolders') or {}) + lst = list(folders.get(surface) or []) + # ⚠ the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the + # built contract; the split doc's first draft said `id`). Converged here rather + # than asking the client to re-emit — see the C4 reconciliation note in the doc. + fid = str(event.get('folderId') or '').strip()[:80] + if kind == 'folder_create': + name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] + if not fid or not name or len(lst) >= _agf.MAX_FOLDERS: + return None + if any(f['id'] == fid for f in lst): + return None + row = {'id': fid, 'name': name, 'order': len(lst)} + icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5) + if icon: + row['icon'] = icon + lst.append(row) + elif kind == 'folder_rename': + name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME] + hit = next((f for f in lst if f['id'] == fid), None) + if not hit or not name: + return None + hit['name'] = name + # wave-9 I15 (C5): the rename pane is also where the icon is changed. An event + # that carries NO icon key leaves the existing one alone (a rename must not + # silently strip a chosen icon); an explicit null clears it back to default. + if 'icon' in event: + icon = _agf.clean_folder_icon(event.get('icon')) + if icon: + hit['icon'] = icon + else: + hit.pop('icon', None) + elif kind == 'folder_delete': + if not any(f['id'] == fid for f in lst): + return None + lst = [f for f in lst if f['id'] != fid] + # CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that: + # an item with no placement renders at the top level. + placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items() + if v != fid} + elif kind == 'folder_reorder': + # ⭐ WAVE 20 (owner item 19, contract C-FOLDER-REORDER) — the user's own folder + # ORDER in the rail. Views could already be dragged between folders; the folders + # themselves could not be dragged past each other. + # + # THE WIRE CARRIES THE FULL ORDER, NEVER A DELTA, and that is the contract's + # doing: a partial list cannot say where an UNNAMED folder went, so a "moved B + # after C" message would leave every other folder's position to be re-derived by + # two sides that can disagree. The client sends the list it is looking at. + # + # ⚠ IDS THIS USER DOES NOT OWN ARE IGNORED, NOT REJECTED. A stale tab can emit an + # order containing a folder that has since been deleted; dropping the whole event + # would make the rail un-reorderable until reload, while dropping the unknown id + # reorders exactly what still exists. Anything the payload omits keeps its + # relative position AFTER the named ones — an order that silently deleted a + # folder it merely failed to mention would be a data loss dressed as a sort. + order = [str(i).strip()[:80] for i in (event.get('order') or [])] + if not order: + return None + known = {f['id']: f for f in lst} + seen, ranked = set(), [] + for fid_ in order: + if fid_ in known and fid_ not in seen: + seen.add(fid_) + ranked.append(known[fid_]) + ranked.extend(f for f in lst if f['id'] not in seen) + if len(ranked) != len(lst): # cannot happen; refuse rather than truncate + return None + for i, f in enumerate(ranked): + f['order'] = i + lst = ranked + elif kind == 'folder_duplicate': + new_id = str(event.get('newId') or '').strip()[:80] + src = next((f for f in lst if f['id'] == fid), None) + if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS: + return None + if any(f['id'] == new_id for f in lst): + return None + lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME], + 'order': len(lst)}) + # ⚠ The folder's CONTENTS are duplicated by the client emitting the ordinary + # per-item duplicate events it already owns (view_upsert with a fresh id, the + # cohort copy path) and then item_move-ing them here. Copying views/cohorts + # inside this handler would be a SECOND implementation of "duplicate an item", + # and the two would drift the first time either one changed. + else: # item_move + item_id = str(event.get('itemId') or '').strip()[:120] + target = event.get('folderId') + if not item_id or item_id not in _own_ids(surface, cur_ws): + return None + cur = dict(placed.get(surface) or {}) + if target is None: + # ⚠ "UNFILE" — the client says nothing about where it went. Popping is right + # here and always was: no placement = render at the top level, and that is the + # state a brand-new item is already in. + cur.pop(item_id, None) # back to the root + elif str(target) == _agf.ROOT_PLACEMENT: + # ⭐⭐ WAVE 32 · OWNER ITEM 20 (`W32-T27`, ASK C-16) — "FILED AT ROOT", STORED. + # + # ⛔ THE THIRD CASE, AND ITS ABSENCE IS THE WHOLE DEFECT. Popping (above) and + # this branch produce the SAME rendering for an ordinary view, which is why + # nobody missed it for nine waves — but they are different FACTS, and for a + # SHARED view the difference is the entire feature: "never filed" sends it to + # the Shared group, "the receiver dragged it out" must not. Absence cannot hold + # two facts, so the second one gets a value. + # + # ⛔ AND IT IS DELIBERATELY EXEMPT FROM THE FOLDER-EXISTS TEST BELOW. This id + # names no folder by design; running it through `any(f['id'] == tid ...)` would + # refuse every root filing, which is the same silent no-op this ticket exists + # to remove. `aios_grid.clean_item_folders` carries the matching exemption on + # the READ side — the two are one change (see its note). + cur[item_id] = _agf.ROOT_PLACEMENT + else: + tid = str(target)[:80] + if not any(f['id'] == tid for f in lst): + return None + cur[item_id] = tid + placed[surface] = cur + folders[surface] = lst + clean = _agf.clean_folders(folders) + return clean, _agf.clean_item_folders( + placed, clean, + {sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES}) + + # ⚠ NO allowed_pids ON PURPOSE — this is app.py's bare `_cl_table_workspace(uname)` + # call at the old :6268, preserved exactly. Omitting the pool drops memberPids from + # any foreign shared view (fail-closed), which is harmless here because the folder + # legs read only view IDS and the folder maps. Passing the pool would be a behavior + # change inside a pure move. + _res = _mutate(table_workspace(ctx, consume_corrections=False)) + if _res is None: + return False # refused: nothing is written + _tops(ctx).save_folders(uname, _res[0], _res[1]) + # True: the sidebars render from server state (which folder holds what), not from local + # optimism — the same contract the cohort panel's membership edits ride. + return True + + if kind == 'view_reorder': + # ⭐ WAVE-27 item 5 (contract C7) — the user drags views up and down the rail. + # + # THE WIRE CARRIES THE FULL ORDER, NEVER A DELTA — the `folder_reorder` contract above, + # verbatim, and for its reason: a partial list cannot say where an unnamed view went. + # + # ⚠ NO OWNERSHIP FILTER, and that is the difference from `item_move`. Filing a view into + # a folder writes a placement keyed by a view this user must own; ARRANGING the rail is + # about the reader's own screen, and the rail legitimately contains views this user does + # not own — the system view, cohort projections, R8's injected Overview, and anything + # SHARED with them. Refusing ids they do not own would make exactly those un-draggable, + # which is the half of sharing people notice. + # + # ⚠ Unresolvable ids are harmless by construction rather than by filtering here: the READ + # side (`aios_grid.views_from_defs`) ranks by this list and appends anything unranked, so + # an id naming a deleted view simply never matches. Validating against today's view set + # at WRITE time would also drop an id whose view is merely not visible in this request. + raw_order = event.get('order') + if not isinstance(raw_order, list): + return False # a non-list order is malformed, not an empty arrangement + order, seen_v = [], set() + for vid in raw_order[:MAX_VIEW_ORDER]: + vid = str(vid or '').strip()[:120] + if vid and vid not in seen_v: + seen_v.add(vid) + order.append(vid) + # ⚠ THE SESSION FALLBACK IS `record_layout`'s, COPIED DELIBERATELY, and it is not just a + # testability convenience. Both keys are per-user PRESENTATION state for one surface, and + # both should keep working while the store is briefly unavailable — a rail that refuses + # to reorder because a write failed is a frozen control with no explanation. It also + # makes the round trip reachable from `verify_grid_events`, which is headless and where + # `store.available()` is False: the store-backed branch of every OTHER stratum writer is + # covered only by the live pass, and this one did not have to be. + if _store_of(ctx).available(): + _tops(ctx).save_view_order(uname, order) + else: + _session_ready() + if order: + ws['viewOrder'] = order + else: + ws.pop('viewOrder', None) # absent = server order, never a stored [] + # True: the rail renders from server state, like the folder sidebars it sits beside. + return True + + if kind == 'field_delete': + # Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS + # the permission: only the created strata (`custom_` overlay fields, `measure_` formula + # columns) are deletable — a base field's key fails the prefix test no matter what the + # browser claims, so the table's contract cannot be deleted from the client. The store + # scrubs the field's stored cell values with it (delete_table_field); views naming the + # key self-heal on their next autosave, the rule every stale colId already rides. + key = str(event.get('key') or '')[:80] + import aios_grid as _ag3 + if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)): + return False + if _store_of(ctx).available(): + _tops(ctx).delete_field(uname, key) + else: + _session_ready() + ws['fields'].pop(key, None) + for _row in ws['overlays'].values(): + if isinstance(_row, dict): + _row.pop(key, None) + # Wave-6 item 3: the client dropped the column optimistically; the store scrub is + # invisible to the next render (the values leave WITH the column). + return False + + if kind == 'choice_rename': + # ⭐ WAVE 20 (owner item 15, contract C-RENAME) — RENAMING AN OPTION CARRIES ITS VALUES. + # + # Owner: *"if I change the option from 'Aaron' to 'Aron' the existing field that has the + # Selected 'Aaron' doesn't change to 'Aron'."* Correct, and it is a data-model fact rather + # than an oversight: a select cell stores the option's LABEL, so renaming the option in + # the field definition orphans every cell holding the old string. They do not render as + # the new name; they render as a value that is no longer an option. + # + # ⛔ THE WIRE CARRIES AN EXPLICIT {from, to} MAPPING, NEVER A DIFF OF THE CHOICE LISTS. + # Diffing is what makes this unsafe: rename A→B while also DELETING C and adding D, and a + # differ sees two removals and two additions with no way to know which pairs up. It would + # cheerfully rewrite every C cell to D. The client knows which row of its editor the user + # typed in; that intent travels, and nothing here has to guess it. + # + # THREE PLACES HOLD THE OLD STRING, and missing any one is its own visible bug: + # 1. the CELLS (`overlays`) — the owner's complaint; + # 2. the FIELD's own `choices` list — else the picker still offers the old name; + # 3. every VIEW that FILTERS or COLOURS BY that value — a saved view filtering + # `Agent is "Aaron"` silently matches NOTHING the moment the value moves, which is + # the failure that looks like data loss ([[aios-shared-views]]). + key = str(event.get('key') or '')[:80] + pairs = [] + for r in (event.get('renames') or ()): + if not isinstance(r, dict): + continue + a, b = str(r.get('from') or '')[:200], str(r.get('to') or '')[:200] + if a and b and a != b: + pairs.append((a, b)) + field = field_by_key.get(key) + if not key or not pairs or not isinstance(field, dict): + return False + if str(field.get('type') or '') not in ('select', 'multiselect', 'status'): + return False + # Only a field this caller may edit. A rename is a write to the field CONTRACT, so it + # rides the same wall `field_upsert` does rather than a looser one of its own. + if key in (ctx.hidden_keys or frozenset()): + return False + mapping = dict(pairs) + + def _rename_in(ws_): + fields_ = ws_.setdefault('fields', {}) + fld = fields_.get(key) + if isinstance(fld, dict) and isinstance(fld.get('choices'), list): + seen, out = set(), [] + for c in fld['choices']: + if isinstance(c, dict): + nm = mapping.get(str(c.get('name') or ''), None) + if nm is not None: + c = {**c, 'name': nm} + ident = str(c.get('name') or '') + else: + c = mapping.get(str(c), str(c)) + ident = str(c) + # A rename ONTO an existing option MERGES rather than duplicating it: the + # user typed a name that already exists, and two identical options is a + # picker bug. Their cells merge too, which is what the value rewrite below + # does anyway. + if ident in seen: + continue + seen.add(ident) + out.append(c) + fld['choices'] = out + fields_[key] = fld + for row in (ws_.get('overlays') or {}).values(): + if not isinstance(row, dict) or key not in row: + continue + val = row[key] + if isinstance(val, list): # multiselect + merged, out2 = set(), [] + for v in val: + nv = mapping.get(str(v), str(v)) + if nv not in merged: + merged.add(nv) + out2.append(nv) + row[key] = out2 + elif isinstance(val, str) and val in mapping: + row[key] = mapping[val] + # Views: filter rule values + colour-by keys naming the old option. + for view in (ws_.get('views') or {}).values(): + if not isinstance(view, dict): + continue + cfg = view.get('config') if isinstance(view.get('config'), dict) else view + + def _walk(nodes): + for n in nodes or (): + if not isinstance(n, dict): + continue + if isinstance(n.get('children'), list): + _walk(n['children']) + continue + if str(n.get('colId') or '') != key: + continue + v = n.get('value') + if isinstance(v, str) and v in mapping: + n['value'] = mapping[v] + elif isinstance(v, list): + n['value'] = [mapping.get(str(x), x) for x in v] + + _walk(cfg.get('filters') or []) + # ⚠ `colorBy` NEEDS NO REWRITE, and the reason is worth stating because the + # opposite looks obvious: it stores a COLUMN KEY (a string, validated against + # `valid_keys` at `view_upsert`), not a {value: colour} map. Per-value colours + # ride the CHOICE OBJECTS themselves, which this function renames in place — + # `{**c, 'name': nm}` keeps every other key on the choice, colour included — so + # a renamed option keeps its swatch for free. An earlier version of this handler + # tried to rewrite `colorBy['rules']` and crashed the whole event pipeline with + # `TypeError: unhashable type: 'dict'` on the FIRST view it touched. + return ws_ + + if _store_of(ctx).available(): + _tops(ctx).rename_choice_values(uname, _rename_in) + else: + _session_ready() + _rename_in(ws) + # True = the host must re-read: the client dropped its optimistic copy of the CHOICES, + # but the rewritten cells and view filters only exist server-side until the next payload. + return True + + if kind == 'field_duplicate': + # Wave-5 item 1. The DESTINATION key is client-generated (the established pattern — + # the client must know it synchronously to place the clone right of the source), and + # both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate, + # because duplicating the source-of-truth into an editable copy would need a + # value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for + # `custom_` sources (a measure clone recomputes; a formula clone re-evaluates). + # The duplicator becomes the clone's creator — it is a NEW field they made. + src_key = str(event.get('sourceKey') or '')[:80] + new_key = str(event.get('key') or '')[:80] + import aios_grid as _ag5 + prefix = ('custom_' if src_key.startswith('custom_') + else _ag5.MEASURE_FIELD_PREFIX + if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None) + if (prefix is None or not new_key.startswith(prefix) or new_key == src_key + or new_key in field_by_key): + return False + src = ws.get('fields', {}).get(src_key) + if not isinstance(src, dict): + return False + clone = dict(src) + clone['key'] = new_key + label = str(event.get('label') or '').strip()[:120] + requested_label = label or (str(src.get('label') or 'Untitled')[:110] + ' copy') + _workspace_field_keys = set((ws.get('fields') or {})) + _reserved_field_names = [ + value.get('label') for candidate_key, value in field_by_key.items() + if (candidate_key != new_key and candidate_key not in _workspace_field_keys + and isinstance(value, dict)) + ] + clone['label'] = requested_label + clone['createdBy'] = uname + # Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort + # page the duplicate surface may explicitly place it instead (same create-time rule + # as field_upsert — a page that is not the cohort page cannot mint a cohort field). + if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'): + if event['scope'] == 'cohort': + clone['scope'] = 'cohort' + else: + clone.pop('scope', None) + if _store_of(ctx).available(): + clone = (_tops(ctx).duplicate_field( + uname, src_key, new_key, clone, + reserved_names=_reserved_field_names, + correction_id=event_id) or clone) + else: + _session_ready() + _field_names = list(_reserved_field_names) + _field_names.extend( + value.get('label') + for candidate_key, value in ws['fields'].items() + if candidate_key != new_key and isinstance(value, dict) + ) + clone.pop('labelCorrectedFrom', None) + clone.pop('labelCorrectionId', None) + clone['label'] = _unique_name(requested_label, _field_names) + if clone['label'] != requested_label and event_id: + clone['labelCorrectedFrom'] = requested_label + clone['labelCorrectionId'] = str(event_id)[:180] + ws['fields'][new_key] = clone + if src_key.startswith('custom_'): + for _row in ws['overlays'].values(): + if isinstance(_row, dict) and src_key in _row: + _row[new_key] = _row[src_key] + # Wave-6 item 3: True only when host-held VALUES must flow back — copied overlay + # cells (custom_ sources) or host-computed measure values. formula/created_time + # clones compute client-side, so the optimistic insert is already complete. + label_changed = clone['label'] != requested_label + return label_changed or src.get('type') not in ('formula', 'created_time') + + if kind == 'add_to_list': + # "Add to list" (owner item 6): a view's current matches become fixed cohort members. + # + # ⚠ `pids` arrives from the BROWSER. It is intersected with `allowed_pids` — the pool this + # user may actually see — before anything is stored. The client computing the set is a + # correctness decision (it is the only engine that already knows the answer, and asking a + # second engine the same question is the drift verify_filter_engine.py exists to prevent); + # it is NOT a trust decision. + raw_pids = event.get('pids') + if not isinstance(raw_pids, list): + return False + pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids] + if not pids: + return False + cohort_id = str(event.get('cohortId') or '').strip()[:120] + name = str(event.get('name') or '').strip()[:120] + if not _store_of(ctx).available(): + return False + # ⭐ WAVE 19 / R9 — THE BUG THIS RULING NAMES. This branch is where a list is BORN, and it + # named `cohort_mod` directly: a list created from the Product surface was written into + # `customer_cohorts` with CRC32 product pids in it, then rendered on the Customer page as + # a list of customers nobody recognised. The set now belongs to the database it was + # created on, and `ctx.scope_key` is the only thing that decides which. + sets = _cohorts(ctx) + try: + if not cohort_id: + if not name: + return False + cohort_id = sets.create(uname, name, pids) + else: + if cohort_id not in sets.all_for(uname): + return False # not this user's cohort — fail closed, do not create + sets.add_members(uname, cohort_id, pids) + except ValueError as e: + _tel.error('cohort:add_to_list', e) + return False + # True: the Cohort page's list and counts must reflect this, and the toast below only + # renders on a fresh run. + # + # ⚠ The NOUN is the TOPIC's, not the literal "customer" this line carried since wave 2. + # "Added 12 customers" over a set of SKUs is the surface stating, in words, that the list + # went somewhere it did not — the user-visible half of the same leak. + _noun = sets.noun + ctx.out.toast = ( + f"Added {len(pids):,} {_noun}{'' if len(pids) == 1 else 's'} to " + f"{name or 'the list'}.") + return True + + if kind == 'overlay_patch': + pid = event.get('pid') + if not isinstance(pid, int) or pid not in allowed_pids: + return False + updates = {} + refused = False + stage_moved = False + row_events = [] # C3 (wave 22): admitted writes, emitted AFTER they apply + raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {} + for key, value in raw_updates.items(): + if key not in overlay_keys or not isinstance(value, (str, int, float)): + refused = True + continue + # C-PERM: a field this caller's permissions HIDE cannot be written, only refused. + # `overlay_keys` asks whether the column exists, not whether this person may touch + # it — so without this line a restricted user who knows a hidden key could PATCH a + # value they are not allowed to read back. Refused LOUDLY (repaint) so the client + # reverts rather than showing a value the store never took. + if key in (ctx.hidden_keys or frozenset()): + refused = True + continue + # Wave-5 item 1: `permissions.edit` is enforced HERE, per key — the client hiding + # its editor is courtesy, this is the wall. Fail-closed: a restricted field with + # no readable createdBy admits only admins. + perms = (field_by_key.get(key) or {}).get('permissions') or {} + edit = perms.get('edit') + if edit == 'admins' and not admin: + refused = True + continue + if edit == 'creator' and not ( + admin or (field_by_key.get(key) or {}).get('createdBy') == uname): + refused = True + continue + # C5-AUTOFIELD (wave 18): an automation cell is MACHINE-WRITTEN. The client marks + # the type read-only (READONLY_CELL_TYPES) but that is courtesy — this is the wall. + # Without it the UI hides the editor while the API still takes a PATCH, which is + # exactly the shape a "read-only" type must not have. + if (field_by_key.get(key) or {}).get('type') == 'automation': + refused = True + continue + # C2 (wave 22): a STAGE FIELD is engine-written EXCEPT at a review gate, where a + # human may advance a card to a member of the review stage's `next` and nothing + # else. The law rides the field definition (`automation.humanMoves` / `arrive`, + # stamped by the engine that derives the stages) because this module must not + # import the engine — the engine states the law, this door enforces it + # mechanically. An ADMITTED move writes THROUGH to the definition row, the shared + # truth the engine and every viewer read: parked in this user's overlay stratum it + # would be a board position only its author could see. Everything else about the + # field is refused — the same wall the automation-cell type gets, extended, not + # replaced. + if str(ctx.scope_key or '').startswith('ut_'): + import core.user_tables as _ut_mod + _tdef = _ut_mod.get(ctx.scope_key, st=ctx.table.st) if ctx.table is not None \ + else None + _fdef = next((f for f in ((_tdef or {}).get('fields') or []) + if f.get('key') == key), None) + # C7 (wave 22): a METRIC cell is engine-computed from the master series — a + # human write would be an invented measurement, refused the way an automation + # cell is (the wall, not the courtesy). + # ⭐ 2026-08-07 — EXTENDED TO THE RELATIONAL PAIR, and read through the field + # layer's own predicate rather than re-tested here. A `rollup` cell is an + # aggregate the server computes; a DERIVED `link` cell is a join the server + # resolves. Both would be overwritten by the next refresh anyway, so accepting + # the write would not merely be wrong — it would look like it worked and then + # silently revert, which is worse than a refusal. + # ⚠ An ORDINARY (user-picked) link is deliberately NOT refused: `is_computed_cell` + # answers per FIELD, not per type, because one kind is read-only in one mode and + # editable in the other. + if _ut_mod.is_computed_cell(_fdef): + refused = True + continue + _auto = (_fdef or {}).get('automation') + # An ordinary Link is a shared database relationship. Validate its selected + # target ids and write it through to the definition row; a personal overlay + # would be invisible to reciprocal links and Rollups. + if (_fdef or {}).get('type') == 'link': + _linked = _ut_mod.patch_link_cell( + ctx.scope_key, pid, key, value, st=ctx.table.st) + if _linked is None: + refused = True + continue + if _linked != str(value): + refused = True + row_events.append((key, _linked)) + continue + # ⭐ WAVE 25 (C3 + amendment C3-A1, owner rulings R7/R6) — THE PROFILE CELL. + # Two things happen here and neither can live anywhere else: + # + # 1. IT WRITES THROUGH TO THE DEFINITION ROW, the stage field's mechanism two + # branches up, for the identical reason. MEASURED before it was written: an + # ordinary `ut_*` cell edit lands in the TYPIST'S OWN overlay stratum, and + # the automation engine reads `t['rows']` — so a handle somebody typed + # would be invisible to the enrich action and to every other viewer of the + # same database. The flag would be decorative. + # 2. BLANKING IT CLEARS THAT ROW'S PRESET CELLS IN THE SAME WRITE (R6) — + # `patch_profile_cell` does both inside ONE store update, because a blank + # that commits without its clear leaves a row with no handle and a live + # follower count, i.e. stale numbers attributed to nobody. + # + # ⛔ NOT VIA THE EMIT GUARD (HARD RULE 5 / D-40). Nothing here emits; this is + # `user_tables`' own write door, reached from the branch that already knows the + # scope is `ut_`. ⚠ `editRole` is deliberately NOT consulted: it governs the + # SCHEMA, and C3 is explicit that a profile field's VALUES stay fully editable — + # that is the whole point of item 5a. + if isinstance((_fdef or {}).get('profile'), dict): + _pw = _ut_mod.patch_profile_cell( + ctx.scope_key, pid, key, value, st=ctx.table.st) + if _pw is None: + # Not a handle we can enrich. Refused LOUDLY so the client repaints to + # the stored truth instead of showing a value the store never took. + refused = True + continue + # A normalisation (`@Name` -> `name`, a pasted URL -> its handle) is a real + # difference from what the client is drawing, so it must repaint. An + # untouched value echoes and stays on the no-blip path. + if _pw['handle'] != str(value): + refused = True + if _pw['cleared']: + stage_moved = True # cells the client never typed changed: re-read + row_events.append((key, _pw['handle'])) + continue + # ⭐ OWNER ITEM 4 (2026-08-06) — A COLUMN AN AUTOMATION FILLS IS NOT A COLUMN A + # PERSON TYPES IN. Owner, verbatim: *"I should not be able to edit any of the + # Pre-set fields on instagram automation, we talked about this extensively."* + # + # ⛔ IT HAD NEVER BEEN ENFORCED ANYWHERE. `editRole: 'admins'` rides on every + # preset column and reads exactly like this wall, which is why it was believed to + # be one — but it governs the SCHEMA (`user_tables.py:535`): who may rename or + # retype the column. The VALUES stayed open to anyone with grid access, and the + # owner typed into one to prove it. The metric branch above is the same rule for + # the same reason, written for a different column kind; this is its sibling. + # + # ⚠ REACHED ONLY AFTER stage AND profile have `continue`d, and the order is the + # rule rather than an accident: both of those columns carry the identical tag and + # both are cells a human is SUPPOSED to drive — the stage because moving a card + # IS the decision, the profile because it is where enrichment gets its subject. + # Refusing on the tag alone would freeze every kanban. + # + # `flowId` and not merely `automation`: a user-configured `automation` COLUMN + # carries its own gear bag (kind/source/urlField/settings) and no flowId, and it + # is not this ruling's subject. + if isinstance(_auto, dict) and (_auto.get('flowId') or _auto.get('preset')): + refused = True + continue + # ⭐ WAVE 23 (C7) — the json wall, BEFORE the generic 10 000-char truncation, which + # would otherwise cut a document in half and store the halves as valid text. A json + # column promises its readers a parseable document; a write that is not one is + # REFUSED with the cell unchanged rather than silently kept as a string nothing can + # open. Blank stays legal — an empty cell is "no document", not a malformed one. + if (field_by_key.get(key) or {}).get('type') == 'json': + raw_j = value if isinstance(value, str) else _json_ev.dumps(value) + if not str(raw_j).strip(): + updates[key] = '' + continue + if len(str(raw_j)) > _ut_limits.MAX_JSON_CELL: + refused = True + continue + try: + _json_ev.loads(raw_j) + except (ValueError, TypeError): + refused = True + continue + updates[key] = str(raw_j) + continue + clean = str(value)[:10000] + if isinstance(value, str) and clean != value: + # Truncated: the stored truth is shorter than the client's copy. (An + # int/float arriving as its string form renders identically — not a refusal.) + refused = True + definition = field_by_key.get(key) or {} + if definition.get('type') in ('select', 'multiselect') and clean: + options = [str(v) for v in definition.get('options') or []] + canonical = {v.strip().lower(): v for v in options} + if definition.get('type') == 'select': + normalized = canonical.get(clean.strip().lower()) + if normalized is None: + refused = True + continue + else: + normalized_parts = [] + seen = set() + invalid = False + for part in clean.split(','): + match = canonical.get(part.strip().lower()) + if match is None: + invalid = True + break + if match.lower() not in seen: + seen.add(match.lower()) + normalized_parts.append(match) + if invalid: + refused = True + continue + normalized = ','.join(normalized_parts) + if normalized != clean: + refused = True + clean = normalized + updates[key] = clean + if updates: + if str(ctx.scope_key or '').startswith('ut_') and ctx.table is not None: + # User-table cells are collaborative record data. Views/layout remain personal, + # but record values live in the definition rows so Links, Rollups, automations, + # and every viewer observe one truth. + import core.user_tables as _ut_values + _ut_values.patch_cells(ctx.scope_key, pid, updates, st=ctx.table.st) + elif _store_of(ctx).available(): + _tops(ctx).patch_overlay(uname, pid, updates) + else: + _session_ready() + ws['overlays'].setdefault(str(pid), {}).update(updates) + # C3 (wave 22): the admitted writes become ROW EVENTS — emitted AFTER they applied, on + # the ut scopes the event triggers watch this wave, through the seam in user_tables so + # this module never imports the engine. Human door only, which IS the loop law. + if str(ctx.scope_key or '').startswith('ut_') and ctx.table is not None: + import core.user_tables as _ut_ev + for _k, _v in list(updates.items()) + row_events: + _ut_ev.emit_row_event({'type': 'event_field', 'table': str(ctx.scope_key), + 'rowId': str(pid), 'field': _k, 'after': _v, + 'st': ctx.table.st, 'user': uname}) + # Wave-6 item 3: the client's cell already shows the typed value — rerun ONLY when + # some of what it shows was refused or clamped and must be repainted to the truth. + # A stage move reruns too: its `arrive` side-effects (the tracked flag) changed cells + # the client never typed. + return refused or stage_moved + + return False diff --git a/platform/core/perms.py b/platform/core/perms.py index 0c26d8098d989330c5259c53943a17fead2087dc..f1f234422737ee5f548bbe22daf91238a4830634 100644 --- a/platform/core/perms.py +++ b/platform/core/perms.py @@ -199,116 +199,6 @@ def nav_pages(user): return out -# ------------------------------------------------------------------ governable modules (C3) -#: ⭐⭐ WAVE 33, OWNER ITEM 11 — *"its an error that ALL fucking tenant can see both Customer and -#: Product database under Manage User, this is only for Royal Imports tenant. Make sure per tenant -#: configuration always work, this is my fourth time I flag it."* -#: -#: The cause was two lines in `routes_admin.py`: `_PERM_MODULES = ("customer_data","product_data")` -#: and a `_MODULE_LABELS` dict beside it. `get_perms` built its answer from the literal and never -#: touched `session.runtime`, so the permission editor was TENANT-BLIND BY CONSTRUCTION — every -#: tenant was handed tenant #0's two grid modules whether or not it is provisioned for either. -#: -#: ⛔ WHY THE DERIVATION LIVES HERE AND NOT AT THE ROUTE. `routes_nav.py::nav` already applies -#: exactly this rule and has since wave 18; the perms route is the ONE surface that skipped it. -#: A second copy of the rule at a second call site is how the two came apart in the first place, -#: so the rule is stated once, as a pure function of (runtime, catalogue), and both the route and -#: `verify_perm_scope` ask THIS. -def tenant_modules(runtime): - """The registry keys this TENANT is provisioned for. None means "every key". - - ⛔ FAIL-CLOSED ON THE EMPTY LIST, and that is not the obvious reading. `routes_admin`'s own - docstring records two shipped fail-OPENs of exactly this shape (`modules: []` and - `bus: []` both read as UNRESTRICTED), so ONLY the literal `'all'` — or an absent - key, which is how the compiled-in tenant #0 record is written — means every module. A - bucket-provisioned tenant gets `{"modules": rec.get("modules", [])}` - (`harness/runtime.py::_builder_from_record`), so `[]` there means the tenant was provisioned - for NOTHING and must be told so, never handed the catalogue. - - Legacy keys are migrated on read for the same reason `allowed_modules` does it: a tenant - record written before a rename must not silently lose the module it names. - """ - cfg = getattr(getattr(runtime, 'tenant', None), 'config', None) or {} - raw = cfg.get('modules', 'all') - if raw is None or raw == 'all': - return None - if isinstance(raw, str): - raw = [raw] - return {_LEGACY_KEYS.get(str(k).strip(), str(k).strip()) - for k in raw if str(k or '').strip()} - - -def tenant_has_module(runtime, key): - """Is `key` in this tenant's provisioned set? PARENT-AWARE, like `may_open`. - - `routes_nav` keeps a page whose `parent` is granted even when the child key is not, and a - governable list that disagreed with the nav would offer an admin a wall for a database the - tenant cannot open — or hide one it can. - """ - allowed = tenant_modules(runtime) - if allowed is None or key in allowed: - return True - parent = registry.BY_KEY.get(key, {}).get('parent') - return parent in allowed if parent else False - - -def tenant_governable_modules(runtime, topics, ut_entries=()): - """The databases the permission editor may render for THIS tenant, in stable order. - - ⚠ NAMED `tenant_*` TO STAY DISTINCT FROM `registry.governable_modules()`, which lane E - published under contract C3 and which this consumes. They answer different questions — - registry's is "which topics are governable at all, platform-wide"; this one is "which - databases does THIS tenant govern" — and two same-named functions one import apart is the - [[one-question-two-normalizers]] shape waiting to happen. - - `topics` — the registry-backed grid modules the editor can build pickers for, as - `[{'key': ..., 'label'?: ...}]`, i.e. `registry.governable_modules()` narrowed - to the topics the caller has a field-schema arm for. The CALLER owns that - narrowing because the schema behind each arm is code - (`routes_admin::_module_fields`), not a policy list. Tenant-blind on the way - in; this function is the only place the tenant is applied. - `ut_entries` — this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape. - - Returns `[{'key', 'label', 'enforced'}]`. - - ⛔⛔ `enforced` IS THE HONEST HALF AND IT IS NOT DECORATION. `perm_scope` walls the REGISTRY - topics — `grid_assembly` calls `may_access`, `visible_fields` and `apply_row_scope` on every - read. It is never consulted for a `ut_*` database: `routes_tables`' visibility is - `user_tables.may_open` (creator, admin, or a `core.shares` grant) and there is no module - grant in that path at all. So a stored `ut_*` wall would be INERT — the editor would say - DENY, the table routes would keep serving, and nothing anywhere would say so. That is the - same class as the empty-dropdown defect (wave 26 item 24): a control that answers when it - cannot. The database is LISTED, because the owner asked to see every database; the wall is - declared unenforced, and `routes_admin::_clean_perms` REFUSES to store one against it rather - than accepting a rule nobody applies. Arming `perm_scope` over `ut_*` is a change to - `routes_tables`/`user_tables` (lane A's fence) — booked, not faked. - """ - out = [] - for t in topics or (): - key = str((t or {}).get('key') or '').strip() - if not key or not tenant_has_module(runtime, key): - continue - # The LABEL comes from the registry, never from a dict beside the list. `_MODULE_LABELS` - # was a hardcoded copy of two registry labels, so wave 33's R2 relabel ("Odoo customers" / - # "Odoo products") would have moved one of them and left the permission editor showing the - # old name with nothing to notice. - label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key) - out.append({'key': key, 'label': label, 'enforced': True}) - seen = {r['key'] for r in out} - for e in ut_entries or (): - key = str((e or {}).get('key') or '').strip() - if not key or key in seen: - continue - seen.add(key) - out.append({'key': key, 'label': (e.get('label') or key), 'enforced': False}) - return out - - -def enforced_module_keys(modules): - """The subset of `governable_modules`' answer a permission block may actually be stored for.""" - return {m['key'] for m in (modules or ()) if m.get('enforced')} - - def landing_page(user): """Where a restricted user lands: the first MAIN-chrome page in their grant. None for unrestricted users (the client lands on the Analyst, as the host does). diff --git a/platform/core/registry.py b/platform/core/registry.py index e65062743bf8cf7a90c80b0accc86b5b7731812c..97a571af0bd97600f4f4ce1080506c6ceff1556f 100644 --- a/platform/core/registry.py +++ b/platform/core/registry.py @@ -22,22 +22,6 @@ Fields: 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement + Collections + Analyst/plumbing): products, assortment, financial, management, pricing, inventory, vendor, warehouse, returns, outreach, complexity, health. - subject set -> the REAL-WORLD POPULATION this row presents, as a source-qualified id - ('odoo:res.partner'). ⭐ W33-T41 / item 12a: it is what makes "two databases present - the same thing" a statement code can refuse, and it exists because NO OTHER FIELD - CAN CARRY THAT TEST. Not the key (`product_data` and `ut_odoo_products` are different - keys by construction — that is the bug, not the check), and NOT THE LABEL: the owner's - own complaint is "why do we have 'Odoo products' ALREADY with the current Products - database?", i.e. the twin was already carrying the label the legacy row is about to - take (W33-T42). Between T42 and T44 the two rows legitimately share a label, so a - label-keyed check would refuse the rename that fixes the bug. - ⚠ A row without `subject` claims nothing and conflicts with nothing — silence is not - a claim, so adding this field cannot retro-refuse anything already shipped. - topic set -> this row is a GOVERNABLE DATABASE and this is the word the rest of the codebase - already calls it by (`routes_grid._SCOPES`, `routes_alerts._TOPICS`, `scope=` on the - grid routes). ⭐ W33-T46 / contract C3: it is what `governable_modules()` keys on, and - it exists so `routes_admin`'s permission list can be DERIVED from the registry instead - of being the hardcoded two-element tuple that made every tenant see Customer + Product. note one-line scope note shown in the UI / docs """ # WAVE-9 I8 — `'source'`: which connected system a module's data comes from. @@ -81,14 +65,7 @@ REGISTRY = [ # myday->customer_list was done. The STORE keys are deliberately NOT renamed: # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them # would orphan every saved view. Label-only changes need none of that machinery. - # LABEL renamed 'Customer' -> 'Odoo customers' (owner item 12, 2026-08-14 / W33-T42): the - # naming convention of every other database from this connector ("Odoo orders", "Odoo vendor - # bills"). ⭐ STILL LABEL-ONLY, and the note above is the precedent — the KEY does not move, so - # no grant, saved view, cohort, formula, shared column or `?page=` deep link is touched. - # ⛔ `nav_meta` could not have done this: `routes_nav.py` raises `400 name_not_allowed` for any - # key that does not start with `ut_`, so a tenant rename cannot reach a registry module at all. - {'key': 'customer_data', 'label': 'Odoo customers', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', - 'subject': 'odoo:res.partner', 'topic': 'customer', + {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer ' 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship ' 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'}, @@ -124,10 +101,7 @@ REGISTRY = [ 'group_only': True, 'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is ' 'not part of this family.'}, - # LABEL renamed 'Product' -> 'Odoo products' (owner item 12, 2026-08-14 / W33-T42) — see the - # customer_data note above; label-only, the key stays `product_data`. - {'key': 'product_data', 'label': 'Odoo products', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family', - 'subject': 'odoo:product.product', 'topic': 'product', + {'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family', 'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) ' 'stock columns, with saved views and custom fields. Identity = the SKU code.'}, {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True, @@ -223,90 +197,3 @@ BY_KEY = {m['key']: m for m in REGISTRY} def brand_filterable(key) -> bool: return bool(BY_KEY.get(key, {}).get('brand')) - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner: *"why do we have 'Odoo products' already with the current Products database? The -# Unique ID is redundant, make sure this never happens. One unique ID per database always."* -# -# ⛔ THE CHECK HAD TO GO WHERE THE BUG WAS MADE, AND THAT IS NOT WHERE ANYONE WOULD LOOK. -# `core.user_tables.create` — the UI door — HAS had a collision loop all along: it slugs the -# label to a key and appends `_2 … _49` until the key is free. It never fired here for two -# reasons, and both matter. (1) It compares KEYS inside ONE tenant's `user_tables` document, -# and `customer_data` / `product_data` are compiled REGISTRY rows that never enter that -# document at all — the two halves of the duplicate live in different namespaces and only meet -# in `routes_nav.py::nav`, where the registry pages and `user_tables.nav_entries` are merged -# for display. (2) The connector path never calls `create()`: `odoo_relational. -# _ensure_table_inplace` writes `cur[key] = {...}` straight into the document. So a check bolted -# onto `create()` would have been green through the entire life of this defect. -# -# ⚠ THE SUFFIX LOOP IS ALSO WHY A RELABEL ALONE WOULD HAVE MADE THIS WORSE, MECHANICALLY: a -# naive "just rename Product to Odoo products" mints `ut_odoo_products_2` — the owner's -# redundant id again, from the fix for it. -# -#: subject -> the ONE registry key entitled to present it. Derived, never typed twice. -SUBJECT_CLAIMS = {m['subject']: m['key'] for m in REGISTRY if m.get('subject')} - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W33-T46 / CONTRACT C3 — THE GOVERNABLE MODULE LIST IS DERIVED, NOT A LITERAL -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner, on his FOURTH flag of the same thing: *"its an error that ALL fucking tenant can see -# both Customer and Product database under Manage User, this is only for Royal Imports tenant. -# Make sure per tenant configuration always work."* -# -# ⛔ THE CAUSE IS A TWO-ELEMENT TUPLE. `routes_admin._PERM_MODULES = ("customer_data", -# "product_data")` — a compiled literal that `get_perms` builds its whole list from without ever -# touching `session.runtime`. It is tenant-blind BY CONSTRUCTION, which is why no amount of -# per-tenant work elsewhere ever fixed it, and why the answer is not "filter the tuple" but -# "delete the tuple": a second hand-typed copy of a fact the registry already holds is a thing -# that CAN go stale, and this one did, four times. -# -# ⚠ THE LINE BETWEEN THE TWO HALVES, stated because both sides need it: this map is -# REGISTRY-SIDE and TENANT-BLIND — it answers *"which compiled modules are governable, and what -# are they called"*, nothing more. Applying the tenant's own `config["modules"]` filter and -# merging that tenant's `ut_*` databases is the CALLER's half (`routes_admin`, D's fence), which -# is exactly what `routes_nav.py` already does correctly and `routes_admin` never did. -def governable_modules(): - """`{topic: {'key','label','subject'}}` — every registry row that is a governable DATABASE. - - A fresh dict per call, deliberately: a module-level mapping handed to a route is one - `.pop()` away from being a different list for every request after the first. - """ - return {m['topic']: {'key': m['key'], 'label': m['label'], 'subject': m.get('subject')} - for m in REGISTRY - if m.get('topic') and not m.get('archived') and not m.get('group_only')} - - -def subject_of(key): - """The subject a REGISTRY row claims, or None. (`ut_*` databases carry theirs on the stored - definition instead — they are per-tenant data, not compiled rows.)""" - return (BY_KEY.get(key) or {}).get('subject') - - -def subject_conflict(subject, key, claimed=None): - """The OTHER key already presenting `subject`, or None. - - `claimed` is an optional {subject: key} of claims outside the registry — in practice the - caller's own tenant document, whose stored definitions carry a `subject` stamp. It is a - PARAMETER rather than a lookup on purpose: the store is per-tenant, two tenants both holding - an "Odoo customers" is entirely legitimate, and a module-level cache of somebody's tables is - how D-169 happened. This function knows nothing about which tenant it is serving. - - ⭐ THE TEST IS "A DIFFERENT KEY", NOT "IS THIS SUBJECT TAKEN". `_ensure_table_inplace` and - `ut_ensure` re-run on every boot and every 1800 s resync, adopting their table BY KEY — so a - check phrased "is this subject already claimed?" goes red on the second boot against the very - table it created on the first, and the container fails to start. Idempotency is not an - exception to the rule here; it is the common case. - """ - if not subject: - return None # silence is not a claim - for holder in (SUBJECT_CLAIMS, claimed or {}): - other = holder.get(subject) - if other and other != key: - return other - return None diff --git a/platform/core/shares.py b/platform/core/shares.py index 146302359d42a22e8e701d7cff8d04d94d5d431b..09e691b9543512a9f41880e0e987cf1a4367a748 100644 --- a/platform/core/shares.py +++ b/platform/core/shares.py @@ -43,32 +43,10 @@ SHARES_KEY = 'object_shares' #: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken". KINDS = ('view', 'folder', 'database') -#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform". -#: Spelled as a single character so it can never collide with a username (usernames are lower-case -#: and non-empty by `core/users.py`, and are checked against this explicitly below). -#: -#: ⛔⛔ **AND WHAT THAT MEANS DEPENDS ON THE KIND — THE LINE THIS NOTE USED TO CARRY WAS FALSE FOR -#: ONE OF THE THREE** (W33-T30, `waves/wave32/sharing-audit.md` S-8). It read *"the module/table -#: wall runs FIRST and this never widens past it"*, flatly, and the audit's own words for that are -#: *"the third docstring in this audit describing a check that is not on the path"*. Corrected -#: here rather than deleted, because the sentence is TRUE of two kinds and the difference is the -#: whole point: -#: * `kind='view'` / `kind='folder'` on a GOVERNED module (`customer_data`, `product_data`) — -#: the sentence holds. `require_session` plus the topic's own gate run first, and the -#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is -#: merged, so a grant can only narrow-or-equal what that account could already reach. -#: * `kind='database'` on a `ut_*` table — **THE SENTENCE IS FALSE AND THIS REGISTRY IS THE -#: ONLY WALL.** `routes_admin._PERM_MODULES` is `("customer_data", "product_data")` and -#: `_clean_perms` 400s anything else, so no row filter and no hidden field can even be -#: DECLARED for a user table; `routes_tables.py` makes zero `perm_scope` calls and passes -#: `hidden_keys=frozenset()`. A `database` grant is therefore ALL-OR-NOTHING — every row, -#: every column — and an `*` database grant admits every account in the tenant to all of it. -#: -#: ⚠ THAT IS A REAL CAPABILITY, DELIBERATELY KEPT, not a hole to plug in passing. What was wrong -#: was a docstring promising a second wall that does not exist for this kind; scoping user tables -#: is booked (S-8), not done. `routes_shares.py`'s module docstring says the same thing at the -#: OTHER door — the audit's fix is "say so at both", and one door saying it is how the next reader -#: gets the confident half [[one-question-two-normalizers]]. +#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform" — +#: the module/table wall runs FIRST and this never widens past it. Spelled as a single character +#: so it can never collide with a username (usernames are lower-case and non-empty by +#: `core/users.py`, and are checked against this explicitly below). EVERYONE = '*' ROLES = ('view', 'edit') diff --git a/platform/core/store.py b/platform/core/store.py index c57dd523f6ff62ae22d94f6db27c8b0fedfc7da2..4a5cf6d22a58450480955fbce134fe4c047d36f2 100644 --- a/platform/core/store.py +++ b/platform/core/store.py @@ -1,838 +1,817 @@ -"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store -(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags. - -Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The -repo is created private on first write. Reads use a short in-process cache that writes refresh, so -within a running instance reads are consistent with writes; a fresh instance re-fetches the latest. - -WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level -functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) — every existing -`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves. -`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo -(nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each -instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's -write burst can never starve another's flush window. -""" -import io -import os -import json -import threading -import time as _time -from pathlib import Path - -try: - from dotenv import load_dotenv - load_dotenv(Path(__file__).resolve().parents[1] / '.env') -except Exception: - pass - -from huggingface_hub import HfApi, hf_hub_download -from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError - -REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data') -_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit -_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget) - -#: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP. -#: -#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a -#: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude -#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a -#: change, which is the truthful answer: the client's copy predates this process entirely. -#: -#: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality. -REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}' - - -def _token(): - return os.environ.get('HF_TOKEN') or None - - -class _Projected(dict): - """One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped. - - ⛔⛔ W32-T01: THE ALTERNATIVE IS A SILENT WRONG ANSWER, WHICH IS WHY THIS CLASS EXISTS AT ALL. - A projection built as a plain dict answers `defn.get('rows') or {}` with `{}` — indistinguishable - from a database that genuinely has no rows. The caller then paints an empty grid, exports an - empty CSV, or (worse) writes that emptiness back. This repo has shipped that exact shape before - ([[empty-answer-vs-unfinished-answer]], and a grouped view that painted zero rows). - - So a dropped key is not absent — it is UNREAD, and asking for it raises with the reason and the - fix in the message. `KeyError` is the type on purpose: `dict.__getitem__`'s own failure mode, so - nothing has to learn a new exception; and the message names `get_projection` so the traceback - points at the caller's mistake rather than at this file. - - ⚠ Still a real `dict` for every OTHER purpose — `json.dumps`, `dict(x)`, `**x`, iteration and - `in` all behave normally, because a projection has to serialise onto the wire like any - definition. Only the named keys are booby-trapped. - """ - - __slots__ = ('_dropped',) - - def __init__(self, data=None, dropped=()): - super().__init__(data or {}) - object.__setattr__(self, '_dropped', frozenset(dropped)) - - def _refuse(self, key): - raise KeyError( - f'{key!r} was DROPPED by a projected store read and is not absent — this document came ' - f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would ' - f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if ' - f'you need {key!r}.') - - def __getitem__(self, key): - if key in self._dropped: - self._refuse(key) - return super().__getitem__(key) - - def get(self, key, default=None): - if key in self._dropped: - self._refuse(key) - return super().get(key, default) - - def pop(self, key, *a): - if key in self._dropped: - self._refuse(key) - return super().pop(key, *a) - - -def _project(doc, drop): - """A deep COPY of `doc` with `drop`'s keys removed from every top-level value. - - ⭐ The order is the optimisation: build the retained structure FIRST (a shallow comprehension - that never touches the dropped values), then deep-copy THAT. One `json.dumps` over 0.1% of the - bytes, instead of one over all of them. - """ - if not isinstance(doc, dict): - return json.loads(json.dumps(doc)) - kept = {k: ({kk: vv for kk, vv in v.items() if kk not in drop} if isinstance(v, dict) else v) - for k, v in doc.items()} - copied = json.loads(json.dumps(kept)) - return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()} - - -def is_projected(value): - """Did this value come back from a PROJECTED read? — the TAKEN-PATH predicate (W33-T01/D-213). - - ⭐⭐ WHY A PREDICATE AND NOT A COUNTER, and it is the whole reason this exists. `all_defs` - FALLS BACK to the whole read on any failure, silently and by design — so a caller that gets the - right answer has learned nothing about which read produced it. A gate asserting only "the - definitions came back" passes identically when the projection worked and when it quietly did - not, which is [[gate-can-report-green-on-nothing]] wearing a green tick. - - ⭐ This is INTRINSIC rather than instrumented: `_project` stamps every top-level value as a - `_Projected`, so the evidence rides the document itself. Nothing has to be wrapped, no counter - can be installed on the wrong object ([[measure-the-real-call]]), and the answer is the same in - production as it is under a gate — a monkey-patched tally is true only while the patch is on. - - ⚠ It answers about ONE VALUE (a table's definition), not about a document: `_project` returns a - plain dict OF `_Projected` values, so `is_projected(doc)` is False and `is_projected(doc[key])` - is True. That is the right grain — the thing a route holds and reads is the definition. - """ - return isinstance(value, _Projected) - - -class Store: - """One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush. - - Every method body is the pre-wave-18 module function, verbatim except globals→fields. - The behaviour contracts that matter are stated on each method and are UNCHANGED: - strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads. - """ - - def __init__(self, repo): - self.repo = str(repo) - self._lock = threading.RLock() - self._cache = {} - # --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------ - # One typed character in a Notes cell used to cost THREE synchronous network calls - # inside the component round-trip. update(..., flush='async') applies the mutation to - # the in-process cache synchronously (read-your-writes: every reader consults the - # cache first) and a background worker coalesces uploads. The restart-wipe protection - # is UNCHANGED where it matters: the FIRST update of a key in a fresh process still - # strict-reads. Known, accepted: with a second concurrent writer the stale window - # widens from ~one round-trip to ~the flush gap. - self._owned = {} # name -> True once cache reflects our own last accepted write - self._dirty = {} # name -> cache is ahead of the hub - self._flushers = {} # name -> live worker thread - self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance - self._last_flush = {} # name -> monotonic time of the last completed upload - # --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------ - # name -> how many accepted writes this process has applied to that bucket, and when. - # Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for - # why that constraint is the whole feature rather than an optimisation. - self._revs = {} - # --- ⭐⭐ W31-T13 (D-177(d)) — THE PER-KEY WRITE LOCK ------------------------------ - # `self._lock` guards the CACHE. It used to be held across a sync `update`'s whole - # read-modify-write — a hub download, the caller's `fn`, and a blocking hub upload — so - # one write froze every reader in the process for two network round trips. `get()` takes - # the same lock, and `/nav`, `/workspace` and every permission check are `get()`. - # ⛔ IT IS NOT SAFE TO SIMPLY DROP THE LOCK. Two sync writers of one key would interleave - # read-modify-write and one update would vanish — corruption, not slowness. So the two - # jobs are separated: this lock serialises WRITERS OF ONE KEY (held across the network, - # never touching readers), and `self._lock` is taken only for the microseconds a cache - # read or write needs. Readers never wait on a network call again. - # ⚠ PER KEY, not one write lock: two tenants' buckets, or `user_tables` and `nav_meta`, - # have no reason to serialise against each other. - self._key_locks = {} - self._key_locks_guard = threading.Lock() - self._rev_at = {} - - def available(self): - """True when a token is configured — callers degrade gracefully without one.""" - return bool(_token()) - - def _download(self, name): - """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent. - Any other failure (network, auth, rate-limit, corrupt JSON) RAISES — a transient read - error must never be mistaken for an empty store, or a caller could overwrite real data - with an empty dict (this is what wiped the user registry on restart).""" - try: - path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset', - token=_token(), force_download=True) - with open(path, encoding='utf-8') as f: - return json.load(f) - except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError): - return {} - except ValueError as e: - # Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in - # ValueError("Force download failed…") with the 404 as __cause__ — so a brand-new - # tenant repo's very first strict read raised here and aborted its very first write. - # A genuinely-absent repo is the same fact as a genuinely-absent file: {}. - cause = getattr(e, '__cause__', None) - if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)): - return {} - raise - - def get(self, name, fresh=False): - """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error, - hand back the last known-good cached value if we have one, else {}. Mutating callers go - through update()/_read_strict, which RAISE instead.""" - with self._lock: - if not fresh and name in self._cache: - return json.loads(json.dumps(self._cache[name])) # hand back a copy - try: - data = self._download(name) - except Exception as e: - try: # lazy import — core stays layer-clean - import harness.telemetry as _tel # noqa: PLC0415 - _tel.error(f'store:get:{name}', e, - fallback='cached' if name in self._cache else 'empty') - except Exception: - pass - if name in self._cache: - return json.loads(json.dumps(self._cache[name])) - return {} - self._cache[name] = data - return json.loads(json.dumps(data)) - - def get_projection(self, name, drop=()): - """Like `get`, but each top-level VALUE comes back WITHOUT the keys named in `drop`. - - ⭐⭐ W32-T01 (D-185) — THE POINT IS THE COPY, NOT THE PAYLOAD SIZE ON THE WIRE. - - `get()` is honest and expensive: it hands back `json.loads(json.dumps(...))` so a caller - cannot mutate the cache, and on tenant #0's `user_tables` that copy is **28.6 MB / 81,330 - rows / ~703 ms warm**, of which **99.9% is `rows`** — which no permission check, no nav - render and no workspace envelope ever reads. `/nav` used to pay ~13 of those copies per - request; W31-T10 made it ONE, and that one is still 703 ms of pure copying. This is the - floor D-185 named: copy only what was asked for. - - ⛔⛔ **THE PROJECTION IS NEVER CACHED, AND THAT LINE IS THE WHOLE SAFETY ARGUMENT.** A - rows-less document in `self._cache` would be served to the next WHOLE reader, and the next - `update()` would upload it — every row in the tenant deleted, by a read. So the cold path - downloads whole, caches WHOLE (exactly as `get` does), and projects the RETURN value only. - `verify_store_projection.py` asserts the cache still contains `rows` after a projected read, - with an NC that reds when the projection is cached instead. - - ⚠ **`self._lock` is held across the copy, same as `get`** — the retained structure is - ~0.1% of the bytes, so this is microseconds where `get` is most of a second. Narrowing it - would let a concurrent writer mutate a value mid-serialise for no measurable gain. - - ⚠ **A dropped key RAISES on read rather than reading as absent** (`_Projected`). A - projection that answered `{}` for `rows` would paint an empty grid — the exact - [[empty-answer-vs-unfinished-answer]] shape this repo has shipped before — and the caller - that most wants rows is the one least able to tell. See `_Projected`. - - ⚠ Lenient, like `get`: a transient fetch error falls back to the cached document (projected) - and then to `{}`. Mutating callers still go through `update()`/`_read_strict`, which never - project — a write must always read whole. - """ - drop = frozenset(str(d) for d in drop) - with self._lock: - if name in self._cache: - return _project(self._cache[name], drop) - try: - data = self._download(name) - except Exception as e: - try: # lazy import — core stays layer-clean - import harness.telemetry as _tel # noqa: PLC0415 - _tel.error(f'store:get_projection:{name}', e, - fallback='cached' if name in self._cache else 'empty') - except Exception: - pass - with self._lock: - if name in self._cache: - return _project(self._cache[name], drop) - return {} - with self._lock: - self._cache[name] = data # ⛔ the WHOLE document, never the projection - return _project(data, drop) - - def _key_lock(self, name): - """The write lock for ONE key (W31-T13). Created on demand; never removed.""" - n = str(name) - with self._key_locks_guard: - lk = self._key_locks.get(n) - if lk is None: - lk = self._key_locks[n] = threading.RLock() - return lk - - def _read_strict(self, name): - """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely - absent file), so a failed read ABORTS a read-modify-write instead of merging into {}. - - ⭐ W31-T13 — THE DOWNLOAD IS OUTSIDE `self._lock` NOW, and the re-check after it is the - whole of what makes that safe. A hub read takes a round trip; holding the cache lock - across it is what froze every reader. But moving it out opens a window in which an ASYNC - writer can accept a row into the cache while our download is in flight — and assigning our - (older) hub copy afterwards would erase it. That is D-177's original symptom exactly - (`POST /rows` → 201, row gone), re-created by the lock split rather than by a call site. - So the dirty mark is re-read AFTER the download, under the short lock, and a key that - became dirty keeps its cache. - """ - data = self._download(name) # network, no lock held - with self._lock: - if self._dirty.get(name) and self._owned.get(name) and name in self._cache: - # Our copy is stale by construction: a pending upload is going to overwrite the - # hub anyway, so the fresh read cannot preserve a remote write — it can only - # destroy a local one we have already acknowledged. Same argument, and the same - # three conditions, as `update`'s pending-async branch below. - return json.loads(json.dumps(self._cache[name])) - self._cache[name] = data - return json.loads(json.dumps(data)) - - def exists(self, name): - """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY - uncertainty — a seeder must never clobber a registry we merely failed to reach.""" - try: - files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset') - return f'{name}.json' in files - except RepositoryNotFoundError: - return False - except Exception: - return True - - # ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6) - def _bump(self, name): - """Advance one bucket's write counter. THE CALLER HOLDS THE LOCK. - - Called from the two places an accepted write becomes visible to readers of this process - — `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT - from `_flush_worker`: that uploads data a bump already accounted for, and counting it - again would report a change to every browser each time the coalescing worker woke. - """ - n = str(name) - self._revs[n] = self._revs.get(n, 0) + 1 - self._rev_at[n] = _time.time() - - def revision(self, name): - """`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else. - - ⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why - this method exists at all. A browser asking "has anything changed" by re-fetching rows - costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way - in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s - per bucket — so two tabs polling saturate the server and every unrelated write queues - behind them. This performs no download, no `get()`, and no copy of the value. - ⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace` - counts `get`/`_read_strict`/`_download` on the serving path and goes red on one). - - `rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend - ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops - being inert the moment D-4 flips, with the wire shape unchanged. - - ⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write - by a SECOND container against the same HF dataset is invisible here. That is not a limit - this introduces — `get()` caches a bucket for the life of the process and never re-reads - it, so the other container's write is invisible to every reader on this backend already. - The token is exactly as fresh as the store it reports on. Postgres has no such gap. - ⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted: - an attachment is fetched on explicit user action, never polled. - """ - n = str(name) - with self._lock: - rev = int(self._revs.get(n, 0)) - at = self._rev_at.get(n) - return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'} - - def _upload(self, name, data): - """The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra - API round-trip on every put(), pure latency after the first success.""" - api = HfApi(token=_token()) - if not self._repo_ok: - api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True) - self._repo_ok = True - buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8')) - api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo, - repo_type='dataset', commit_message=f'update {name}') - - def upload_bytes(self, path_in_repo, data, message=None): - """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents). - Bytes are never cached in-process: large, cold, fetched on explicit user action.""" - if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') - with self._lock: - api = HfApi(token=_token()) - if not self._repo_ok: - api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True) - self._repo_ok = True - api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo, - repo_id=self.repo, repo_type='dataset', - commit_message=message or f'add {path_in_repo}') - - def download_bytes(self, path_in_repo): - """Raw bytes back, or None when absent/unreachable — a missing attachment degrades to - "that file is gone", never to a broken page.""" - if not self.available(): - return None - try: - local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset', - token=_token()) - return Path(local).read_bytes() - except (EntryNotFoundError, RepositoryNotFoundError): - return None - except Exception: - return None - - def delete_path(self, path_in_repo): - """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal.""" - if not self.available(): - return False - try: - HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo, - repo_type='dataset', - commit_message=f'delete {path_in_repo}') - return True - except (EntryNotFoundError, RepositoryNotFoundError): - return True - except Exception: - return False - - def put(self, name, data): - """Persist {name}.json (creates the private repo on first write). Updates the cache.""" - if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') - with self._lock: - self._upload(name, data) - self._cache[name] = json.loads(json.dumps(data)) - self._owned[name] = True - self._last_flush[name] = _time.monotonic() - self._bump(name) # C6 — AFTER the upload: a raise here is not a write - return self._cache[name] - - def _flush_worker(self, name): - """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure — - the cache stays the truth and the dirty mark survives, so nothing is lost silently. - - ⭐⭐ W31-T13 — THIS WORKER IS THE SECOND WRITER, AND THE PER-KEY LOCK MUST COVER IT. - ⛔ THE RACE THE LOCK SPLIT OPENED, spelled out because it is silent and survives a restart - as real data loss. `update`'s big lock used to span `_read_strict` + `fn` + the upload, so - this worker could never snapshot mid-update. With the cache lock narrowed it can: - async write -> `_dirty=True`, this worker scheduled, sleeps `_FLUSH_DELAY` - sync update t+0.5 -> takes the dirty branch, runs `fn`, enters `_upload` (no cache - lock held — that is the whole point of the ticket) - worker t+2.0 -> `self._lock` is FREE and `_dirty` is still True (the sync commit - has not happened yet), so it snapshots the PRE-UPDATE cache, - clears the mark, and uploads it AFTER the sync upload lands - result -> cache has the new value, the hub has the old one, `_dirty` is - False, and nothing will ever re-upload. Divergence with no - symptom until the next process boots and reads the hub. - ⚠ `user_tables` has ~40 sync writers against 2 async ones, so that window is the COMMON - case, not an exotic one — it is D-177's own shape, re-created one layer down. - ⛔ LOCK ORDER IS KEY-THEN-CACHE, EVERYWHERE. `_update_locked` takes them in that order, so - this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay - OUTSIDE the key lock — a worker that dozed while holding it would block every writer of - that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove. - """ - backoff = 2.0 - while True: - _time.sleep(_FLUSH_DELAY) - wait, err, snapshot = None, None, None - with self._key_lock(name): - with self._lock: - gap = _time.monotonic() - self._last_flush.get(name, 0.0) - if gap < _FLUSH_MIN_GAP: - wait = _FLUSH_MIN_GAP - gap - elif not self._dirty.get(name): - self._flushers.pop(name, None) - return - else: - self._dirty[name] = False - snapshot = json.loads(json.dumps(self._cache.get(name, {}))) - if wait is None: - try: - self._upload(name, snapshot) - with self._lock: - self._last_flush[name] = _time.monotonic() - backoff = 2.0 - except Exception as e: # noqa: BLE001 - with self._lock: - self._dirty[name] = True # keep it queued; cache is still the truth - err = e - if wait is not None: - _time.sleep(wait) - continue - if err is not None: - try: - import harness.telemetry as _tel # noqa: PLC0415 - _tel.error(f'store:flush:{name}', err) - except Exception: - pass - _time.sleep(backoff) - backoff = min(backoff * 2, 60.0) - - def _schedule_flush(self, name): - """Mark dirty + ensure exactly one live worker for the key. Caller holds the lock.""" - self._dirty[name] = True - t = self._flushers.get(name) - if t is not None and t.is_alive(): - return - t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True, - name=f'store-flush-{self.repo}:{name}') - self._flushers[name] = t - t.start() - - def flush(self, name=None, timeout=30.0): - """Block until pending async writes for `name` (or ALL keys) reach the hub — QA gates - and shutdown hooks use this; the app itself never needs to.""" - deadline = _time.monotonic() + timeout - names = [name] if name else None - while _time.monotonic() < deadline: - with self._lock: - pending = {k for k, v in self._dirty.items() if v} | { - k for k, t in self._flushers.items() if t.is_alive()} - if names is not None: - pending &= set(names) - if not pending: - return True - _time.sleep(0.2) - return False - - def _flush_now_at_exit(self): - """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing - gap — a Space restart must not eat the last seconds of the async window. - - ⚠ W31-T13 — DELIBERATELY NOT KEY-LOCKED, stated so the omission is not read as one. This - runs at interpreter exit with the process going away: a writer mid-`update` will never - finish, so waiting for its key lock could only turn a partial save into no save at all. - Best-effort is the correct contract here, and it is the one case where taking the lock - would be worse than skipping it. - """ - with self._lock: - dirty = [k for k, v in self._dirty.items() if v] - snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty} - for k in dirty: - self._dirty[k] = False - for k, d in snaps.items(): - try: - self._upload(k, d) - except Exception: - pass # exiting anyway — nothing left to reschedule - - def update(self, name, fn, flush='sync'): - """Read-modify-write. STRICT first read (raises on a transient error rather than - starting from an empty dict), apply fn(data)->data, persist. - - flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the - table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and - returns immediately; a coalescing worker commits. The strict read still guards the - FIRST async update of a key per process. - - ⭐⭐ W31-T13 (D-177(d)) — TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET. - `self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key - still cannot interleave (last-write-wins is the contract; a lost update is not). - `self._lock` — the one every `get()` takes — is now held only for the microseconds a cache - read or write needs, never across a download or an upload. Before this, one sync write - froze every reader in the process for two network round trips, on a store whose warm - `get()` already costs 703 ms of deep copy on tenant #0. - ⚠ Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly - that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`. - """ - with self._key_lock(name): - return self._update_locked(name, fn, flush) - - def _update_locked(self, name, fn, flush): - """`update`'s body, with this key's write lock already held. See its docstring. - - ⚠ THE CACHE LOCK IS TAKEN THREE TIMES AND RELEASED BETWEEN THEM, which is only safe - because THIS key's write lock is held throughout: nothing else can be mid-write on it, so - the state read in the first section is still ours in the third. A caller reaching this - without the key lock would have the old interleaving bug back. - """ - # ⚠ A FLAG, NOT A `None` SENTINEL. `None` can be a real cached value, and a sentinel that - # can equal real data is not a sentinel — this repo has paid for that exact shape before - # (`routes_nav`'s locked-mode default). The flag says "the cache answered", full stop. - data, from_cache = None, False - with self._lock: - if flush == 'async' and self._owned.get(name) and name in self._cache: - data, from_cache = json.loads(json.dumps(self._cache[name])), True - elif self._dirty.get(name) and self._owned.get(name) and name in self._cache: - # ⛔ A SYNC READ-MODIFY-WRITE MUST NOT DISCARD A PENDING ASYNC WRITE. - # - # THE BUG THIS LINE EXISTS FOR (owner report 2026-08-09, MEASURED on staging): - # `POST /rows` answered `201 {"rid":"100"}` and the row never existed. - # `user_tables.add_row` commits `flush='async'` — the row lives ONLY in - # `self._cache` until the coalescing worker uploads it, which is `_FLUSH_DELAY` - # (2s) away and up to `_FLUSH_MIN_GAP` (20s) if that key committed recently. - # Inside that window ANY sync writer of the SAME key took the `else` branch - # below, and `_read_strict` does `self._cache[name] = self._download(name)` — - # it REPLACES the cache with the hub's copy, erasing the only place the new row - # existed, and then `put()` uploaded that. Nothing raised. The 201 was truthful - # about `add_row`'s return value and false about the outcome. - # - # ⚠ `user_tables` has ~40 sync writers against 2 async ones, so this was never - # one bad call site — it is a property of the KEY. Wave 27 only made it fire - # every time, by putting the relation refresh on the add path. - # - # ⛔ AND RE-DOWNLOADING WAS PROTECTING NOTHING. Once a key is dirty our cache - # has already diverged and the pending upload was going to overwrite the hub - # anyway — so the fresh read cannot preserve a remote write, it can only destroy - # a local one we already acknowledged. This is the class's OWN documented - # contract, restored: see `_owned` above — *"read-your-writes: every reader - # consults the cache first"* — and `_flush_worker`'s *"the cache stays the truth - # and the dirty mark survives, so nothing is lost silently."* - # - # ⚠ NOT a weakening of `_read_strict`'s purpose. That exists so a FAILED read - # aborts the RMW instead of merging into `{}`; the cache is populated - # known-good local state, and this branch is only reachable when it is. - data, from_cache = json.loads(json.dumps(self._cache[name])), True - # else: the strict hub read happens BELOW, outside this lock (W31-T13) - - # ⭐ W31-T13 — THE HUB READ AND THE CALLER'S `fn` RUN WITH NO CACHE LOCK HELD. `fn` is - # arbitrary caller code — `automation_engine`'s relation pass has run whole-tenant work - # inside one — and it used to execute with every reader in the process blocked behind it. - if not from_cache: - data = self._read_strict(name) # network; re-checks the dirty mark on the way back - result = fn(data) - data = result if result is not None else data - - if flush == 'async': - if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') - with self._lock: - self._cache[name] = json.loads(json.dumps(data)) - self._owned[name] = True - self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time - self._schedule_flush(name) - return data - - # ⛔ THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE. - # `put()` holds `self._lock` across its own `_upload` — which is exactly the freeze this - # ticket removes — but it is PUBLIC with seven callers outside this file (`users.py`, - # `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`, - # `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency - # semantics at once. Duplicating four assignments with this comment beside them is the - # smaller blast radius; if `put` is ever split, this collapses back into a call. - if not self.available(): - raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') - self._upload(name, data) # network, no cache lock held - with self._lock: - self._cache[name] = json.loads(json.dumps(data)) - self._owned[name] = True - self._last_flush[name] = _time.monotonic() - # ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_upload` raises on failure, - # and a bump on that path would have `revision()` report a change that never landed. - self._bump(name) - # The upload above CARRIED whatever was pending, so the key is no longer ahead of - # the hub. Without this the flush worker wakes on a still-dirty mark and re-uploads - # identical content — a second commit against the 256-commits/hr budget for nothing. - # ⚠ AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache - # is still the truth and the row must still be committed by the worker). Clearing it - # first would turn a transient upload error into the very silent loss the branch above - # exists to stop — which is why it sits here and not before `_upload`. - self._dirty[name] = False - return data - - -# ------------------------------------------------------------------- instances + module API -_INSTANCES = {} -_INSTANCES_LOCK = threading.RLock() - - -def for_repo(repo_id): - """The HF Store bound to `repo_id` — one instance per repo per process (each with its own - cache and flush state). - - ⚠ WAVE 20: this is now the HF-SPECIFIC factory. Callers that want "the right store for this - tenant, whatever the backend is" call `handle()` below. `for_repo` keeps its exact old - behaviour because the seed/migration tooling has to be able to name the FILE store - explicitly while the app runs on Postgres — a migration that could only reach the active - backend could not copy between them. - """ - rid = str(repo_id or '').strip() or REPO - with _INSTANCES_LOCK: - inst = _INSTANCES.get(rid) - if inst is None: - inst = Store(rid) - _INSTANCES[rid] = inst - return inst - - -# ============================================================================================= -# WAVE 20 (owner ruling R1, closes DEBT D-4) — THE BACKEND SEAM. -# -# `core/store_backend.py` has carried this warning since EXIT-2b, and it was correct at the time: -# -# "Flipping STORE_BACKEND=pg does NOT redirect the ~40 existing callers that say -# `import core.store as store` — they are bound to the HF module directly. Rewiring them is -# task C-4 … THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise." -# -# **R1 IS C-4.** The owner ruled the cutover on 2026-08-05 (the D-4 trigger that fired: a SECOND -# server process — `royal-imports/cfo-os` came back as a pinned LIVE environment beside staging, -# and two containers on one last-write-wins file store is the race B-3 was always about). -# -# THE REWIRE IS HERE RATHER THAN IN 28 FILES, and that is a deliberate choice over the obvious -# alternative of `sed`-ing every `import core.store as store` to `import core.store_backend`: -# * every one of those callers means "the store for the tenant I am serving", which is exactly -# what this module has always meant. The BACKEND is not their concern and making it their -# concern is how one of them gets missed; -# * a missed caller under a search-and-replace does not fail — it silently keeps writing to the -# file store while everything else writes to Postgres. That is the split-brain this whole -# ruling exists to end, reintroduced by the fix for it; -# * `store_backend.py`'s stated reason for refusing to do it this way — "no dual-read window and -# no way to compare the two stores' contents first" — is satisfied: `ops/seed_pg_from_hf.py` -# copies, then `--verify` diffs the two stores key-by-key before anything flips. -# -# ⛔ THE IMPORT IS LAZY AND MUST STAY LAZY. `core/store_pg.py` imports psycopg only inside -# `_pool()`, so an `hf` deployment installs no driver; importing it at module scope here would -# undo that and make the default backend depend on the optional dependency. -# ============================================================================================= - -def backend(): - """`'hf'` | `'pg'` — validated, resolved per call so a test can flip the env var. - - Mirrors `core.store_backend.name()` deliberately rather than importing it: that module - imports THIS one, so reaching back would be a cycle. `verify_store_pg` asserts the two agree - on every value, which is the guard against them drifting apart. - """ - raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower() - if raw not in ('hf', 'pg'): - raise RuntimeError( - f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store) or 'pg' " - f"(Postgres, needs DATABASE_URL). Refusing to guess: a typo that silently served the " - f"other store is how data ends up in two places.") - return raw - - -def handle(repo_id=None, slug=None): - """THE tenant-bound store handle for the ACTIVE backend — the one seam every caller crosses. - - `repo_id` addresses the HF backend (a dataset repo); `slug` addresses Postgres (a schema). - Both are passed by `harness.runtime.get_runtime`, which knows both facts, so flipping the - backend never needs a lookup table between them — and neither identifier has to be invented - for the backend that does not use it. - """ - if backend() == 'pg': - import core.store_pg as _pg # noqa: PLC0415 — lazy: see the banner above - return _pg.PgStore(slug or os.environ.get('AIOS_TENANT') or 'royal-imports') - return for_repo(repo_id) - - -#: The HF default instance. Kept as a module global (not a property) because the seed/migration -#: tooling and `_flush_all_at_exit` both need the FILE store by name even when pg is active. -_DEFAULT = for_repo(REPO) - - -def _d(): - """The default tenant's store on the active backend — what every module function below uses. - - ⚠ Resolved PER CALL, never cached. A cached default would freeze the backend at import time, - and import order is exactly what nobody controls: `api/main.py` imports half the platform - before it has read a single environment variable it did not inherit. - """ - return handle() if backend() == 'pg' else _DEFAULT - - -def available(): - return _d().available() - - -def get(name, fresh=False): - return _d().get(name, fresh=fresh) - - -def _read_strict(name): - return _d()._read_strict(name) - - -def exists(name): - return _d().exists(name) - - -def upload_bytes(path_in_repo, data, message=None): - return _d().upload_bytes(path_in_repo, data, message=message) - - -def download_bytes(path_in_repo): - return _d().download_bytes(path_in_repo) - - -def delete_path(path_in_repo): - return _d().delete_path(path_in_repo) - - -def put(name, data): - return _d().put(name, data) - - -def flush(name=None, timeout=30.0): - return _d().flush(name=name, timeout=timeout) - - -def update(name, fn, flush='sync'): - return _d().update(name, fn, flush=flush) - - -def get_projection(name, drop=()): - """A projected read — see `Store.get_projection`. Wave 32, W32-T01 (D-185). - - ⛔ AN ELEVENTH STORE OPERATION, AND IT HAD TO BECOME ONE — the method alone was NOT enough, - which is the opposite of what it looks like from `revision()`'s precedent. `revision` could - ship method-only for a few hours because its caller held a Store. This one's caller is - `harness.runtime.TenantRuntime`, and `TenantRuntime._store()` returns **the MODULE** whenever - `store_handle is None` — which is precisely tenant #0, the tenant whose 703 ms `/nav` copy this - exists to remove. A method-only projection would have been unreachable from the only place that - needed it, while every gate stayed green. - - ⛔ SO `verify_store_pg.IFACE` GAINS IT IN THE SAME CHANGE, and `core/store_pg.py` gains the - twin below it. That gate's `extra:` check would red on this function otherwise — correctly: - this addresses a key and reads it, so it is a store operation and both backends must answer. - `backend`/`handle` are exempt because they SELECT a store; this one reads one. - """ - return _d().get_projection(name, drop=drop) - - -def revision(name): - """The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing. - - ⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS - HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely - so a capability one backend lacks cannot be reached through the shared door and die at the - cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change - and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half - had no owner: a method is a capability the HF store has, a module function is a promise both - backends keep. - """ - return _d().revision(name) - - -def _flush_all_at_exit(): - with _INSTANCES_LOCK: - instances = list(_INSTANCES.values()) - for inst in instances: - inst._flush_now_at_exit() - - -import atexit # noqa: E402 (registered after the class it needs) -atexit.register(_flush_all_at_exit) +"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store +(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags. + +Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The +repo is created private on first write. Reads use a short in-process cache that writes refresh, so +within a running instance reads are consistent with writes; a fresh instance re-fetches the latest. + +WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level +functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) — every existing +`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves. +`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo +(nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each +instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's +write burst can never starve another's flush window. +""" +import io +import os +import json +import threading +import time as _time +from pathlib import Path + +try: + from dotenv import load_dotenv + load_dotenv(Path(__file__).resolve().parents[1] / '.env') +except Exception: + pass + +from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError + +REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data') +_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit +_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget) + +#: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP. +#: +#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a +#: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude +#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a +#: change, which is the truthful answer: the client's copy predates this process entirely. +#: +#: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality. +REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}' + + +def _token(): + return os.environ.get('HF_TOKEN') or None + + +class _Projected(dict): + """One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped. + + ⛔⛔ W32-T01: THE ALTERNATIVE IS A SILENT WRONG ANSWER, WHICH IS WHY THIS CLASS EXISTS AT ALL. + A projection built as a plain dict answers `defn.get('rows') or {}` with `{}` — indistinguishable + from a database that genuinely has no rows. The caller then paints an empty grid, exports an + empty CSV, or (worse) writes that emptiness back. This repo has shipped that exact shape before + ([[empty-answer-vs-unfinished-answer]], and a grouped view that painted zero rows). + + So a dropped key is not absent — it is UNREAD, and asking for it raises with the reason and the + fix in the message. `KeyError` is the type on purpose: `dict.__getitem__`'s own failure mode, so + nothing has to learn a new exception; and the message names `get_projection` so the traceback + points at the caller's mistake rather than at this file. + + ⚠ Still a real `dict` for every OTHER purpose — `json.dumps`, `dict(x)`, `**x`, iteration and + `in` all behave normally, because a projection has to serialise onto the wire like any + definition. Only the named keys are booby-trapped. + """ + + __slots__ = ('_dropped',) + + def __init__(self, data=None, dropped=()): + super().__init__(data or {}) + object.__setattr__(self, '_dropped', frozenset(dropped)) + + def _refuse(self, key): + raise KeyError( + f'{key!r} was DROPPED by a projected store read and is not absent — this document came ' + f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would ' + f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if ' + f'you need {key!r}.') + + def __getitem__(self, key): + if key in self._dropped: + self._refuse(key) + return super().__getitem__(key) + + def get(self, key, default=None): + if key in self._dropped: + self._refuse(key) + return super().get(key, default) + + def pop(self, key, *a): + if key in self._dropped: + self._refuse(key) + return super().pop(key, *a) + + +def _project(doc, drop): + """A deep COPY of `doc` with `drop`'s keys removed from every top-level value. + + ⭐ The order is the optimisation: build the retained structure FIRST (a shallow comprehension + that never touches the dropped values), then deep-copy THAT. One `json.dumps` over 0.1% of the + bytes, instead of one over all of them. + """ + if not isinstance(doc, dict): + return json.loads(json.dumps(doc)) + kept = {k: ({kk: vv for kk, vv in v.items() if kk not in drop} if isinstance(v, dict) else v) + for k, v in doc.items()} + copied = json.loads(json.dumps(kept)) + return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()} + + +class Store: + """One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush. + + Every method body is the pre-wave-18 module function, verbatim except globals→fields. + The behaviour contracts that matter are stated on each method and are UNCHANGED: + strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads. + """ + + def __init__(self, repo): + self.repo = str(repo) + self._lock = threading.RLock() + self._cache = {} + # --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------ + # One typed character in a Notes cell used to cost THREE synchronous network calls + # inside the component round-trip. update(..., flush='async') applies the mutation to + # the in-process cache synchronously (read-your-writes: every reader consults the + # cache first) and a background worker coalesces uploads. The restart-wipe protection + # is UNCHANGED where it matters: the FIRST update of a key in a fresh process still + # strict-reads. Known, accepted: with a second concurrent writer the stale window + # widens from ~one round-trip to ~the flush gap. + self._owned = {} # name -> True once cache reflects our own last accepted write + self._dirty = {} # name -> cache is ahead of the hub + self._flushers = {} # name -> live worker thread + self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance + self._last_flush = {} # name -> monotonic time of the last completed upload + # --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------ + # name -> how many accepted writes this process has applied to that bucket, and when. + # Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for + # why that constraint is the whole feature rather than an optimisation. + self._revs = {} + # --- ⭐⭐ W31-T13 (D-177(d)) — THE PER-KEY WRITE LOCK ------------------------------ + # `self._lock` guards the CACHE. It used to be held across a sync `update`'s whole + # read-modify-write — a hub download, the caller's `fn`, and a blocking hub upload — so + # one write froze every reader in the process for two network round trips. `get()` takes + # the same lock, and `/nav`, `/workspace` and every permission check are `get()`. + # ⛔ IT IS NOT SAFE TO SIMPLY DROP THE LOCK. Two sync writers of one key would interleave + # read-modify-write and one update would vanish — corruption, not slowness. So the two + # jobs are separated: this lock serialises WRITERS OF ONE KEY (held across the network, + # never touching readers), and `self._lock` is taken only for the microseconds a cache + # read or write needs. Readers never wait on a network call again. + # ⚠ PER KEY, not one write lock: two tenants' buckets, or `user_tables` and `nav_meta`, + # have no reason to serialise against each other. + self._key_locks = {} + self._key_locks_guard = threading.Lock() + self._rev_at = {} + + def available(self): + """True when a token is configured — callers degrade gracefully without one.""" + return bool(_token()) + + def _download(self, name): + """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent. + Any other failure (network, auth, rate-limit, corrupt JSON) RAISES — a transient read + error must never be mistaken for an empty store, or a caller could overwrite real data + with an empty dict (this is what wiped the user registry on restart).""" + try: + path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset', + token=_token(), force_download=True) + with open(path, encoding='utf-8') as f: + return json.load(f) + except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError): + return {} + except ValueError as e: + # Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in + # ValueError("Force download failed…") with the 404 as __cause__ — so a brand-new + # tenant repo's very first strict read raised here and aborted its very first write. + # A genuinely-absent repo is the same fact as a genuinely-absent file: {}. + cause = getattr(e, '__cause__', None) + if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)): + return {} + raise + + def get(self, name, fresh=False): + """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error, + hand back the last known-good cached value if we have one, else {}. Mutating callers go + through update()/_read_strict, which RAISE instead.""" + with self._lock: + if not fresh and name in self._cache: + return json.loads(json.dumps(self._cache[name])) # hand back a copy + try: + data = self._download(name) + except Exception as e: + try: # lazy import — core stays layer-clean + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:get:{name}', e, + fallback='cached' if name in self._cache else 'empty') + except Exception: + pass + if name in self._cache: + return json.loads(json.dumps(self._cache[name])) + return {} + self._cache[name] = data + return json.loads(json.dumps(data)) + + def get_projection(self, name, drop=()): + """Like `get`, but each top-level VALUE comes back WITHOUT the keys named in `drop`. + + ⭐⭐ W32-T01 (D-185) — THE POINT IS THE COPY, NOT THE PAYLOAD SIZE ON THE WIRE. + + `get()` is honest and expensive: it hands back `json.loads(json.dumps(...))` so a caller + cannot mutate the cache, and on tenant #0's `user_tables` that copy is **28.6 MB / 81,330 + rows / ~703 ms warm**, of which **99.9% is `rows`** — which no permission check, no nav + render and no workspace envelope ever reads. `/nav` used to pay ~13 of those copies per + request; W31-T10 made it ONE, and that one is still 703 ms of pure copying. This is the + floor D-185 named: copy only what was asked for. + + ⛔⛔ **THE PROJECTION IS NEVER CACHED, AND THAT LINE IS THE WHOLE SAFETY ARGUMENT.** A + rows-less document in `self._cache` would be served to the next WHOLE reader, and the next + `update()` would upload it — every row in the tenant deleted, by a read. So the cold path + downloads whole, caches WHOLE (exactly as `get` does), and projects the RETURN value only. + `verify_store_projection.py` asserts the cache still contains `rows` after a projected read, + with an NC that reds when the projection is cached instead. + + ⚠ **`self._lock` is held across the copy, same as `get`** — the retained structure is + ~0.1% of the bytes, so this is microseconds where `get` is most of a second. Narrowing it + would let a concurrent writer mutate a value mid-serialise for no measurable gain. + + ⚠ **A dropped key RAISES on read rather than reading as absent** (`_Projected`). A + projection that answered `{}` for `rows` would paint an empty grid — the exact + [[empty-answer-vs-unfinished-answer]] shape this repo has shipped before — and the caller + that most wants rows is the one least able to tell. See `_Projected`. + + ⚠ Lenient, like `get`: a transient fetch error falls back to the cached document (projected) + and then to `{}`. Mutating callers still go through `update()`/`_read_strict`, which never + project — a write must always read whole. + """ + drop = frozenset(str(d) for d in drop) + with self._lock: + if name in self._cache: + return _project(self._cache[name], drop) + try: + data = self._download(name) + except Exception as e: + try: # lazy import — core stays layer-clean + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:get_projection:{name}', e, + fallback='cached' if name in self._cache else 'empty') + except Exception: + pass + with self._lock: + if name in self._cache: + return _project(self._cache[name], drop) + return {} + with self._lock: + self._cache[name] = data # ⛔ the WHOLE document, never the projection + return _project(data, drop) + + def _key_lock(self, name): + """The write lock for ONE key (W31-T13). Created on demand; never removed.""" + n = str(name) + with self._key_locks_guard: + lk = self._key_locks.get(n) + if lk is None: + lk = self._key_locks[n] = threading.RLock() + return lk + + def _read_strict(self, name): + """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely + absent file), so a failed read ABORTS a read-modify-write instead of merging into {}. + + ⭐ W31-T13 — THE DOWNLOAD IS OUTSIDE `self._lock` NOW, and the re-check after it is the + whole of what makes that safe. A hub read takes a round trip; holding the cache lock + across it is what froze every reader. But moving it out opens a window in which an ASYNC + writer can accept a row into the cache while our download is in flight — and assigning our + (older) hub copy afterwards would erase it. That is D-177's original symptom exactly + (`POST /rows` → 201, row gone), re-created by the lock split rather than by a call site. + So the dirty mark is re-read AFTER the download, under the short lock, and a key that + became dirty keeps its cache. + """ + data = self._download(name) # network, no lock held + with self._lock: + if self._dirty.get(name) and self._owned.get(name) and name in self._cache: + # Our copy is stale by construction: a pending upload is going to overwrite the + # hub anyway, so the fresh read cannot preserve a remote write — it can only + # destroy a local one we have already acknowledged. Same argument, and the same + # three conditions, as `update`'s pending-async branch below. + return json.loads(json.dumps(self._cache[name])) + self._cache[name] = data + return json.loads(json.dumps(data)) + + def exists(self, name): + """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY + uncertainty — a seeder must never clobber a registry we merely failed to reach.""" + try: + files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset') + return f'{name}.json' in files + except RepositoryNotFoundError: + return False + except Exception: + return True + + # ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6) + def _bump(self, name): + """Advance one bucket's write counter. THE CALLER HOLDS THE LOCK. + + Called from the two places an accepted write becomes visible to readers of this process + — `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT + from `_flush_worker`: that uploads data a bump already accounted for, and counting it + again would report a change to every browser each time the coalescing worker woke. + """ + n = str(name) + self._revs[n] = self._revs.get(n, 0) + 1 + self._rev_at[n] = _time.time() + + def revision(self, name): + """`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else. + + ⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why + this method exists at all. A browser asking "has anything changed" by re-fetching rows + costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way + in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s + per bucket — so two tabs polling saturate the server and every unrelated write queues + behind them. This performs no download, no `get()`, and no copy of the value. + ⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace` + counts `get`/`_read_strict`/`_download` on the serving path and goes red on one). + + `rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend + ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops + being inert the moment D-4 flips, with the wire shape unchanged. + + ⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write + by a SECOND container against the same HF dataset is invisible here. That is not a limit + this introduces — `get()` caches a bucket for the life of the process and never re-reads + it, so the other container's write is invisible to every reader on this backend already. + The token is exactly as fresh as the store it reports on. Postgres has no such gap. + ⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted: + an attachment is fetched on explicit user action, never polled. + """ + n = str(name) + with self._lock: + rev = int(self._revs.get(n, 0)) + at = self._rev_at.get(n) + return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'} + + def _upload(self, name, data): + """The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra + API round-trip on every put(), pure latency after the first success.""" + api = HfApi(token=_token()) + if not self._repo_ok: + api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True) + self._repo_ok = True + buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8')) + api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo, + repo_type='dataset', commit_message=f'update {name}') + + def upload_bytes(self, path_in_repo, data, message=None): + """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents). + Bytes are never cached in-process: large, cold, fetched on explicit user action.""" + if not self.available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + with self._lock: + api = HfApi(token=_token()) + if not self._repo_ok: + api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True) + self._repo_ok = True + api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo, + repo_id=self.repo, repo_type='dataset', + commit_message=message or f'add {path_in_repo}') + + def download_bytes(self, path_in_repo): + """Raw bytes back, or None when absent/unreachable — a missing attachment degrades to + "that file is gone", never to a broken page.""" + if not self.available(): + return None + try: + local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset', + token=_token()) + return Path(local).read_bytes() + except (EntryNotFoundError, RepositoryNotFoundError): + return None + except Exception: + return None + + def delete_path(self, path_in_repo): + """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal.""" + if not self.available(): + return False + try: + HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo, + repo_type='dataset', + commit_message=f'delete {path_in_repo}') + return True + except (EntryNotFoundError, RepositoryNotFoundError): + return True + except Exception: + return False + + def put(self, name, data): + """Persist {name}.json (creates the private repo on first write). Updates the cache.""" + if not self.available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + with self._lock: + self._upload(name, data) + self._cache[name] = json.loads(json.dumps(data)) + self._owned[name] = True + self._last_flush[name] = _time.monotonic() + self._bump(name) # C6 — AFTER the upload: a raise here is not a write + return self._cache[name] + + def _flush_worker(self, name): + """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure — + the cache stays the truth and the dirty mark survives, so nothing is lost silently. + + ⭐⭐ W31-T13 — THIS WORKER IS THE SECOND WRITER, AND THE PER-KEY LOCK MUST COVER IT. + ⛔ THE RACE THE LOCK SPLIT OPENED, spelled out because it is silent and survives a restart + as real data loss. `update`'s big lock used to span `_read_strict` + `fn` + the upload, so + this worker could never snapshot mid-update. With the cache lock narrowed it can: + async write -> `_dirty=True`, this worker scheduled, sleeps `_FLUSH_DELAY` + sync update t+0.5 -> takes the dirty branch, runs `fn`, enters `_upload` (no cache + lock held — that is the whole point of the ticket) + worker t+2.0 -> `self._lock` is FREE and `_dirty` is still True (the sync commit + has not happened yet), so it snapshots the PRE-UPDATE cache, + clears the mark, and uploads it AFTER the sync upload lands + result -> cache has the new value, the hub has the old one, `_dirty` is + False, and nothing will ever re-upload. Divergence with no + symptom until the next process boots and reads the hub. + ⚠ `user_tables` has ~40 sync writers against 2 async ones, so that window is the COMMON + case, not an exotic one — it is D-177's own shape, re-created one layer down. + ⛔ LOCK ORDER IS KEY-THEN-CACHE, EVERYWHERE. `_update_locked` takes them in that order, so + this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay + OUTSIDE the key lock — a worker that dozed while holding it would block every writer of + that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove. + """ + backoff = 2.0 + while True: + _time.sleep(_FLUSH_DELAY) + wait, err, snapshot = None, None, None + with self._key_lock(name): + with self._lock: + gap = _time.monotonic() - self._last_flush.get(name, 0.0) + if gap < _FLUSH_MIN_GAP: + wait = _FLUSH_MIN_GAP - gap + elif not self._dirty.get(name): + self._flushers.pop(name, None) + return + else: + self._dirty[name] = False + snapshot = json.loads(json.dumps(self._cache.get(name, {}))) + if wait is None: + try: + self._upload(name, snapshot) + with self._lock: + self._last_flush[name] = _time.monotonic() + backoff = 2.0 + except Exception as e: # noqa: BLE001 + with self._lock: + self._dirty[name] = True # keep it queued; cache is still the truth + err = e + if wait is not None: + _time.sleep(wait) + continue + if err is not None: + try: + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:flush:{name}', err) + except Exception: + pass + _time.sleep(backoff) + backoff = min(backoff * 2, 60.0) + + def _schedule_flush(self, name): + """Mark dirty + ensure exactly one live worker for the key. Caller holds the lock.""" + self._dirty[name] = True + t = self._flushers.get(name) + if t is not None and t.is_alive(): + return + t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True, + name=f'store-flush-{self.repo}:{name}') + self._flushers[name] = t + t.start() + + def flush(self, name=None, timeout=30.0): + """Block until pending async writes for `name` (or ALL keys) reach the hub — QA gates + and shutdown hooks use this; the app itself never needs to.""" + deadline = _time.monotonic() + timeout + names = [name] if name else None + while _time.monotonic() < deadline: + with self._lock: + pending = {k for k, v in self._dirty.items() if v} | { + k for k, t in self._flushers.items() if t.is_alive()} + if names is not None: + pending &= set(names) + if not pending: + return True + _time.sleep(0.2) + return False + + def _flush_now_at_exit(self): + """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing + gap — a Space restart must not eat the last seconds of the async window. + + ⚠ W31-T13 — DELIBERATELY NOT KEY-LOCKED, stated so the omission is not read as one. This + runs at interpreter exit with the process going away: a writer mid-`update` will never + finish, so waiting for its key lock could only turn a partial save into no save at all. + Best-effort is the correct contract here, and it is the one case where taking the lock + would be worse than skipping it. + """ + with self._lock: + dirty = [k for k, v in self._dirty.items() if v] + snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty} + for k in dirty: + self._dirty[k] = False + for k, d in snaps.items(): + try: + self._upload(k, d) + except Exception: + pass # exiting anyway — nothing left to reschedule + + def update(self, name, fn, flush='sync'): + """Read-modify-write. STRICT first read (raises on a transient error rather than + starting from an empty dict), apply fn(data)->data, persist. + + flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the + table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and + returns immediately; a coalescing worker commits. The strict read still guards the + FIRST async update of a key per process. + + ⭐⭐ W31-T13 (D-177(d)) — TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET. + `self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key + still cannot interleave (last-write-wins is the contract; a lost update is not). + `self._lock` — the one every `get()` takes — is now held only for the microseconds a cache + read or write needs, never across a download or an upload. Before this, one sync write + froze every reader in the process for two network round trips, on a store whose warm + `get()` already costs 703 ms of deep copy on tenant #0. + ⚠ Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly + that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`. + """ + with self._key_lock(name): + return self._update_locked(name, fn, flush) + + def _update_locked(self, name, fn, flush): + """`update`'s body, with this key's write lock already held. See its docstring. + + ⚠ THE CACHE LOCK IS TAKEN THREE TIMES AND RELEASED BETWEEN THEM, which is only safe + because THIS key's write lock is held throughout: nothing else can be mid-write on it, so + the state read in the first section is still ours in the third. A caller reaching this + without the key lock would have the old interleaving bug back. + """ + # ⚠ A FLAG, NOT A `None` SENTINEL. `None` can be a real cached value, and a sentinel that + # can equal real data is not a sentinel — this repo has paid for that exact shape before + # (`routes_nav`'s locked-mode default). The flag says "the cache answered", full stop. + data, from_cache = None, False + with self._lock: + if flush == 'async' and self._owned.get(name) and name in self._cache: + data, from_cache = json.loads(json.dumps(self._cache[name])), True + elif self._dirty.get(name) and self._owned.get(name) and name in self._cache: + # ⛔ A SYNC READ-MODIFY-WRITE MUST NOT DISCARD A PENDING ASYNC WRITE. + # + # THE BUG THIS LINE EXISTS FOR (owner report 2026-08-09, MEASURED on staging): + # `POST /rows` answered `201 {"rid":"100"}` and the row never existed. + # `user_tables.add_row` commits `flush='async'` — the row lives ONLY in + # `self._cache` until the coalescing worker uploads it, which is `_FLUSH_DELAY` + # (2s) away and up to `_FLUSH_MIN_GAP` (20s) if that key committed recently. + # Inside that window ANY sync writer of the SAME key took the `else` branch + # below, and `_read_strict` does `self._cache[name] = self._download(name)` — + # it REPLACES the cache with the hub's copy, erasing the only place the new row + # existed, and then `put()` uploaded that. Nothing raised. The 201 was truthful + # about `add_row`'s return value and false about the outcome. + # + # ⚠ `user_tables` has ~40 sync writers against 2 async ones, so this was never + # one bad call site — it is a property of the KEY. Wave 27 only made it fire + # every time, by putting the relation refresh on the add path. + # + # ⛔ AND RE-DOWNLOADING WAS PROTECTING NOTHING. Once a key is dirty our cache + # has already diverged and the pending upload was going to overwrite the hub + # anyway — so the fresh read cannot preserve a remote write, it can only destroy + # a local one we already acknowledged. This is the class's OWN documented + # contract, restored: see `_owned` above — *"read-your-writes: every reader + # consults the cache first"* — and `_flush_worker`'s *"the cache stays the truth + # and the dirty mark survives, so nothing is lost silently."* + # + # ⚠ NOT a weakening of `_read_strict`'s purpose. That exists so a FAILED read + # aborts the RMW instead of merging into `{}`; the cache is populated + # known-good local state, and this branch is only reachable when it is. + data, from_cache = json.loads(json.dumps(self._cache[name])), True + # else: the strict hub read happens BELOW, outside this lock (W31-T13) + + # ⭐ W31-T13 — THE HUB READ AND THE CALLER'S `fn` RUN WITH NO CACHE LOCK HELD. `fn` is + # arbitrary caller code — `automation_engine`'s relation pass has run whole-tenant work + # inside one — and it used to execute with every reader in the process blocked behind it. + if not from_cache: + data = self._read_strict(name) # network; re-checks the dirty mark on the way back + result = fn(data) + data = result if result is not None else data + + if flush == 'async': + if not self.available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + with self._lock: + self._cache[name] = json.loads(json.dumps(data)) + self._owned[name] = True + self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time + self._schedule_flush(name) + return data + + # ⛔ THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE. + # `put()` holds `self._lock` across its own `_upload` — which is exactly the freeze this + # ticket removes — but it is PUBLIC with seven callers outside this file (`users.py`, + # `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`, + # `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency + # semantics at once. Duplicating four assignments with this comment beside them is the + # smaller blast radius; if `put` is ever split, this collapses back into a call. + if not self.available(): + raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + self._upload(name, data) # network, no cache lock held + with self._lock: + self._cache[name] = json.loads(json.dumps(data)) + self._owned[name] = True + self._last_flush[name] = _time.monotonic() + # ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_upload` raises on failure, + # and a bump on that path would have `revision()` report a change that never landed. + self._bump(name) + # The upload above CARRIED whatever was pending, so the key is no longer ahead of + # the hub. Without this the flush worker wakes on a still-dirty mark and re-uploads + # identical content — a second commit against the 256-commits/hr budget for nothing. + # ⚠ AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache + # is still the truth and the row must still be committed by the worker). Clearing it + # first would turn a transient upload error into the very silent loss the branch above + # exists to stop — which is why it sits here and not before `_upload`. + self._dirty[name] = False + return data + + +# ------------------------------------------------------------------- instances + module API +_INSTANCES = {} +_INSTANCES_LOCK = threading.RLock() + + +def for_repo(repo_id): + """The HF Store bound to `repo_id` — one instance per repo per process (each with its own + cache and flush state). + + ⚠ WAVE 20: this is now the HF-SPECIFIC factory. Callers that want "the right store for this + tenant, whatever the backend is" call `handle()` below. `for_repo` keeps its exact old + behaviour because the seed/migration tooling has to be able to name the FILE store + explicitly while the app runs on Postgres — a migration that could only reach the active + backend could not copy between them. + """ + rid = str(repo_id or '').strip() or REPO + with _INSTANCES_LOCK: + inst = _INSTANCES.get(rid) + if inst is None: + inst = Store(rid) + _INSTANCES[rid] = inst + return inst + + +# ============================================================================================= +# WAVE 20 (owner ruling R1, closes DEBT D-4) — THE BACKEND SEAM. +# +# `core/store_backend.py` has carried this warning since EXIT-2b, and it was correct at the time: +# +# "Flipping STORE_BACKEND=pg does NOT redirect the ~40 existing callers that say +# `import core.store as store` — they are bound to the HF module directly. Rewiring them is +# task C-4 … THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise." +# +# **R1 IS C-4.** The owner ruled the cutover on 2026-08-05 (the D-4 trigger that fired: a SECOND +# server process — `royal-imports/cfo-os` came back as a pinned LIVE environment beside staging, +# and two containers on one last-write-wins file store is the race B-3 was always about). +# +# THE REWIRE IS HERE RATHER THAN IN 28 FILES, and that is a deliberate choice over the obvious +# alternative of `sed`-ing every `import core.store as store` to `import core.store_backend`: +# * every one of those callers means "the store for the tenant I am serving", which is exactly +# what this module has always meant. The BACKEND is not their concern and making it their +# concern is how one of them gets missed; +# * a missed caller under a search-and-replace does not fail — it silently keeps writing to the +# file store while everything else writes to Postgres. That is the split-brain this whole +# ruling exists to end, reintroduced by the fix for it; +# * `store_backend.py`'s stated reason for refusing to do it this way — "no dual-read window and +# no way to compare the two stores' contents first" — is satisfied: `ops/seed_pg_from_hf.py` +# copies, then `--verify` diffs the two stores key-by-key before anything flips. +# +# ⛔ THE IMPORT IS LAZY AND MUST STAY LAZY. `core/store_pg.py` imports psycopg only inside +# `_pool()`, so an `hf` deployment installs no driver; importing it at module scope here would +# undo that and make the default backend depend on the optional dependency. +# ============================================================================================= + +def backend(): + """`'hf'` | `'pg'` — validated, resolved per call so a test can flip the env var. + + Mirrors `core.store_backend.name()` deliberately rather than importing it: that module + imports THIS one, so reaching back would be a cycle. `verify_store_pg` asserts the two agree + on every value, which is the guard against them drifting apart. + """ + raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower() + if raw not in ('hf', 'pg'): + raise RuntimeError( + f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store) or 'pg' " + f"(Postgres, needs DATABASE_URL). Refusing to guess: a typo that silently served the " + f"other store is how data ends up in two places.") + return raw + + +def handle(repo_id=None, slug=None): + """THE tenant-bound store handle for the ACTIVE backend — the one seam every caller crosses. + + `repo_id` addresses the HF backend (a dataset repo); `slug` addresses Postgres (a schema). + Both are passed by `harness.runtime.get_runtime`, which knows both facts, so flipping the + backend never needs a lookup table between them — and neither identifier has to be invented + for the backend that does not use it. + """ + if backend() == 'pg': + import core.store_pg as _pg # noqa: PLC0415 — lazy: see the banner above + return _pg.PgStore(slug or os.environ.get('AIOS_TENANT') or 'royal-imports') + return for_repo(repo_id) + + +#: The HF default instance. Kept as a module global (not a property) because the seed/migration +#: tooling and `_flush_all_at_exit` both need the FILE store by name even when pg is active. +_DEFAULT = for_repo(REPO) + + +def _d(): + """The default tenant's store on the active backend — what every module function below uses. + + ⚠ Resolved PER CALL, never cached. A cached default would freeze the backend at import time, + and import order is exactly what nobody controls: `api/main.py` imports half the platform + before it has read a single environment variable it did not inherit. + """ + return handle() if backend() == 'pg' else _DEFAULT + + +def available(): + return _d().available() + + +def get(name, fresh=False): + return _d().get(name, fresh=fresh) + + +def _read_strict(name): + return _d()._read_strict(name) + + +def exists(name): + return _d().exists(name) + + +def upload_bytes(path_in_repo, data, message=None): + return _d().upload_bytes(path_in_repo, data, message=message) + + +def download_bytes(path_in_repo): + return _d().download_bytes(path_in_repo) + + +def delete_path(path_in_repo): + return _d().delete_path(path_in_repo) + + +def put(name, data): + return _d().put(name, data) + + +def flush(name=None, timeout=30.0): + return _d().flush(name=name, timeout=timeout) + + +def update(name, fn, flush='sync'): + return _d().update(name, fn, flush=flush) + + +def get_projection(name, drop=()): + """A projected read — see `Store.get_projection`. Wave 32, W32-T01 (D-185). + + ⛔ AN ELEVENTH STORE OPERATION, AND IT HAD TO BECOME ONE — the method alone was NOT enough, + which is the opposite of what it looks like from `revision()`'s precedent. `revision` could + ship method-only for a few hours because its caller held a Store. This one's caller is + `harness.runtime.TenantRuntime`, and `TenantRuntime._store()` returns **the MODULE** whenever + `store_handle is None` — which is precisely tenant #0, the tenant whose 703 ms `/nav` copy this + exists to remove. A method-only projection would have been unreachable from the only place that + needed it, while every gate stayed green. + + ⛔ SO `verify_store_pg.IFACE` GAINS IT IN THE SAME CHANGE, and `core/store_pg.py` gains the + twin below it. That gate's `extra:` check would red on this function otherwise — correctly: + this addresses a key and reads it, so it is a store operation and both backends must answer. + `backend`/`handle` are exempt because they SELECT a store; this one reads one. + """ + return _d().get_projection(name, drop=drop) + + +def revision(name): + """The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing. + + ⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS + HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely + so a capability one backend lacks cannot be reached through the shared door and die at the + cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change + and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half + had no owner: a method is a capability the HF store has, a module function is a promise both + backends keep. + """ + return _d().revision(name) + + +def _flush_all_at_exit(): + with _INSTANCES_LOCK: + instances = list(_INSTANCES.values()) + for inst in instances: + inst._flush_now_at_exit() + + +import atexit # noqa: E402 (registered after the class it needs) +atexit.register(_flush_all_at_exit) diff --git a/platform/core/table_store.py b/platform/core/table_store.py index 78f64bf597fcce4ae5f9d9b9e3225a443fc29881..e33559f04b5799c0b4945f5b7e1b2b31bf51cc91 100644 --- a/platform/core/table_store.py +++ b/platform/core/table_store.py @@ -1,618 +1,618 @@ -"""The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory. - -One durable store key holds one table OBJECT's per-user Airtable-style state: - - {username: {'views': {view_id: SavedView}, - 'fields': {field_key: Field}, # notes + custom_ + measure_ strata - 'overlays': {str(pid): {field_key: value}}}} - -`make(table_key)` returns the six operations a table page's host loop needs, closed over that -key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are -these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table -pattern to a new object is a registry row + a config, not a copy of the store plumbing -(owner directive: the table-page factory). - -A LIST (membership/formula semantics) is deliberately a different store from a VIEW -(presentation/query state) — see modules/customer_data.py's customer_lists key. -""" -import core.store as store - -#: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but -#: their creator live here instead of in a personal workspace, under a key that cannot collide -#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared` -#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from -#: here, and a view shared is removed from its creator's workspace. Two homes would mean two -#: divergent copies the moment either was edited. -SHARED_KEY = '__shared__' - - -def _may_see(view, viewer, is_admin=False): - """Visibility for ONE shared view, fail-closed. - - 'collaborative' = everyone who can already open the module (the caller has gated that). - 'users' = the named users, plus the creator, plus admins — an admin who could not - see a view could not administer it either. - Anything unrecognised returns False rather than defaulting open: an unreadable permission - must never widen access ([[aios-permissioning]] — no fail-open defaults). - """ - if not isinstance(view, dict): - return False - if view.get('createdBy') == viewer or is_admin: - return True - perms = view.get('permissions') or {} - edit = perms.get('edit') - if edit == 'collaborative': - return True - if edit == 'users': - return viewer in set(perms.get('users') or ()) - return False # 'personal', absent, or junk - - -def _may_edit(view, viewer, is_admin=False): - """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who - can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a - separate function so they can diverge (a future read-only share) without hunting callers.""" - return _may_see(view, viewer, is_admin) - - -def _may_administer(view, viewer, is_admin=False): - """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY. - - Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they - could grant themselves sole ownership of somebody else's view, or quietly widen a - users-scoped view to everyone — the classic privilege-escalation-by-edit hole. - """ - if not isinstance(view, dict): - return False - return bool(is_admin) or view.get('createdBy') == viewer - - -def is_shared(view): - """A view belongs in the shared bucket when its permissions reach beyond its creator.""" - return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users') - - -def source_override_is_empty(payload): - """Does this stored definition of a SOURCE (non-custom) column carry any user state? - - A cleared note on an immutable source field returns to the canonical schema instead of - leaving a meaningless override row. Custom fields remain even with an empty note — and so - does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format - override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY. - - ⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently - stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days` - built an override whose only content was that summary, so this rule threw the whole row away - on write while the menu went on reading "Summary: Average" from the client's own optimistic - copy until the next login — a discarded WRITE wearing the face of a failed read - ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores - its definitions through another door entirely, kept it. - - ⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a - discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]). - ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the - column goes back to whatever the contract declares for it. - """ - payload = payload or {} - if payload.get('custom'): - return False - return (not str(payload.get('note') or '').strip() - and not isinstance(payload.get('measure'), dict) - and not isinstance(payload.get('format'), dict) - and not str(payload.get('agg') or '').strip()) - - -def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120): - """Allocate one human-facing name inside a store.update transaction. - - Keys/ids remain structural identity. Names compare case-insensitively after collapsing - whitespace, because those variants are indistinguishable in the UI. This helper belongs - in the store layer: allocating from a pre-write snapshot lets two concurrent requests both - choose the same free name before either write lands. - """ - limit = max(1, int(max_len)) - - def _clean(value): - return ' '.join(str(value or '').split()) - - base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip() - taken = {_clean(value).casefold() for value in existing if _clean(value)} - if base.casefold() not in taken: - return base - index = 2 - while True: - suffix = f' {index}' - stem = base[:max(0, limit - len(suffix))].rstrip() - candidate = f'{stem}{suffix}' if stem else str(index)[-limit:] - if candidate.casefold() not in taken: - return candidate - index += 1 - - -class TableStore: - """The six store operations for one table object's workspace, closed over its store key. - - `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` / - `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller, - zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors - apply the tenant prefix / repo binding — which is what makes a user table created by a - Nurilab admin land in Nurilab's store instead of Royal's. - """ - - def __init__(self, table_key, st=None): - self.table_key = table_key - self._st = st if st is not None else store - - @property - def st(self): - """The bound store handle — for SIBLING registries (core/shares) that must read the - same tenant's buckets this workspace lives in (wave 21, C1).""" - return self._st - - def find_view(self, view_id): - """`(owner_username, view)` for a view living in ANY personal stratum, else None. - - ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted - view means locating the OWNER's record inside this topic's bucket. Personal strata - only — the `__shared__` bucket has its own read path (`shared_views`), and serving one - view from two finders is how two copies drift.""" - vid = str(view_id or '').strip() - if not vid: - return None - try: - data = self._st.get(self.table_key) or {} - except Exception: - return None - for username, ws in data.items(): - if username == SHARED_KEY or not isinstance(ws, dict): - continue - v = (ws.get('views') or {}).get(vid) - if isinstance(v, dict): - return str(username), dict(v) - return None - - def find_folder(self, folder_id): - """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any - personal stratum, else None. `find_view`'s sibling, and here for the same reason. - - ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind - `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share - this folder with Karen" recorded a row, listed under Shared with me, and put nothing on - Karen's screen. Projecting a folder means two lookups the view path does not need: WHO - owns it, and WHICH views are filed in it. Folder membership lives in the owner's - `itemFolders` map (item id -> folder id), never on the view record, so the views are - found by asking that map rather than by reading a list off the folder. - - Views only (`folders['views']`): the cohort surface has its own store and its own - sharing question, and answering both here would make one function mean two things. - """ - fid = str(folder_id or '').strip() - if not fid: - return None - try: - data = self._st.get(self.table_key) or {} - except Exception: - return None - for username, ws in data.items(): - if username == SHARED_KEY or not isinstance(ws, dict): - continue - rows = (ws.get('folders') or {}).get('views') or [] - hit = next((f for f in rows - if isinstance(f, dict) and str(f.get('id') or '') == fid), None) - if not hit: - continue - # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) — - # reading item ids off the top level finds the surface names instead and matches - # nothing, so the projection silently returns an EMPTY folder and the feature looks - # exactly as broken as it was before the fix. Caught by this change's own gate, - # which is the entire argument for writing one. - placed = (ws.get('itemFolders') or {}).get('views') or {} - views = ws.get('views') or {} - inside = {str(vid): dict(v) for vid, v in views.items() - if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid} - return str(username), dict(hit), inside - return None - - # ---------------------------------------------------------------- read - def workspace(self, username, consume_corrections=True): - """One user's durable workspace: always the full three-strata shape.""" - try: - data = self._st.get(self.table_key) or {} - ws = data.get(username, {}) or {} - # A collision acknowledgement is protocol state, not part of a field definition. - # Consume it with the first fresh workspace payload after the correcting write, - # then splice a bounded copy into that payload only. Keeping it out of `fields` - # prevents an old request id surviving forever and overriding a later rename. - corrections = {} - if consume_corrections and ws.get('fieldCorrections'): - def _take(current): - current_ws = current.get(username) or {} - pending = current_ws.get('fieldCorrections') or {} - corrections.update({ - str(key)[:80]: dict(value) - for key, value in pending.items() - if isinstance(value, dict) - }) - current_ws.pop('fieldCorrections', None) - return current - - data = self._st.update(self.table_key, _take, flush='async') - ws = (data or {}).get(username, {}) or {} - except Exception: - ws = {} - corrections = {} - fields = { - key: dict(value) if isinstance(value, dict) else value - for key, value in (ws.get('fields') or {}).items() - } - for key, ack in corrections.items(): - field = fields.get(key) - accepted_label = str(ack.get('label') or '')[:120] - requested_label = str(ack.get('labelCorrectedFrom') or '')[:120] - correction_id = str(ack.get('labelCorrectionId') or '')[:180] - # A newer field write clears/replaces the pending ack in the SAME transaction. - # The label check is an extra belt against ever attaching a stale ack to a newer - # definition if a future store implementation weakens that ordering. - if (isinstance(field, dict) and accepted_label - and str(field.get('label') or '') == accepted_label - and requested_label and correction_id): - field['labelCorrectedFrom'] = requested_label - field['labelCorrectionId'] = correction_id - out = { - 'views': dict(ws.get('views') or {}), - 'fields': fields, - 'overlays': dict(ws.get('overlays') or {}), - # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH - # stratum rather than a key on each item — see aios_grid.clean_folders for why - # (a cohort lives in another store, and filing is an organising act, not part of - # what a view is). Absent for every workspace saved before this wave, which is - # exactly "no folders yet". - 'folders': dict(ws.get('folders') or {}), - 'itemFolders': dict(ws.get('itemFolders') or {}), - } - # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The - # client's localStorage copy wins when present; this is the server's answer for a - # fresh profile, which used to fall all the way to the system default view. - if ws.get('activeViewId'): - out['activeViewId'] = str(ws['activeViewId']) - # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth - # stratum, absent until the user first reorders — exactly "default order". - if isinstance(ws.get('recordLayout'), dict): - out['recordLayout'] = dict(ws['recordLayout']) - return out - - # ---------------------------------------------------------------- write - def _update(self, username, change): - def _up(data): - ws = data.setdefault(username, {}) - ws.setdefault('views', {}) - ws.setdefault('fields', {}) - ws.setdefault('overlays', {}) - ws.setdefault('folders', {}) - ws.setdefault('itemFolders', {}) - change(ws) - return data - # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak, - # column note and typed overlay cell lands here inside the component round-trip, and - # the historical synchronous hub commit cost seconds per edit. The mutation applies to - # the in-process cache (read-your-writes for every subsequent render); the hub write - # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'. - return self._st.update(self.table_key, _up, flush='async') - - def rename_choice_values(self, username, change): - """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME). - - ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A - public "do anything to the workspace" method is an invitation to put write logic in - callers instead of here, and every OTHER method on this class exists precisely because - that logic belongs in one place. Renaming a choice is the one operation that must touch - three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that - filter or colour by the old value — inside a SINGLE transaction, because a rename that - updated the cells and not the filters would leave a saved view matching nothing. - - `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`). - """ - return self._update(username, change) - - def save_active_view(self, username, view_id): - """Remember which view this user last opened (owner item 3, 2026-07-31). - - Presentation state, not authorisation: the READ side re-validates the id against what - the caller may actually see, so a stale or foreign id degrades to the default view - rather than granting anything. Stored per user like every other stratum. - """ - vid = str(view_id or '').strip()[:120] - if not vid or username == SHARED_KEY: - return - - def _set(ws): - ws['activeViewId'] = vid - self._update(username, _set) - - def save_record_layout(self, username, order): - """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT). - - Presentation state for ONE surface — the record modal. Deliberately not view config: - the owner's ask is per-user, not per-view, and it must never reorder grid columns. - The event handler validated keys against the live field set; the wire re-validates at - serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here. - An empty order clears the stratum back to "default order". - """ - if username == SHARED_KEY: - return - clean, seen = [], set() - for key in (order or [])[:200]: - key = str(key or '').strip()[:80] - if key and key not in seen: - seen.add(key) - clean.append(key) - - def _set(ws): - if clean: - ws['recordLayout'] = {'order': clean} - else: - ws.pop('recordLayout', None) - - self._update(username, _set) - - def save_folders(self, username, folders, item_folders): - """Replace the folder stratum wholesale (wave-8 I11). - - Wholesale rather than per-folder because the caller has ALREADY validated the complete - picture through aios_grid.clean_folders / clean_item_folders, and those two are - interdependent: a placement is only legal while its folder exists, so committing them - separately would leave a window where a reader sees an item filed into a folder that is - not there yet. One write, one consistent state. - """ - def _set(ws): - ws['folders'] = dict(folders or {}) - ws['itemFolders'] = dict(item_folders or {}) - self._update(username, _set) - - def save_view_order(self, username, order): - """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail. - - Wholesale, like `save_folders` above and for the same reason: the client sends the full - list it is looking at, not a delta, because a partial order cannot say where an UNNAMED - view went. - - ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves. - `aios_grid`'s own folder note argues it out for placements and every word applies: an - arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out - of the view config means duplicating, sharing or exporting a view does not drag one - person's rail position along with it. It also means a SHARED view can sit in a different - place for each person who can see it, which is the only coherent answer once two people - share one view. - - An empty list CLEARS the arrangement (back to server order) rather than storing `[]`. - """ - def _set(ws): - clean = [] - seen = set() - for vid in (order or []): - vid = str(vid).strip()[:120] - if vid and vid not in seen: - seen.add(vid) - clean.append(vid) - if clean: - ws['viewOrder'] = clean - else: - ws.pop('viewOrder', None) - self._update(username, _set) - - def shared_views(self, viewer, is_admin=False): - """Every SHARED view this viewer may see, by id (wave-9 I17). - - Read-only and independent of the viewer's own workspace: the caller merges. Returns - only what `_may_see` allows, so a caller cannot accidentally render somebody else's - personal view by forgetting to filter. - """ - try: - bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {} - except Exception: - return {} - return {vid: dict(v) for vid, v in bucket.items() - if _may_see(v, viewer, is_admin)} - - def shared_view(self, view_id): - """One shared view RAW — no visibility filter. For authorisation decisions only: a - caller must know a view exists and who owns it before it can decide whether the actor - may touch it. Never hand the result to a renderer without checking `_may_see`.""" - try: - return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {} - ).get('views', {}).get(str(view_id)) - except Exception: - return None - - def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False): - """Upsert a SavedView into its ONE home — personal workspace or the shared bucket. - - `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home - it lands in, the view is REMOVED from the other, so a view can never exist as two - copies that diverge on the next edit. - - ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer - moves data and does not know who is asking. `_cl_handle_one` is the wall. - """ - view_id = str((view or {}).get('id') or '').strip() - if not view_id: - raise ValueError('view id is required') - if username == SHARED_KEY: - raise ValueError('reserved username') - to_shared = is_shared(view) if shared is None else bool(shared) - requested = dict(view) - accepted = {} - - def _up(data): - # View names are tenant-global: every personal workspace plus the shared bucket. - # This deliberately includes views the actor cannot see. The only disclosed fact - # is that a display name is already taken, while the categorical "no duplicate - # view names" contract remains true when a personal view is later shared. - names = list(reserved_names or ()) - for workspace in data.values(): - if not isinstance(workspace, dict): - continue - names.extend( - value.get('name') - for candidate_id, value in (workspace.get('views') or {}).items() - if candidate_id != view_id and isinstance(value, dict) - ) - payload = dict(requested) - payload['name'] = _unique_name(payload.get('name'), names) - accepted.clear() - accepted.update(payload) - if to_shared: - bucket = data.setdefault(SHARED_KEY, {}) - bucket.setdefault('views', {})[view_id] = payload - # it may have lived in the creator's workspace before being shared - owner = data.get(payload.get('createdBy') or username) or {} - (owner.get('views') or {}).pop(view_id, None) - else: - ws = data.setdefault(username, {}) - ws.setdefault('views', {})[view_id] = payload - (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None) - return data - - self._st.update(self.table_key, _up, flush='async') - return dict(accepted) - - def delete_view(self, username, view_id): - """Delete a custom/list view override. The system all-rows view is guarded by caller. - - Removes from BOTH homes: the caller has already authorised the delete, and leaving a - stale copy in the other bucket would resurrect the view on the next read. - """ - vid = str(view_id) - - def _up(data): - (data.get(username, {}).get('views') or {}).pop(vid, None) - (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None) - return data - - self._st.update(self.table_key, _up, flush='async') - - def save_field(self, username, field, reserved_names=(), correction_id=None): - """Persist a column note or a user-created (custom_/measure_) field definition.""" - key = str((field or {}).get('key') or '').strip() - if not key: - raise ValueError('field key is required') - requested = dict(field) - accepted = {} - - def _save(ws): - names = list(reserved_names or ()) - names.extend( - value.get('label') - for candidate_key, value in (ws.get('fields') or {}).items() - if candidate_key != key and isinstance(value, dict) - ) - payload = dict(requested) - payload.pop('labelCorrectedFrom', None) - payload.pop('labelCorrectionId', None) - requested_label = ' '.join( - str(payload.get('label') or 'Untitled').split())[:120].rstrip() - payload['label'] = _unique_name(requested_label, names) - corrections = ws.setdefault('fieldCorrections', {}) - corrections.pop(key, None) - if payload['label'] != requested_label and correction_id: - corrections[key] = { - 'label': payload['label'], - 'labelCorrectedFrom': requested_label, - 'labelCorrectionId': str(correction_id)[:180], - } - if not corrections: - ws.pop('fieldCorrections', None) - accepted.clear() - accepted.update(payload) - if source_override_is_empty(payload): - ws['fields'].pop(key, None) - else: - ws['fields'][key] = payload - - self._update(username, _save) - return dict(accepted) - - def duplicate_field(self, username, source_key, new_key, field, - reserved_names=(), correction_id=None): - """Clone a user-created field in ONE store transaction (wave-5 item 1): the new - definition plus — for `custom_` overlay sources only — every stored cell value under - the source key. One transaction, because a def without its values (or values without a - def) is exactly the orphan state delete_field exists to prevent, in reverse. - The caller validated both keys (same created stratum) and stamped the clone's - createdBy; this layer only moves data.""" - source_key = str(source_key or '').strip() - new_key = str(new_key or '').strip() - if not source_key or not new_key or source_key == new_key: - raise ValueError('duplicate_field needs two distinct keys') - requested = dict(field) - accepted = {} - - def _dup(ws): - names = list(reserved_names or ()) - names.extend( - value.get('label') - for candidate_key, value in (ws.get('fields') or {}).items() - if candidate_key != new_key and isinstance(value, dict) - ) - payload = dict(requested) - payload.pop('labelCorrectedFrom', None) - payload.pop('labelCorrectionId', None) - requested_label = ' '.join( - str(payload.get('label') or 'Untitled').split())[:120].rstrip() - payload['label'] = _unique_name(requested_label, names) - corrections = ws.setdefault('fieldCorrections', {}) - corrections.pop(new_key, None) - if payload['label'] != requested_label and correction_id: - corrections[new_key] = { - 'label': payload['label'], - 'labelCorrectedFrom': requested_label, - 'labelCorrectionId': str(correction_id)[:180], - } - if not corrections: - ws.pop('fieldCorrections', None) - accepted.clear() - accepted.update(payload) - ws['fields'][new_key] = payload - if source_key.startswith('custom_'): - for row in ws['overlays'].values(): - if isinstance(row, dict) and source_key in row: - row[new_key] = row[source_key] - - self._update(username, _dup) - return dict(accepted) - - def delete_field(self, username, key): - """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27). - - Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula - columns — the caller enforces the prefix). The stored overlay VALUES for the key are - scrubbed with it: a deleted column's cells must not linger as orphan data that would - silently resurface if the key were ever reused. Views referencing the key self-heal on - their next autosave (an unknown colId is dropped) — the rule every stale key rides. - """ - key = str(key or '').strip() - if not key: - return - - def _drop(ws): - ws['fields'].pop(key, None) - for row in ws['overlays'].values(): - if isinstance(row, dict): - row.pop(key, None) - - self._update(username, _drop) - - def patch_overlay(self, username, pid, updates): - """Patch only the external editable stratum; never writes to the source system.""" - clean = dict(updates or {}) - if not clean: - return - - def _patch(ws): - ws['overlays'].setdefault(str(int(pid)), {}).update(clean) - - self._update(username, _patch) - - -def make(table_key, st=None): - return TableStore(table_key, st=st) +"""The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory. + +One durable store key holds one table OBJECT's per-user Airtable-style state: + + {username: {'views': {view_id: SavedView}, + 'fields': {field_key: Field}, # notes + custom_ + measure_ strata + 'overlays': {str(pid): {field_key: value}}}} + +`make(table_key)` returns the six operations a table page's host loop needs, closed over that +key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are +these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table +pattern to a new object is a registry row + a config, not a copy of the store plumbing +(owner directive: the table-page factory). + +A LIST (membership/formula semantics) is deliberately a different store from a VIEW +(presentation/query state) — see modules/customer_data.py's customer_lists key. +""" +import core.store as store + +#: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but +#: their creator live here instead of in a personal workspace, under a key that cannot collide +#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared` +#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from +#: here, and a view shared is removed from its creator's workspace. Two homes would mean two +#: divergent copies the moment either was edited. +SHARED_KEY = '__shared__' + + +def _may_see(view, viewer, is_admin=False): + """Visibility for ONE shared view, fail-closed. + + 'collaborative' = everyone who can already open the module (the caller has gated that). + 'users' = the named users, plus the creator, plus admins — an admin who could not + see a view could not administer it either. + Anything unrecognised returns False rather than defaulting open: an unreadable permission + must never widen access ([[aios-permissioning]] — no fail-open defaults). + """ + if not isinstance(view, dict): + return False + if view.get('createdBy') == viewer or is_admin: + return True + perms = view.get('permissions') or {} + edit = perms.get('edit') + if edit == 'collaborative': + return True + if edit == 'users': + return viewer in set(perms.get('users') or ()) + return False # 'personal', absent, or junk + + +def _may_edit(view, viewer, is_admin=False): + """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who + can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a + separate function so they can diverge (a future read-only share) without hunting callers.""" + return _may_see(view, viewer, is_admin) + + +def _may_administer(view, viewer, is_admin=False): + """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY. + + Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they + could grant themselves sole ownership of somebody else's view, or quietly widen a + users-scoped view to everyone — the classic privilege-escalation-by-edit hole. + """ + if not isinstance(view, dict): + return False + return bool(is_admin) or view.get('createdBy') == viewer + + +def is_shared(view): + """A view belongs in the shared bucket when its permissions reach beyond its creator.""" + return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users') + + +def source_override_is_empty(payload): + """Does this stored definition of a SOURCE (non-custom) column carry any user state? + + A cleared note on an immutable source field returns to the canonical schema instead of + leaving a meaningless override row. Custom fields remain even with an empty note — and so + does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format + override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY. + + ⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently + stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days` + built an override whose only content was that summary, so this rule threw the whole row away + on write while the menu went on reading "Summary: Average" from the client's own optimistic + copy until the next login — a discarded WRITE wearing the face of a failed read + ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores + its definitions through another door entirely, kept it. + + ⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a + discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]). + ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the + column goes back to whatever the contract declares for it. + """ + payload = payload or {} + if payload.get('custom'): + return False + return (not str(payload.get('note') or '').strip() + and not isinstance(payload.get('measure'), dict) + and not isinstance(payload.get('format'), dict) + and not str(payload.get('agg') or '').strip()) + + +def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120): + """Allocate one human-facing name inside a store.update transaction. + + Keys/ids remain structural identity. Names compare case-insensitively after collapsing + whitespace, because those variants are indistinguishable in the UI. This helper belongs + in the store layer: allocating from a pre-write snapshot lets two concurrent requests both + choose the same free name before either write lands. + """ + limit = max(1, int(max_len)) + + def _clean(value): + return ' '.join(str(value or '').split()) + + base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip() + taken = {_clean(value).casefold() for value in existing if _clean(value)} + if base.casefold() not in taken: + return base + index = 2 + while True: + suffix = f' {index}' + stem = base[:max(0, limit - len(suffix))].rstrip() + candidate = f'{stem}{suffix}' if stem else str(index)[-limit:] + if candidate.casefold() not in taken: + return candidate + index += 1 + + +class TableStore: + """The six store operations for one table object's workspace, closed over its store key. + + `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` / + `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller, + zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors + apply the tenant prefix / repo binding — which is what makes a user table created by a + Nurilab admin land in Nurilab's store instead of Royal's. + """ + + def __init__(self, table_key, st=None): + self.table_key = table_key + self._st = st if st is not None else store + + @property + def st(self): + """The bound store handle — for SIBLING registries (core/shares) that must read the + same tenant's buckets this workspace lives in (wave 21, C1).""" + return self._st + + def find_view(self, view_id): + """`(owner_username, view)` for a view living in ANY personal stratum, else None. + + ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted + view means locating the OWNER's record inside this topic's bucket. Personal strata + only — the `__shared__` bucket has its own read path (`shared_views`), and serving one + view from two finders is how two copies drift.""" + vid = str(view_id or '').strip() + if not vid: + return None + try: + data = self._st.get(self.table_key) or {} + except Exception: + return None + for username, ws in data.items(): + if username == SHARED_KEY or not isinstance(ws, dict): + continue + v = (ws.get('views') or {}).get(vid) + if isinstance(v, dict): + return str(username), dict(v) + return None + + def find_folder(self, folder_id): + """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any + personal stratum, else None. `find_view`'s sibling, and here for the same reason. + + ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind + `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share + this folder with Karen" recorded a row, listed under Shared with me, and put nothing on + Karen's screen. Projecting a folder means two lookups the view path does not need: WHO + owns it, and WHICH views are filed in it. Folder membership lives in the owner's + `itemFolders` map (item id -> folder id), never on the view record, so the views are + found by asking that map rather than by reading a list off the folder. + + Views only (`folders['views']`): the cohort surface has its own store and its own + sharing question, and answering both here would make one function mean two things. + """ + fid = str(folder_id or '').strip() + if not fid: + return None + try: + data = self._st.get(self.table_key) or {} + except Exception: + return None + for username, ws in data.items(): + if username == SHARED_KEY or not isinstance(ws, dict): + continue + rows = (ws.get('folders') or {}).get('views') or [] + hit = next((f for f in rows + if isinstance(f, dict) and str(f.get('id') or '') == fid), None) + if not hit: + continue + # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) — + # reading item ids off the top level finds the surface names instead and matches + # nothing, so the projection silently returns an EMPTY folder and the feature looks + # exactly as broken as it was before the fix. Caught by this change's own gate, + # which is the entire argument for writing one. + placed = (ws.get('itemFolders') or {}).get('views') or {} + views = ws.get('views') or {} + inside = {str(vid): dict(v) for vid, v in views.items() + if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid} + return str(username), dict(hit), inside + return None + + # ---------------------------------------------------------------- read + def workspace(self, username, consume_corrections=True): + """One user's durable workspace: always the full three-strata shape.""" + try: + data = self._st.get(self.table_key) or {} + ws = data.get(username, {}) or {} + # A collision acknowledgement is protocol state, not part of a field definition. + # Consume it with the first fresh workspace payload after the correcting write, + # then splice a bounded copy into that payload only. Keeping it out of `fields` + # prevents an old request id surviving forever and overriding a later rename. + corrections = {} + if consume_corrections and ws.get('fieldCorrections'): + def _take(current): + current_ws = current.get(username) or {} + pending = current_ws.get('fieldCorrections') or {} + corrections.update({ + str(key)[:80]: dict(value) + for key, value in pending.items() + if isinstance(value, dict) + }) + current_ws.pop('fieldCorrections', None) + return current + + data = self._st.update(self.table_key, _take, flush='async') + ws = (data or {}).get(username, {}) or {} + except Exception: + ws = {} + corrections = {} + fields = { + key: dict(value) if isinstance(value, dict) else value + for key, value in (ws.get('fields') or {}).items() + } + for key, ack in corrections.items(): + field = fields.get(key) + accepted_label = str(ack.get('label') or '')[:120] + requested_label = str(ack.get('labelCorrectedFrom') or '')[:120] + correction_id = str(ack.get('labelCorrectionId') or '')[:180] + # A newer field write clears/replaces the pending ack in the SAME transaction. + # The label check is an extra belt against ever attaching a stale ack to a newer + # definition if a future store implementation weakens that ordering. + if (isinstance(field, dict) and accepted_label + and str(field.get('label') or '') == accepted_label + and requested_label and correction_id): + field['labelCorrectedFrom'] = requested_label + field['labelCorrectionId'] = correction_id + out = { + 'views': dict(ws.get('views') or {}), + 'fields': fields, + 'overlays': dict(ws.get('overlays') or {}), + # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH + # stratum rather than a key on each item — see aios_grid.clean_folders for why + # (a cohort lives in another store, and filing is an organising act, not part of + # what a view is). Absent for every workspace saved before this wave, which is + # exactly "no folders yet". + 'folders': dict(ws.get('folders') or {}), + 'itemFolders': dict(ws.get('itemFolders') or {}), + } + # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The + # client's localStorage copy wins when present; this is the server's answer for a + # fresh profile, which used to fall all the way to the system default view. + if ws.get('activeViewId'): + out['activeViewId'] = str(ws['activeViewId']) + # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth + # stratum, absent until the user first reorders — exactly "default order". + if isinstance(ws.get('recordLayout'), dict): + out['recordLayout'] = dict(ws['recordLayout']) + return out + + # ---------------------------------------------------------------- write + def _update(self, username, change): + def _up(data): + ws = data.setdefault(username, {}) + ws.setdefault('views', {}) + ws.setdefault('fields', {}) + ws.setdefault('overlays', {}) + ws.setdefault('folders', {}) + ws.setdefault('itemFolders', {}) + change(ws) + return data + # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak, + # column note and typed overlay cell lands here inside the component round-trip, and + # the historical synchronous hub commit cost seconds per edit. The mutation applies to + # the in-process cache (read-your-writes for every subsequent render); the hub write + # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'. + return self._st.update(self.table_key, _up, flush='async') + + def rename_choice_values(self, username, change): + """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME). + + ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A + public "do anything to the workspace" method is an invitation to put write logic in + callers instead of here, and every OTHER method on this class exists precisely because + that logic belongs in one place. Renaming a choice is the one operation that must touch + three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that + filter or colour by the old value — inside a SINGLE transaction, because a rename that + updated the cells and not the filters would leave a saved view matching nothing. + + `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`). + """ + return self._update(username, change) + + def save_active_view(self, username, view_id): + """Remember which view this user last opened (owner item 3, 2026-07-31). + + Presentation state, not authorisation: the READ side re-validates the id against what + the caller may actually see, so a stale or foreign id degrades to the default view + rather than granting anything. Stored per user like every other stratum. + """ + vid = str(view_id or '').strip()[:120] + if not vid or username == SHARED_KEY: + return + + def _set(ws): + ws['activeViewId'] = vid + self._update(username, _set) + + def save_record_layout(self, username, order): + """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT). + + Presentation state for ONE surface — the record modal. Deliberately not view config: + the owner's ask is per-user, not per-view, and it must never reorder grid columns. + The event handler validated keys against the live field set; the wire re-validates at + serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here. + An empty order clears the stratum back to "default order". + """ + if username == SHARED_KEY: + return + clean, seen = [], set() + for key in (order or [])[:200]: + key = str(key or '').strip()[:80] + if key and key not in seen: + seen.add(key) + clean.append(key) + + def _set(ws): + if clean: + ws['recordLayout'] = {'order': clean} + else: + ws.pop('recordLayout', None) + + self._update(username, _set) + + def save_folders(self, username, folders, item_folders): + """Replace the folder stratum wholesale (wave-8 I11). + + Wholesale rather than per-folder because the caller has ALREADY validated the complete + picture through aios_grid.clean_folders / clean_item_folders, and those two are + interdependent: a placement is only legal while its folder exists, so committing them + separately would leave a window where a reader sees an item filed into a folder that is + not there yet. One write, one consistent state. + """ + def _set(ws): + ws['folders'] = dict(folders or {}) + ws['itemFolders'] = dict(item_folders or {}) + self._update(username, _set) + + def save_view_order(self, username, order): + """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail. + + Wholesale, like `save_folders` above and for the same reason: the client sends the full + list it is looking at, not a delta, because a partial order cannot say where an UNNAMED + view went. + + ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves. + `aios_grid`'s own folder note argues it out for placements and every word applies: an + arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out + of the view config means duplicating, sharing or exporting a view does not drag one + person's rail position along with it. It also means a SHARED view can sit in a different + place for each person who can see it, which is the only coherent answer once two people + share one view. + + An empty list CLEARS the arrangement (back to server order) rather than storing `[]`. + """ + def _set(ws): + clean = [] + seen = set() + for vid in (order or []): + vid = str(vid).strip()[:120] + if vid and vid not in seen: + seen.add(vid) + clean.append(vid) + if clean: + ws['viewOrder'] = clean + else: + ws.pop('viewOrder', None) + self._update(username, _set) + + def shared_views(self, viewer, is_admin=False): + """Every SHARED view this viewer may see, by id (wave-9 I17). + + Read-only and independent of the viewer's own workspace: the caller merges. Returns + only what `_may_see` allows, so a caller cannot accidentally render somebody else's + personal view by forgetting to filter. + """ + try: + bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {} + except Exception: + return {} + return {vid: dict(v) for vid, v in bucket.items() + if _may_see(v, viewer, is_admin)} + + def shared_view(self, view_id): + """One shared view RAW — no visibility filter. For authorisation decisions only: a + caller must know a view exists and who owns it before it can decide whether the actor + may touch it. Never hand the result to a renderer without checking `_may_see`.""" + try: + return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {} + ).get('views', {}).get(str(view_id)) + except Exception: + return None + + def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False): + """Upsert a SavedView into its ONE home — personal workspace or the shared bucket. + + `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home + it lands in, the view is REMOVED from the other, so a view can never exist as two + copies that diverge on the next edit. + + ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer + moves data and does not know who is asking. `_cl_handle_one` is the wall. + """ + view_id = str((view or {}).get('id') or '').strip() + if not view_id: + raise ValueError('view id is required') + if username == SHARED_KEY: + raise ValueError('reserved username') + to_shared = is_shared(view) if shared is None else bool(shared) + requested = dict(view) + accepted = {} + + def _up(data): + # View names are tenant-global: every personal workspace plus the shared bucket. + # This deliberately includes views the actor cannot see. The only disclosed fact + # is that a display name is already taken, while the categorical "no duplicate + # view names" contract remains true when a personal view is later shared. + names = list(reserved_names or ()) + for workspace in data.values(): + if not isinstance(workspace, dict): + continue + names.extend( + value.get('name') + for candidate_id, value in (workspace.get('views') or {}).items() + if candidate_id != view_id and isinstance(value, dict) + ) + payload = dict(requested) + payload['name'] = _unique_name(payload.get('name'), names) + accepted.clear() + accepted.update(payload) + if to_shared: + bucket = data.setdefault(SHARED_KEY, {}) + bucket.setdefault('views', {})[view_id] = payload + # it may have lived in the creator's workspace before being shared + owner = data.get(payload.get('createdBy') or username) or {} + (owner.get('views') or {}).pop(view_id, None) + else: + ws = data.setdefault(username, {}) + ws.setdefault('views', {})[view_id] = payload + (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None) + return data + + self._st.update(self.table_key, _up, flush='async') + return dict(accepted) + + def delete_view(self, username, view_id): + """Delete a custom/list view override. The system all-rows view is guarded by caller. + + Removes from BOTH homes: the caller has already authorised the delete, and leaving a + stale copy in the other bucket would resurrect the view on the next read. + """ + vid = str(view_id) + + def _up(data): + (data.get(username, {}).get('views') or {}).pop(vid, None) + (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None) + return data + + self._st.update(self.table_key, _up, flush='async') + + def save_field(self, username, field, reserved_names=(), correction_id=None): + """Persist a column note or a user-created (custom_/measure_) field definition.""" + key = str((field or {}).get('key') or '').strip() + if not key: + raise ValueError('field key is required') + requested = dict(field) + accepted = {} + + def _save(ws): + names = list(reserved_names or ()) + names.extend( + value.get('label') + for candidate_key, value in (ws.get('fields') or {}).items() + if candidate_key != key and isinstance(value, dict) + ) + payload = dict(requested) + payload.pop('labelCorrectedFrom', None) + payload.pop('labelCorrectionId', None) + requested_label = ' '.join( + str(payload.get('label') or 'Untitled').split())[:120].rstrip() + payload['label'] = _unique_name(requested_label, names) + corrections = ws.setdefault('fieldCorrections', {}) + corrections.pop(key, None) + if payload['label'] != requested_label and correction_id: + corrections[key] = { + 'label': payload['label'], + 'labelCorrectedFrom': requested_label, + 'labelCorrectionId': str(correction_id)[:180], + } + if not corrections: + ws.pop('fieldCorrections', None) + accepted.clear() + accepted.update(payload) + if source_override_is_empty(payload): + ws['fields'].pop(key, None) + else: + ws['fields'][key] = payload + + self._update(username, _save) + return dict(accepted) + + def duplicate_field(self, username, source_key, new_key, field, + reserved_names=(), correction_id=None): + """Clone a user-created field in ONE store transaction (wave-5 item 1): the new + definition plus — for `custom_` overlay sources only — every stored cell value under + the source key. One transaction, because a def without its values (or values without a + def) is exactly the orphan state delete_field exists to prevent, in reverse. + The caller validated both keys (same created stratum) and stamped the clone's + createdBy; this layer only moves data.""" + source_key = str(source_key or '').strip() + new_key = str(new_key or '').strip() + if not source_key or not new_key or source_key == new_key: + raise ValueError('duplicate_field needs two distinct keys') + requested = dict(field) + accepted = {} + + def _dup(ws): + names = list(reserved_names or ()) + names.extend( + value.get('label') + for candidate_key, value in (ws.get('fields') or {}).items() + if candidate_key != new_key and isinstance(value, dict) + ) + payload = dict(requested) + payload.pop('labelCorrectedFrom', None) + payload.pop('labelCorrectionId', None) + requested_label = ' '.join( + str(payload.get('label') or 'Untitled').split())[:120].rstrip() + payload['label'] = _unique_name(requested_label, names) + corrections = ws.setdefault('fieldCorrections', {}) + corrections.pop(new_key, None) + if payload['label'] != requested_label and correction_id: + corrections[new_key] = { + 'label': payload['label'], + 'labelCorrectedFrom': requested_label, + 'labelCorrectionId': str(correction_id)[:180], + } + if not corrections: + ws.pop('fieldCorrections', None) + accepted.clear() + accepted.update(payload) + ws['fields'][new_key] = payload + if source_key.startswith('custom_'): + for row in ws['overlays'].values(): + if isinstance(row, dict) and source_key in row: + row[new_key] = row[source_key] + + self._update(username, _dup) + return dict(accepted) + + def delete_field(self, username, key): + """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27). + + Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula + columns — the caller enforces the prefix). The stored overlay VALUES for the key are + scrubbed with it: a deleted column's cells must not linger as orphan data that would + silently resurface if the key were ever reused. Views referencing the key self-heal on + their next autosave (an unknown colId is dropped) — the rule every stale key rides. + """ + key = str(key or '').strip() + if not key: + return + + def _drop(ws): + ws['fields'].pop(key, None) + for row in ws['overlays'].values(): + if isinstance(row, dict): + row.pop(key, None) + + self._update(username, _drop) + + def patch_overlay(self, username, pid, updates): + """Patch only the external editable stratum; never writes to the source system.""" + clean = dict(updates or {}) + if not clean: + return + + def _patch(ws): + ws['overlays'].setdefault(str(int(pid)), {}).update(clean) + + self._update(username, _patch) + + +def make(table_key, st=None): + return TableStore(table_key, st=st) diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 0b48b615762f56360bd9dad54e5537c43465bcd8..0b49961db6b89aa6ae624121bad8a4896aff9d2c 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -1,2713 +1,2678 @@ -"""USER-CREATED TABLES (wave-9 I8 / contract C6) — databases that are not a connector. - -The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely -new database blank, where this time the user can actually insert row that they want."* The first -branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second. - -WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at -import time — it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it -at runtime without editing code. A user-created table is tenant data, so it lives in the tenant -store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed -machinery honest: a user table is never "archived", it is deleted, and it is never in -`allowed_modules` because it is not a module. - -SHAPE (one store key, `user_tables`): - - {table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}} - -`source` is deliberately NOT 'Odoo' — the nav badge is generated from it, so a blank table reads -"Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`, -which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new -storage concept. - -⚠ ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and the -row endpoints refuse unless `is_user_table(key)` — a user must not be able to invent a customer -in Odoo by typing into a grid. (Wave 18 note: the events seam has NO row event types; row -add/delete are REST endpoints on `aios-web/api/routes_tables.py`, and this predicate is their -wall.) - -WAVE 18 (C3-UT): every function takes an optional `st` STORE HANDLE — anything exposing -`get(name)` / `update(name, fn, flush=)` / `exists(name)`. Default = `core.store` (tenant #0, -the Streamlit host, unchanged). The API passes the session's `TenantRuntime`, whose accessors -apply the tenant prefix / repo binding, so a Nurilab table lands in Nurilab's store. -""" -import datetime as _dt -import hashlib -import json as _json -import re - -import core.store as store - -STORE_KEY = 'user_tables' -MAX_TABLES = 40 -MAX_LABEL = 60 - -#: ⭐ THE PER-TABLE ROW CEILING — 5,000 until 2026-08-09, and it was never a property of the -#: substrate. Owner: *"Why do we keep having this MAX_ROWS problem, 5000 is too little. We need to -#: exceed it… solve the rootcause."* -#: -#: ⛔ WHAT 5,000 ACTUALLY WAS. Two things wore the same number and neither justified it: -#: * `add_row`'s human-insert door (below) — a guard against a runaway paste, not a measurement; -#: * `odoo_relational.plan()`'s own refusal, which is what kept the Odoo databases at OPEN AR -#: only and is the reason the owner could not find every Odoo id in them. -#: `_ensure_table` writes rows straight through `rt.update(STORE_KEY, …)` and never crosses either, -#: so the cap was never even enforced on the population it was blamed for. And `ig_master.py` has -#: run a **500,000**-row bucket the whole time, which settles the question of whether the store can -#: hold more than five thousand of anything. -#: -#: ⭐ THE REAL CONSTRAINT, MEASURED 2026-08-09 against tenant #0's own mirror, is the one this -#: number now expresses: every `ut_*` table lives in ONE `user_tables` document that -#: `Store.get`/`Store.update` copy with `json.loads(json.dumps(...))` on every call. Built from -#: real Odoo rows, one table costs: -#: -#: customers 2,465 rows 0.43 MB 8 ms invoices 31,418 rows 8.68 MB 158 ms -#: products 5,829 rows 1.20 MB 26 ms orders 32,700 rows 7.50 MB 129 ms -#: -#: ⭐ THE NUMBER IS DERIVED FROM TWO HEADROOMS, and both are checked rather than asserted by -#: `verify_odoo_relational._prove_row_ceiling` ([[rules-need-gates]]): -#: * BYTES — the widest table the product ships measures **0.318 KB/row** (`ut_odoo_invoices`, -#: 13 columns), so 60,000 rows is **19.1 MB** against a 32 MB per-table budget: 40% spare. -#: * GROWTH — the largest shipped population is 32,700 confirmed orders, growing ~8k/year, so -#: the cap is 83% above what exists and does not bind for roughly three years. -#: ⚠ 100,000 WAS THE FIRST ANSWER AND IT WAS WRONG BY ITS OWN GATE: at the measured row width it -#: is 31.8 MB against the same budget — 0.6% of headroom, i.e. a ceiling that one extra column -#: turns red. A cap chosen so it *just* passes its own control is a cap chosen to look justified. -#: ⛔ WHEN THIS BINDS, THE FIX IS NOT A BIGGER NUMBER. It is D-87's per-table row-key split, after -#: which the budget applies to a table on its own rather than to a share of one shared document. -#: -#: ⚠ IT IS A PER-TABLE CEILING AND THE TENANT PAYS THE SUM. The four Odoo databases together are -#: 20.6 MB / ≈ 370 ms per composite copy — real, and the reason the per-table row KEY split is -#: booked as the next increment (DEBT D-87) rather than claimed here. Order lines (256,810 rows, -#: 63.9 MB, 2.57 s) stay out of the store under ANY JSON substrate; they are answered by the -#: read-through rollup, which never copies a row. -MAX_ROWS = 60_000 - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# ⭐⭐ WAVE 30 / OWNER RULING R6 — THE CEILING ABOVE DOES NOT APPLY TO A CONNECTED SOURCE. -# -# Verbatim: *"there is no cap in how many data from the API source (as long as its from a -# connected source like Odoo) that can be pulled into the app. Make this a standing rule. Now if -# there is lag or it can't be done, you need to explicitly tell me why and recommend a fix."* -# -# ⭐ BOTH SENTENCES ARE THE RULE. The first removes the cap; the second says a cap that CANNOT be -# removed must be reported with its cause and a recommendation — `limit_report()` below is that -# sentence, and it is served on the wire rather than left in a comment. A silent limit is the -# failure mode, not a limit. -# -# ⚠ AND THE EDITABLE SUBSTRATE KEEPS ITS BOUND. `MAX_ROWS` is not a rule about data, it is a -# measurement of THIS document (the header above derives it from 0.318 KB/row against a 32 MB -# per-table budget). A `ut_*` database somebody types into still lives here and still pays that -# cost. What changes is that a CONNECTED table's rows stop living here at all — they are served -# read-through from the mirror (R7, `routes_odoo_tables`), where 971,034 GL lines already sit and -# are counted in ~24 ms. Conflating the two turns R6 into a rewrite of user tables. - -#: Table keys DECLARED connected by the app layer. Populated by `routes_odoo_tables._sources()`; -#: `core` never imports up, so the app tells this layer rather than being interrogated by it. -_CONNECTED = set() - - -def register_connected(*table_keys): - """Declare table keys as CONNECTED-SOURCE — their rows come from a connector and are served - read-through, so this document's ceiling is not a fact about them.""" - for key in table_keys: - k = str(key or '').strip() - if k: - _CONNECTED.add(k) - return frozenset(_CONNECTED) - - -#: ⭐⭐ W31-T25 / CONTRACT C2 — the KEY-CONVENTION leg's prefixes are a REGISTRY, not a literal. -#: -#: `is_connected`'s third leg used to read `key.startswith('ut_odoo_')` inline. Meta Ads is the -#: second connector to spawn locked databases (`ut_meta_*`), and the alternative was a second -#: predicate in the app layer answering the same question — one question, two normalizers, which -#: is a recorded defect class here ([[one-question-two-normalizers]]). So the connector layer -#: DECLARES its prefix and this module keeps the single evaluator. -#: -#: ⛔ `'ut_odoo_'` IS SEEDED RATHER THAN REGISTERED, AND THAT IS THE POINT OF THE LEG. The whole -#: reason a naming convention sits below the registry is to answer in a process where nothing has -#: registered anything — a cold worker that never imported `routes_odoo_tables` still owes the -#: right answer for `ut_odoo_gl_lines`, and an empty seed would have it fall through to -#: `MAX_ROWS`, i.e. re-cap a connected source against R6. A newly registered prefix carries the -#: weaker guarantee that `register_connected` already carries: it holds once its registrar has run. -_CONNECTED_PREFIXES = {'ut_odoo_'} - - -def register_connected_prefix(*prefixes): - """Declare a table-key prefix whose automation-sourced, records-locked tables are CONNECTED. - - The connector layer calls this beside `register_connected` (which names exact keys): the - prefix answers for tables this process has not enumerated, the exact keys answer for the ones - it has. Returns the whole registered set, so a caller can assert what it just joined. - - ⚠ A PREFIX ALONE NEVER MAKES A TABLE CONNECTED. `is_connected`'s third leg still demands - `source == AUTOMATION_SOURCE` and `recordMode == AUTOMATION_RECORD_MODE`, because a person who - names a database "Meta ads spend" gets a `ut_meta_…`-shaped key too — and exempting it from - the ceiling on the strength of its NAME would be exactly the silent hole the fence exists for. - """ - for prefix in prefixes: - p = str(prefix or '').strip() - if p: - _CONNECTED_PREFIXES.add(p) - return connected_prefixes() - - -def connected_prefixes(): - """The registered key prefixes — the ONE list a caller may read to name the same set. - - C2's parity gate reads this rather than re-spelling `ut_odoo_` / `ut_meta_` on the far side: - a second literal is a second normalizer wearing an assertion's clothes. - - ⚠ AND IT IS THE REGISTRAR'S OWN RETURN VALUE, which is not tidiness — `verify_reachability` - went RED the hour this shipped, correctly: a public function whose only callers are `verify_*` - files is a feature no user can reach ([[reachable-is-not-the-same-as-built]]), and "another - lane's gate will call it" is not a caller. Routing the write door's answer through the read - door makes one construction site of the set instead of two, and makes the accessor - load-bearing rather than decorative. - """ - return frozenset(_CONNECTED_PREFIXES) - - -def is_connected(table_key, st=None): - """Is this database backed by a connected source rather than by typing? - - THREE ANSWERS, in falling order of authority, and the third is fenced deliberately: - 1. the app layer registered it (`register_connected`) — the live path; - 2. the stored definition says so (`connected: True`) — survives a process with no registrar; - 3. ⚠ a REGISTERED KEY PREFIX (`register_connected_prefix`; seeded `ut_odoo_`, and Meta Ads - joins it as `ut_meta_`), AND ONLY on a table that is automation-sourced with records - locked. A person who names a database "Odoo foo" gets exactly that key, and exempting it - from the ceiling on the strength of its name would be a silent hole. Their table is - `Blank`-sourced and records-mutable, so it cannot reach this leg. - """ - key = str(table_key or '').strip() - if key in _CONNECTED: - return True - table = get(key, st) or {} - if not table: - return False - if table.get('connected') is True: - return True - return (any(key.startswith(p) for p in _CONNECTED_PREFIXES) - and table.get('source') == AUTOMATION_SOURCE - and table.get('recordMode') == AUTOMATION_RECORD_MODE) - - -#: ⭐⭐ W30-T31 (D-87) — table keys whose rows DO NOT LIVE HERE AT ALL. -#: -#: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the -#: DATA came from, so `MAX_ROWS` is not a fact about it (R6). *Read-through* says where the rows -#: are SERVED FROM, so they are not in this document — which is the only thing that makes the -#: per-request copy cheaper. The eight Odoo grids are all connected; only the ones the app layer -#: has actually bound to the mirror AND proven safe to un-materialise are registered here. -#: -#: ⛔ THE APP LAYER DECIDES, and it is not merely "is there a binding". `routes_odoo_tables` -#: checks three conditions, because getting any one of them wrong is a silent wrong answer rather -#: than an error: a binding must exist; NOTHING may fold or link at the table's stored rows (a -#: rollup over a table with no rows writes ZEROS, and `compute_relation_cells` reads the raw -#: document, not this module); and the population must fit a window while the client still asks -#: for whole tables. Every table that fails one is REPORTED, per table, on the status door. -_READ_THROUGH = set() - - -def register_read_through(*table_keys): - """Declare that these databases are SERVED from the mirror and store no rows here.""" - for key in table_keys: - k = str(key or '').strip() - if k: - _READ_THROUGH.add(k) - _CONNECTED.add(k) # read-through implies connected; never one without the other - return frozenset(_READ_THROUGH) - - -def materialises(table_key, st=None, defn=None): - """Does this database keep its rows in the shared `user_tables` document? - - True for everything except a read-through grid — including every editable table, every - automation table, and a connected table whose conversion has not happened yet. - - ⛔ THE STORED FLAG IS NOT A CACHE OF THE REGISTRY, IT IS THE DURABLE HALF, and without it this - predicate would be a live hazard rather than a lookup. `_READ_THROUGH` is filled by an app-layer - route; a worker process that has never served one would answer True for a table whose rows were - stripped weeks ago, read `rows: {}` and serve an EMPTY GRID with nothing going red. So the strip - STAMPS the definition and the stamp is what a cold process reads. Same three-tier shape as - `is_connected` above, minus the naming convention — there is no safe way to guess this one. - - ⚠ `defn` LETS A CALLER LEND THE DEFINITION IT IS ALREADY HOLDING, and on the row path that is - not a micro-optimisation: every `get()` here is another whole-document deep copy under - `Store._lock`, so asking this question inside `scoped_pool` without lending would have added a - third copy to the very request this ticket exists to make cheaper. - """ - key = str(table_key or '').strip() - if key in _READ_THROUGH: - return False - if defn is None: - defn = get(key, st) or {} - return not bool((defn or {}).get('readThrough')) - - -def strip_materialised(st=None): - """Remove the stored rows of every read-through database. Idempotent; returns what moved. - - ⭐ THIS IS THE HALF THAT MAKES THE INVARIANT TRUE RATHER THAN INTENDED. The spawn - (`odoo_relational._ensure_table_inplace`) writes rows by mutating this document inside its own - `rt.update(STORE_KEY, …)`, so it never passes through any function here — a guard on the write - doors below would simply not be on that path. Re-running this after a refresh is therefore the - enforcement, and being idempotent is what lets every door call it without coordinating. - - ⚠ The definition STAYS. A read-through table is a real database — its fields, label, views, - grants and lock all live here; only the rows are elsewhere. Deleting the definition would take - the permission wall and the nav entry with it. - """ - out = {'dropped': {}, 'before': 0, 'after': 0} - if not _READ_THROUGH: - return out - s = _st(st) - try: - cur = s.get(STORE_KEY) or {} - except Exception: # noqa: BLE001 - return out - fat = {k: len((cur.get(k) or {}).get('rows') or {}) for k in _READ_THROUGH} - fat = {k: n for k, n in fat.items() if n} - if not fat: - return out # the common case: nothing to do, and nothing measured - # Only now is the measurement worth its own cost — `json.dumps` of a 20 MB document is not - # free, and this exists to be reported (R6's second sentence), not to run on every request. - out['before'] = len(_json.dumps(cur, default=str)) - - def _strip(doc): - for key in list(_READ_THROUGH): - t = (doc or {}).get(key) - if isinstance(t, dict) and t.get('rows'): - out['dropped'][key] = len(t['rows']) - t['rows'] = {} - # ⛔ THE STAMP IS THE POINT — see `materialises`. Without it the next process to - # boot reads an empty `rows` and calls that the answer. - t['readThrough'] = True - return doc - - s.update(STORE_KEY, _strip, flush='sync') - try: - out['after'] = len(_json.dumps(s.get(STORE_KEY) or {}, default=str)) - except Exception: # noqa: BLE001 - out['after'] = 0 - return out - - -def row_limit(table_key, st=None): - """The row ceiling that applies to this database: 0, None, or `MAX_ROWS`. - - ONE evaluator, so the write doors, the spawn's refusal and the wire's report cannot disagree - about whether a table is capped ([[one-evaluator-per-question]]). - - ⛔ THE THREE ANSWERS ARE THREE DIFFERENT STATEMENTS AND 0 IS NOT "A VERY SMALL CAP": - * **0** — this database stores no rows HERE at all; it is served read-through from the - mirror. A caller that builds rows for it is doing wasted, dangerous work: 963,783 GL lines - is ~240 MB of python dicts in one process. `odoo_relational.plan` reads this to decide - whether to build a bucket at all. - * **None** — connected, materialised, and UNCAPPED (R6): however many rows Odoo has. - * **MAX_ROWS** — the editable substrate, which really is bounded by this document. - """ - if not materialises(table_key, st): - return 0 - return None if is_connected(table_key, st) else MAX_ROWS - - -def limit_report(table_key, st=None): - """R6's second sentence as data: `{subject, cause, recommendation, effect}` or None. - - ⛔ `effect` is never `truncated`. Every enforcement site REFUSES — a truncated table - understates every total it feeds while looking exactly like a complete one, which is the - failure this whole module is arranged against. - """ - cap = row_limit(table_key, st) - if cap is None: - return None - if cap == 0: - return { - 'subject': 'rows', 'limit': 0, 'effect': 'read_through', - 'cause': 'this database is served THROUGH the connector mirror, so its rows are ' - 'never copied into this tenant document and no ceiling applies to them', - 'recommendation': 'read it with a window (`/odoo-tables/{key}/rows`); the row count ' - 'you see there is a SQL count over the whole table', - } - return { - 'subject': 'rows', 'limit': cap, 'effect': 'refused', - 'cause': f'this database is EDITABLE, so its rows live in the shared `{STORE_KEY}` ' - f'document that every request copies; at the widest shipped row (0.318 KB) ' - f'{cap:,} rows is 19.1 MB against a 32 MB per-table budget', - 'recommendation': 'connect it to a source instead — connected data is served ' - 'read-through from the mirror and is not bounded (R6); for typed data ' - 'the increment is D-87, a per-table row key, not a bigger number', - } - - -MAX_FIELDS = 60 -#: the badge the nav paints for a table with no connector behind it -BLANK_SOURCE = 'Blank' -#: automation-created tables carry their maker instead (wave 18 C4-AUTO) -AUTOMATION_SOURCE = 'Automation' -AUTOMATION_RECORD_MODE = 'automation' -#: every user table's key is prefixed, so it can never collide with a registry module key -KEY_PREFIX = 'ut_' - - -#: ⭐⭐ W31 QA — KEYS WHOSE RECORDS ARE MACHINE-OWNED **BY DECLARATION**, registered by the layer -#: that owns the connector (`automation_engine` at import), exactly as `register_connected_prefix` -#: is. `core` never imports up, so the engine pushes its list down here rather than this module -#: reaching for it. -#: -#: ⛔ WHY A REGISTRY AND NOT THE STORED FLAG ALONE — THE OWNER FOUND THIS IN PRODUCTION. -#: `recordMode` is stamped by `ut_ensure`, so a table only became locked THE NEXT TIME AN -#: AUTOMATION WROTE TO IT. W31-T32 proved that migration works and read it as done; it means every -#: TikTok database already sitting in a tenant stayed unlocked — offering "+ New record" — until -#: somebody re-ran a TikTok automation, which nobody had. Owner, verbatim: *"the databases for -#: Tiktok do not have the small lock icon as I asked."* A lock that arrives on the next write is a -#: lock that is absent exactly while a person is looking at it. -#: ⚠ THE REGISTRY IS THE AUTHORITY AND THE STORED FLAG STILL COUNTS — either one locks. The flag -#: keeps working for tables nobody declares, and the declaration needs no store write, no boot -#: order and no per-tenant migration, so it is true for every tenant the moment the module loads. -#: ⚠ LOCKED DATABASE, NOT READ-ONLY (DESIGN.md §4): this predicate governs RECORDS. Adding a FIELD -#: must still work, and `may_edit_field` is a different wall. -_LOCKED_RECORD_KEYS = set() - - -def register_locked_records(*keys): - """Declare table keys whose records are machine-owned. Idempotent; -> the registered set.""" - for k in keys: - for one in (k if isinstance(k, (set, frozenset, list, tuple)) else (k,)): - if str(one or '').strip(): - _LOCKED_RECORD_KEYS.add(str(one).strip()) - return frozenset(_LOCKED_RECORD_KEYS) - - - -def records_mutable(table_key, st=None): - """May a human add/edit/delete records in this database? - - Ordinary and Profile databases default open. Automation-owned child datasets opt out — by - DECLARATION (`register_locked_records`, which needs no write) or by the stored table-level - mode; engine writers use their direct coalesced path and are intentionally not routed through - this human-door predicate. - """ - table = get(table_key, st) or {} - if not table: - return False - if str(table_key) in _LOCKED_RECORD_KEYS: - return False - return table.get('recordMode') != AUTOMATION_RECORD_MODE - -#: The field types a user table may declare. ⚠ Kept a SUBSET of -#: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `created_time` stays -#: out (a row-datum kind — a base column of it would have no author); -#: `image` is unchanged this wave. `automation` JOINED in WAVE 21 (item 7a, -#: contract C2): the engine and the automation-column picker read the DEFINITION, so a -#: grid-created automation column must be able to live there — the column's CELLS stay -#: machine-written (the engine is the writer). Before this, the grid's automation columns -#: landed in the per-user workspace stratum and the picker could not see them — the owner's -#: "Choose a column..." stays empty. Local literal rather than an aios_grid import so this -#: module stays dependency-light for the API's boot path. -#: ⭐ `json` JOINED IN WAVE 23 (C7) — and it belongs here rather than staying grid-only the way -#: `image` did, because the databases that need it are exactly these: a scrape lands a comment -#: thread or a webhook payload against a row, and the automation engine writes those cells. -#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They belong here for -#: `metric`'s reason, not `formula`'s: their cells are MACHINE-COMPUTED SERVER-SIDE and -#: materialised into the row, because a rollup reads ANOTHER TABLE'S ROWS — data the client has -#: not loaded and must not have to. -#: ⭐⭐ 2026-08-10 — `formula` JOINED, and the argument that kept it out was answered rather than -#: overruled. "A base column of it would have no author" was about its VALUES, and that half is -#: still true: a formula cell is computed in the browser from the row's other cells and is stored -#: NOWHERE, here or anywhere else. What joins this set is the DEFINITION — the expression, the -#: label, the column's existence. Excluded, those landed in `_table_workspace`, the -#: PER-USER overlay: measured on nurilab as `Trimmed reel views`, a column only its creator could -#: see on a database four people share. That is not a narrower feature, it is the same shape as -#: the rollup defect one stratum over ([[rollup-lives-in-the-definition-not-the-overlay]]) with a -#: different consequence — not "nothing can compute it" but "nobody else can SEE it". -#: ⛔ SAY THE LIMIT OUT LOUD, because this line will be read as a bigger promise than it is: the -#: values still do not exist server-side, so an automation, a rollup, a `where` clause and an -#: export still cannot read a formula column. Making the definition shared does not make the -#: number stored. A number the server must read is a `rollup` (link fold) or a `metric`. -#: ⚠ Its cells are therefore in `is_computed_cell` — the WRITE wall. Without that, `grid_events` -#: would accept a paste into a column whose renderer overwrites it on the next paint. -#: ⛔ `image` WAS MISSING FROM THIS SET UNTIL WAVE 27, AND THAT WAS A LIVE DEFECT — recorded -#: here because the shape repeats and the next kind will be added by somebody reading this line. -#: Wave-19 R7 landed `image` on FOUR of the five surfaces (the `FieldType` union, `CREATABLE_TYPES`, -#: `aios_grid.CUSTOM_FIELD_TYPES`, the shape/label tables) and missed this one. The client -#: therefore offered an Image column in EVERY column menu, including on `ut_*` databases, while -#: `_clean_field` returned None for it — so on a user database the column was created, named, -#: configured, and GONE on the next read, with nothing anywhere going red. It survived three -#: waves because `verify_fields_contract` diffed the client's offer against the ODOO grid's -#: acceptance and never against this set. Wave-27 item 13 adds that derived leg, which is what -#: found it (measured, not reviewed: `_clean_field({'type':'image'})` returned None). -UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date', - 'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json', - 'link', 'rollup', 'code', 'image', 'formula'} -#: A cell is still a scalar on the wire (`Row` values are strings/numbers) — what this bounds is -#: the DOCUMENT inside it. Source data is one already-paid provider response; keep it whole so a -#: provider field is never silently discarded merely because the response is rich. 32 MiB matches -#: the automation transport's guarded maximum response size. -MAX_JSON_CELL = 32 * 1024 * 1024 - -# --------------------------------------------------------------------------------------------- -# THE PROFILE FLAG (wave 25, contract C3 / owner ruling R7) -# --------------------------------------------------------------------------------------------- -# R7: *"the Profile field is a FLAG on an ordinary text field, not a new field kind"* — so it -# works on databases that already exist, and every reader that does not know about it goes on -# rendering a text column. `profile: {source: 'instagram'}` present = flagged; absent = ordinary. -# -# ⛔ THE FLAG IS ONLY WORTH HAVING BECAUSE OF THE TWO MITIGATIONS R7 ACCEPTED AS OPEN HOLES, and -# they are what this block is: AT MOST ONE per table (`_profile_of`, enforced at both write -# doors) and the flag VALIDATES ITS CELL (`normalize_profile`). Without the first, "the profile -# column" is not a thing the enrich action can resolve; without the second, the flag is a label. - -#: Where a profile handle points. A CLOSED vocabulary, for the reason every other closed enum -#: here is closed: a source nobody wrote a reader for renders as a column that promises a link. -#: -#: ⭐ WAVE-29 CONTRACT C2 (owner R1/R2, 2026-08-11): `tiktok` joins it, so a `ut_tt_*` row can -#: carry a profile flag the enrich action resolves. **A source is not a NAME, it is three rules** -#: (below) — the vocabulary and the rules move together, because adding the word alone would -#: accept a TikTok handle and then hand it an `instagram.com` link from `profile_url`. -PROFILE_SOURCES = ('instagram', 'tiktok') - -#: Hosts whose `//` path IS an Instagram profile. `www.` is stripped before the lookup. -_IG_HOSTS = ('instagram.com', 'instagr.am') - -#: URL path segments that are NOT handles. `instagram.com/p/Cxyz` is a POST and -#: `instagram.com/explore/tags/silk` is a search — both are things a person pastes, and both -#: would otherwise normalise to a handle (`p`, `explore`) that can never resolve. -_IG_RESERVED = frozenset({'p', 'reel', 'reels', 'tv', 'stories', 'explore', 'accounts', - 'direct', 'about', 'developer', 'legal', 'privacy', 'terms'}) - -#: Instagram's own handle rule: 1–30 of letters, digits, period, underscore. -_IG_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{1,30}$') - -#: TikTok's profile hosts. ⛔ `vm.tiktok.com` is deliberately ABSENT: those are short links that -#: usually resolve to a VIDEO, and accepting one would store a "handle" (the opaque code) that can -#: never enrich — the same failure `_IG_RESERVED` exists to prevent, one layer earlier. -_TT_HOSTS = ('tiktok.com', 'm.tiktok.com') - -#: TikTok's non-handle path segments. `tiktok.com/@user/video/123` is already refused by the -#: one-segment rule; these are the ONE-segment paths that are still not people. -_TT_RESERVED = frozenset({'video', 'tag', 'music', 'discover', 'search', 'foryou', 'explore', - 'live', 'upload', 'about', 'legal', 'privacy', 'terms', 'effect', - 'business', 'ads', 'node', 'login', 'signup'}) - -#: TikTok's own handle rule: 2–24 of letters, digits, period, underscore. ⚠ NOT Instagram's — -#: the charset matches but the bounds do not, and a shared regex would accept a 30-character -#: TikTok "handle" that the vendor answers nothing for. -_TT_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{2,24}$') - -#: ⭐ C2 — the per-source rules, ONE table, so `PROFILE_SOURCES` cannot grow a member that no -#: normaliser knows. Every function below reads THIS rather than branching on the word: a source -#: with no entry refuses at the door instead of silently taking Instagram's rules and Instagram's -#: URL, which is what "add the word and ship" would have produced. -#: -#: `at_handle` = the profile path wears a leading `@` (TikTok's `/@name`; Instagram's `/name`). -#: It is stripped before validation and re-added by `profile_url`, so the STORED truth is the bare -#: handle in both families and a filter on the column cannot miss half of it. -_PROFILE_RULES = { - 'instagram': { - 'hosts': _IG_HOSTS, - 'reserved': _IG_RESERVED, - 'handle': _IG_HANDLE_RE, - 'at_handle': False, - 'url': 'https://www.instagram.com/{handle}/', - }, - 'tiktok': { - 'hosts': _TT_HOSTS, - 'reserved': _TT_RESERVED, - 'handle': _TT_HANDLE_RE, - 'at_handle': True, - 'url': 'https://www.tiktok.com/@{handle}', - }, -} - - -def _profile_rules(source): - """The rule bag for a source, or None — the ONE resolution of "is this source real".""" - key = str(source or '').strip().lower() - if key not in PROFILE_SOURCES: - return None - return _PROFILE_RULES.get(key) - -#: The C1 preset keys a profile field OWNS — the cells R6 clears when its handle is blanked. -#: -#: MIRRORED from the engine's C1 set as local literals, the same discipline (and for the same -#: boot-path reason) as `METRIC_MEASURES` below; the pair is held in step by a parity check -#: rather than by an import, because `core/` must stay importable without the API layer. -#: -#: Human-written columns are deliberately absent. `ut_ensure` stamps automation provenance on -#: generated fields, so clearing by that tag alone would erase unrelated machine-owned columns. -#: This is therefore an intersection of the known Instagram preset keys and the provenance tag. -PROFILE_PRESET_KEYS = ( - # the profile facts themselves (CANDIDATE_FIELDS + the enrichment run_field_instagram pulls) - 'full_name', 'followers', 'following', 'posts_count', 'avg_engagement', 'bio', - 'external_url', 'verified', 'category', 'business_category', 'is_business', - 'is_professional', 'is_private', 'highlights_count', 'bio_hashtags', - 'pronouns', 'ig_id', 'profile_url', - # ⭐ 2026-08-07 — the rest of the Bright Data profile schema, promoted to preset columns by - # owner instruction. They are profile FACTS like the nineteen above, so blanking the handle - # clears them for the same reason (R6): they describe an account this row no longer names. - 'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title', - 'fbid', 'related_accounts', 'country_code', 'source_payload', - # relational projections and summaries owned by the Instagram graph - 'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link', - 'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12', - 'posts_captured', 'profile_reads', 'post_measurements_captured', 'comments_captured', - # R3's LATEST-value stamp. The full series stays in `ut_ig_snapshots` — one store for one - # series — which is why clearing here can never be a history delete. - 'enriched_at', -) - - -def _clean_profile(raw): - """One `profile` bag → the stored shape, or None (refused). - - Shaped exactly like `_clean_metric`: a mis-shaped flag REFUSES THE FIELD rather than being - dropped, because a column stored without the flag it was created with is a column whose - cells nothing will ever validate — the silent half of the same mistake. - """ - if not isinstance(raw, dict): - return None - source = str(raw.get('source') or '').strip().lower() - if source not in PROFILE_SOURCES: - return None - return {'source': source} - - -def normalize_profile(value, source='instagram'): - """A typed profile cell → `(handle, ok)`. THE cell validator the flag promises (C3). - - Accepts a bare handle (`name`), an at-handle (`@name`) or a profile URL of THIS SOURCE in any - of the forms a person actually pastes (`https://www.instagram.com/name/?hl=en`, - `instagram.com/name`, `https://www.tiktok.com/@name`). Returns the BARE HANDLE — the URL form - is derived for display, so the stored truth has exactly one spelling and a filter on it cannot - miss half the column. - - ⛔ C2: the host, the reserved segments and the handle rule all come from `_PROFILE_RULES[source]`. - A TikTok URL pasted into an Instagram-flagged column is REFUSED, and vice versa — a cell that - names the wrong network is not a handle this row's automation can enrich, and quietly accepting - it would defer the failure to a run that then answers nothing. - - `('', True)` for a blank: clearing a profile cell is legal and is the whole of R6. - `(None, False)` for anything else, and the caller refuses the write with a sentence — a - value that is neither is not a handle we can enrich, and storing it would put the failure - off until the automation runs and answers nothing. - """ - rules = _profile_rules(source) - if rules is None: - return (None, False) - raw = str(value if value is not None else '').strip() - if not raw: - return ('', True) # the clear — R6's whole subject - cand = raw.lstrip('@') - # ⚠ THE URL TEST IS `/`, AND NOTHING CLEVERER. An earlier version also treated a DOT as a - # sign of a URL, which refused `@nuri.lab_1` — a period is legal in an Instagram handle and - # common in real ones, so that test rejected a whole class of valid input while every URL - # case still passed. A bare handle can never contain a slash; every URL form does. - if '/' in cand: - # Parse rather than regex it: a hand-rolled pattern is how - # `instagram.com.evil.test/name` gets read as Instagram. - probe = cand if '//' in cand else 'https://' + cand.lstrip('/') - try: - from urllib.parse import urlsplit - parts = urlsplit(probe) - except Exception: - return (None, False) - host = (parts.netloc or '').split('@')[-1].split(':')[0].strip().lower() - if host.startswith('www.'): - host = host[4:] - if host in rules['hosts']: - segs = [s for s in (parts.path or '').split('/') if s] - # C2: TikTok's profile path is `/@name`; Instagram's is `/name`. The `@` is stripped - # here and re-added by `profile_url`, so both families STORE a bare handle. - first = segs[0].lstrip('@') if segs else '' - if len(segs) != 1 or not first or first.lower() in rules['reserved']: - # Two segments is a post/reel/video, zero is the site itself. Refused rather - # than taking the first segment, which is what turns `/p/Cxyz` into handle `p`. - return (None, False) - cand = first - elif host: - return (None, False) # a URL, but not one of THIS source's - if not rules['handle'].match(cand) or cand.lower() in rules['hosts']: - # The host check catches a bare `instagram.com` — it matches the handle charset, so - # without this it would normalise to a "handle" that can only ever resolve to nothing. - return (None, False) - return (cand.lower(), True) - - -def profile_url(handle, source='instagram'): - """The DERIVED display form of a stored handle (C3: *"the URL form is derived for display"*). - Derived and never stored, so the two spellings cannot drift apart. - - ⛔ C2 — the template comes from the SOURCE's own rules. This function used to hardcode - `instagram.com` while its guard merely checked membership of `PROFILE_SOURCES`, so the moment - that tuple grew a second member it would have handed every TikTok handle an Instagram link — - a guard that passes and a body that lies ([[hardcoded-fallback-not-hardcoded-key]]). - """ - h = str(handle or '').strip().lstrip('@') - rules = _profile_rules(source) - if not h or rules is None: - return '' - return rules['url'].format(handle=h) - - -def _profile_of(fields, exclude=None): - """The profile-flagged field in this list, or None. `exclude` skips one key — a PATCH of the - profile field itself must not read as "a second one already exists".""" - for f in (fields or []): - if (isinstance(f, dict) and isinstance(f.get('profile'), dict) - and f.get('key') != exclude): - return f - return None - - -def profile_field(table_key, st=None): - """THE profile column of one table, or None. ONE resolver, so "which column holds the - handle" has exactly one answer for the write door, the clear, the grid and the engine.""" - return _profile_of((get(table_key, st) or {}).get('fields') or []) - - -def _st(st): - return st if st is not None else store - - -#: Wave 22 (contract C3) — the ROW-EVENT seam. The API layer APPENDS listeners (the automation -#: engine's trigger hook, registered by `routes_automation` — the one module that imports both -#: sides); platform code only ever EMITS. This is what keeps the Notion loop-prevention law -#: structural: the emit sites are the HUMAN write doors (`add_row` here, `overlay_patch` in -#: grid_events) and nothing the engine itself writes through, so an automation's write cannot -#: fire an event trigger — its own or a sibling's — by construction. -ROW_HOOKS = [] - - -def emit_row_event(evt): - """Fan one row event out to every registered listener. A listener failure is swallowed — - a broken trigger must never break typing into a cell — but it is swallowed LOUDLY.""" - for hook in list(ROW_HOOKS): - try: - hook(evt) - except Exception as e: # noqa: BLE001 - print(f'[user-tables] row hook failed: {type(e).__name__}: {e}') - - -def _slug(label): - s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_') - return (s or 'table')[:40] - - -def _ag_formula(raw): - """`aios_grid._clean_formula`, reached the way this module reaches everything one layer up. - - ⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and - `_clean_field`). `core` must not import `aios_grid` at module level — the API's boot path - depends on this module staying dependency-light — and the `code` branch already pays that - cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up - calling a different validator. - - ⚠ `valid_keys` IS DELIBERATELY NOT PASSED. See the note at `_clean_field`'s formula branch: - neither door has reliable sibling-field context, and a second, weaker copy of the ref rule is - worse than leaving refs to the read-time behaviour `_clean_formula` already documents. - """ - import aios_grid as _agf - return _agf._clean_formula(raw) - - -#: Display formats a field may declare. ⛔ DISPLAY ONLY — none of these touches the stored value, -#: which stays the scalar the fold or the mapper wrote. That separation is the whole safety of the -#: feature: a `thousands` setting can never make a number wrong, only easier to read. -#: ⚠ `tz`/`time` belong to the date family and `thousands`/`decimals`/`abbrev` to the number family; -#: they are validated together because ONE bag rides one field and the renderer already reads only -#: the keys its own type understands (`display.numberText` / `display.dateTimeText`). -FORMAT_TZ = ('local', 'utc') -FORMAT_MAX_DECIMALS = 4 - - -#: ⭐ WAVE-29 CONTRACT C7 — the COLUMN-SUMMARY vocabulary a field's `agg` may take. -#: -#: MIRRORED from `aios_grid.FIELD_AGGS` (session C publishes it) and from the client's -#: `aggregations.FIELD_AGGS`, as a local literal for the reason `METRIC_MEASURES` and -#: `PROFILE_PRESET_KEYS` are local literals here: `core/` must stay importable without the layers -#: above it. Three copies, one law, held in step by C's parity gate rather than by an import. -#: -#: ⛔ NOT `CHART_AGGS` (which spells it `avg` and gatekeeps STORED chart values with live data) -#: and NOT `ROLLUP_FNS` (16 fold names, overlapping but not the same list). Merging any two of the -#: three silently turns live stored charts into sums. -FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') - -#: A choice colour, as the client's editor spells one. `#RRGGBB`, upper-cased on the way in so a -#: stored value has exactly ONE spelling and two fields cannot disagree about `#ebf6ef`. -_OPTION_COLOR_RE = re.compile(r'#[0-9A-F]{6}') - - -def _clean_option_colors(raw, options): - """`{option label → #RRGGBB}`, capped to the field's own option vocabulary (wave-29 C1). - - ⛔ MIRRORED from `aios_grid._clean_option_colors` as a LOCAL function rather than imported, for - the reason `METRIC_MEASURES` and `PROFILE_PRESET_KEYS` are local literals here: `core/` must - stay importable without the layers above it, and `aios_grid` sits above this module. The pair - is held in step by a gate leg, never by an import. - - ⚠ THE CAP IS THE POINT, not tidiness: colours are keyed by LABEL, so a renamed or deleted - option would otherwise leave a colour behind that nothing renders and no editor can reach — - and on the next rename that orphan could re-attach to a different option with the same name. - Case-insensitive on the way in, canonical on the way out, so `Done` and `done` cannot both - claim the same option. - """ - if not isinstance(raw, dict): - return {} - supplied = {} - for label, color in raw.items(): - if not isinstance(label, str) or not isinstance(color, str): - continue - clean = color.strip().upper() - if _OPTION_COLOR_RE.fullmatch(clean): - supplied[label.strip().lower()] = clean - out = {} - for option in options or []: - color = supplied.get(str(option).strip().lower()) - if color: - out[str(option)] = color - return out - - -def _clean_format(raw): - """One `format` bag → the stored shape, or None. - - ⛔⛔ THIS FUNCTION EXISTS BECAUSE THE BAG HAD NO STORAGE AT ALL. The column menu has shipped a - "Field format" pane (thousands separator, decimal places, abbreviate) since wave 5, and on a - `ut_*` database BOTH write doors built their entry dict key-by-key and never copied `format` — - so the pane saved, the request succeeded, and the setting was gone by the next read. It looked - like a rendering bug and was a persistence one. Same shape as - [[flag-shipped-without-its-writer]] from the other end: here the writer exists and the STORE - does not. - - ⚠ EVERY KEY IS OPTIONAL AND AN EMPTY BAG IS None, so a field that declares no format is stored - exactly as it was before this existed — the wave-5 note's own parity rule. - """ - if not isinstance(raw, dict): - return None - out = {} - if isinstance(raw.get('thousands'), bool): - out['thousands'] = raw['thousands'] - if isinstance(raw.get('abbrev'), bool): - out['abbrev'] = raw['abbrev'] - if isinstance(raw.get('time'), bool): - out['time'] = raw['time'] - decimals = raw.get('decimals') - # `bool` is an `int` subclass, so `True` would otherwise store as 1 decimal place — a checkbox - # arriving in a numeric slot should be refused, not interpreted (the `sigmas` lesson). - if isinstance(decimals, int) and not isinstance(decimals, bool): - if 0 <= decimals <= FORMAT_MAX_DECIMALS: - out['decimals'] = decimals - tz = str(raw.get('tz') or '').strip().lower() - if tz in FORMAT_TZ: - out['tz'] = tz - return out or None - - -def clean_fields(raw): - """Validate a caller-supplied field list into the stored shape — refuse junk, never store - it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every - base column of a user table is user-editable by construction), or None when nothing - survives. Unknown types are DROPPED, not coerced — a column silently retyped is a lie.""" - out, seen = [], set() - for f in (raw or [])[:MAX_FIELDS]: - if not isinstance(f, dict): - continue - label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80] - key = re.sub(r'[^a-z0-9_]+', '_', str(f.get('key') or _slug(label)).strip().lower()) - key = key.strip('_')[:60] - ftype = str(f.get('type') or 'text').strip().lower() - if not key or key in seen or ftype not in UT_FIELD_TYPES or not label: - continue - seen.add(key) - entry = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'} - # ⭐ 2026-08-10 — the DISPLAY format, carried through the CREATE door. See `_clean_format`: - # this is the fifth time an entry built key-by-key has silently dropped a bag the editor - # sends, and the fourth of those was `agg` on this very line one wave ago. - fmt = _clean_format(f.get('format')) - if fmt: - entry['format'] = fmt - description = ' '.join(str(f.get('description') or '').split())[:300] - if description: - entry['description'] = description - if ftype in ('select', 'multiselect'): - opts = [' '.join(str(o).split())[:60] for o in (f.get('options') or []) - if str(o).strip()][:50] - if opts: - entry['options'] = opts - if f.get('default') is True: - entry['default'] = True - # ⭐ WAVE-27 item 19 (R15) + D-80 — THE PRIMARY COLUMN, CARRIED THROUGH THE CREATE DOOR. - # - # `_clean_field` has honoured `pinned` since 2026-08-07; THIS door silently dropped it, - # and the split the comment below names is why. It did not matter while `create()` minted - # one column — with nothing pinned, `fields.find(f => f.pinned) ?? fields[0]` made the - # only column the primary either way. R15 seeds FOUR at once, so "whichever came first" - # stops being a coincidence that happens to be right, and D-80 is the register entry for - # exactly that fallback quietly being the whole rule. MEASURED before fixing: - # `clean_fields([{...'pinned': True}])` returned the field without the key. - if f.get('pinned') is True: - entry['pinned'] = True - # ⭐ WAVE-27 item 13 — the code language, for the same reason: a `code` column created - # through THIS door would otherwise arrive without the bag `_clean_field` preserves, and - # a column whose language depends on which door made it is the divergence the note below - # warns about, arriving through a different key. - if ftype == 'code' and f.get('code') is not None: - import aios_grid as _agc - c = _agc._clean_code(f.get('code')) - if c: - entry['code'] = c - # C3 (wave 25): the profile flag rides the CREATE path too, and the one-per-table law - # holds inside a single list. ⚠ This validator and `_clean_field` are two doors onto one - # contract (a pre-existing split — `create`/`set_fields` come through here, `add_field`/ - # `patch_field` through the other), so a rule taught to only one of them is a rule the - # other silently permits: a table could be CREATED with two profile columns and then - # refuse to accept a third, which is the shape a user reads as random. - if f.get('profile') is not None: - p = _clean_profile(f.get('profile')) - if p is None or ftype != 'text' or _profile_of(out): - continue # refused: dropped, never stored unvalidated - entry['profile'] = p - # ⭐ 2026-08-09 — THE RELATIONAL PAIR RIDES THIS DOOR TOO, and it is the same divergence - # `pinned` and `code` were fixed for above, arriving through a third key. `_clean_field` - # has carried `link`/`rollup` since 2026-08-07; THIS validator dropped both, so a `link` - # or `rollup` column created through `create()` / `set_fields()` arrived with its bag - # stripped — a column that renders, is typed `rollup`, and computes nothing forever. - # ⚠ IT WAS LATENT, NOT LIVE (no API route calls `set_fields`, and `odoo_relational` writes - # its definitions directly), and it stops being latent the moment a user can BUILD a - # rollup from the field editor — which is exactly what this wave ships. Caught by - # `verify_odoo_relational`'s own NOTE line, which had been printing the list for two days. - # ⛔ THE TYPE/BAG PAIRING IS ENFORCED BOTH WAYS, as it is in `_clean_field`: a `link` with - # no bag can never resolve a row, and a bag on a column of another type is a declaration - # nothing reads. Either half alone is the silent kind of broken. - for bag_key, cleaner in (('link', _clean_link), ('rollup', _clean_rollup)): - bag = cleaner(f.get(bag_key)) if f.get(bag_key) is not None else None - if ftype == bag_key and bag is None: - entry = None - break - if bag is not None: - if ftype != bag_key: - entry = None - break - entry[bag_key] = bag - # ⭐⭐ 2026-08-10 — the FORMULA expression rides this door too, for the fourth time in the - # same seam (`pinned`, `code`, `link`/`rollup`, now this). A formula column created through - # `create()`/`set_fields()` with its expression dropped is a permanently blank column that - # renders, sorts and filters — the exact silent shape the paragraph above describes. - if entry is not None: - fx = (_ag_formula(f.get('formula')) if f.get('formula') is not None else None) - if (ftype == 'formula') != (fx is not None): - entry = None - elif fx is not None: - entry['formula'] = fx - if entry is None: - seen.discard(key) - continue - # `agg` decides HOW the grid's totals row summarises the column. A rollup that declares it - # through one door and not the other summarises differently depending on who made it. - # ⭐ WAVE-29 C7: the vocabulary is `FIELD_AGGS`, not the single word `sum` — see its note. - agg_in = str(f.get('agg') or '').strip() - if agg_in in FIELD_AGGS: - entry['agg'] = agg_in - out.append(entry) - return out or None - - -def all_tables(st=None): - """{key: definition} — every user-created table in this tenant. - - ⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that - only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below. - """ - try: - return dict(_st(st).get(STORE_KEY) or {}) - except Exception: - return {} - - -#: The keys `all_defs` refuses to read. `rows` is 99.9% of tenant #0's document by bytes and is -#: touched by no permission check, no nav render and no workspace envelope (D-185, W32-T01). -DEF_ONLY_DROPS = ('rows',) - - -def all_defs(st=None): - """{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row. - - ⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for - every definition key at ~0.1% of the bytes. `/nav` is the caller that made it worth building: - `nav_entries`, `may_open` and the manage/canDelete/locked loop read `label`, `source`, - `createdBy` and `recordMode` — and nothing else. - - ⛔ **A DROPPED KEY RAISES, IT DOES NOT READ AS EMPTY.** `defn.get('rows')` on one of these - raises `KeyError` naming the projection, because the alternative — `{}` — is indistinguishable - from a database that genuinely has no rows, and the caller least able to notice is the one that - wanted rows. If you get that error you wanted `all_tables`. - - ⚠ Falls back to `all_tables` on ANY failure, including a store handle too old to project (a - fake store installed by a gate, a `_Lent` handle serving a document somebody else read). The - fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for - — which is the one direction a fallback here is allowed to be wrong in. - - ⛔⛔ W33-T01 (D-213) — **THAT PARAGRAPH WAS FALSE ON THIS FUNCTION'S OWN BODY UNTIL 2026-08-14, - AND THE LIE WAS THE DANGEROUS DIRECTION.** Only `AttributeError` fell back; every other - exception returned `{}` — the exact silent-empty answer the paragraph ABOVE it forbids, three - lines earlier, in bold. `/nav`, `/nav/prefs` and `routes_shares` would have rendered ZERO - databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]). - Nothing had noticed because the projection does not fail on a healthy store — a fallback is - dead code until the day it is the only code running. **Every failure now falls back to the - whole read**, which keeps `all_tables`' own `except: return {}` as the single backstop instead - of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on - this function, so an empty answer here stops being a blank nav and becomes a 404 on every - database in the tenant. - """ - try: - return dict(_st(st).get_projection(STORE_KEY, drop=DEF_ONLY_DROPS) or {}) - except Exception: - # ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was - # only ever ONE of the ways this can fail, and singling it out is what made the other ways - # silent. `all_tables` is correct for all of them. - return all_tables(st) - - -def get(table_key, st=None): - return all_tables(st).get(str(table_key)) - - -def is_user_table(table_key, st=None): - """THE permission predicate for row writes. Cheap and total: a key that is not in this store - is not a user table, so it cannot accept invented rows.""" - return str(table_key or '').startswith(KEY_PREFIX) and get(table_key, st) is not None - - -def create(label, username, fields=None, source=None, st=None): - """Create a blank table. Returns its key, or None if refused. - - Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) — - the caller is a UI form and a refusal is a message, not an exception. - """ - label = str(label or '').strip()[:MAX_LABEL] - if not label: - return None - existing = all_tables(st) - if len(existing) >= MAX_TABLES: - return None - key = KEY_PREFIX + _slug(label) - if key in existing: # a second table of the same name gets a suffix - for n in range(2, 50): - if f'{key}_{n}' not in existing: - key = f'{key}_{n}' - break - else: - return None - # A blank table still needs ONE column to be a table at all: a row with no fields cannot be - # displayed, edited or identified. 'name' is the identity column, the same role `customer` - # plays in the Customer table. - defn = { - 'key': key, - 'label': label, - 'source': (source if source in (BLANK_SOURCE, AUTOMATION_SOURCE) else BLANK_SOURCE), - 'createdBy': username, - 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), - 'fields': clean_fields(fields) or [{'key': 'name', 'label': 'Name', 'type': 'text', - 'source': 'overlay', 'default': True}], - 'rows': {}, - } - - def _add(cur): - cur[key] = defn - return cur - - _st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it - return key - - -def delete(table_key, st=None): - """Delete a user table AND its artifact families (wave 21, item 6a / C3). - - Wave 20 popped the definition alone and booked ten orphan families; the owner then ruled a - real Delete (R3) with the footprint disclosed in the confirm dialog first, so the cleanup - is no longer optional. Cleaned here, in one place: definition (`user_tables`) · the - workspace bucket (`_table_workspace`, every stratum incl. `__shared__` — views, - custom fields, overlays, folders) · cohorts (`_cohorts`) · record comments - (`_record_comments`) · docs metadata (`_docs`) AND the dataset BYTES each - metadata row names (D-36, closed wave 23 — wave 21 left those unreachable-but-present) · - `nav_meta[]` · share grants (the database grant plus a view grant per view that - lived in this bucket, via `shares.drop_objects`) · alert definitions on this topic. - Bound AUTOMATIONS are the API layer's to disable (platform must not import the engine). - Every family after the definition is best-effort: a store blip mid-sweep must not resurrect - the table, and an orphaned empty bucket is residue, not a leak.""" - key = str(table_key) - s = _st(st) - - def _drop(cur): - cur.pop(key, None) - return cur - s.update(STORE_KEY, _drop, flush='sync') - - ws_key = f'{key}_table_workspace' - view_ids = set() - try: - bucket = s.get(ws_key) or {} - for _u, ws in bucket.items(): - if isinstance(ws, dict): - view_ids |= set((ws.get('views') or {}).keys()) - except Exception: - pass - # ⭐ D-36 CLOSED (wave 23) — the document BYTES, not just their metadata. Wave 21 cleaned - # `_docs` and disclosed that the blobs under `docs//…` were left on the dataset; - # "unreachable" is not "deleted", and for a tenant asking us to delete a database the - # difference is the whole promise. Read the metadata FIRST (it carries each blob's exact - # stored `path`, so nothing is guessed from a naming convention that could drift), then - # clear the bucket, then delete the blobs. Best-effort per file: a storage blip must not - # resurrect the table or abort the rest of the sweep — an undeleted blob is residue we can - # sweep again, an aborted delete is a table the user asked to be gone. - doc_paths = [] - try: - for _pid, _rows in (s.get(f'{key}_docs') or {}).items(): - for _doc in (_rows or []): - p = str((_doc or {}).get('path') or '') - if p: - doc_paths.append(p) - except Exception: - pass - # ⚠ `__shared` JOINED THIS LIST IN WAVE 30 AND ITS ABSENCE WAS A REAL LEAK. The - # tenant-wide overlay (W30-T28) is a bucket like the four beside it, and the docstring above - # promises the artifact FAMILIES go with the database. Left out, a deleted database's shared - # columns and their values survived in the store — invisible, and resurrected wholesale if - # anybody ever re-created a database under the same key, which is exactly the reuse - # `drop_field` scrubs cells to prevent one column at a time. - try: - import core.shared_overlay as _so - shared_key = _so.bucket(key) - except Exception: # noqa: BLE001 - shared_key = f'{key}__shared' - for b in (ws_key, f'{key}_cohorts', f'{key}_record_comments', f'{key}_docs', shared_key): - try: - s.update(b, lambda cur: {}, flush='async') - except Exception: - pass - for p in doc_paths: - try: - import core.store as _store_bytes - _store_bytes.delete_path(p) - except Exception: - pass - - def _drop_meta(cur): - if isinstance(cur, dict): - cur.pop(key, None) - return cur - try: - s.update('nav_meta', _drop_meta, flush='async') - except Exception: - pass - try: - import core.shares as shares - shares.drop_objects([('database', key)] + [('view', v) for v in view_ids], st=s) - except Exception: - pass - try: - import core.alerts as alerts - alerts.drop_topic(key, st=s) - except Exception: - pass - - -def set_fields(table_key, fields, st=None): - """Replace a table's base-field contract (wave 18 — the automation engine grows tables it - creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless.""" - clean = clean_fields(fields) - if not clean or not is_user_table(table_key, st): - return False - - def _set(cur): - t = cur.get(str(table_key)) - if t is not None: - t['fields'] = clean - return cur - - _st(st).update(STORE_KEY, _set, flush='sync') - return True - - -def add_row(table_key, values=None, username=None, st=None, rid=None): - """Append one row. Returns the new row id, or None if refused. - - `rid` RESTORES A ROW UNDER ITS OLD ID (contract C-ADDROW / C-UNDO). Undo has to put a deleted - row back where it was: a restore under a fresh id would break every cohort, comment and view - filter that named the original, so the id is part of what is being undone. It is honoured - only when that id is FREE — an undo can never overwrite a row somebody has since created in - the gap, and it never invents a non-numeric id, because `scoped_pool` reads row ids as ints. - """ - if not is_user_table(table_key, st) or not records_mutable(table_key, st): - return None # ⚠ never on a connector-backed table - defn = get(table_key, st) or {} - rows = defn.get('rows') or {} - # R6: `row_limit` is None for a connected source. `add_row` cannot reach one anyway — the - # `records_mutable` guard above refuses first — but reading the ceiling through the ONE - # evaluator is what stops a later relaxation of that guard quietly re-imposing this cap. - cap = row_limit(table_key, st) - if cap is not None and len(rows) >= cap: - return None - want = str(rid or '').strip() - if want and want.isdigit() and want not in rows: - rid = want - else: - rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1) - valid = {f['key'] for f in (defn.get('fields') or [])} - clean = {k: str(v) for k, v in (values or {}).items() if k in valid} - # C3 (wave 25): a PROFILE cell is validated and normalised on the insert door too, not only - # on the edit door. A row born with `https://instagram.com/p/Cxyz` in its handle column is a - # row the enrich action can never answer, and the failure would otherwise surface as an - # automation that quietly returns nothing rather than as a refused write. - pf = _profile_of(defn.get('fields') or []) - if pf and pf['key'] in clean: - handle, ok = normalize_profile(clean[pf['key']], pf['profile'].get('source')) - if not ok: - return None - clean[pf['key']] = handle - - def _add(cur): - t = cur.get(str(table_key)) - if t is not None: - t.setdefault('rows', {})[rid] = clean - return cur - - _st(st).update(STORE_KEY, _add, flush='async') - # C3 (wave 22): a HUMAN-door insert is a row event — the record_created trigger's whole - # substrate. The engine's own writers never come through here, which is the loop law. - emit_row_event({'type': 'record_created', 'table': str(table_key), 'rowId': rid, - 'st': _st(st), 'user': str(username or '')}) - return rid - - -def add_rows(table_key, rows_in, username=None, st=None): - """⭐⭐ WAVE-29 T25 (owner item 6) — APPEND MANY ROWS IN ONE STORE WRITE. Returns the list of - new row ids, or None if the table refuses records at all. - - ⛔ WHY THIS EXISTS RATHER THAN A LOOP OVER `add_row`. Every `add_row` is a read-modify-write of - the WHOLE table document plus a row event; importing a 2,000-row spreadsheet that way is 2,000 - full-document copies under one lock, which is the same shape as the 1.4 s-per-bucket problem - item 20 exists to avoid — on the one uvicorn process this product runs. One `update()` writes - them all. - - ⛔ AND WHY NOT `automation_engine.upsert_rows`, which this module's own note (below) says to - reuse: that function is UPSERT-BY-KEY shaped, and v1 of the import door is deliberately - APPEND-ONLY (no key to merge on, no silent overwrite of a row somebody edited). It also lives - in the API layer, and `core/` may not import upward. When the import door grows an "update - matching rows" mode, the engine's is the one to lift — the note stands, it just is not this. - - ⚠ THE CAP IS CHECKED AGAINST THE WHOLE BATCH, not per row: a partial import that stops at - `MAX_ROWS` leaves the user reconciling a spreadsheet against a table, which is exactly what - the refusal-with-a-sentence exists to prevent. Nothing is written if the batch does not fit. - - ⚠ Ids continue the table's own sequence, and a PROFILE column validates every cell exactly as - `add_row` does — a row born with an unusable handle is a row the enrich action can never - answer, and the failure would otherwise surface as an automation quietly returning nothing. - """ - if not is_user_table(table_key, st) or not records_mutable(table_key, st): - return None - batch = [dict(r) for r in (rows_in or []) if isinstance(r, dict)] - if not batch: - return [] - defn = get(table_key, st) or {} - rows = defn.get('rows') or {} - cap = row_limit(table_key, st) # R6: None for a connected source - if cap is not None and len(rows) + len(batch) > cap: - return None - valid = {f['key'] for f in (defn.get('fields') or [])} - pf = _profile_of(defn.get('fields') or []) - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) - made = [] - for values in batch: - clean = {k: str(v) for k, v in values.items() if k in valid} - if pf and pf['key'] in clean: - handle, ok = normalize_profile(clean[pf['key']], pf['profile'].get('source')) - if not ok: - return None # atomic: one bad handle refuses the FILE - clean[pf['key']] = handle - next_id += 1 - made.append((str(next_id), clean)) - - def _add(cur): - t = cur.get(str(table_key)) - if t is not None: - bucket = t.setdefault('rows', {}) - for rid, clean in made: - bucket[rid] = clean - return cur - - _st(st).update(STORE_KEY, _add, flush='sync') - # ⚠ ONE EVENT PER ROW, deliberately: `record_created` is a trigger substrate, and an import of - # 500 rows genuinely IS 500 created records. The engine's own flood hold is what decides - # whether that runs an automation 500 times — a decision that belongs there, not to a writer - # that quietly emits less than it did. - for rid, _clean in made: - emit_row_event({'type': 'record_created', 'table': str(table_key), 'rowId': rid, - 'st': _st(st), 'user': str(username or '')}) - return [rid for rid, _ in made] - - -# ⛔ `upsert_rows` USED TO LIVE HERE, AND DELETING IT IS THE FIX (DEBT D-6, wave 20). -# -# There were TWO bulk-upsert implementations — this one and `automation_engine.upsert_rows` — -# with different counts (`{updated, inserted, orphans}` vs the engine's seven, including the -# `capped` count that makes a full table LOUD), a different signature (a dict keyed by value vs -# a list of rows) and a different cap story. D-6 called it a "silent drift risk"; the honest -# measurement is worse and simpler: **this one had zero callers, anywhere in the repo.** It was -# not drifting from the engine, it was a second answer nobody had ever asked. -# -# So there is now one implementation because there is one implementation — no parity gate to -# maintain, no second definition to keep in step. If a host-side bulk upsert is ever wanted, the -# engine's is the one that has been exercised (verify_automation section B) and it is PURE: -# `(existing, incoming, key_field, cap) -> (rows, counts)`, so it can be lifted here without its -# store half coming along. Do not re-derive a new one. - - -def set_label(table_key, label, st=None): - """Rename a table IN ITS DEFINITION. Returns the stored label, or None if refused. - - ⚠ WAVE 20, item 6a — THE RENAME USED TO WRITE ONLY `nav_meta`. That bucket is the nav's - display layer, so the rail showed the new name while everything reading the DEFINITION — - the automation editor's database picker above all — went on showing the old one. A rename - that only some surfaces can see is worse than no rename: the picker was not stale-looking, - it was confidently wrong, and a user choosing "Influencers" there could be choosing the - table they had renamed to something else months earlier. - """ - label = ' '.join(str(label or '').split())[:MAX_LABEL] - if not label or not is_user_table(table_key, st): - return None - - def _set(cur): - t = cur.get(str(table_key)) - if t is not None: - t['label'] = label - return cur - - _st(st).update(STORE_KEY, _set, flush='sync') - return label - - -# --------------------------------------------------------------------------------------------- -# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) -# --------------------------------------------------------------------------------------------- -# R2 reverses wave 17's "fields are per-user" law FOR THIS PATH: a `ut_*` table's fields are the -# TABLE'S SCHEMA, the way they are in Airtable — everyone with access to the database sees the -# same columns, and the creator or an admin edits them. Per-field `editRole` narrows or widens -# who may edit ONE column's definition without handing over the whole table. -# -# ⚠ `editRole` GOVERNS THE SCHEMA, NEVER THE VALUES. 'everyone' means anybody with access may -# rename this column or change its options; it does not decide who may type in its cells. Those -# are different questions and conflating them would let a column's own settings quietly become a -# data-permission system nobody wrote. - -#: Who may edit ONE field's definition. Fail-closed default: admins (= creator or admin). -FIELD_EDIT_ROLES = ('admins', 'everyone') - -#: Wave 22 (C7) — the metric-field vocabulary, MIRRORED from `automation_engine` as local -#: literals for the boot-path reason `MACHINE_OWNERS` states; the pair is held in step by a -#: verify_automation parity check, the same discipline. -METRIC_MEASURES = ('followers', 'avg_engagement', 'likes', 'comments') -METRIC_WINDOWS = ('latest', 'last_3_posts', 'last_7d', 'last_30d') -METRIC_PROFILE_MEASURES = ('followers', 'avg_engagement') -METRIC_AGGS = ('avg', 'sum', 'latest') - - -def _clean_metric(raw): - """One `metric` bag → the stored shape, or None (refused). A profile count over a - post-count window — or a SUM of a follower count — is a question the series cannot - answer; refused at write, never bent into a number that looks plausible (C7).""" - if not isinstance(raw, dict): - return None - measure = str(raw.get('measure') or '').strip() - window = str(raw.get('window') or '').strip() - agg = str(raw.get('agg') or '').strip() - if measure not in METRIC_MEASURES or window not in METRIC_WINDOWS: - return None - if measure in METRIC_PROFILE_MEASURES and (window == 'last_3_posts' or agg == 'sum'): - return None - if agg and agg not in METRIC_AGGS: - return None - out = {'source': 'ig', 'measure': measure, 'window': window} - if agg: - out['agg'] = agg - return out - - -# --------------------------------------------------------------------------------------------- -# ⭐⭐ THE RELATIONAL PAIR (2026-08-07, owner instruction) — `link` and `rollup` -# --------------------------------------------------------------------------------------------- -# Owner: *"adding relational database function with links and rollups… exactly like how Airtable -# does it… and this rollup needs to have formula that we can use to calculate things like average -# Views over last N posts."* -# -# Airtable's own contract, fetched 2026-08-07 (SPEC): -# multipleRecordLinks.options : linkedTableId · isReversed · prefersSingleRecordLink · -# inverseLinkFieldId · viewIdForRecordSelection -# rollup.options : recordLinkFieldId · fieldIdInLinkedTable · result · -# referencedFieldIds · isValid -# 17 functions, and a rollup is exactly ONE HOP. -# -# ⛔ THE CELL IS A SCALAR, WHICH IS NOT A PREFERENCE. Airtable's link cell is an ARRAY of record -# ids; the `Row` contract here is scalars end to end and `grid_events` refuses a non-scalar write. -# So a link cell holds a COMMA-JOINED list of linked row ids — the shape `multiselect` already -# uses for a set, so the grouping/filter/copy paths already know what to do with it. -# -# ⭐ THREE MODES, ONE KIND: -# `on` DECLARED -> a DERIVED link. The linked rows are those whose `on` column equals this -# row's `from` column. Machine-maintained, read-only, recomputed on the same -# pass that already recomputes `metric` cells. THIS IS THE INSTAGRAM CASE, and -# it is why the mode exists: `ut_ig_posts.influencer_key` ALREADY holds the -# relation and is rewritten by the engine on every pull, so a second -# user-editable copy of the same relation could only ever drift away from it. -# `on` ABSENT -> an ORDINARY link. The user picks records; the cell is editable; `single` is -# Airtable's `prefersSingleRecordLink`. Its value is the source of truth. -# `inverse` -> the COMPUTED reciprocal of one ordinary source link. It never fans writes -# into target rows: the relation pass derives it from the source cells, so the -# same shared-row truth powers both sides and every dependent Rollup. - -#: How many linked ids one link cell may carry. Row ids are short numeric strings, so 500 ids is -#: ~3 KB — comfortably inside a text cell, and `MAX_ROWS` bounds the absolute worst case anyway. -LINK_MAX_IDS = 500 - -#: Airtable's 17 aggregations, minus the two that are meaningless over scalar cells. -#: ⛔ `arrayflatten` and `arrayslice` are NOT here: both operate on NESTED arrays, and a cell in -#: this product is a scalar — they could only ever return their own input, so offering them would -#: mint columns that compute nothing. `arrayslice`'s real use ("just the first N") is what `limit` -#: does below, honestly and by declared rank. -#: ⭐ `stdev` is the ONE addition beyond Airtable's list (wave 28, owner ruling R4), and it is here -#: for a request no combination of the other fourteen can express: *"the average of the last 10 -#: posts, with anything beyond 2 standard deviations removed"*. `average` alone cannot say which -#: rows are outliers, and a client that computed the threshold itself would be a second definition -#: of the same statistic over a row set only the server can see (the fold receives the UNCAPPED -#: linked set; a link CELL shows at most `LINK_MAX_IDS`). -#: ⚠ SAMPLE standard deviation (n-1) — see `_rollup_fold`. The population form (n) is the wrong -#: default here: a rollup folds the rows that happen to be linked, which is a sample of an -#: account's posting history, not its entirety. -ROLLUP_FNS = ('sum', 'average', 'stdev', 'min', 'max', 'latest', 'count', 'counta', 'countall', - 'and', 'or', 'xor', 'concatenate', 'arrayjoin', 'arraycompact', 'arrayunique') -ROLLUP_SORT_DIRS = ('asc', 'desc') -ROLLUP_CONDITION_OPS = ('eq', 'neq', 'contains', 'not_contains', 'is_empty', 'is_not_empty', - 'gt', 'gte', 'lt', 'lte') -ROLLUP_CONDITION_CONJ = ('and', 'or') -ROLLUP_MAX_CONDITIONS = 20 - -#: ⭐⭐ THE SET-STATISTIC THRESHOLD (wave 28, R4 — contract C1). A condition may compare a field -#: against a LITERAL (`value`) or against a statistic OF THE SCOPED SET ITSELF -#: (`ref: {sigmas: k}` → mean + k*stdev of that same field). It is what makes "beyond 2 sigma" -#: expressible as configuration instead of code, and it composes with everything already here: -#: the scope (`sortBy` + `limit`) picks the window, THEN the threshold is computed over that -#: window, THEN the conditions filter it. -#: -#: ⛔ ORDER IS THE WHOLE CONTRACT AND IT IS NOT NEGOTIABLE: the statistic is computed over the -#: SCOPED set (after ranking and `limit`, before filtering). Computing it over the unfiltered -#: table would answer "2 sigma of everything this account ever posted" while the column says "of -#: the last 10", and both readings produce a plausible number — which is the failure this module -#: refuses everywhere else. -#: -#: ⛔ ORDERING OPS ONLY. `eq`/`neq` against a computed float is a coin flip on floating-point -#: representation, and `contains` against a number is meaningless — offering either would mint a -#: condition that silently never matches. -ROLLUP_REF_OPS = ('gt', 'gte', 'lt', 'lte') -#: A sanity bound, not a statistical one. Beyond ±10 sigma nothing is being selected on any real -#: distribution, so a value out here is a typo or a unit mistake, and refusing it is kinder than -#: storing a filter that can only ever match nothing. ⭐ It also does the NaN/infinity work for -#: free: `not (-10 <= NaN <= 10)` is True, and infinity fails the same comparison. -ROLLUP_MAX_SIGMAS = 10.0 - -#: The date-window KINDS a source-backed rollup may name. MIRRORS `harness/windows.py`'s -#: `WINDOW_KINDS`, deliberately as a local literal — `core` stays dependency-light and does not -#: import up, the same reason `UT_FIELD_TYPES` is a literal rather than an `aios_grid` import. -#: ⚠ This list VALIDATES; it never RESOLVES. `harness.windows.resolve(spec, today)` owns turning a -#: kind into a date pair, so there is exactly one implementation of what "ytd" means and this -#: module cannot drift into a second one. A kind added there and not here is simply not offerable -#: on a rollup yet — the fail-closed direction. -ROLLUP_SOURCE_WINDOWS = ( - 'all_time', 'today', 'yesterday', 'this_week', 'last_week', 'this_month', 'last_month', - 'this_quarter', 'last_quarter', 'this_year', 'last_year', 'ytd', 'ytd_last_year', - 'ltm', 'past_week', 'past_month', 'past_year', -) -#: ⛔ DELIBERATELY ABSENT: `last_n_days`, `next_n_days`, `custom`. Each needs a PARAMETER (an `n`, -#: or a date pair) that this bag has nowhere to carry, and a parameterised kind accepted without -#: its parameter would resolve to something arbitrary — a window that looks configured and -#: measures the wrong period. They land when the bag learns to carry the parameter, not before. -#: The ceiling on `limit`. Not a performance bound — `MAX_ROWS` is that — but a refusal to let a -#: column claim a window bigger than a table can hold. It therefore MOVES WITH the cap, which is -#: why it is written as `MAX_ROWS` and not as a number. -#: ⚠ THIS COMMENT SAID "100,000" UNTIL 2026-08-09 (wave 28) while sitting eight lines above -#: `ROLLUP_MAX_LIMIT = MAX_ROWS`, where `MAX_ROWS` is 60,000 — a doc contradicting the constant it -#: annotates, in the same screenful. Two sibling comments said it too (`odoo_relational`, -#: `rollup_sql`). 100,000 was a candidate cap REJECTED during its own derivation for clearing the -#: memory budget by 0.6%; the prose was written before the number lost. -#: ⛔ IT IS NOT `LINK_MAX_IDS`, and the near-miss is worth keeping. 500 looks like the real bound — -#: a rollup folds what a link names, and a link cell holds at most 500 ids. But that cap is a -#: DISPLAY cap: `compute_relation_cells` projects the first 500 ids into the cell and hands every -#: rollup the UNCAPPED `resolved` set, so a customer with 3,000 invoices shows 500 and still sums -#: all 3,000. Tightening this to 500 would have silently truncated the fold's declared window on -#: exactly the tables this wave exists to build ([[measure-the-real-call]]). -ROLLUP_MAX_LIMIT = MAX_ROWS - - -def _field_key(raw): - """A field/table key as this module spells them, or '' — the same normalisation - `_clean_field` applies to `key`, so a bag can never name a key the field layer would store - differently.""" - k = re.sub(r'[^a-z0-9_]+', '_', str(raw or '').strip().lower()).strip('_')[:60] - return k - - -def _clean_link(raw): - """One `link` bag → the stored shape, or None (refused). - - Same posture as `_clean_metric`: a mis-shaped bag REFUSES THE FIELD. A link column stored - without a usable target is a column promising a relation that nothing can ever resolve. - """ - if not isinstance(raw, dict): - return None - table = _field_key(raw.get('table')) - # ⛔ It must NAME A USER TABLE. A link into a connector-backed module key (`customer`, - # `product`) would promise a join across two strata with different row-id namespaces and - # different permission walls — a different feature, not a smaller version of this one. - if not table.startswith(KEY_PREFIX) or len(table) <= len(KEY_PREFIX): - return None - out = {'table': table} - on = _field_key(raw.get('on')) - frm = _field_key(raw.get('from')) - inverse = _field_key(raw.get('inverse')) - reciprocal = _field_key(raw.get('reciprocal')) - if inverse: - # Airtable's reciprocal field: rows in `table` whose ordinary `inverse` link includes - # this row. It is computed, so `on`/`from` cannot simultaneously configure another join. - if on or frm: - return None - out['inverse'] = inverse - elif on: - out['on'] = on - # `from` is OPTIONAL and resolved at compute time (the profile-flagged column, then the - # pinned one) — which is what makes an Instagram database link up with no configuration - # at all, i.e. the "automatically" in the instruction. - if frm: - out['from'] = frm - elif frm: - # ⛔ A `from` with no `on` configures NOTHING — there is no join to drive. Refused rather - # than dropped: a bag half of which is silently ignored is [[wrong-parent-not-broken-control]] - # with the control still on screen. - return None - if reciprocal: - out['reciprocal'] = reciprocal - if raw.get('single') is True: - out['single'] = True - return out - - -def _clean_rollup(raw): - """One `rollup` bag → the stored shape, or None (refused). - - ⭐ `limit` + `sortBy` ARE THE DELIBERATE SUPERSET OF AIRTABLE, and they are the owner's - headline: *"average Views over last N posts"*. Airtable's rollup conditions filter by - PREDICATE, never by RANK, so "the last 12" is not expressible there at all. - - ⛔ `limit` REQUIRES `sortBy`, and that refusal is the whole care in this function. "The last - N" with no declared order is not a measurement — it is whichever N rows happen to sit first - in the store, which for an append table is capture order and for a merged one is nothing at - all. A wrong number that looks right is the failure this module refuses everywhere else. - """ - if not isinstance(raw, dict): - return None - # ⭐ A SOURCE-BACKED ROLLUP — the read-through kind (owner 2026-08-09: *"make rollup be able to - # capture up to 1 million rows of a linked database … ok to push rollup filter etc to SQL"*). - # - # ⛔ WHY IT IS A DIFFERENT SHAPE RATHER THAN A BIGGER `link`. A linked rollup folds ROWS THAT - # EXIST IN THE STORE, and `MAX_ROWS` bounds them because every `ut_*` table lives in one JSON - # blob that is parsed and copied per request. ⚠ RAISING THE CAP TO 60,000 DOES NOT RETIRE THIS - # KIND, and the arithmetic is why: 256,810 order lines are 63.9 MB and 2.57 s PER COPY, so they - # cannot live there under any JSON substrate — 4.3x the cap and 8x a tolerable copy. This kind - # never COPIES the rows at all: it names a governed semantic TOPIC and a METRIC KEY, and one - # grouped SQL query answers every parent row at once (MEASURED: 1,748 customers over 256,810 - # lines in 232 ms). - # - # ⚠ A METRIC KEY, NEVER A FILTER FRAGMENT, and that is the load-bearing decision. `model/ - # metrics/*.yml` already carries each metric's scope, its `store_filter_sql` AND the matching - # `live_domain` — whose own comment says *"BOTH or store_parity compares two different - # questions"*. Binding to the key inherits the scope and the live-parity oracle; letting a - # rollup carry SQL would mint a second definition of a number the semantic layer exists to - # define once. - # - # ⚠ `window` is a date-window KIND (`ytd`, `this_month`, …) resolved against `today` at - # COMPUTE time, never a literal date pair — a hand-computed year start is correct until - # 1 January and wrong after it, with nothing to notice ([[date-window-vocabulary]]). - src = raw.get('source') - if isinstance(src, dict): - topic = _field_key(src.get('topic')) - measure = _field_key(src.get('measure')) - group_by = _field_key(src.get('groupBy')) - on = _field_key(src.get('on')) - if not (topic and measure and group_by and on): - return None - bag = {'topic': topic, 'measure': measure, 'groupBy': group_by, 'on': on} - window = str(src.get('window') or '').strip().lower() - if window: - if window not in ROLLUP_SOURCE_WINDOWS: - return None - bag['window'] = window - return {'source': bag} - link = _field_key(raw.get('link')) - field = _field_key(raw.get('field')) - fn = str(raw.get('fn') or '').strip().lower() - if not link or fn not in ROLLUP_FNS: - return None - # `countall` counts LINKED RECORDS, so it alone needs no field to read (Airtable's rule). - if not field and fn != 'countall': - return None - out = {'link': link, 'fn': fn} - if field: - out['field'] = field - try: - limit = int(str(raw.get('limit') or '0').strip() or 0) - except (TypeError, ValueError): - return None - sort_by = _field_key(raw.get('sortBy')) - distinct_by = _field_key(raw.get('distinctBy')) - sort_dir = str(raw.get('sortDir') or '').strip().lower() - if limit < 0 or limit > ROLLUP_MAX_LIMIT: - return None - if (limit or fn == 'latest') and not sort_by: - return None - if sort_dir and sort_dir not in ROLLUP_SORT_DIRS: - return None - if sort_by: - out['sortBy'] = sort_by - # Newest-first is the default because every window this feature exists for is "the LAST - # N", and a default that silently means "the first N" would answer a different question - # with the same column name. - out['sortDir'] = sort_dir or 'desc' - if limit: - out['limit'] = limit - elif sort_dir: - return None - if distinct_by: - # A rollup may cross a table that already contains duplicate logical records. Keep the - # first row after ranking for each identity so "last 12 posts" means twelve POSTS, while - # a snapshot table can still keep every timestamped observation by omitting this option. - out['distinctBy'] = distinct_by - # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, and it is a SECOND list rather than a flag on the - # first. Owner: *"instead of last 12 posts, we also want to make it so its last N record, - # where the record's Status is video."* - # - # ⛔ WHY TWO LISTS AND NOT ONE. `conditions` is applied AFTER the window (C1 fixed that order - # deliberately, so a `ref: {sigmas}` threshold is a statistic OF the rows being folded). That - # makes "the last 10 REELS" inexpressible with it: the window would take the last 10 posts of - # any kind and the filter would then keep whichever of those happened to be video — on this - # tenant's live data, 12 captured posts per profile of which ~8 are video, so the column would - # silently answer "the reels among the last 12" and wear the name "the last 10 reels". - # `where` runs BEFORE the ranking, so the window is spent on rows that already qualify. - # - # ⛔ `ref` IS REFUSED HERE, and that refusal is what keeps C1 coherent. A sigma computed over - # the pre-filtered-but-unwindowed set is a statistic of a DIFFERENT set from the one the fold - # reports on — the exact incoherence C1's own note describes ("the set being described and the - # set doing the describing would be different sets"). Fail closed: the threshold belongs in - # the post-filter, where the window is already fixed. - for bag_key, conj_key, allow_ref in (('where', 'whereConj', False), - ('conditions', 'conditionConj', True)): - conditions = [] - raw_conditions = raw.get(bag_key) or [] - if not isinstance(raw_conditions, list) or len(raw_conditions) > ROLLUP_MAX_CONDITIONS: - return None - for condition in raw_conditions: - if not isinstance(condition, dict): - return None - condition_field = _field_key(condition.get('field')) - op = str(condition.get('op') or '').strip().lower() - if not condition_field or op not in ROLLUP_CONDITION_OPS: - return None - item = {'field': condition_field, 'op': op} - if op not in ('is_empty', 'is_not_empty'): - ref = condition.get('ref') - if ref is not None: - # ⭐ THE SET-STATISTIC THRESHOLD (R4 / contract C1). `ref` replaces `value`; - # the number this leaf compares against is computed by the FOLD, over the - # scoped set, at compute time — so it moves with the data instead of freezing - # a threshold that was true the day somebody typed it. - # ⛔ NEVER BOTH. A leaf carrying `value` AND `ref` has two answers to one - # question and whichever the fold happened to read would be silent. Refuse - # the field. - if not allow_ref: - return None - if not isinstance(ref, dict) or condition.get('value') is not None: - return None - if op not in ROLLUP_REF_OPS: - return None - sigmas = ref.get('sigmas') - # `bool` is an `int` subclass, so `True` would otherwise validate as 1.0 - # sigma — a checkbox arriving in a numeric slot should be refused, not - # interpreted. - if isinstance(sigmas, bool) or not isinstance(sigmas, (int, float)): - return None - sigmas = float(sigmas) - if not (-ROLLUP_MAX_SIGMAS <= sigmas <= ROLLUP_MAX_SIGMAS): - return None # also catches NaN and +/-inf: every comparison is False - item['ref'] = {'sigmas': sigmas} - else: - value = condition.get('value') - if not isinstance(value, (str, int, float, bool)): - return None - item['value'] = str(value)[:1000] - conditions.append(item) - if conditions: - conj = str(raw.get(conj_key) or 'and').strip().lower() - if conj not in ROLLUP_CONDITION_CONJ: - return None - out[bag_key] = conditions - out[conj_key] = conj - return out - - -def _clean_field(raw, previous=None): - """One field dict → the stored shape, or None. The single validator for create AND patch, so - a column cannot be typed one way on the way in and another on the way back.""" - prev = previous or {} - raw = raw if isinstance(raw, dict) else {} - label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80] - key = re.sub(r'[^a-z0-9_]+', '_', - str(raw.get('key') or prev.get('key') or _slug(label)).strip().lower()) - key = key.strip('_')[:60] - ftype = str(raw.get('type') or prev.get('type') or 'text').strip().lower() - if not key or not label or ftype not in UT_FIELD_TYPES: - return None - out = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'} - # ⭐ 2026-08-10 — the DISPLAY format, on the add/patch door too. ⚠ `'format' in raw` rather than - # `raw.get('format')`, exactly like `description` below: an explicit `{}`/None CLEARS the - # format, while omitting the key keeps what is stored. A PATCH that sends only a label must not - # silently strip a formatting choice — that is how `agg` stopped totalling a money column. - fmt = _clean_format(raw.get('format') if 'format' in raw else prev.get('format')) - if fmt: - out['format'] = fmt - # Canonical descriptions are schema metadata, not the user's per-view note. Preserve them - # through every field write so product-owned presets can explain themselves in the header - # tooltip and schema drawer. An explicit blank clears a user-created field's description; - # omission keeps the previous declaration. - description_raw = (raw.get('description') if 'description' in raw - else prev.get('description')) - description = ' '.join(str(description_raw or '').split())[:300] - if description: - out['description'] = description - opts_raw = raw.get('options') if 'options' in raw else prev.get('options') - if ftype in ('select', 'multiselect'): - opts = [' '.join(str(o).split())[:60] for o in (opts_raw or []) if str(o).strip()][:50] - if opts: - out['options'] = opts - # ⭐⭐ WAVE-29 CONTRACT C1 (owner item 13) — **THE OPTION COLOURS, WHICH THIS DOOR DROPPED.** - # - # The client has carried `optionColors`/`colorCodeOptions` since wave 14 (`types.ts`, the - # `BRAND_SWATCHES` editor, `cells.optionTint`, the kanban card) and `aios_grid._clean_field` - # keeps both — but THIS validator, the one every `ut_*` create and patch goes through, had - # zero occurrences of either key. So the owner picked a colour on a user database, the pane - # saved, and the colour was gone on the next read: the same crossed-validator shape as item 2a, - # in a different pair of doors. `grid_events._FIELD_ECHO_PROPS` already carried both, so the - # echo wire was plumbed for a value nothing stored. - # - # ⛔ SIBLING KEYS, and `options` IS NOT RESHAPED. Every consumer treats options as `string[]` - # (this function flattens each to a bare `str` twelve lines up); folding colour into the option - # entries would be a wire change across the grid, the kanban, the editor and the filter panel. - # - # ⛔ OMIT-WHEN-ABSENT, exactly like `format`/`metric`/`link`, because `verify_api.py` and - # `verify_automation.py` assert `_clean_field(f) == f` over every engine-seeded field list — - # a branch that emitted `optionColors: {}` would redden both gates for every preset column in - # the product. ([[default-must-pass-its-own-guard]]) - # - # ⚠ INHERITED ACROSS A PATCH, for `agg`'s reason: a PATCH that sends only a label must not - # silently strip the colours somebody chose, which is precisely how a money column stopped - # totalling. - # - # ⚠ A HEX IS VALIDATED; BRAND MEMBERSHIP IS NOT, and that is deliberate. `_clean_option_colors` - # (`aios_grid.py:200`) — the sibling this mirrors — accepts any `#RRGGBB`, and the client - # renders a stored colour through `choiceColors.brandedTint`, which maps it to its NEAREST - # brand swatch at DISPLAY time while leaving what is stored byte-stable. Refusing a non-brand - # hex here would refuse values the other validator accepts on the same UI, and would break a - # column that moved between the two. - if ftype in ('select', 'multiselect'): - colors_raw = (raw.get('optionColors') if 'optionColors' in raw - else prev.get('optionColors')) - colors = _clean_option_colors(colors_raw, out.get('options') or []) - if colors: - out['optionColors'] = colors - color_code = (raw.get('colorCodeOptions') if 'colorCodeOptions' in raw - else prev.get('colorCodeOptions')) - if isinstance(color_code, bool): - out['colorCodeOptions'] = color_code - # ⭐ WAVE-29 T22's own repair: a `rating` column's star count. Grid-created columns on a `ut_*` - # database now come through this door rather than the per-user overlay, and this validator had - # no `max` — so a 3-star or 10-star column silently became 5-star on the way in. Bounded rather - # than refused, mirroring `aios_grid._clean_rating_max`: the field still holds its 1..max - # integers whatever the number was, so an unusable value is a default and never a lost column. - if ftype == 'rating': - max_raw = raw.get('max') if 'max' in raw else prev.get('max') - if max_raw is not None: - try: - out['max'] = max(2, min(int(max_raw), 10)) - except (TypeError, ValueError): - pass - role = str(raw.get('editRole') or prev.get('editRole') or 'admins').strip().lower() - out['editRole'] = role if role in FIELD_EDIT_ROLES else 'admins' - if prev.get('default') is True or raw.get('default') is True: - out['default'] = True - # ⭐ 2026-08-07 (owner ruling) — THE PRIMARY COLUMN, DECLARED RATHER THAN POSITIONAL. - # - # The grid's identity column is `fields.find(f => f.pinned)?.key ?? fields[0]?.key` - # (`useGridColumns.reconcileOrder` / `permsModel.identityKey`), forced to position 0 and - # force-shown. No `ut_*` field had ever declared `pinned`, so the fallback was the whole rule: - # whichever column happened to be created FIRST became the locked primary forever. On a - # database somebody made by hand and later pointed at an Instagram automation, that is the - # `name` column `create()` mints — which nothing ever writes, so the table's identity column - # was permanently blank while `handle` sat six columns to the right. - # - # ⚠ STICKY ACROSS A PATCH, like `default` one line up: the engine declares it (the C1 preset - # set pins `handle`), and a client PATCH that simply omits the key must not silently unpin the - # column it is editing. Clearing it is deliberate work, not a side effect of renaming a header. - if prev.get('pinned') is True or raw.get('pinned') is True: - out['pinned'] = True - # ⭐ 2026-08-09 (wave 28) — `agg` DECIDES WHETHER THE GRID'S TOTALS ROW SUMS THE COLUMN, and - # this door silently dropped it while `clean_fields` (:402) kept it. Two validators, one - # question, opposite answers — D-91's exact class, found from the other end: the register's row - # described `clean_fields` as the lax door, and by the time it was read that half had been - # fixed while THIS one still diverged. - # ⛔ THE CONSEQUENCE IS QUIET, WHICH IS WHY IT SURVIVED. Every currency and money-rollup column - # in the four Odoo databases declares `agg: "sum"`; the spawn writes definitions straight to - # the store, so the totals row worked — until anyone PATCHED one of those fields (a rename, a - # width, a description), at which point the column silently stopped totalling and nothing - # anywhere went red. Caught by a gate leg written to assert the LINK bag survives this door, - # which found `agg` instead. - # ⚠ Sticky across a patch for the same reason as `default` and `pinned`: a client PATCH that - # omits the key must not un-total a column as a side effect of renaming its header. - # ⭐ WAVE-29 C7 — the vocabulary widened from the single word `sum` to `FIELD_AGGS`. The - # STICKINESS is unchanged and now applies to every name: a PATCH that omits `agg` keeps - # whatever the column had, because dropping it is how a money column silently stopped - # totalling. ⛔ An unknown name is DROPPED rather than stored — a summary nothing can compute - # would render blank under a picker that says otherwise. - # ⚠ `'agg' in raw`, not `raw.get('agg') or …` — the same idiom `format` and `description` use - # two screens up, and it is what makes the difference between OMITTING the key (keep what is - # stored: the wave-28 stickiness that stops a rename un-totalling a money column) and sending - # it EMPTY (clear the summary, which the picker's "None" row has to be able to do). Written - # the loose way, "None" would have been a no-op with the menu cheerfully showing the change. - agg_name = str((raw.get('agg') if 'agg' in raw else prev.get('agg')) or '').strip() - if agg_name in FIELD_AGGS: - out['agg'] = agg_name - # An automation column carries the binding the engine reads; it is preserved across a patch - # rather than re-declared, because the automation editor owns it and this route does not. - auto = raw.get('automation') if 'automation' in raw else prev.get('automation') - if isinstance(auto, dict): - out['automation'] = auto - # C7 (wave 22): a metric bag is validated or the FIELD is refused — a mis-shaped metric - # column stored anyway would be a column of invented blanks wearing a number's name. - metric_raw = raw.get('metric') if 'metric' in raw else prev.get('metric') - if metric_raw is not None: - m = _clean_metric(metric_raw) - if m is None: - return None - out['metric'] = m - # C3 (wave 25, R7): the PROFILE FLAG. Same posture as the metric bag one line up — a - # mis-shaped flag refuses the FIELD, and an explicit `profile: None` CLEARS it, so the flag - # can be taken off a column without deleting the column. ⚠ `type: 'text'` is enforced here - # rather than left to the caller: R7 is *"a flag on an ordinary text field"*, and a flag on - # a select/int would promise a validated handle to a cell this module never validates. - profile_raw = raw.get('profile') if 'profile' in raw else prev.get('profile') - if profile_raw is not None: - p = _clean_profile(profile_raw) - if p is None or ftype != 'text': - return None - out['profile'] = p - # ⭐ 2026-08-07 — the relational pair. Same posture as the two bags above: a mis-shaped bag - # refuses the FIELD, and the bag is required for its own type. ⛔ THE TYPE/BAG PAIRING IS - # ENFORCED BOTH WAYS — a `link` with no `link` bag is a column that can never resolve a row, - # and a `link` bag on an `int` column is a declaration nothing reads. Either half alone is the - # silent kind of broken: the column exists, renders, and means nothing. - # ⚠ AN INHERITED BAG IS ONLY INHERITED WHILE THE TYPE STILL WANTS IT (2026-08-09, owner: - # *"everything is custom and changeable always"*). `prev`'s bag used to be picked up on ANY - # patch, so retyping a rollup column to text sent `{'type': 'text'}`, inherited the rollup bag - # from `prev`, hit the `ftype != 'rollup'` arm and returned None — i.e. the server refused to - # let a rollup become anything else, with a sentence about the name and type. Changing the - # type is the one act that MEANS "drop this bag", so a mismatched inherited bag is dropped. - # ⛔ AN EXPLICIT bag on a mismatched type STILL REFUSES: that is a caller declaring something - # nothing will ever read, which is the silent-broken half this pairing exists to catch. - prev_type = str(prev.get('type') or '').strip().lower() - link_raw = (raw.get('link') if 'link' in raw - else (prev.get('link') if prev_type == ftype else None)) - if link_raw is not None: - lk = _clean_link(link_raw) - if lk is None or ftype != 'link': - return None - out['link'] = lk - elif ftype == 'link': - return None - rollup_raw = (raw.get('rollup') if 'rollup' in raw - else (prev.get('rollup') if prev_type == ftype else None)) - if rollup_raw is not None: - rl = _clean_rollup(rollup_raw) - if rl is None or ftype != 'rollup': - return None - out['rollup'] = rl - elif ftype == 'rollup': - return None - # ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` column's language bag. - # - # ⚠ Deliberately the WEAKEST posture of the four bags above, and the asymmetry is the whole - # point rather than an inconsistency. `metric`/`link`/`rollup` bags DEFINE what their column - # is: without one the cell can never resolve a value, so a missing or mis-shaped bag refuses - # the field. A `language` chooses a HIGHLIGHTER over a string the column already holds - # perfectly well — so it is optional (an unconfigured code column renders plain), it survives - # a patch that omits it (the `automation` rule: the gear owns it, a header rename must not - # clear it), and an unknown language degrades to `plain` instead of destroying the column. - # R13's "NO execution engine" is why this can be so relaxed: the value selects a renderer and - # can never select an interpreter. - # ⚠ Function-local import, the convention `grid_events` already uses for the same module - # (`import aios_grid as _ag2`, five call sites): `core` must not import the layer above it at - # MODULE level. One vocabulary, one validator — declaring the language list a second time - # here is exactly the drift the fields contract exists to catch. - # ⚠ OMIT = KEEP, `plain` = CLEAR. `_clean_code` returns None for plain (see its docstring), - # so an explicit `{'language': 'plain'}` from the editor drops the key here while a patch - # that never mentions `code` inherits the previous bag. Without that split, switching a - # column back to Plain text would be a control that silently does not save. - code_raw = raw.get('code') if 'code' in raw else prev.get('code') - if code_raw is not None and ftype == 'code': - import aios_grid as _agc - c = _agc._clean_code(code_raw) - if c: - out['code'] = c - # ⭐⭐ 2026-08-10 — THE FORMULA EXPRESSION. Posture: `link`/`rollup`'s, not `code`'s — a - # formula column with no expression is a column that can never show a value, so the pairing is - # enforced BOTH WAYS and either half alone refuses the field. - # - # ⚠ `valid_keys` IS DELIBERATELY OMITTED, and it is the one real asymmetry with the overlay - # door. `grid_events` passes the workspace's own key set, so a `{ref}` naming no column is - # refused at write time there. This function is handed ONE field with no sibling context — - # `add_field`/`patch_field` could supply the table's keys, but `clean_fields` validates a list - # mid-construction where half the referenced columns do not exist yet, and `clean_machine_ - # fields` judges a list that has no table at all. A second, weaker copy of the ref rule in one - # of the three would be the divergence this module keeps paying for. STRUCTURE is checked here - # (charset, length, balanced parens and quotes, well-formed refs); an unresolvable ref blanks - # the CELL client-side, which is `_clean_formula`'s own documented read-time behaviour and the - # same answer a later-deleted column already produces. - # ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import. - formula_raw = (raw.get('formula') if 'formula' in raw - else (prev.get('formula') if prev_type == ftype else None)) - if formula_raw is not None: - fx = _ag_formula(formula_raw) - if fx is None or ftype != 'formula': - return None - out['formula'] = fx - elif ftype == 'formula': - return None - return out - - -def is_derived_link(field): - """Is this a link whose cell the ENGINE owns? (an `on` join, not a user-picked set) - - ⛔ ONE definition, read by the write wall, the compute pass and the wire. The alternative is - three places independently asking "does the bag have an `on`", which is how one of them - silently keeps taking writes after the other two stop. - """ - lk = (field or {}).get('link') - return isinstance(lk, dict) and bool(lk.get('on') or lk.get('inverse')) - - -def is_computed_cell(field): - """Is this field's cell computed by the SERVER, so a human write is an invented value? - - The `ut_*` twin of `aios_grid.READONLY_CUSTOM_TYPES`, and deliberately a PREDICATE rather - than a type set: a `link` is read-only when it is derived and editable when it is not, which - a set of type names cannot express. `grid_events` is the wall that reads it. - - ⭐⭐ 2026-08-10 — `formula` JOINS, and it is the one member here whose value the server does - NOT compute. Every other member is refused because the server owns the number; a formula cell - is refused because NOBODY owns a stored number — the browser recomputes the column on every - paint from the row's other cells, so a value written here is overwritten on screen before it - is ever read and sits in the store as a fossil no reader agrees with. The client already - declines to offer an editor (`READONLY_CELL_TYPES`) and declines the paste - (`coerceClipboardValue`); both are courtesy, and this is the wall - ([[schema-role-is-not-a-value-wall]] — a courtesy in front of no wall is the shape that - keeps shipping). - """ - f = field or {} - return (isinstance(f.get('metric'), dict) or f.get('type') in ('rollup', 'formula') - or is_derived_link(f)) - - -#: The numeric family and the date family, named once. Membership decides which cells the -#: refusal below can speak about at all — a `text` column takes any string by definition. -_NUMERIC_CELL_TYPES = ('int', 'currency', 'pct') -_DATE_CELL_TYPES = ('date',) - - -def cell_type_refusal(field, value): - """Why this value cannot go in this column, or None if it can. ⛔ REFUSES; never rewrites. - - ⭐⭐ W29-T81 — THE SERVER-SIDE HALF of "can this text be this field's value". Until now the - only evaluator was `coerceClipboardValue`, which lives in the BROWSER — so the import door - took `{"qa_count": "seventeen-ish"}` at an `int` column, answered `201 {"imported": 1}`, and - stored it verbatim. Measured against production on `bac40c2`. A rule enforced only by the - client is a rule that holds for exactly one client ([[one-evaluator-per-question]]). - - ⚠ DELIBERATELY NARROWER THAN THE CLIENT'S COERCER, and it must stay that way: this asks only - whether a NUMBER is a number and a DATE is a date, the two families where an unusable string - is later rendered as a fabricated figure. It accepts everything the client's coercer emits - (canonical ISO dates, plain decimals) plus the human spellings that coercer also takes - ("$1,234.50"), so no UI path can trip a wall the UI cannot see. Choice vocabularies stay - client-side for now: their options change under a stored row, and refusing an import against - a vocabulary edited yesterday would reject data that is merely out of date. - - ⚠ BLANK IS ALWAYS ACCEPTABLE. An empty cell is an empty cell in every column; the import - plan already omits them, and refusing one would make a ragged spreadsheet unimportable. - """ - f = field or {} - ftype = f.get('type') - text = '' if value is None else str(value).strip() - if not text: - return None - label = f.get('label') or f.get('key') or 'that column' - if ftype in _NUMERIC_CELL_TYPES: - try: - float(text.replace('$', '').replace(',', '').replace('%', '').replace(' ', '')) - except (TypeError, ValueError): - return f'{text[:40]!r} is not a number, and {label!r} holds numbers' - return None - if ftype == 'rating': - try: - n = int(text) - except (TypeError, ValueError): - return f'{text[:40]!r} is not a whole number, and {label!r} holds a rating' - top = f.get('max') if isinstance(f.get('max'), int) and f.get('max') else 5 - if not 1 <= n <= top: - return f'{label!r} takes a rating from 1 to {top}, and {text[:40]!r} is outside it' - return None - if ftype in _DATE_CELL_TYPES: - # ISO-prefixed, which is exactly what the client's coercer produces and what every - # reader of a stored date assumes. A month name is not refused as invalid English — it - # is refused because nothing downstream would parse it either. - if not re.match(r'^\d{4}-\d{2}-\d{2}', text): - return (f'{text[:40]!r} is not a date, and {label!r} holds dates ' - f'(write it as YYYY-MM-DD)') - return None - return None - - -#: ⭐⭐ 2026-08-09 (owner) — THE STAMP THAT SURVIVES A RECONCILER. -#: -#: Owner: *"No rollup field should be uneditable, everything is custom and changeable always."* -#: Unlocking `may_edit_field` alone would have been a lie with a delay on it: BOTH preset -#: reconcilers (`automation_engine._reconcile_ig_graph_fields` and `odoo_relational`'s forward -#: migration) OVERWRITE every contract key — `label`, `type`, `link`, `rollup` — on any field -#: still carrying `automation.preset`. So a reconfigured "Avg views · last 12 posts" would have -#: saved, rendered, recomputed, and then silently reverted to the shipped contract at the next -#: enrichment run or store resync. The edit would look like it worked, which is worse than a -#: refusal ([[flag-shipped-without-its-writer]]). -#: -#: ⛔ IT DOES NOT CLEAR `preset`, and that is deliberate. `preset` records PROVENANCE (this column -#: was spawned by the product, and `ut_ensure` re-stamps it on every run anyway — clearing it -#: would be undone within one tick). This stamp records CUSTODY: a human has taken this column -#: over, so the machine stops rewriting its definition while still owning its VALUES. -USER_EDITED_KEY = 'userEdited' - - -def user_edited(field): - """Has a human taken over this preset column's DEFINITION? (the reconcilers' skip signal)""" - auto = (field or {}).get('automation') - return isinstance(auto, dict) and auto.get(USER_EDITED_KEY) is True - - -def preset_editable(field): - """Does the pre-set lock let this column's DEFINITION be changed anyway? - - ⛔ ONE PREDICATE FOR ONE QUESTION, read by all three walls that enforce the lock - (`may_edit_field` here, `routes_tables._field_or_refuse`, and the client's `schemaLocked`). - Three independent copies of "is this preset?" is how one of them keeps refusing after the - other two stop — the shape D-107 wore, and the reason `is_derived_link` above is a function - rather than an expression repeated three times. - - Today it answers YES for exactly one thing: a ROLLUP. The owner's ruling names rollups, and - the argument is specific to them rather than a general softening of the lock — a rollup holds - no data of its own. It is a QUESTION asked of other rows ("average views over the last N - posts"), it can be re-asked at any time, and re-asking it costs nothing because the answer is - recomputed from the authoritative store on the next pass. Retyping a preset `followers` - column, by contrast, would strand real measurements in a column that can no longer read them. - """ - return str((field or {}).get('type') or '') == 'rollup' - - -def clean_machine_fields(fields): - """`(clean, refused)` — the SANCTIONED way for a machine writer to seed a table's schema - (wave 25, doc order step 2). `clean` is the list as this module would store it; `refused` - names every field that did not survive, so a caller can be LOUD rather than short. - - ⛔ WHY THIS EXISTS. `automation_engine.ut_ensure` writes field dicts STRAIGHT into the - `user_tables` bucket with `rt.update(UT_STORE_KEY, …)` — it never routes through this - module, so **the field layer's own validator has never judged a single column the engine has - ever spawned.** That is [[default-must-pass-its-own-guard]] with a live subject: the product - seeds fields against a law it does not run. Nothing has broken yet only because the drift is - one key wide and in the fail-closed direction (a field with no `editRole` reads as - not-`everyone`), which is luck, not design. - - ⚠ THIS IS THE "IMPOSSIBLE" HALF AND IT IS OPT-IN; the LOUD half is `verify_api`'s derived - section, which walks every field list the engine declares and refuses to let one through - that this validator would change in any way but adding `editRole`. A helper the caller may - ignore is not a control — the gate is the control, and it covers a list nobody remembered - to route through here. - """ - clean, refused = [], [] - for f in (fields or []): - got = _clean_field(f) - if got is None: - refused.append(str((f or {}).get('key') or f)) - else: - clean.append(got) - return clean, refused - - -def may_edit_field(table_key, fkey, viewer, is_admin=False, st=None): - """May `viewer` change THIS column's definition? Creator/admin always; others only when the - field itself says `editRole: 'everyone'`. Fail-closed on an unknown field.""" - table = get(table_key, st) or {} - field = next((f for f in (table.get('fields') or []) if f.get('key') == str(fkey)), None) - # Instagram's pre-set schema is product contract, not tenant configuration. Even an admin - # may sort/filter/hide it, but cannot rename, retype, duplicate or delete it. Ordinary fields - # on the same database remain user-owned, including user-created Links and Rollups. - if isinstance((field or {}).get('automation'), dict) \ - and field['automation'].get('preset') is True \ - and not preset_editable(field): - return False - if may_open(table_key, viewer, is_admin, st) and ( - bool(is_admin) or table.get('createdBy') == viewer): - return True - for f in (table.get('fields') or []): - if f.get('key') == str(fkey): - return f.get('editRole') == 'everyone' - return False - - -def flow_bound(bag, st=None): - """C8 (wave 22, owner item 5) — may this `automation` bag be STORED? Only when its - `flowId` names an automation definition that exists in this tenant's `automations` bucket. - Fail-closed both ways: no flowId is a column that silently never runs, and a flowId naming - a deleted flow is the same thing one delete later. (Read tolerance for pre-law fields - lives in `aios_grid._clean_automation` — this guards the WRITE doors only.)""" - if not isinstance(bag, dict): - return True # no bag, no law — an ordinary column - flow = str(bag.get('flowId') or '').strip() - if not flow: - return False - try: - return flow in (_st(st).get('automations') or {}) - except Exception: - return False - - -def add_field(table_key, raw, st=None): - """Append one column to the shared schema. Returns the stored field, or None if refused.""" - if not is_user_table(table_key, st): - return None - defn = get(table_key, st) or {} - have = [f for f in (defn.get('fields') or [])] - if len(have) >= MAX_FIELDS: - return None - field = _clean_field(raw) - if not field or any(f.get('key') == field['key'] for f in have): - return None - if not flow_bound(field.get('automation'), st): - return None # C8: no field without a flow (400 at the route) - # C3 (wave 25, R7's first mitigation): AT MOST ONE profile field per table. Without it the - # flag means nothing — "the profile column" stops being resolvable the moment there are two, - # and the enrich action would have to guess which handle it was pointed at. The route names - # the existing column in its refusal (`_refusal_sentence`); the LAW is here. - if field.get('profile') and _profile_of(have): - return None - - def _add(cur): - t = cur.get(str(table_key)) - if t is not None: - t.setdefault('fields', []).append(field) - return cur - - _st(st).update(STORE_KEY, _add, flush='sync') - return field - - -def _reciprocal_link_key(table_key, field_key): - digest = hashlib.sha1(f'{table_key}:{field_key}'.encode('utf-8')).hexdigest()[:10] - return f'linked_{digest}' - - -def sync_reciprocal_link(table_key, field_key, st=None): - """Create/repair Airtable's reciprocal link field for one ordinary link. - - The source cell stores picked target row ids. The reciprocal is a computed inverse link on - the target database; it lists source rows that include the current target id. Retargeting or - retyping the source removes the obsolete inverse in the same store update. - """ - table_key, field_key = str(table_key), str(field_key) - result = {'field': None, 'reciprocal': None} - - def _sync(cur): - cur = cur if isinstance(cur, dict) else {} - source = cur.get(table_key) or {} - source_field = next((f for f in (source.get('fields') or []) - if f.get('key') == field_key), None) - # Remove every old inverse for this source identity first. This makes retargeting and - # deleting deterministic instead of leaving a live-looking backlink on the old table. - for candidate in cur.values(): - if not isinstance(candidate, dict): - continue - candidate['fields'] = [f for f in (candidate.get('fields') or []) - if not (isinstance(f.get('link'), dict) - and f['link'].get('table') == table_key - and f['link'].get('inverse') == field_key)] - if not source_field or source_field.get('type') != 'link': - return cur - bag = dict(source_field.get('link') or {}) - if bag.get('on') or bag.get('inverse'): - return cur - target_key = str(bag.get('table') or '') - target = cur.get(target_key) - if target is None: - return cur - # ⛔ 2026-08-10 — THE COLUMN CAP APPLIES TO A COLUMN NOBODY ASKED FOR, TOO. - # - # `add_field` refuses at `MAX_FIELDS`; this function appended straight past it, and the - # field it appends lands on a table the user is not even looking at. The IG preset set is - # 41 columns against a cap of 60, so a Posts or Comments database is genuinely reachable — - # and the failure mode is the worst kind: the source link saves, the target silently grows - # its 61st column, and every LATER `add_field` on that table is refused with a sentence - # about a cap the user never crossed on purpose. - # ⚠ THE SOURCE LINK STILL SAVES. Refusing the whole relation because the OTHER database is - # full would be a refusal the user cannot act on from where they are standing; an ordinary - # link with no reciprocal is exactly what every link was before this function existed, and - # it still resolves its own cell. `reciprocal` is simply not stamped on the bag, so the - # next `sync` on a table with room creates it — this is a skip, not a tombstone. - if len(target.get('fields') or []) >= MAX_FIELDS: - return cur - reciprocal_key = _reciprocal_link_key(table_key, field_key) - reciprocal = _clean_field({ - 'key': reciprocal_key, - 'label': str(source.get('label') or table_key)[:80], - 'type': 'link', - 'link': {'table': table_key, 'inverse': field_key, - 'reciprocal': field_key}, - 'editRole': 'admins', - }) - if reciprocal is None: - return cur - target.setdefault('fields', []).append(reciprocal) - bag['reciprocal'] = reciprocal_key - source_field['link'] = bag - result['field'] = dict(source_field) - result['reciprocal'] = dict(reciprocal) - return cur - - _st(st).update(STORE_KEY, _sync, flush='sync') - return result - - -def patch_field(table_key, fkey, raw, st=None): - """Edit one column's definition IN PLACE. Returns the stored field, or None if refused. - - ⚠ THE KEY NEVER MOVES. A field's key is what every stored cell is filed under, so accepting - a new one here would orphan every value in the column while looking like a rename. The - LABEL is the renameable thing; the key is identity. - """ - if not is_user_table(table_key, st): - return None - fields = [dict(f) for f in ((get(table_key, st) or {}).get('fields') or [])] - idx = next((i for i, f in enumerate(fields) if f.get('key') == str(fkey)), -1) - if idx < 0: - return None - merged = dict(raw or {}) - merged['key'] = str(fkey) - field = _clean_field(merged, fields[idx]) - if not field: - return None - if 'automation' in merged and not flow_bound(field.get('automation'), st): - return None # C8: a patch cannot un-bind a column either - # C3: the one-per-table law on the PATCH door too — flagging a second column is the same act - # as adding one. `exclude=fkey` so re-saving the profile column does not collide with itself. - if field.get('profile') and _profile_of(fields, exclude=str(fkey)): - return None - # ⭐⭐ 2026-08-09 — THE CUSTODY STAMP IS WRITTEN HERE, at the ONE door a human reaches. - # - # `USER_EDITED_KEY`'s note has the why. It is written here rather than inside `_clean_field` - # because `_clean_field` is also the machine's validator (`clean_fields`, `clean_machine_ - # fields`): stamping there would mark the product's OWN spawn as user-edited and switch the - # reconcilers off for every preset column on the first tick, which is the inverse of the bug - # it exists to prevent. - # ⚠ Only when the definition ACTUALLY differs. A no-op save (open the editor, press Save) - # must not quietly take a column out of the product's contract. - if user_edited(fields[idx]) or (field != fields[idx] - and isinstance(fields[idx].get('automation'), dict) - and fields[idx]['automation'].get('preset') is True): - automation = dict(field.get('automation') or fields[idx].get('automation') or {}) - automation[USER_EDITED_KEY] = True - field['automation'] = automation - - def _set(cur): - t = cur.get(str(table_key)) - if t is not None: - for i, f in enumerate(t.get('fields') or []): - if f.get('key') == str(fkey): - t['fields'][i] = field - break - return cur - - _st(st).update(STORE_KEY, _set, flush='sync') - return field - - -def delete_field(table_key, fkey, st=None): - """Drop one column from the shared schema. Refuses to leave a table fieldless. - - ⚠ THE CELLS ARE LEFT IN THE ROWS ON PURPOSE. A deleted column whose values were also - scrubbed makes an accidental delete unrecoverable; the values are invisible without a field - declaring them, and re-adding the column with the same key brings them back. Same reasoning - as `routes_tables.delete_table` leaving the workspace bucket in place. Booked, not hidden. - """ - if not is_user_table(table_key, st): - return False - fields = (get(table_key, st) or {}).get('fields') or [] - if len(fields) <= 1 or not any(f.get('key') == str(fkey) for f in fields): - return False - doomed = next((f for f in fields if f.get('key') == str(fkey)), {}) - doomed_link = doomed.get('link') if isinstance(doomed.get('link'), dict) else {} - inverse_source = None - if doomed_link.get('inverse'): - source_table = str(doomed_link.get('table') or '') - source_field = str(doomed_link.get('inverse') or '') - source = get(source_table, st) or {} - if len(source.get('fields') or []) <= 1: - return False - inverse_source = (source_table, source_field) - - def _drop(cur): - t = cur.get(str(table_key)) - if t is not None: - t['fields'] = [f for f in (t.get('fields') or []) if f.get('key') != str(fkey)] - if inverse_source: - source_table, source_field = inverse_source - source = cur.get(source_table) - if isinstance(source, dict): - source['fields'] = [f for f in (source.get('fields') or []) - if f.get('key') != source_field] - # An ordinary link owns its reciprocal field. Delete the reciprocal in the same schema - # update so no target database can retain a live-looking backlink to a missing source. - for candidate in cur.values(): - if not isinstance(candidate, dict): - continue - candidate['fields'] = [f for f in (candidate.get('fields') or []) - if not (isinstance(f.get('link'), dict) - and ((f['link'].get('table') == str(table_key) - and f['link'].get('inverse') == str(fkey)) - or (inverse_source - and f['link'].get('table') == inverse_source[0] - and f['link'].get('inverse') == inverse_source[1])))] - return cur - - _st(st).update(STORE_KEY, _drop, flush='sync') - return True - - -def rename_choice_values(table_key, fkey, renames, st=None): - """Rename select/multiselect OPTIONS **and migrate every stored cell** (contract C-RENAME). - - `renames` = `[{'from': old, 'to': new}, …]` — an EXPLICIT MAPPING, never a diff. A diff - cannot tell "renamed Blue to Navy" from "deleted Blue and added Navy", and guessing wrong - empties a column silently. - - Returns `{'options': n, 'cells': n}`. This is the DEFINITION half — the base options list and - the base row values. A user's per-user overlay stratum and any view filter naming the old - value are `core.table_store`'s half of the same contract; the caller runs both. - """ - pairs = [] - for r in (renames or [])[:50]: - if not isinstance(r, dict): - continue - a = ' '.join(str(r.get('from') or '').split())[:60] - b = ' '.join(str(r.get('to') or '').split())[:60] - if a and b and a != b: - pairs.append((a, b)) - if not pairs or not is_user_table(table_key, st): - return {'options': 0, 'cells': 0} - mapping = dict(pairs) - counts = {'options': 0, 'cells': 0} - - def _apply(cur): - t = cur.get(str(table_key)) - if t is None: - return cur - multi = False - for f in (t.get('fields') or []): - if f.get('key') != str(fkey): - continue - multi = f.get('type') == 'multiselect' - opts = f.get('options') or [] - new_opts, seen = [], set() - for o in opts: - v = mapping.get(o, o) - if v not in seen: # a rename ONTO an existing option merges them - seen.add(v) - new_opts.append(v) - if o in mapping: - counts['options'] += 1 - if opts: - f['options'] = new_opts - for row in (t.get('rows') or {}).values(): - if not isinstance(row, dict) or str(fkey) not in row: - continue - cell = str(row.get(str(fkey)) or '') - if not cell: - continue - if multi: - # ⚠ MULTISELECT CELLS ARE COMMA-JOINED LABEL STRINGS, so a rename has to walk the - # parts. Rewriting the whole string would only ever hit a single-value cell. - parts = [p.strip() for p in cell.split(',')] - nxt = [mapping.get(p, p) for p in parts] - if nxt != parts: - row[str(fkey)] = ', '.join(dict.fromkeys(nxt)) - counts['cells'] += 1 - elif cell in mapping: - row[str(fkey)] = mapping[cell] - counts['cells'] += 1 - return cur - - _st(st).update(STORE_KEY, _apply, flush='sync') - return counts - - -def patch_cells(table_key, row_id, values, st=None): - """Write shared definition-row cells for trusted server-side writers.""" - if not is_user_table(table_key, st): - return False - rows = (get(table_key, st) or {}).get('rows') or {} - if str(row_id) not in rows: - return False - - def _set(cur): - t = cur.get(str(table_key)) - if t is not None: - t.setdefault('rows', {}).setdefault(str(row_id), {}).update( - {str(k): str(v) for k, v in (values or {}).items()}) - return cur - - _st(st).update(STORE_KEY, _set, flush='sync') - return True - - -def patch_link_cell(table_key, row_id, fkey, value, st=None): - """Persist one user-picked Link cell in the shared row and return its canonical id string. - - A Link is a relationship in the database schema, not one user's visual overlay. Persisting - it here makes the reciprocal field and every Rollup see the same source of truth. Derived - joins and inverse fields are engine-owned and are refused by this door. - """ - table_key, row_id, fkey = str(table_key), str(row_id), str(fkey) - if not is_user_table(table_key, st) or not records_mutable(table_key, st): - return None - table = get(table_key, st) or {} - if row_id not in (table.get('rows') or {}): - return None - field = next((f for f in (table.get('fields') or []) if f.get('key') == fkey), None) - bag = (field or {}).get('link') - if (field or {}).get('type') != 'link' or not isinstance(bag, dict) \ - or bag.get('on') or bag.get('inverse'): - return None - target = get(str(bag.get('table') or ''), st) or {} - valid = set((target.get('rows') or {}).keys()) - raw_ids = value if isinstance(value, (list, tuple, set)) else str(value or '').split(',') - picked, seen = [], set() - for raw_id in raw_ids: - rid = str(raw_id).strip() - if not rid or rid in seen: - continue - if rid not in valid or len(picked) >= LINK_MAX_IDS: - return None - seen.add(rid) - picked.append(rid) - if bag.get('single'): - break - canonical = ','.join(picked) - - def _set(cur): - current = cur.get(table_key) - if current is not None and row_id in (current.get('rows') or {}): - current['rows'][row_id][fkey] = canonical - return cur - - _st(st).update(STORE_KEY, _set, flush='sync') - return canonical - - -def patch_profile_cell(table_key, row_id, fkey, value, st=None): - """Write a PROFILE cell — and, when it is blanked, clear that row's preset cells IN THE SAME - WRITE (wave 25, contract C3 + owner ruling R6). Returns - `{'handle', 'cleared': [key, …]}`, or None when the write is refused. - - ⭐ THIS IS THE `ut_*` WRITE DOOR R6 NAMES, and it is deliberately NOT the event seam. - HARD RULE 5: item 5a's clearing *looks* exactly like an event-trigger feature, and - implementing it by widening `grid_events`' emit guard is D-40 — deferred by ruling A-15 - because customer/product `EventCtx` carries no scoped handle and a nurilab user's edit would - fire against royal-imports' automations. Nothing here emits anything. - - ⚠ IT WRITES THE DEFINITION ROW, NOT AN OVERLAY (amendment C3-A1, MEASURED). An ordinary - `ut_*` cell edit lands in the typist's own overlay stratum, where the automation engine — - which reads `t['rows']` — can never see it. A handle nobody can enrich is the flag being - decorative, so a profile cell writes THROUGH to the shared row, exactly as the stage field - does for exactly the same reason. - - ⚠ ONE `update`, not two. "In the same write" is the contract, and it is also the only safe - shape: a blank committed without its clear leaves a row whose handle is gone and whose - follower count still reads 41,000 — stale numbers attributed to nobody, which is worse than - either the old row or the empty one. - - ⛔ IT DELETES NO FIELDS AND TOUCHES NO HISTORY. The columns stay (they are the table's - schema, and the next handle refills them); `ut_ig_snapshots` is a different table this - module never opens. R3's *"one store for one series"* is why that separation holds — the - cleared cells are the LATEST-value stamp, and the series itself was never in them. - """ - if not is_user_table(table_key, st): - return None - defn = get(table_key, st) or {} - rows = defn.get('rows') or {} - if str(row_id) not in rows: - return None - fields = defn.get('fields') or [] - fdef = next((f for f in fields if f.get('key') == str(fkey)), None) - if not isinstance(fdef, dict) or not isinstance(fdef.get('profile'), dict): - return None # not a profile column — the caller's wall failed open - handle, ok = normalize_profile(value, fdef['profile'].get('source')) - if not ok: - return None # refused with a sentence at the route - # Clear only the known Instagram presets that the automation declared, never every field with - # automation provenance. - cleared = [] - if not handle: - cleared = [f['key'] for f in fields - if f.get('key') in PROFILE_PRESET_KEYS - and isinstance(f.get('automation'), dict) - and str(rows.get(str(row_id), {}).get(f['key']) or '') != ''] - - def _set(cur): - t = cur.get(str(table_key)) - if t is not None: - row = t.setdefault('rows', {}).setdefault(str(row_id), {}) - row[str(fkey)] = handle - for k in cleared: - row[k] = '' - return cur - - _st(st).update(STORE_KEY, _set, flush='sync') - return {'handle': handle, 'cleared': cleared} - - -def delete_row(table_key, row_id, st=None): - """⛔ SYNC, AND IT WAS `async` — owner report 2026-08-10: *"it took me forever to delete a - record… it took me like 3 tries"*. - - `flush='async'` applies the drop to the in-process CACHE and defers the upload by - `store._FLUSH_DELAY` (2 s) so a burst coalesces into one commit. That trade was made for - `add_row`, and made there for a stated reason — the owner's *"adding a new record visually - takes too long, I need to be able to spam it"*. **This function inherited the flag and none of - the argument.** Nobody spams a delete: it is rare, it is destructive, and it is the one - mutation where "eventually consistent" is indistinguishable from "it did not work". Inside - that window the row is gone from this process and still present in the store, so anything that - re-reads the hub copy — a restart, an eviction, another reader — brings it back, and the user - deletes it again. - - ⚠ ONE COMMIT PER DELETE is the whole cost, and it is the correct one to pay: the alternative - is a destructive action whose durability depends on the container staying up for two seconds. - Adds keep their async path untouched — the spam case is real and is theirs. - ⚠ THIS IS NOT THE WHOLE OF D-118. A sync commit closes the lost-delete window; it does not make - two writers safe, which is D-4's job. - """ - if not is_user_table(table_key, st) or not records_mutable(table_key, st): - return False - - def _drop(cur): - t = cur.get(str(table_key)) - if t is not None: - (t.get('rows') or {}).pop(str(row_id), None) - return cur - - _st(st).update(STORE_KEY, _drop, flush='sync') - return True - - -#: Names the ENGINE uses for itself when no human is on the other end of a run. ⛔ A table -#: stamped with one of these has NO HUMAN OWNER, so `may_open` can only admit an admin — which -#: is why the automation engine must never mint one (see `ut_ensure`'s owner rule, wave 20). -#: Named here rather than inline so the two modules that care about it read the SAME list. -MACHINE_OWNERS = ('automation', 'scheduler') - - -def may_open(table_key, viewer, is_admin=False, st=None): - """FAIL-CLOSED visibility: the creator, or an admin. Nothing else. **THE one resolver.** - - A user table cannot be gated by `allowed_modules` — a module grant written last month cannot - describe a table created this morning — so it needs its own rule, and the safe rule is the - narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including - a BU-scoped sales agent) could open a table somebody else created just by knowing its key. - Sharing a user table with named colleagues is a follow-on, and it should reuse the shared- - VIEW vocabulary rather than inventing a second one. - - ⛔ WAVE 20 — THERE WAS A SECOND, WIDER RULE, AND THE TWO DISAGREED. `routes_automation`'s - table picker admitted `createdBy in (uname, 'automation', 'scheduler')`, so a non-admin could - SEE an automation-created database in the picker and then be refused when they opened it, - edited it, or tried to delete it. Two ideas of who owns a table is the same defect class as - the two ideas of an agent's book (D-30): both surfaces look right in isolation and only - disagree in front of a user. The picker now calls THIS function, and the engine stamps a - real owner (see `MACHINE_OWNERS`) so nothing legitimate is narrowed by the merge. - """ - t = get(table_key, st) - if not t: - return False - if bool(is_admin) or t.get('createdBy') == viewer: - return True - # ⭐ WAVE 20 (R10, closes DEBT D-32) — A GRANT ADMITS TOO. `core/shares.py` is the ONE registry - # for view/folder/database grants, and this is the line that makes a shared DATABASE actually - # open: without it the share was RECORDED and the receiver's nav never changed, which reads as - # "sharing is broken" rather than as a missing wall. - # - # ⚠ PURELY ADDITIVE, and deliberately last: it can only ever admit somebody a grant NAMES, and - # can never deny anyone the creator-or-admin rule already admits. `role_for` is fail-closed on - # a junk bucket or an unreadable store, so an unreachable registry degrades to today's rule - # rather than to an open door. - try: - import core.shares as shares - return shares.may_see('database', table_key, viewer, is_admin=is_admin, st=st) - except Exception: - return False - - -#: ⭐⭐ W31-T10 / CONTRACT C1 — THE TWO BUCKETS `may_open` READS, LENT FOR THE LENGTH OF ONE PASS. -#: -#: `core/store.py:Store.get` deep-copies the WHOLE tenant document on every call, hit or miss, -#: under `self._lock` — a documented 35.8 MB ceiling. So asking `may_open` about N databases cost -#: `1 + N` full copies of `user_tables`, plus up to another N of `object_shares` (its last leg), -#: and `GET /nav` measured a median of **9,548 ms live / 24,741 ms in-process** on tenant #0. -#: -#: ⛔ THE OBVIOUS FIX IS THE FORBIDDEN ONE, and `may_open`'s own docstring is the record of why: -#: wave 20 found TWO ideas of who owns a table (`routes_automation` admitted -#: `createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a database in a picker -#: and was refused on opening it. Inlining creator-or-admin at a call site re-creates exactly -#: that. **`may_open` stays THE one resolver, unmodified and still asked about every table** — -#: it is simply no longer charged for documents the caller is already holding. -#: -#: ⚠ WHY THE SHARES BUCKET IS IN HERE TOO, when `routes_automation._LentTables` intercepts only -#: `user_tables`: `may_open`'s admit-by-grant leg calls `shares.may_see` → `role_for` → `grants`, -#: which is `st.get('object_shares')` PER TABLE. It is a different key, so a `user_tables`-only -#: lend passes it straight through — the exact residual C1 says needs its own answer. It is read -#: LAZILY (an admin, or a table's creator, is admitted before that leg runs and never pays for -#: it) and memoised for the pass. -#: -#: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through -#: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the -#: runtime through `__getattr__` — so this can serve a stale value for exactly two keys, for -#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level -#: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS -#: `createdBy`). -_LENDABLE = (STORE_KEY, 'object_shares') - - -class _Lent: - """A read-only `st` that serves already-read documents and passes everything else through.""" - - def __init__(self, runtime, docs): - self._runtime, self._docs = runtime, dict(docs or {}) - - def get(self, name, fresh=False): - if fresh or name not in _LENDABLE: - return self._runtime.get(name, fresh) if fresh else self._runtime.get(name) - if name not in self._docs: - self._docs[name] = self._runtime.get(name) # lazily, once, for this pass - return self._docs[name] - - def get_projection(self, name, drop=()): - """⭐ W32-T01: serve the LENT document here too, or a projection re-reads what the pass - already holds — `__getattr__` would forward this straight to the runtime and the lend - would buy nothing for exactly the callers built to use it. - - ⚠ THE TWO MISMATCHES ARE BOTH SAFE, AND IN OPPOSITE WAYS — which is why this serves the - lent document as-is rather than re-projecting it: - • lent WHOLE, asked for a projection ⇒ the caller gets MORE than it asked for. Correct, - just not cheap — the one direction a shortcut here is allowed to be wrong in. - • lent PROJECTED, asked for the whole thing via `get` ⇒ the caller gets a `_Projected`, - and reading a dropped key RAISES with the reason. Loud, not silent. - A re-projection would cost a second copy to convert the first case and change nothing - about the second. - """ - if name not in _LENDABLE: - return self._runtime.get_projection(name, drop=drop) - if name not in self._docs: - self._docs[name] = self._runtime.get_projection(name, drop=drop) - return self._docs[name] - - def __getattr__(self, name): - return getattr(self._runtime, name) - - -def lend(st=None, **docs): - """A handle that serves `docs` for `_LENDABLE` keys and defers everything else to `st`. - - `lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and - hand the result to every predicate in that pass; see the note above for what makes it safe. - """ - return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None}) - - -def lend_defs(st=None): - """`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows. - - ⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02 - (`all_defs` then `lend(st, user_tables=tables)`); this is that pair named, so the TABLE WALL - can take it too. `routes_tables._defn_or_refuse` is the wall a database SWITCH goes through and - it read the whole 28.6 MB document to answer two questions — "does this key exist" and - "may this session open it" — neither of which has ever looked at a row. - - ⛔ **HAND THIS TO A READ, NEVER TO A WRITE.** Every value it serves is a `_Projected`, so a - caller that reaches `defn['rows']` gets a `KeyError` naming the projection — loud, which is the - point — and a read-modify-write handed this document would upload a tenant with its rows - removed. `store.update`/`_read_strict` are unaffected (they never project), and the six write - walls in `routes_tables` deliberately do not opt in. C5's second clause is the same rule from - the other end: a projected snapshot is never handed to a post-write read-back. - - ⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY - should take the whole read once rather than this plus that. - """ - return lend(st, **{STORE_KEY: all_defs(st)}) - - -def nav_entries(viewer=None, is_admin=False, st=None): - """Registry-SHAPED dicts so the nav can render user tables beside real modules with no - special-casing: the same keys the flat nav reads (`key`, `label`, `source`). - - Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse. - - ⭐ W31-T10 (C1): ONE document read for the whole listing. The filter is unchanged — the same - `may_open`, asked about the same tables, in the same order — it is handed the document this - function has already read instead of taking a fresh 35.8 MB copy per table. A caller that - already holds the document passes its own `lend(...)` as `st` and this pays nothing at all. - - ⭐⭐ W32-T02 (R8/D-175): and that ONE read is now a PROJECTED one. This function reads `label`, - `source` and `key`, and `may_open` reads `createdBy` (then the shares registry) — **no caller - of this listing has ever touched a row.** On tenant #0 that turns a 1,750 ms copy into 1.4 ms, - for both consumers: `nav()` and `_placeable_top_keys`, i.e. `GET /nav` and `GET /nav/prefs` — - the two that `Shell.tsx` fires CONCURRENTLY on the same `Store._lock`, which is why they added - up rather than overlapped and why the rail rows arrived seconds apart (owner item 5). - """ - tables = all_defs(st) - lent = lend(st, **{STORE_KEY: tables}) - out = [] - for key, t in sorted(tables.items(), - key=lambda kv: (kv[1].get('label') or '').lower()): - if viewer is not None and not may_open(key, viewer, is_admin, st=lent): - continue - out.append({'key': key, 'label': t.get('label') or key, - 'source': t.get('source') or BLANK_SOURCE, - 'user_table': True, 'nav': True, 'validate': False}) - return out +"""USER-CREATED TABLES (wave-9 I8 / contract C6) — databases that are not a connector. + +The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely +new database blank, where this time the user can actually insert row that they want."* The first +branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second. + +WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at +import time — it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it +at runtime without editing code. A user-created table is tenant data, so it lives in the tenant +store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed +machinery honest: a user table is never "archived", it is deleted, and it is never in +`allowed_modules` because it is not a module. + +SHAPE (one store key, `user_tables`): + + {table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}} + +`source` is deliberately NOT 'Odoo' — the nav badge is generated from it, so a blank table reads +"Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`, +which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new +storage concept. + +⚠ ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and the +row endpoints refuse unless `is_user_table(key)` — a user must not be able to invent a customer +in Odoo by typing into a grid. (Wave 18 note: the events seam has NO row event types; row +add/delete are REST endpoints on `aios-web/api/routes_tables.py`, and this predicate is their +wall.) + +WAVE 18 (C3-UT): every function takes an optional `st` STORE HANDLE — anything exposing +`get(name)` / `update(name, fn, flush=)` / `exists(name)`. Default = `core.store` (tenant #0, +the Streamlit host, unchanged). The API passes the session's `TenantRuntime`, whose accessors +apply the tenant prefix / repo binding, so a Nurilab table lands in Nurilab's store. +""" +import datetime as _dt +import hashlib +import json as _json +import re + +import core.store as store + +STORE_KEY = 'user_tables' +MAX_TABLES = 40 +MAX_LABEL = 60 + +#: ⭐ THE PER-TABLE ROW CEILING — 5,000 until 2026-08-09, and it was never a property of the +#: substrate. Owner: *"Why do we keep having this MAX_ROWS problem, 5000 is too little. We need to +#: exceed it… solve the rootcause."* +#: +#: ⛔ WHAT 5,000 ACTUALLY WAS. Two things wore the same number and neither justified it: +#: * `add_row`'s human-insert door (below) — a guard against a runaway paste, not a measurement; +#: * `odoo_relational.plan()`'s own refusal, which is what kept the Odoo databases at OPEN AR +#: only and is the reason the owner could not find every Odoo id in them. +#: `_ensure_table` writes rows straight through `rt.update(STORE_KEY, …)` and never crosses either, +#: so the cap was never even enforced on the population it was blamed for. And `ig_master.py` has +#: run a **500,000**-row bucket the whole time, which settles the question of whether the store can +#: hold more than five thousand of anything. +#: +#: ⭐ THE REAL CONSTRAINT, MEASURED 2026-08-09 against tenant #0's own mirror, is the one this +#: number now expresses: every `ut_*` table lives in ONE `user_tables` document that +#: `Store.get`/`Store.update` copy with `json.loads(json.dumps(...))` on every call. Built from +#: real Odoo rows, one table costs: +#: +#: customers 2,465 rows 0.43 MB 8 ms invoices 31,418 rows 8.68 MB 158 ms +#: products 5,829 rows 1.20 MB 26 ms orders 32,700 rows 7.50 MB 129 ms +#: +#: ⭐ THE NUMBER IS DERIVED FROM TWO HEADROOMS, and both are checked rather than asserted by +#: `verify_odoo_relational._prove_row_ceiling` ([[rules-need-gates]]): +#: * BYTES — the widest table the product ships measures **0.318 KB/row** (`ut_odoo_invoices`, +#: 13 columns), so 60,000 rows is **19.1 MB** against a 32 MB per-table budget: 40% spare. +#: * GROWTH — the largest shipped population is 32,700 confirmed orders, growing ~8k/year, so +#: the cap is 83% above what exists and does not bind for roughly three years. +#: ⚠ 100,000 WAS THE FIRST ANSWER AND IT WAS WRONG BY ITS OWN GATE: at the measured row width it +#: is 31.8 MB against the same budget — 0.6% of headroom, i.e. a ceiling that one extra column +#: turns red. A cap chosen so it *just* passes its own control is a cap chosen to look justified. +#: ⛔ WHEN THIS BINDS, THE FIX IS NOT A BIGGER NUMBER. It is D-87's per-table row-key split, after +#: which the budget applies to a table on its own rather than to a share of one shared document. +#: +#: ⚠ IT IS A PER-TABLE CEILING AND THE TENANT PAYS THE SUM. The four Odoo databases together are +#: 20.6 MB / ≈ 370 ms per composite copy — real, and the reason the per-table row KEY split is +#: booked as the next increment (DEBT D-87) rather than claimed here. Order lines (256,810 rows, +#: 63.9 MB, 2.57 s) stay out of the store under ANY JSON substrate; they are answered by the +#: read-through rollup, which never copies a row. +MAX_ROWS = 60_000 + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# ⭐⭐ WAVE 30 / OWNER RULING R6 — THE CEILING ABOVE DOES NOT APPLY TO A CONNECTED SOURCE. +# +# Verbatim: *"there is no cap in how many data from the API source (as long as its from a +# connected source like Odoo) that can be pulled into the app. Make this a standing rule. Now if +# there is lag or it can't be done, you need to explicitly tell me why and recommend a fix."* +# +# ⭐ BOTH SENTENCES ARE THE RULE. The first removes the cap; the second says a cap that CANNOT be +# removed must be reported with its cause and a recommendation — `limit_report()` below is that +# sentence, and it is served on the wire rather than left in a comment. A silent limit is the +# failure mode, not a limit. +# +# ⚠ AND THE EDITABLE SUBSTRATE KEEPS ITS BOUND. `MAX_ROWS` is not a rule about data, it is a +# measurement of THIS document (the header above derives it from 0.318 KB/row against a 32 MB +# per-table budget). A `ut_*` database somebody types into still lives here and still pays that +# cost. What changes is that a CONNECTED table's rows stop living here at all — they are served +# read-through from the mirror (R7, `routes_odoo_tables`), where 971,034 GL lines already sit and +# are counted in ~24 ms. Conflating the two turns R6 into a rewrite of user tables. + +#: Table keys DECLARED connected by the app layer. Populated by `routes_odoo_tables._sources()`; +#: `core` never imports up, so the app tells this layer rather than being interrogated by it. +_CONNECTED = set() + + +def register_connected(*table_keys): + """Declare table keys as CONNECTED-SOURCE — their rows come from a connector and are served + read-through, so this document's ceiling is not a fact about them.""" + for key in table_keys: + k = str(key or '').strip() + if k: + _CONNECTED.add(k) + return frozenset(_CONNECTED) + + +#: ⭐⭐ W31-T25 / CONTRACT C2 — the KEY-CONVENTION leg's prefixes are a REGISTRY, not a literal. +#: +#: `is_connected`'s third leg used to read `key.startswith('ut_odoo_')` inline. Meta Ads is the +#: second connector to spawn locked databases (`ut_meta_*`), and the alternative was a second +#: predicate in the app layer answering the same question — one question, two normalizers, which +#: is a recorded defect class here ([[one-question-two-normalizers]]). So the connector layer +#: DECLARES its prefix and this module keeps the single evaluator. +#: +#: ⛔ `'ut_odoo_'` IS SEEDED RATHER THAN REGISTERED, AND THAT IS THE POINT OF THE LEG. The whole +#: reason a naming convention sits below the registry is to answer in a process where nothing has +#: registered anything — a cold worker that never imported `routes_odoo_tables` still owes the +#: right answer for `ut_odoo_gl_lines`, and an empty seed would have it fall through to +#: `MAX_ROWS`, i.e. re-cap a connected source against R6. A newly registered prefix carries the +#: weaker guarantee that `register_connected` already carries: it holds once its registrar has run. +_CONNECTED_PREFIXES = {'ut_odoo_'} + + +def register_connected_prefix(*prefixes): + """Declare a table-key prefix whose automation-sourced, records-locked tables are CONNECTED. + + The connector layer calls this beside `register_connected` (which names exact keys): the + prefix answers for tables this process has not enumerated, the exact keys answer for the ones + it has. Returns the whole registered set, so a caller can assert what it just joined. + + ⚠ A PREFIX ALONE NEVER MAKES A TABLE CONNECTED. `is_connected`'s third leg still demands + `source == AUTOMATION_SOURCE` and `recordMode == AUTOMATION_RECORD_MODE`, because a person who + names a database "Meta ads spend" gets a `ut_meta_…`-shaped key too — and exempting it from + the ceiling on the strength of its NAME would be exactly the silent hole the fence exists for. + """ + for prefix in prefixes: + p = str(prefix or '').strip() + if p: + _CONNECTED_PREFIXES.add(p) + return connected_prefixes() + + +def connected_prefixes(): + """The registered key prefixes — the ONE list a caller may read to name the same set. + + C2's parity gate reads this rather than re-spelling `ut_odoo_` / `ut_meta_` on the far side: + a second literal is a second normalizer wearing an assertion's clothes. + + ⚠ AND IT IS THE REGISTRAR'S OWN RETURN VALUE, which is not tidiness — `verify_reachability` + went RED the hour this shipped, correctly: a public function whose only callers are `verify_*` + files is a feature no user can reach ([[reachable-is-not-the-same-as-built]]), and "another + lane's gate will call it" is not a caller. Routing the write door's answer through the read + door makes one construction site of the set instead of two, and makes the accessor + load-bearing rather than decorative. + """ + return frozenset(_CONNECTED_PREFIXES) + + +def is_connected(table_key, st=None): + """Is this database backed by a connected source rather than by typing? + + THREE ANSWERS, in falling order of authority, and the third is fenced deliberately: + 1. the app layer registered it (`register_connected`) — the live path; + 2. the stored definition says so (`connected: True`) — survives a process with no registrar; + 3. ⚠ a REGISTERED KEY PREFIX (`register_connected_prefix`; seeded `ut_odoo_`, and Meta Ads + joins it as `ut_meta_`), AND ONLY on a table that is automation-sourced with records + locked. A person who names a database "Odoo foo" gets exactly that key, and exempting it + from the ceiling on the strength of its name would be a silent hole. Their table is + `Blank`-sourced and records-mutable, so it cannot reach this leg. + """ + key = str(table_key or '').strip() + if key in _CONNECTED: + return True + table = get(key, st) or {} + if not table: + return False + if table.get('connected') is True: + return True + return (any(key.startswith(p) for p in _CONNECTED_PREFIXES) + and table.get('source') == AUTOMATION_SOURCE + and table.get('recordMode') == AUTOMATION_RECORD_MODE) + + +#: ⭐⭐ W30-T31 (D-87) — table keys whose rows DO NOT LIVE HERE AT ALL. +#: +#: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the +#: DATA came from, so `MAX_ROWS` is not a fact about it (R6). *Read-through* says where the rows +#: are SERVED FROM, so they are not in this document — which is the only thing that makes the +#: per-request copy cheaper. The eight Odoo grids are all connected; only the ones the app layer +#: has actually bound to the mirror AND proven safe to un-materialise are registered here. +#: +#: ⛔ THE APP LAYER DECIDES, and it is not merely "is there a binding". `routes_odoo_tables` +#: checks three conditions, because getting any one of them wrong is a silent wrong answer rather +#: than an error: a binding must exist; NOTHING may fold or link at the table's stored rows (a +#: rollup over a table with no rows writes ZEROS, and `compute_relation_cells` reads the raw +#: document, not this module); and the population must fit a window while the client still asks +#: for whole tables. Every table that fails one is REPORTED, per table, on the status door. +_READ_THROUGH = set() + + +def register_read_through(*table_keys): + """Declare that these databases are SERVED from the mirror and store no rows here.""" + for key in table_keys: + k = str(key or '').strip() + if k: + _READ_THROUGH.add(k) + _CONNECTED.add(k) # read-through implies connected; never one without the other + return frozenset(_READ_THROUGH) + + +def materialises(table_key, st=None, defn=None): + """Does this database keep its rows in the shared `user_tables` document? + + True for everything except a read-through grid — including every editable table, every + automation table, and a connected table whose conversion has not happened yet. + + ⛔ THE STORED FLAG IS NOT A CACHE OF THE REGISTRY, IT IS THE DURABLE HALF, and without it this + predicate would be a live hazard rather than a lookup. `_READ_THROUGH` is filled by an app-layer + route; a worker process that has never served one would answer True for a table whose rows were + stripped weeks ago, read `rows: {}` and serve an EMPTY GRID with nothing going red. So the strip + STAMPS the definition and the stamp is what a cold process reads. Same three-tier shape as + `is_connected` above, minus the naming convention — there is no safe way to guess this one. + + ⚠ `defn` LETS A CALLER LEND THE DEFINITION IT IS ALREADY HOLDING, and on the row path that is + not a micro-optimisation: every `get()` here is another whole-document deep copy under + `Store._lock`, so asking this question inside `scoped_pool` without lending would have added a + third copy to the very request this ticket exists to make cheaper. + """ + key = str(table_key or '').strip() + if key in _READ_THROUGH: + return False + if defn is None: + defn = get(key, st) or {} + return not bool((defn or {}).get('readThrough')) + + +def strip_materialised(st=None): + """Remove the stored rows of every read-through database. Idempotent; returns what moved. + + ⭐ THIS IS THE HALF THAT MAKES THE INVARIANT TRUE RATHER THAN INTENDED. The spawn + (`odoo_relational._ensure_table_inplace`) writes rows by mutating this document inside its own + `rt.update(STORE_KEY, …)`, so it never passes through any function here — a guard on the write + doors below would simply not be on that path. Re-running this after a refresh is therefore the + enforcement, and being idempotent is what lets every door call it without coordinating. + + ⚠ The definition STAYS. A read-through table is a real database — its fields, label, views, + grants and lock all live here; only the rows are elsewhere. Deleting the definition would take + the permission wall and the nav entry with it. + """ + out = {'dropped': {}, 'before': 0, 'after': 0} + if not _READ_THROUGH: + return out + s = _st(st) + try: + cur = s.get(STORE_KEY) or {} + except Exception: # noqa: BLE001 + return out + fat = {k: len((cur.get(k) or {}).get('rows') or {}) for k in _READ_THROUGH} + fat = {k: n for k, n in fat.items() if n} + if not fat: + return out # the common case: nothing to do, and nothing measured + # Only now is the measurement worth its own cost — `json.dumps` of a 20 MB document is not + # free, and this exists to be reported (R6's second sentence), not to run on every request. + out['before'] = len(_json.dumps(cur, default=str)) + + def _strip(doc): + for key in list(_READ_THROUGH): + t = (doc or {}).get(key) + if isinstance(t, dict) and t.get('rows'): + out['dropped'][key] = len(t['rows']) + t['rows'] = {} + # ⛔ THE STAMP IS THE POINT — see `materialises`. Without it the next process to + # boot reads an empty `rows` and calls that the answer. + t['readThrough'] = True + return doc + + s.update(STORE_KEY, _strip, flush='sync') + try: + out['after'] = len(_json.dumps(s.get(STORE_KEY) or {}, default=str)) + except Exception: # noqa: BLE001 + out['after'] = 0 + return out + + +def row_limit(table_key, st=None): + """The row ceiling that applies to this database: 0, None, or `MAX_ROWS`. + + ONE evaluator, so the write doors, the spawn's refusal and the wire's report cannot disagree + about whether a table is capped ([[one-evaluator-per-question]]). + + ⛔ THE THREE ANSWERS ARE THREE DIFFERENT STATEMENTS AND 0 IS NOT "A VERY SMALL CAP": + * **0** — this database stores no rows HERE at all; it is served read-through from the + mirror. A caller that builds rows for it is doing wasted, dangerous work: 963,783 GL lines + is ~240 MB of python dicts in one process. `odoo_relational.plan` reads this to decide + whether to build a bucket at all. + * **None** — connected, materialised, and UNCAPPED (R6): however many rows Odoo has. + * **MAX_ROWS** — the editable substrate, which really is bounded by this document. + """ + if not materialises(table_key, st): + return 0 + return None if is_connected(table_key, st) else MAX_ROWS + + +def limit_report(table_key, st=None): + """R6's second sentence as data: `{subject, cause, recommendation, effect}` or None. + + ⛔ `effect` is never `truncated`. Every enforcement site REFUSES — a truncated table + understates every total it feeds while looking exactly like a complete one, which is the + failure this whole module is arranged against. + """ + cap = row_limit(table_key, st) + if cap is None: + return None + if cap == 0: + return { + 'subject': 'rows', 'limit': 0, 'effect': 'read_through', + 'cause': 'this database is served THROUGH the connector mirror, so its rows are ' + 'never copied into this tenant document and no ceiling applies to them', + 'recommendation': 'read it with a window (`/odoo-tables/{key}/rows`); the row count ' + 'you see there is a SQL count over the whole table', + } + return { + 'subject': 'rows', 'limit': cap, 'effect': 'refused', + 'cause': f'this database is EDITABLE, so its rows live in the shared `{STORE_KEY}` ' + f'document that every request copies; at the widest shipped row (0.318 KB) ' + f'{cap:,} rows is 19.1 MB against a 32 MB per-table budget', + 'recommendation': 'connect it to a source instead — connected data is served ' + 'read-through from the mirror and is not bounded (R6); for typed data ' + 'the increment is D-87, a per-table row key, not a bigger number', + } + + +MAX_FIELDS = 60 +#: the badge the nav paints for a table with no connector behind it +BLANK_SOURCE = 'Blank' +#: automation-created tables carry their maker instead (wave 18 C4-AUTO) +AUTOMATION_SOURCE = 'Automation' +AUTOMATION_RECORD_MODE = 'automation' +#: every user table's key is prefixed, so it can never collide with a registry module key +KEY_PREFIX = 'ut_' + + +#: ⭐⭐ W31 QA — KEYS WHOSE RECORDS ARE MACHINE-OWNED **BY DECLARATION**, registered by the layer +#: that owns the connector (`automation_engine` at import), exactly as `register_connected_prefix` +#: is. `core` never imports up, so the engine pushes its list down here rather than this module +#: reaching for it. +#: +#: ⛔ WHY A REGISTRY AND NOT THE STORED FLAG ALONE — THE OWNER FOUND THIS IN PRODUCTION. +#: `recordMode` is stamped by `ut_ensure`, so a table only became locked THE NEXT TIME AN +#: AUTOMATION WROTE TO IT. W31-T32 proved that migration works and read it as done; it means every +#: TikTok database already sitting in a tenant stayed unlocked — offering "+ New record" — until +#: somebody re-ran a TikTok automation, which nobody had. Owner, verbatim: *"the databases for +#: Tiktok do not have the small lock icon as I asked."* A lock that arrives on the next write is a +#: lock that is absent exactly while a person is looking at it. +#: ⚠ THE REGISTRY IS THE AUTHORITY AND THE STORED FLAG STILL COUNTS — either one locks. The flag +#: keeps working for tables nobody declares, and the declaration needs no store write, no boot +#: order and no per-tenant migration, so it is true for every tenant the moment the module loads. +#: ⚠ LOCKED DATABASE, NOT READ-ONLY (DESIGN.md §4): this predicate governs RECORDS. Adding a FIELD +#: must still work, and `may_edit_field` is a different wall. +_LOCKED_RECORD_KEYS = set() + + +def register_locked_records(*keys): + """Declare table keys whose records are machine-owned. Idempotent; -> the registered set.""" + for k in keys: + for one in (k if isinstance(k, (set, frozenset, list, tuple)) else (k,)): + if str(one or '').strip(): + _LOCKED_RECORD_KEYS.add(str(one).strip()) + return frozenset(_LOCKED_RECORD_KEYS) + + + +def records_mutable(table_key, st=None): + """May a human add/edit/delete records in this database? + + Ordinary and Profile databases default open. Automation-owned child datasets opt out — by + DECLARATION (`register_locked_records`, which needs no write) or by the stored table-level + mode; engine writers use their direct coalesced path and are intentionally not routed through + this human-door predicate. + """ + table = get(table_key, st) or {} + if not table: + return False + if str(table_key) in _LOCKED_RECORD_KEYS: + return False + return table.get('recordMode') != AUTOMATION_RECORD_MODE + +#: The field types a user table may declare. ⚠ Kept a SUBSET of +#: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section). `created_time` stays +#: out (a row-datum kind — a base column of it would have no author); +#: `image` is unchanged this wave. `automation` JOINED in WAVE 21 (item 7a, +#: contract C2): the engine and the automation-column picker read the DEFINITION, so a +#: grid-created automation column must be able to live there — the column's CELLS stay +#: machine-written (the engine is the writer). Before this, the grid's automation columns +#: landed in the per-user workspace stratum and the picker could not see them — the owner's +#: "Choose a column..." stays empty. Local literal rather than an aios_grid import so this +#: module stays dependency-light for the API's boot path. +#: ⭐ `json` JOINED IN WAVE 23 (C7) — and it belongs here rather than staying grid-only the way +#: `image` did, because the databases that need it are exactly these: a scrape lands a comment +#: thread or a webhook payload against a row, and the automation engine writes those cells. +#: ⭐ 2026-08-07 — `link` and `rollup` JOINED (the relational wave). They belong here for +#: `metric`'s reason, not `formula`'s: their cells are MACHINE-COMPUTED SERVER-SIDE and +#: materialised into the row, because a rollup reads ANOTHER TABLE'S ROWS — data the client has +#: not loaded and must not have to. +#: ⭐⭐ 2026-08-10 — `formula` JOINED, and the argument that kept it out was answered rather than +#: overruled. "A base column of it would have no author" was about its VALUES, and that half is +#: still true: a formula cell is computed in the browser from the row's other cells and is stored +#: NOWHERE, here or anywhere else. What joins this set is the DEFINITION — the expression, the +#: label, the column's existence. Excluded, those landed in `
_table_workspace`, the +#: PER-USER overlay: measured on nurilab as `Trimmed reel views`, a column only its creator could +#: see on a database four people share. That is not a narrower feature, it is the same shape as +#: the rollup defect one stratum over ([[rollup-lives-in-the-definition-not-the-overlay]]) with a +#: different consequence — not "nothing can compute it" but "nobody else can SEE it". +#: ⛔ SAY THE LIMIT OUT LOUD, because this line will be read as a bigger promise than it is: the +#: values still do not exist server-side, so an automation, a rollup, a `where` clause and an +#: export still cannot read a formula column. Making the definition shared does not make the +#: number stored. A number the server must read is a `rollup` (link fold) or a `metric`. +#: ⚠ Its cells are therefore in `is_computed_cell` — the WRITE wall. Without that, `grid_events` +#: would accept a paste into a column whose renderer overwrites it on the next paint. +#: ⛔ `image` WAS MISSING FROM THIS SET UNTIL WAVE 27, AND THAT WAS A LIVE DEFECT — recorded +#: here because the shape repeats and the next kind will be added by somebody reading this line. +#: Wave-19 R7 landed `image` on FOUR of the five surfaces (the `FieldType` union, `CREATABLE_TYPES`, +#: `aios_grid.CUSTOM_FIELD_TYPES`, the shape/label tables) and missed this one. The client +#: therefore offered an Image column in EVERY column menu, including on `ut_*` databases, while +#: `_clean_field` returned None for it — so on a user database the column was created, named, +#: configured, and GONE on the next read, with nothing anywhere going red. It survived three +#: waves because `verify_fields_contract` diffed the client's offer against the ODOO grid's +#: acceptance and never against this set. Wave-27 item 13 adds that derived leg, which is what +#: found it (measured, not reviewed: `_clean_field({'type':'image'})` returned None). +UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date', + 'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json', + 'link', 'rollup', 'code', 'image', 'formula'} +#: A cell is still a scalar on the wire (`Row` values are strings/numbers) — what this bounds is +#: the DOCUMENT inside it. Source data is one already-paid provider response; keep it whole so a +#: provider field is never silently discarded merely because the response is rich. 32 MiB matches +#: the automation transport's guarded maximum response size. +MAX_JSON_CELL = 32 * 1024 * 1024 + +# --------------------------------------------------------------------------------------------- +# THE PROFILE FLAG (wave 25, contract C3 / owner ruling R7) +# --------------------------------------------------------------------------------------------- +# R7: *"the Profile field is a FLAG on an ordinary text field, not a new field kind"* — so it +# works on databases that already exist, and every reader that does not know about it goes on +# rendering a text column. `profile: {source: 'instagram'}` present = flagged; absent = ordinary. +# +# ⛔ THE FLAG IS ONLY WORTH HAVING BECAUSE OF THE TWO MITIGATIONS R7 ACCEPTED AS OPEN HOLES, and +# they are what this block is: AT MOST ONE per table (`_profile_of`, enforced at both write +# doors) and the flag VALIDATES ITS CELL (`normalize_profile`). Without the first, "the profile +# column" is not a thing the enrich action can resolve; without the second, the flag is a label. + +#: Where a profile handle points. A CLOSED vocabulary, for the reason every other closed enum +#: here is closed: a source nobody wrote a reader for renders as a column that promises a link. +#: +#: ⭐ WAVE-29 CONTRACT C2 (owner R1/R2, 2026-08-11): `tiktok` joins it, so a `ut_tt_*` row can +#: carry a profile flag the enrich action resolves. **A source is not a NAME, it is three rules** +#: (below) — the vocabulary and the rules move together, because adding the word alone would +#: accept a TikTok handle and then hand it an `instagram.com` link from `profile_url`. +PROFILE_SOURCES = ('instagram', 'tiktok') + +#: Hosts whose `//` path IS an Instagram profile. `www.` is stripped before the lookup. +_IG_HOSTS = ('instagram.com', 'instagr.am') + +#: URL path segments that are NOT handles. `instagram.com/p/Cxyz` is a POST and +#: `instagram.com/explore/tags/silk` is a search — both are things a person pastes, and both +#: would otherwise normalise to a handle (`p`, `explore`) that can never resolve. +_IG_RESERVED = frozenset({'p', 'reel', 'reels', 'tv', 'stories', 'explore', 'accounts', + 'direct', 'about', 'developer', 'legal', 'privacy', 'terms'}) + +#: Instagram's own handle rule: 1–30 of letters, digits, period, underscore. +_IG_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{1,30}$') + +#: TikTok's profile hosts. ⛔ `vm.tiktok.com` is deliberately ABSENT: those are short links that +#: usually resolve to a VIDEO, and accepting one would store a "handle" (the opaque code) that can +#: never enrich — the same failure `_IG_RESERVED` exists to prevent, one layer earlier. +_TT_HOSTS = ('tiktok.com', 'm.tiktok.com') + +#: TikTok's non-handle path segments. `tiktok.com/@user/video/123` is already refused by the +#: one-segment rule; these are the ONE-segment paths that are still not people. +_TT_RESERVED = frozenset({'video', 'tag', 'music', 'discover', 'search', 'foryou', 'explore', + 'live', 'upload', 'about', 'legal', 'privacy', 'terms', 'effect', + 'business', 'ads', 'node', 'login', 'signup'}) + +#: TikTok's own handle rule: 2–24 of letters, digits, period, underscore. ⚠ NOT Instagram's — +#: the charset matches but the bounds do not, and a shared regex would accept a 30-character +#: TikTok "handle" that the vendor answers nothing for. +_TT_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{2,24}$') + +#: ⭐ C2 — the per-source rules, ONE table, so `PROFILE_SOURCES` cannot grow a member that no +#: normaliser knows. Every function below reads THIS rather than branching on the word: a source +#: with no entry refuses at the door instead of silently taking Instagram's rules and Instagram's +#: URL, which is what "add the word and ship" would have produced. +#: +#: `at_handle` = the profile path wears a leading `@` (TikTok's `/@name`; Instagram's `/name`). +#: It is stripped before validation and re-added by `profile_url`, so the STORED truth is the bare +#: handle in both families and a filter on the column cannot miss half of it. +_PROFILE_RULES = { + 'instagram': { + 'hosts': _IG_HOSTS, + 'reserved': _IG_RESERVED, + 'handle': _IG_HANDLE_RE, + 'at_handle': False, + 'url': 'https://www.instagram.com/{handle}/', + }, + 'tiktok': { + 'hosts': _TT_HOSTS, + 'reserved': _TT_RESERVED, + 'handle': _TT_HANDLE_RE, + 'at_handle': True, + 'url': 'https://www.tiktok.com/@{handle}', + }, +} + + +def _profile_rules(source): + """The rule bag for a source, or None — the ONE resolution of "is this source real".""" + key = str(source or '').strip().lower() + if key not in PROFILE_SOURCES: + return None + return _PROFILE_RULES.get(key) + +#: The C1 preset keys a profile field OWNS — the cells R6 clears when its handle is blanked. +#: +#: MIRRORED from the engine's C1 set as local literals, the same discipline (and for the same +#: boot-path reason) as `METRIC_MEASURES` below; the pair is held in step by a parity check +#: rather than by an import, because `core/` must stay importable without the API layer. +#: +#: Human-written columns are deliberately absent. `ut_ensure` stamps automation provenance on +#: generated fields, so clearing by that tag alone would erase unrelated machine-owned columns. +#: This is therefore an intersection of the known Instagram preset keys and the provenance tag. +PROFILE_PRESET_KEYS = ( + # the profile facts themselves (CANDIDATE_FIELDS + the enrichment run_field_instagram pulls) + 'full_name', 'followers', 'following', 'posts_count', 'avg_engagement', 'bio', + 'external_url', 'verified', 'category', 'business_category', 'is_business', + 'is_professional', 'is_private', 'highlights_count', 'bio_hashtags', + 'pronouns', 'ig_id', 'profile_url', + # ⭐ 2026-08-07 — the rest of the Bright Data profile schema, promoted to preset columns by + # owner instruction. They are profile FACTS like the nineteen above, so blanking the handle + # clears them for the same reason (R6): they describe an account this row no longer names. + 'profile_name', 'is_joined_recently', 'has_channel', 'partner_id', 'external_url_title', + 'fbid', 'related_accounts', 'country_code', 'source_payload', + # relational projections and summaries owned by the Instagram graph + 'posts_link', 'profile_snapshots_link', 'post_snapshots_link', 'comments_link', + 'avg_views_12', 'avg_plays_12', 'avg_likes_12', 'avg_comments_12', + 'posts_captured', 'profile_reads', 'post_measurements_captured', 'comments_captured', + # R3's LATEST-value stamp. The full series stays in `ut_ig_snapshots` — one store for one + # series — which is why clearing here can never be a history delete. + 'enriched_at', +) + + +def _clean_profile(raw): + """One `profile` bag → the stored shape, or None (refused). + + Shaped exactly like `_clean_metric`: a mis-shaped flag REFUSES THE FIELD rather than being + dropped, because a column stored without the flag it was created with is a column whose + cells nothing will ever validate — the silent half of the same mistake. + """ + if not isinstance(raw, dict): + return None + source = str(raw.get('source') or '').strip().lower() + if source not in PROFILE_SOURCES: + return None + return {'source': source} + + +def normalize_profile(value, source='instagram'): + """A typed profile cell → `(handle, ok)`. THE cell validator the flag promises (C3). + + Accepts a bare handle (`name`), an at-handle (`@name`) or a profile URL of THIS SOURCE in any + of the forms a person actually pastes (`https://www.instagram.com/name/?hl=en`, + `instagram.com/name`, `https://www.tiktok.com/@name`). Returns the BARE HANDLE — the URL form + is derived for display, so the stored truth has exactly one spelling and a filter on it cannot + miss half the column. + + ⛔ C2: the host, the reserved segments and the handle rule all come from `_PROFILE_RULES[source]`. + A TikTok URL pasted into an Instagram-flagged column is REFUSED, and vice versa — a cell that + names the wrong network is not a handle this row's automation can enrich, and quietly accepting + it would defer the failure to a run that then answers nothing. + + `('', True)` for a blank: clearing a profile cell is legal and is the whole of R6. + `(None, False)` for anything else, and the caller refuses the write with a sentence — a + value that is neither is not a handle we can enrich, and storing it would put the failure + off until the automation runs and answers nothing. + """ + rules = _profile_rules(source) + if rules is None: + return (None, False) + raw = str(value if value is not None else '').strip() + if not raw: + return ('', True) # the clear — R6's whole subject + cand = raw.lstrip('@') + # ⚠ THE URL TEST IS `/`, AND NOTHING CLEVERER. An earlier version also treated a DOT as a + # sign of a URL, which refused `@nuri.lab_1` — a period is legal in an Instagram handle and + # common in real ones, so that test rejected a whole class of valid input while every URL + # case still passed. A bare handle can never contain a slash; every URL form does. + if '/' in cand: + # Parse rather than regex it: a hand-rolled pattern is how + # `instagram.com.evil.test/name` gets read as Instagram. + probe = cand if '//' in cand else 'https://' + cand.lstrip('/') + try: + from urllib.parse import urlsplit + parts = urlsplit(probe) + except Exception: + return (None, False) + host = (parts.netloc or '').split('@')[-1].split(':')[0].strip().lower() + if host.startswith('www.'): + host = host[4:] + if host in rules['hosts']: + segs = [s for s in (parts.path or '').split('/') if s] + # C2: TikTok's profile path is `/@name`; Instagram's is `/name`. The `@` is stripped + # here and re-added by `profile_url`, so both families STORE a bare handle. + first = segs[0].lstrip('@') if segs else '' + if len(segs) != 1 or not first or first.lower() in rules['reserved']: + # Two segments is a post/reel/video, zero is the site itself. Refused rather + # than taking the first segment, which is what turns `/p/Cxyz` into handle `p`. + return (None, False) + cand = first + elif host: + return (None, False) # a URL, but not one of THIS source's + if not rules['handle'].match(cand) or cand.lower() in rules['hosts']: + # The host check catches a bare `instagram.com` — it matches the handle charset, so + # without this it would normalise to a "handle" that can only ever resolve to nothing. + return (None, False) + return (cand.lower(), True) + + +def profile_url(handle, source='instagram'): + """The DERIVED display form of a stored handle (C3: *"the URL form is derived for display"*). + Derived and never stored, so the two spellings cannot drift apart. + + ⛔ C2 — the template comes from the SOURCE's own rules. This function used to hardcode + `instagram.com` while its guard merely checked membership of `PROFILE_SOURCES`, so the moment + that tuple grew a second member it would have handed every TikTok handle an Instagram link — + a guard that passes and a body that lies ([[hardcoded-fallback-not-hardcoded-key]]). + """ + h = str(handle or '').strip().lstrip('@') + rules = _profile_rules(source) + if not h or rules is None: + return '' + return rules['url'].format(handle=h) + + +def _profile_of(fields, exclude=None): + """The profile-flagged field in this list, or None. `exclude` skips one key — a PATCH of the + profile field itself must not read as "a second one already exists".""" + for f in (fields or []): + if (isinstance(f, dict) and isinstance(f.get('profile'), dict) + and f.get('key') != exclude): + return f + return None + + +def profile_field(table_key, st=None): + """THE profile column of one table, or None. ONE resolver, so "which column holds the + handle" has exactly one answer for the write door, the clear, the grid and the engine.""" + return _profile_of((get(table_key, st) or {}).get('fields') or []) + + +def _st(st): + return st if st is not None else store + + +#: Wave 22 (contract C3) — the ROW-EVENT seam. The API layer APPENDS listeners (the automation +#: engine's trigger hook, registered by `routes_automation` — the one module that imports both +#: sides); platform code only ever EMITS. This is what keeps the Notion loop-prevention law +#: structural: the emit sites are the HUMAN write doors (`add_row` here, `overlay_patch` in +#: grid_events) and nothing the engine itself writes through, so an automation's write cannot +#: fire an event trigger — its own or a sibling's — by construction. +ROW_HOOKS = [] + + +def emit_row_event(evt): + """Fan one row event out to every registered listener. A listener failure is swallowed — + a broken trigger must never break typing into a cell — but it is swallowed LOUDLY.""" + for hook in list(ROW_HOOKS): + try: + hook(evt) + except Exception as e: # noqa: BLE001 + print(f'[user-tables] row hook failed: {type(e).__name__}: {e}') + + +def _slug(label): + s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_') + return (s or 'table')[:40] + + +def _ag_formula(raw): + """`aios_grid._clean_formula`, reached the way this module reaches everything one layer up. + + ⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and + `_clean_field`). `core` must not import `aios_grid` at module level — the API's boot path + depends on this module staying dependency-light — and the `code` branch already pays that + cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up + calling a different validator. + + ⚠ `valid_keys` IS DELIBERATELY NOT PASSED. See the note at `_clean_field`'s formula branch: + neither door has reliable sibling-field context, and a second, weaker copy of the ref rule is + worse than leaving refs to the read-time behaviour `_clean_formula` already documents. + """ + import aios_grid as _agf + return _agf._clean_formula(raw) + + +#: Display formats a field may declare. ⛔ DISPLAY ONLY — none of these touches the stored value, +#: which stays the scalar the fold or the mapper wrote. That separation is the whole safety of the +#: feature: a `thousands` setting can never make a number wrong, only easier to read. +#: ⚠ `tz`/`time` belong to the date family and `thousands`/`decimals`/`abbrev` to the number family; +#: they are validated together because ONE bag rides one field and the renderer already reads only +#: the keys its own type understands (`display.numberText` / `display.dateTimeText`). +FORMAT_TZ = ('local', 'utc') +FORMAT_MAX_DECIMALS = 4 + + +#: ⭐ WAVE-29 CONTRACT C7 — the COLUMN-SUMMARY vocabulary a field's `agg` may take. +#: +#: MIRRORED from `aios_grid.FIELD_AGGS` (session C publishes it) and from the client's +#: `aggregations.FIELD_AGGS`, as a local literal for the reason `METRIC_MEASURES` and +#: `PROFILE_PRESET_KEYS` are local literals here: `core/` must stay importable without the layers +#: above it. Three copies, one law, held in step by C's parity gate rather than by an import. +#: +#: ⛔ NOT `CHART_AGGS` (which spells it `avg` and gatekeeps STORED chart values with live data) +#: and NOT `ROLLUP_FNS` (16 fold names, overlapping but not the same list). Merging any two of the +#: three silently turns live stored charts into sums. +FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') + +#: A choice colour, as the client's editor spells one. `#RRGGBB`, upper-cased on the way in so a +#: stored value has exactly ONE spelling and two fields cannot disagree about `#ebf6ef`. +_OPTION_COLOR_RE = re.compile(r'#[0-9A-F]{6}') + + +def _clean_option_colors(raw, options): + """`{option label → #RRGGBB}`, capped to the field's own option vocabulary (wave-29 C1). + + ⛔ MIRRORED from `aios_grid._clean_option_colors` as a LOCAL function rather than imported, for + the reason `METRIC_MEASURES` and `PROFILE_PRESET_KEYS` are local literals here: `core/` must + stay importable without the layers above it, and `aios_grid` sits above this module. The pair + is held in step by a gate leg, never by an import. + + ⚠ THE CAP IS THE POINT, not tidiness: colours are keyed by LABEL, so a renamed or deleted + option would otherwise leave a colour behind that nothing renders and no editor can reach — + and on the next rename that orphan could re-attach to a different option with the same name. + Case-insensitive on the way in, canonical on the way out, so `Done` and `done` cannot both + claim the same option. + """ + if not isinstance(raw, dict): + return {} + supplied = {} + for label, color in raw.items(): + if not isinstance(label, str) or not isinstance(color, str): + continue + clean = color.strip().upper() + if _OPTION_COLOR_RE.fullmatch(clean): + supplied[label.strip().lower()] = clean + out = {} + for option in options or []: + color = supplied.get(str(option).strip().lower()) + if color: + out[str(option)] = color + return out + + +def _clean_format(raw): + """One `format` bag → the stored shape, or None. + + ⛔⛔ THIS FUNCTION EXISTS BECAUSE THE BAG HAD NO STORAGE AT ALL. The column menu has shipped a + "Field format" pane (thousands separator, decimal places, abbreviate) since wave 5, and on a + `ut_*` database BOTH write doors built their entry dict key-by-key and never copied `format` — + so the pane saved, the request succeeded, and the setting was gone by the next read. It looked + like a rendering bug and was a persistence one. Same shape as + [[flag-shipped-without-its-writer]] from the other end: here the writer exists and the STORE + does not. + + ⚠ EVERY KEY IS OPTIONAL AND AN EMPTY BAG IS None, so a field that declares no format is stored + exactly as it was before this existed — the wave-5 note's own parity rule. + """ + if not isinstance(raw, dict): + return None + out = {} + if isinstance(raw.get('thousands'), bool): + out['thousands'] = raw['thousands'] + if isinstance(raw.get('abbrev'), bool): + out['abbrev'] = raw['abbrev'] + if isinstance(raw.get('time'), bool): + out['time'] = raw['time'] + decimals = raw.get('decimals') + # `bool` is an `int` subclass, so `True` would otherwise store as 1 decimal place — a checkbox + # arriving in a numeric slot should be refused, not interpreted (the `sigmas` lesson). + if isinstance(decimals, int) and not isinstance(decimals, bool): + if 0 <= decimals <= FORMAT_MAX_DECIMALS: + out['decimals'] = decimals + tz = str(raw.get('tz') or '').strip().lower() + if tz in FORMAT_TZ: + out['tz'] = tz + return out or None + + +def clean_fields(raw): + """Validate a caller-supplied field list into the stored shape — refuse junk, never store + it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every + base column of a user table is user-editable by construction), or None when nothing + survives. Unknown types are DROPPED, not coerced — a column silently retyped is a lie.""" + out, seen = [], set() + for f in (raw or [])[:MAX_FIELDS]: + if not isinstance(f, dict): + continue + label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80] + key = re.sub(r'[^a-z0-9_]+', '_', str(f.get('key') or _slug(label)).strip().lower()) + key = key.strip('_')[:60] + ftype = str(f.get('type') or 'text').strip().lower() + if not key or key in seen or ftype not in UT_FIELD_TYPES or not label: + continue + seen.add(key) + entry = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'} + # ⭐ 2026-08-10 — the DISPLAY format, carried through the CREATE door. See `_clean_format`: + # this is the fifth time an entry built key-by-key has silently dropped a bag the editor + # sends, and the fourth of those was `agg` on this very line one wave ago. + fmt = _clean_format(f.get('format')) + if fmt: + entry['format'] = fmt + description = ' '.join(str(f.get('description') or '').split())[:300] + if description: + entry['description'] = description + if ftype in ('select', 'multiselect'): + opts = [' '.join(str(o).split())[:60] for o in (f.get('options') or []) + if str(o).strip()][:50] + if opts: + entry['options'] = opts + if f.get('default') is True: + entry['default'] = True + # ⭐ WAVE-27 item 19 (R15) + D-80 — THE PRIMARY COLUMN, CARRIED THROUGH THE CREATE DOOR. + # + # `_clean_field` has honoured `pinned` since 2026-08-07; THIS door silently dropped it, + # and the split the comment below names is why. It did not matter while `create()` minted + # one column — with nothing pinned, `fields.find(f => f.pinned) ?? fields[0]` made the + # only column the primary either way. R15 seeds FOUR at once, so "whichever came first" + # stops being a coincidence that happens to be right, and D-80 is the register entry for + # exactly that fallback quietly being the whole rule. MEASURED before fixing: + # `clean_fields([{...'pinned': True}])` returned the field without the key. + if f.get('pinned') is True: + entry['pinned'] = True + # ⭐ WAVE-27 item 13 — the code language, for the same reason: a `code` column created + # through THIS door would otherwise arrive without the bag `_clean_field` preserves, and + # a column whose language depends on which door made it is the divergence the note below + # warns about, arriving through a different key. + if ftype == 'code' and f.get('code') is not None: + import aios_grid as _agc + c = _agc._clean_code(f.get('code')) + if c: + entry['code'] = c + # C3 (wave 25): the profile flag rides the CREATE path too, and the one-per-table law + # holds inside a single list. ⚠ This validator and `_clean_field` are two doors onto one + # contract (a pre-existing split — `create`/`set_fields` come through here, `add_field`/ + # `patch_field` through the other), so a rule taught to only one of them is a rule the + # other silently permits: a table could be CREATED with two profile columns and then + # refuse to accept a third, which is the shape a user reads as random. + if f.get('profile') is not None: + p = _clean_profile(f.get('profile')) + if p is None or ftype != 'text' or _profile_of(out): + continue # refused: dropped, never stored unvalidated + entry['profile'] = p + # ⭐ 2026-08-09 — THE RELATIONAL PAIR RIDES THIS DOOR TOO, and it is the same divergence + # `pinned` and `code` were fixed for above, arriving through a third key. `_clean_field` + # has carried `link`/`rollup` since 2026-08-07; THIS validator dropped both, so a `link` + # or `rollup` column created through `create()` / `set_fields()` arrived with its bag + # stripped — a column that renders, is typed `rollup`, and computes nothing forever. + # ⚠ IT WAS LATENT, NOT LIVE (no API route calls `set_fields`, and `odoo_relational` writes + # its definitions directly), and it stops being latent the moment a user can BUILD a + # rollup from the field editor — which is exactly what this wave ships. Caught by + # `verify_odoo_relational`'s own NOTE line, which had been printing the list for two days. + # ⛔ THE TYPE/BAG PAIRING IS ENFORCED BOTH WAYS, as it is in `_clean_field`: a `link` with + # no bag can never resolve a row, and a bag on a column of another type is a declaration + # nothing reads. Either half alone is the silent kind of broken. + for bag_key, cleaner in (('link', _clean_link), ('rollup', _clean_rollup)): + bag = cleaner(f.get(bag_key)) if f.get(bag_key) is not None else None + if ftype == bag_key and bag is None: + entry = None + break + if bag is not None: + if ftype != bag_key: + entry = None + break + entry[bag_key] = bag + # ⭐⭐ 2026-08-10 — the FORMULA expression rides this door too, for the fourth time in the + # same seam (`pinned`, `code`, `link`/`rollup`, now this). A formula column created through + # `create()`/`set_fields()` with its expression dropped is a permanently blank column that + # renders, sorts and filters — the exact silent shape the paragraph above describes. + if entry is not None: + fx = (_ag_formula(f.get('formula')) if f.get('formula') is not None else None) + if (ftype == 'formula') != (fx is not None): + entry = None + elif fx is not None: + entry['formula'] = fx + if entry is None: + seen.discard(key) + continue + # `agg` decides HOW the grid's totals row summarises the column. A rollup that declares it + # through one door and not the other summarises differently depending on who made it. + # ⭐ WAVE-29 C7: the vocabulary is `FIELD_AGGS`, not the single word `sum` — see its note. + agg_in = str(f.get('agg') or '').strip() + if agg_in in FIELD_AGGS: + entry['agg'] = agg_in + out.append(entry) + return out or None + + +def all_tables(st=None): + """{key: definition} — every user-created table in this tenant. + + ⚠ THIS READS THE ROWS TOO, and on tenant #0 that is 28.6 MB / ~703 ms per call. Callers that + only need DEFINITIONS (labels, `createdBy`, `recordMode`, `fields`) want `all_defs` below. + """ + try: + return dict(_st(st).get(STORE_KEY) or {}) + except Exception: + return {} + + +#: The keys `all_defs` refuses to read. `rows` is 99.9% of tenant #0's document by bytes and is +#: touched by no permission check, no nav render and no workspace envelope (D-185, W32-T01). +DEF_ONLY_DROPS = ('rows',) + + +def all_defs(st=None): + """{key: definition WITHOUT `rows`} — the same listing, for readers that never open a row. + + ⭐⭐ W32-T01 (D-185). `all_tables` is the whole 28.6 MB deep copy; this is the same answer for + every definition key at ~0.1% of the bytes. `/nav` is the caller that made it worth building: + `nav_entries`, `may_open` and the manage/canDelete/locked loop read `label`, `source`, + `createdBy` and `recordMode` — and nothing else. + + ⛔ **A DROPPED KEY RAISES, IT DOES NOT READ AS EMPTY.** `defn.get('rows')` on one of these + raises `KeyError` naming the projection, because the alternative — `{}` — is indistinguishable + from a database that genuinely has no rows, and the caller least able to notice is the one that + wanted rows. If you get that error you wanted `all_tables`. + + ⚠ Falls back to `all_tables` on ANY failure, including a store handle too old to project (a + fake store installed by a gate, a `_Lent` handle serving a document somebody else read). The + fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for + — which is the one direction a fallback here is allowed to be wrong in. + """ + try: + return dict(_st(st).get_projection(STORE_KEY, drop=DEF_ONLY_DROPS) or {}) + except AttributeError: + return all_tables(st) + except Exception: + return {} + + +def get(table_key, st=None): + return all_tables(st).get(str(table_key)) + + +def is_user_table(table_key, st=None): + """THE permission predicate for row writes. Cheap and total: a key that is not in this store + is not a user table, so it cannot accept invented rows.""" + return str(table_key or '').startswith(KEY_PREFIX) and get(table_key, st) is not None + + +def create(label, username, fields=None, source=None, st=None): + """Create a blank table. Returns its key, or None if refused. + + Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) — + the caller is a UI form and a refusal is a message, not an exception. + """ + label = str(label or '').strip()[:MAX_LABEL] + if not label: + return None + existing = all_tables(st) + if len(existing) >= MAX_TABLES: + return None + key = KEY_PREFIX + _slug(label) + if key in existing: # a second table of the same name gets a suffix + for n in range(2, 50): + if f'{key}_{n}' not in existing: + key = f'{key}_{n}' + break + else: + return None + # A blank table still needs ONE column to be a table at all: a row with no fields cannot be + # displayed, edited or identified. 'name' is the identity column, the same role `customer` + # plays in the Customer table. + defn = { + 'key': key, + 'label': label, + 'source': (source if source in (BLANK_SOURCE, AUTOMATION_SOURCE) else BLANK_SOURCE), + 'createdBy': username, + 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), + 'fields': clean_fields(fields) or [{'key': 'name', 'label': 'Name', 'type': 'text', + 'source': 'overlay', 'default': True}], + 'rows': {}, + } + + def _add(cur): + cur[key] = defn + return cur + + _st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it + return key + + +def delete(table_key, st=None): + """Delete a user table AND its artifact families (wave 21, item 6a / C3). + + Wave 20 popped the definition alone and booked ten orphan families; the owner then ruled a + real Delete (R3) with the footprint disclosed in the confirm dialog first, so the cleanup + is no longer optional. Cleaned here, in one place: definition (`user_tables`) · the + workspace bucket (`_table_workspace`, every stratum incl. `__shared__` — views, + custom fields, overlays, folders) · cohorts (`_cohorts`) · record comments + (`_record_comments`) · docs metadata (`_docs`) AND the dataset BYTES each + metadata row names (D-36, closed wave 23 — wave 21 left those unreachable-but-present) · + `nav_meta[]` · share grants (the database grant plus a view grant per view that + lived in this bucket, via `shares.drop_objects`) · alert definitions on this topic. + Bound AUTOMATIONS are the API layer's to disable (platform must not import the engine). + Every family after the definition is best-effort: a store blip mid-sweep must not resurrect + the table, and an orphaned empty bucket is residue, not a leak.""" + key = str(table_key) + s = _st(st) + + def _drop(cur): + cur.pop(key, None) + return cur + s.update(STORE_KEY, _drop, flush='sync') + + ws_key = f'{key}_table_workspace' + view_ids = set() + try: + bucket = s.get(ws_key) or {} + for _u, ws in bucket.items(): + if isinstance(ws, dict): + view_ids |= set((ws.get('views') or {}).keys()) + except Exception: + pass + # ⭐ D-36 CLOSED (wave 23) — the document BYTES, not just their metadata. Wave 21 cleaned + # `_docs` and disclosed that the blobs under `docs//…` were left on the dataset; + # "unreachable" is not "deleted", and for a tenant asking us to delete a database the + # difference is the whole promise. Read the metadata FIRST (it carries each blob's exact + # stored `path`, so nothing is guessed from a naming convention that could drift), then + # clear the bucket, then delete the blobs. Best-effort per file: a storage blip must not + # resurrect the table or abort the rest of the sweep — an undeleted blob is residue we can + # sweep again, an aborted delete is a table the user asked to be gone. + doc_paths = [] + try: + for _pid, _rows in (s.get(f'{key}_docs') or {}).items(): + for _doc in (_rows or []): + p = str((_doc or {}).get('path') or '') + if p: + doc_paths.append(p) + except Exception: + pass + # ⚠ `__shared` JOINED THIS LIST IN WAVE 30 AND ITS ABSENCE WAS A REAL LEAK. The + # tenant-wide overlay (W30-T28) is a bucket like the four beside it, and the docstring above + # promises the artifact FAMILIES go with the database. Left out, a deleted database's shared + # columns and their values survived in the store — invisible, and resurrected wholesale if + # anybody ever re-created a database under the same key, which is exactly the reuse + # `drop_field` scrubs cells to prevent one column at a time. + try: + import core.shared_overlay as _so + shared_key = _so.bucket(key) + except Exception: # noqa: BLE001 + shared_key = f'{key}__shared' + for b in (ws_key, f'{key}_cohorts', f'{key}_record_comments', f'{key}_docs', shared_key): + try: + s.update(b, lambda cur: {}, flush='async') + except Exception: + pass + for p in doc_paths: + try: + import core.store as _store_bytes + _store_bytes.delete_path(p) + except Exception: + pass + + def _drop_meta(cur): + if isinstance(cur, dict): + cur.pop(key, None) + return cur + try: + s.update('nav_meta', _drop_meta, flush='async') + except Exception: + pass + try: + import core.shares as shares + shares.drop_objects([('database', key)] + [('view', v) for v in view_ids], st=s) + except Exception: + pass + try: + import core.alerts as alerts + alerts.drop_topic(key, st=s) + except Exception: + pass + + +def set_fields(table_key, fields, st=None): + """Replace a table's base-field contract (wave 18 — the automation engine grows tables it + creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless.""" + clean = clean_fields(fields) + if not clean or not is_user_table(table_key, st): + return False + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + t['fields'] = clean + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return True + + +def add_row(table_key, values=None, username=None, st=None, rid=None): + """Append one row. Returns the new row id, or None if refused. + + `rid` RESTORES A ROW UNDER ITS OLD ID (contract C-ADDROW / C-UNDO). Undo has to put a deleted + row back where it was: a restore under a fresh id would break every cohort, comment and view + filter that named the original, so the id is part of what is being undone. It is honoured + only when that id is FREE — an undo can never overwrite a row somebody has since created in + the gap, and it never invents a non-numeric id, because `scoped_pool` reads row ids as ints. + """ + if not is_user_table(table_key, st) or not records_mutable(table_key, st): + return None # ⚠ never on a connector-backed table + defn = get(table_key, st) or {} + rows = defn.get('rows') or {} + # R6: `row_limit` is None for a connected source. `add_row` cannot reach one anyway — the + # `records_mutable` guard above refuses first — but reading the ceiling through the ONE + # evaluator is what stops a later relaxation of that guard quietly re-imposing this cap. + cap = row_limit(table_key, st) + if cap is not None and len(rows) >= cap: + return None + want = str(rid or '').strip() + if want and want.isdigit() and want not in rows: + rid = want + else: + rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1) + valid = {f['key'] for f in (defn.get('fields') or [])} + clean = {k: str(v) for k, v in (values or {}).items() if k in valid} + # C3 (wave 25): a PROFILE cell is validated and normalised on the insert door too, not only + # on the edit door. A row born with `https://instagram.com/p/Cxyz` in its handle column is a + # row the enrich action can never answer, and the failure would otherwise surface as an + # automation that quietly returns nothing rather than as a refused write. + pf = _profile_of(defn.get('fields') or []) + if pf and pf['key'] in clean: + handle, ok = normalize_profile(clean[pf['key']], pf['profile'].get('source')) + if not ok: + return None + clean[pf['key']] = handle + + def _add(cur): + t = cur.get(str(table_key)) + if t is not None: + t.setdefault('rows', {})[rid] = clean + return cur + + _st(st).update(STORE_KEY, _add, flush='async') + # C3 (wave 22): a HUMAN-door insert is a row event — the record_created trigger's whole + # substrate. The engine's own writers never come through here, which is the loop law. + emit_row_event({'type': 'record_created', 'table': str(table_key), 'rowId': rid, + 'st': _st(st), 'user': str(username or '')}) + return rid + + +def add_rows(table_key, rows_in, username=None, st=None): + """⭐⭐ WAVE-29 T25 (owner item 6) — APPEND MANY ROWS IN ONE STORE WRITE. Returns the list of + new row ids, or None if the table refuses records at all. + + ⛔ WHY THIS EXISTS RATHER THAN A LOOP OVER `add_row`. Every `add_row` is a read-modify-write of + the WHOLE table document plus a row event; importing a 2,000-row spreadsheet that way is 2,000 + full-document copies under one lock, which is the same shape as the 1.4 s-per-bucket problem + item 20 exists to avoid — on the one uvicorn process this product runs. One `update()` writes + them all. + + ⛔ AND WHY NOT `automation_engine.upsert_rows`, which this module's own note (below) says to + reuse: that function is UPSERT-BY-KEY shaped, and v1 of the import door is deliberately + APPEND-ONLY (no key to merge on, no silent overwrite of a row somebody edited). It also lives + in the API layer, and `core/` may not import upward. When the import door grows an "update + matching rows" mode, the engine's is the one to lift — the note stands, it just is not this. + + ⚠ THE CAP IS CHECKED AGAINST THE WHOLE BATCH, not per row: a partial import that stops at + `MAX_ROWS` leaves the user reconciling a spreadsheet against a table, which is exactly what + the refusal-with-a-sentence exists to prevent. Nothing is written if the batch does not fit. + + ⚠ Ids continue the table's own sequence, and a PROFILE column validates every cell exactly as + `add_row` does — a row born with an unusable handle is a row the enrich action can never + answer, and the failure would otherwise surface as an automation quietly returning nothing. + """ + if not is_user_table(table_key, st) or not records_mutable(table_key, st): + return None + batch = [dict(r) for r in (rows_in or []) if isinstance(r, dict)] + if not batch: + return [] + defn = get(table_key, st) or {} + rows = defn.get('rows') or {} + cap = row_limit(table_key, st) # R6: None for a connected source + if cap is not None and len(rows) + len(batch) > cap: + return None + valid = {f['key'] for f in (defn.get('fields') or [])} + pf = _profile_of(defn.get('fields') or []) + next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + made = [] + for values in batch: + clean = {k: str(v) for k, v in values.items() if k in valid} + if pf and pf['key'] in clean: + handle, ok = normalize_profile(clean[pf['key']], pf['profile'].get('source')) + if not ok: + return None # atomic: one bad handle refuses the FILE + clean[pf['key']] = handle + next_id += 1 + made.append((str(next_id), clean)) + + def _add(cur): + t = cur.get(str(table_key)) + if t is not None: + bucket = t.setdefault('rows', {}) + for rid, clean in made: + bucket[rid] = clean + return cur + + _st(st).update(STORE_KEY, _add, flush='sync') + # ⚠ ONE EVENT PER ROW, deliberately: `record_created` is a trigger substrate, and an import of + # 500 rows genuinely IS 500 created records. The engine's own flood hold is what decides + # whether that runs an automation 500 times — a decision that belongs there, not to a writer + # that quietly emits less than it did. + for rid, _clean in made: + emit_row_event({'type': 'record_created', 'table': str(table_key), 'rowId': rid, + 'st': _st(st), 'user': str(username or '')}) + return [rid for rid, _ in made] + + +# ⛔ `upsert_rows` USED TO LIVE HERE, AND DELETING IT IS THE FIX (DEBT D-6, wave 20). +# +# There were TWO bulk-upsert implementations — this one and `automation_engine.upsert_rows` — +# with different counts (`{updated, inserted, orphans}` vs the engine's seven, including the +# `capped` count that makes a full table LOUD), a different signature (a dict keyed by value vs +# a list of rows) and a different cap story. D-6 called it a "silent drift risk"; the honest +# measurement is worse and simpler: **this one had zero callers, anywhere in the repo.** It was +# not drifting from the engine, it was a second answer nobody had ever asked. +# +# So there is now one implementation because there is one implementation — no parity gate to +# maintain, no second definition to keep in step. If a host-side bulk upsert is ever wanted, the +# engine's is the one that has been exercised (verify_automation section B) and it is PURE: +# `(existing, incoming, key_field, cap) -> (rows, counts)`, so it can be lifted here without its +# store half coming along. Do not re-derive a new one. + + +def set_label(table_key, label, st=None): + """Rename a table IN ITS DEFINITION. Returns the stored label, or None if refused. + + ⚠ WAVE 20, item 6a — THE RENAME USED TO WRITE ONLY `nav_meta`. That bucket is the nav's + display layer, so the rail showed the new name while everything reading the DEFINITION — + the automation editor's database picker above all — went on showing the old one. A rename + that only some surfaces can see is worse than no rename: the picker was not stale-looking, + it was confidently wrong, and a user choosing "Influencers" there could be choosing the + table they had renamed to something else months earlier. + """ + label = ' '.join(str(label or '').split())[:MAX_LABEL] + if not label or not is_user_table(table_key, st): + return None + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + t['label'] = label + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return label + + +# --------------------------------------------------------------------------------------------- +# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) +# --------------------------------------------------------------------------------------------- +# R2 reverses wave 17's "fields are per-user" law FOR THIS PATH: a `ut_*` table's fields are the +# TABLE'S SCHEMA, the way they are in Airtable — everyone with access to the database sees the +# same columns, and the creator or an admin edits them. Per-field `editRole` narrows or widens +# who may edit ONE column's definition without handing over the whole table. +# +# ⚠ `editRole` GOVERNS THE SCHEMA, NEVER THE VALUES. 'everyone' means anybody with access may +# rename this column or change its options; it does not decide who may type in its cells. Those +# are different questions and conflating them would let a column's own settings quietly become a +# data-permission system nobody wrote. + +#: Who may edit ONE field's definition. Fail-closed default: admins (= creator or admin). +FIELD_EDIT_ROLES = ('admins', 'everyone') + +#: Wave 22 (C7) — the metric-field vocabulary, MIRRORED from `automation_engine` as local +#: literals for the boot-path reason `MACHINE_OWNERS` states; the pair is held in step by a +#: verify_automation parity check, the same discipline. +METRIC_MEASURES = ('followers', 'avg_engagement', 'likes', 'comments') +METRIC_WINDOWS = ('latest', 'last_3_posts', 'last_7d', 'last_30d') +METRIC_PROFILE_MEASURES = ('followers', 'avg_engagement') +METRIC_AGGS = ('avg', 'sum', 'latest') + + +def _clean_metric(raw): + """One `metric` bag → the stored shape, or None (refused). A profile count over a + post-count window — or a SUM of a follower count — is a question the series cannot + answer; refused at write, never bent into a number that looks plausible (C7).""" + if not isinstance(raw, dict): + return None + measure = str(raw.get('measure') or '').strip() + window = str(raw.get('window') or '').strip() + agg = str(raw.get('agg') or '').strip() + if measure not in METRIC_MEASURES or window not in METRIC_WINDOWS: + return None + if measure in METRIC_PROFILE_MEASURES and (window == 'last_3_posts' or agg == 'sum'): + return None + if agg and agg not in METRIC_AGGS: + return None + out = {'source': 'ig', 'measure': measure, 'window': window} + if agg: + out['agg'] = agg + return out + + +# --------------------------------------------------------------------------------------------- +# ⭐⭐ THE RELATIONAL PAIR (2026-08-07, owner instruction) — `link` and `rollup` +# --------------------------------------------------------------------------------------------- +# Owner: *"adding relational database function with links and rollups… exactly like how Airtable +# does it… and this rollup needs to have formula that we can use to calculate things like average +# Views over last N posts."* +# +# Airtable's own contract, fetched 2026-08-07 (SPEC): +# multipleRecordLinks.options : linkedTableId · isReversed · prefersSingleRecordLink · +# inverseLinkFieldId · viewIdForRecordSelection +# rollup.options : recordLinkFieldId · fieldIdInLinkedTable · result · +# referencedFieldIds · isValid +# 17 functions, and a rollup is exactly ONE HOP. +# +# ⛔ THE CELL IS A SCALAR, WHICH IS NOT A PREFERENCE. Airtable's link cell is an ARRAY of record +# ids; the `Row` contract here is scalars end to end and `grid_events` refuses a non-scalar write. +# So a link cell holds a COMMA-JOINED list of linked row ids — the shape `multiselect` already +# uses for a set, so the grouping/filter/copy paths already know what to do with it. +# +# ⭐ THREE MODES, ONE KIND: +# `on` DECLARED -> a DERIVED link. The linked rows are those whose `on` column equals this +# row's `from` column. Machine-maintained, read-only, recomputed on the same +# pass that already recomputes `metric` cells. THIS IS THE INSTAGRAM CASE, and +# it is why the mode exists: `ut_ig_posts.influencer_key` ALREADY holds the +# relation and is rewritten by the engine on every pull, so a second +# user-editable copy of the same relation could only ever drift away from it. +# `on` ABSENT -> an ORDINARY link. The user picks records; the cell is editable; `single` is +# Airtable's `prefersSingleRecordLink`. Its value is the source of truth. +# `inverse` -> the COMPUTED reciprocal of one ordinary source link. It never fans writes +# into target rows: the relation pass derives it from the source cells, so the +# same shared-row truth powers both sides and every dependent Rollup. + +#: How many linked ids one link cell may carry. Row ids are short numeric strings, so 500 ids is +#: ~3 KB — comfortably inside a text cell, and `MAX_ROWS` bounds the absolute worst case anyway. +LINK_MAX_IDS = 500 + +#: Airtable's 17 aggregations, minus the two that are meaningless over scalar cells. +#: ⛔ `arrayflatten` and `arrayslice` are NOT here: both operate on NESTED arrays, and a cell in +#: this product is a scalar — they could only ever return their own input, so offering them would +#: mint columns that compute nothing. `arrayslice`'s real use ("just the first N") is what `limit` +#: does below, honestly and by declared rank. +#: ⭐ `stdev` is the ONE addition beyond Airtable's list (wave 28, owner ruling R4), and it is here +#: for a request no combination of the other fourteen can express: *"the average of the last 10 +#: posts, with anything beyond 2 standard deviations removed"*. `average` alone cannot say which +#: rows are outliers, and a client that computed the threshold itself would be a second definition +#: of the same statistic over a row set only the server can see (the fold receives the UNCAPPED +#: linked set; a link CELL shows at most `LINK_MAX_IDS`). +#: ⚠ SAMPLE standard deviation (n-1) — see `_rollup_fold`. The population form (n) is the wrong +#: default here: a rollup folds the rows that happen to be linked, which is a sample of an +#: account's posting history, not its entirety. +ROLLUP_FNS = ('sum', 'average', 'stdev', 'min', 'max', 'latest', 'count', 'counta', 'countall', + 'and', 'or', 'xor', 'concatenate', 'arrayjoin', 'arraycompact', 'arrayunique') +ROLLUP_SORT_DIRS = ('asc', 'desc') +ROLLUP_CONDITION_OPS = ('eq', 'neq', 'contains', 'not_contains', 'is_empty', 'is_not_empty', + 'gt', 'gte', 'lt', 'lte') +ROLLUP_CONDITION_CONJ = ('and', 'or') +ROLLUP_MAX_CONDITIONS = 20 + +#: ⭐⭐ THE SET-STATISTIC THRESHOLD (wave 28, R4 — contract C1). A condition may compare a field +#: against a LITERAL (`value`) or against a statistic OF THE SCOPED SET ITSELF +#: (`ref: {sigmas: k}` → mean + k*stdev of that same field). It is what makes "beyond 2 sigma" +#: expressible as configuration instead of code, and it composes with everything already here: +#: the scope (`sortBy` + `limit`) picks the window, THEN the threshold is computed over that +#: window, THEN the conditions filter it. +#: +#: ⛔ ORDER IS THE WHOLE CONTRACT AND IT IS NOT NEGOTIABLE: the statistic is computed over the +#: SCOPED set (after ranking and `limit`, before filtering). Computing it over the unfiltered +#: table would answer "2 sigma of everything this account ever posted" while the column says "of +#: the last 10", and both readings produce a plausible number — which is the failure this module +#: refuses everywhere else. +#: +#: ⛔ ORDERING OPS ONLY. `eq`/`neq` against a computed float is a coin flip on floating-point +#: representation, and `contains` against a number is meaningless — offering either would mint a +#: condition that silently never matches. +ROLLUP_REF_OPS = ('gt', 'gte', 'lt', 'lte') +#: A sanity bound, not a statistical one. Beyond ±10 sigma nothing is being selected on any real +#: distribution, so a value out here is a typo or a unit mistake, and refusing it is kinder than +#: storing a filter that can only ever match nothing. ⭐ It also does the NaN/infinity work for +#: free: `not (-10 <= NaN <= 10)` is True, and infinity fails the same comparison. +ROLLUP_MAX_SIGMAS = 10.0 + +#: The date-window KINDS a source-backed rollup may name. MIRRORS `harness/windows.py`'s +#: `WINDOW_KINDS`, deliberately as a local literal — `core` stays dependency-light and does not +#: import up, the same reason `UT_FIELD_TYPES` is a literal rather than an `aios_grid` import. +#: ⚠ This list VALIDATES; it never RESOLVES. `harness.windows.resolve(spec, today)` owns turning a +#: kind into a date pair, so there is exactly one implementation of what "ytd" means and this +#: module cannot drift into a second one. A kind added there and not here is simply not offerable +#: on a rollup yet — the fail-closed direction. +ROLLUP_SOURCE_WINDOWS = ( + 'all_time', 'today', 'yesterday', 'this_week', 'last_week', 'this_month', 'last_month', + 'this_quarter', 'last_quarter', 'this_year', 'last_year', 'ytd', 'ytd_last_year', + 'ltm', 'past_week', 'past_month', 'past_year', +) +#: ⛔ DELIBERATELY ABSENT: `last_n_days`, `next_n_days`, `custom`. Each needs a PARAMETER (an `n`, +#: or a date pair) that this bag has nowhere to carry, and a parameterised kind accepted without +#: its parameter would resolve to something arbitrary — a window that looks configured and +#: measures the wrong period. They land when the bag learns to carry the parameter, not before. +#: The ceiling on `limit`. Not a performance bound — `MAX_ROWS` is that — but a refusal to let a +#: column claim a window bigger than a table can hold. It therefore MOVES WITH the cap, which is +#: why it is written as `MAX_ROWS` and not as a number. +#: ⚠ THIS COMMENT SAID "100,000" UNTIL 2026-08-09 (wave 28) while sitting eight lines above +#: `ROLLUP_MAX_LIMIT = MAX_ROWS`, where `MAX_ROWS` is 60,000 — a doc contradicting the constant it +#: annotates, in the same screenful. Two sibling comments said it too (`odoo_relational`, +#: `rollup_sql`). 100,000 was a candidate cap REJECTED during its own derivation for clearing the +#: memory budget by 0.6%; the prose was written before the number lost. +#: ⛔ IT IS NOT `LINK_MAX_IDS`, and the near-miss is worth keeping. 500 looks like the real bound — +#: a rollup folds what a link names, and a link cell holds at most 500 ids. But that cap is a +#: DISPLAY cap: `compute_relation_cells` projects the first 500 ids into the cell and hands every +#: rollup the UNCAPPED `resolved` set, so a customer with 3,000 invoices shows 500 and still sums +#: all 3,000. Tightening this to 500 would have silently truncated the fold's declared window on +#: exactly the tables this wave exists to build ([[measure-the-real-call]]). +ROLLUP_MAX_LIMIT = MAX_ROWS + + +def _field_key(raw): + """A field/table key as this module spells them, or '' — the same normalisation + `_clean_field` applies to `key`, so a bag can never name a key the field layer would store + differently.""" + k = re.sub(r'[^a-z0-9_]+', '_', str(raw or '').strip().lower()).strip('_')[:60] + return k + + +def _clean_link(raw): + """One `link` bag → the stored shape, or None (refused). + + Same posture as `_clean_metric`: a mis-shaped bag REFUSES THE FIELD. A link column stored + without a usable target is a column promising a relation that nothing can ever resolve. + """ + if not isinstance(raw, dict): + return None + table = _field_key(raw.get('table')) + # ⛔ It must NAME A USER TABLE. A link into a connector-backed module key (`customer`, + # `product`) would promise a join across two strata with different row-id namespaces and + # different permission walls — a different feature, not a smaller version of this one. + if not table.startswith(KEY_PREFIX) or len(table) <= len(KEY_PREFIX): + return None + out = {'table': table} + on = _field_key(raw.get('on')) + frm = _field_key(raw.get('from')) + inverse = _field_key(raw.get('inverse')) + reciprocal = _field_key(raw.get('reciprocal')) + if inverse: + # Airtable's reciprocal field: rows in `table` whose ordinary `inverse` link includes + # this row. It is computed, so `on`/`from` cannot simultaneously configure another join. + if on or frm: + return None + out['inverse'] = inverse + elif on: + out['on'] = on + # `from` is OPTIONAL and resolved at compute time (the profile-flagged column, then the + # pinned one) — which is what makes an Instagram database link up with no configuration + # at all, i.e. the "automatically" in the instruction. + if frm: + out['from'] = frm + elif frm: + # ⛔ A `from` with no `on` configures NOTHING — there is no join to drive. Refused rather + # than dropped: a bag half of which is silently ignored is [[wrong-parent-not-broken-control]] + # with the control still on screen. + return None + if reciprocal: + out['reciprocal'] = reciprocal + if raw.get('single') is True: + out['single'] = True + return out + + +def _clean_rollup(raw): + """One `rollup` bag → the stored shape, or None (refused). + + ⭐ `limit` + `sortBy` ARE THE DELIBERATE SUPERSET OF AIRTABLE, and they are the owner's + headline: *"average Views over last N posts"*. Airtable's rollup conditions filter by + PREDICATE, never by RANK, so "the last 12" is not expressible there at all. + + ⛔ `limit` REQUIRES `sortBy`, and that refusal is the whole care in this function. "The last + N" with no declared order is not a measurement — it is whichever N rows happen to sit first + in the store, which for an append table is capture order and for a merged one is nothing at + all. A wrong number that looks right is the failure this module refuses everywhere else. + """ + if not isinstance(raw, dict): + return None + # ⭐ A SOURCE-BACKED ROLLUP — the read-through kind (owner 2026-08-09: *"make rollup be able to + # capture up to 1 million rows of a linked database … ok to push rollup filter etc to SQL"*). + # + # ⛔ WHY IT IS A DIFFERENT SHAPE RATHER THAN A BIGGER `link`. A linked rollup folds ROWS THAT + # EXIST IN THE STORE, and `MAX_ROWS` bounds them because every `ut_*` table lives in one JSON + # blob that is parsed and copied per request. ⚠ RAISING THE CAP TO 60,000 DOES NOT RETIRE THIS + # KIND, and the arithmetic is why: 256,810 order lines are 63.9 MB and 2.57 s PER COPY, so they + # cannot live there under any JSON substrate — 4.3x the cap and 8x a tolerable copy. This kind + # never COPIES the rows at all: it names a governed semantic TOPIC and a METRIC KEY, and one + # grouped SQL query answers every parent row at once (MEASURED: 1,748 customers over 256,810 + # lines in 232 ms). + # + # ⚠ A METRIC KEY, NEVER A FILTER FRAGMENT, and that is the load-bearing decision. `model/ + # metrics/*.yml` already carries each metric's scope, its `store_filter_sql` AND the matching + # `live_domain` — whose own comment says *"BOTH or store_parity compares two different + # questions"*. Binding to the key inherits the scope and the live-parity oracle; letting a + # rollup carry SQL would mint a second definition of a number the semantic layer exists to + # define once. + # + # ⚠ `window` is a date-window KIND (`ytd`, `this_month`, …) resolved against `today` at + # COMPUTE time, never a literal date pair — a hand-computed year start is correct until + # 1 January and wrong after it, with nothing to notice ([[date-window-vocabulary]]). + src = raw.get('source') + if isinstance(src, dict): + topic = _field_key(src.get('topic')) + measure = _field_key(src.get('measure')) + group_by = _field_key(src.get('groupBy')) + on = _field_key(src.get('on')) + if not (topic and measure and group_by and on): + return None + bag = {'topic': topic, 'measure': measure, 'groupBy': group_by, 'on': on} + window = str(src.get('window') or '').strip().lower() + if window: + if window not in ROLLUP_SOURCE_WINDOWS: + return None + bag['window'] = window + return {'source': bag} + link = _field_key(raw.get('link')) + field = _field_key(raw.get('field')) + fn = str(raw.get('fn') or '').strip().lower() + if not link or fn not in ROLLUP_FNS: + return None + # `countall` counts LINKED RECORDS, so it alone needs no field to read (Airtable's rule). + if not field and fn != 'countall': + return None + out = {'link': link, 'fn': fn} + if field: + out['field'] = field + try: + limit = int(str(raw.get('limit') or '0').strip() or 0) + except (TypeError, ValueError): + return None + sort_by = _field_key(raw.get('sortBy')) + distinct_by = _field_key(raw.get('distinctBy')) + sort_dir = str(raw.get('sortDir') or '').strip().lower() + if limit < 0 or limit > ROLLUP_MAX_LIMIT: + return None + if (limit or fn == 'latest') and not sort_by: + return None + if sort_dir and sort_dir not in ROLLUP_SORT_DIRS: + return None + if sort_by: + out['sortBy'] = sort_by + # Newest-first is the default because every window this feature exists for is "the LAST + # N", and a default that silently means "the first N" would answer a different question + # with the same column name. + out['sortDir'] = sort_dir or 'desc' + if limit: + out['limit'] = limit + elif sort_dir: + return None + if distinct_by: + # A rollup may cross a table that already contains duplicate logical records. Keep the + # first row after ranking for each identity so "last 12 posts" means twelve POSTS, while + # a snapshot table can still keep every timestamped observation by omitting this option. + out['distinctBy'] = distinct_by + # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, and it is a SECOND list rather than a flag on the + # first. Owner: *"instead of last 12 posts, we also want to make it so its last N record, + # where the record's Status is video."* + # + # ⛔ WHY TWO LISTS AND NOT ONE. `conditions` is applied AFTER the window (C1 fixed that order + # deliberately, so a `ref: {sigmas}` threshold is a statistic OF the rows being folded). That + # makes "the last 10 REELS" inexpressible with it: the window would take the last 10 posts of + # any kind and the filter would then keep whichever of those happened to be video — on this + # tenant's live data, 12 captured posts per profile of which ~8 are video, so the column would + # silently answer "the reels among the last 12" and wear the name "the last 10 reels". + # `where` runs BEFORE the ranking, so the window is spent on rows that already qualify. + # + # ⛔ `ref` IS REFUSED HERE, and that refusal is what keeps C1 coherent. A sigma computed over + # the pre-filtered-but-unwindowed set is a statistic of a DIFFERENT set from the one the fold + # reports on — the exact incoherence C1's own note describes ("the set being described and the + # set doing the describing would be different sets"). Fail closed: the threshold belongs in + # the post-filter, where the window is already fixed. + for bag_key, conj_key, allow_ref in (('where', 'whereConj', False), + ('conditions', 'conditionConj', True)): + conditions = [] + raw_conditions = raw.get(bag_key) or [] + if not isinstance(raw_conditions, list) or len(raw_conditions) > ROLLUP_MAX_CONDITIONS: + return None + for condition in raw_conditions: + if not isinstance(condition, dict): + return None + condition_field = _field_key(condition.get('field')) + op = str(condition.get('op') or '').strip().lower() + if not condition_field or op not in ROLLUP_CONDITION_OPS: + return None + item = {'field': condition_field, 'op': op} + if op not in ('is_empty', 'is_not_empty'): + ref = condition.get('ref') + if ref is not None: + # ⭐ THE SET-STATISTIC THRESHOLD (R4 / contract C1). `ref` replaces `value`; + # the number this leaf compares against is computed by the FOLD, over the + # scoped set, at compute time — so it moves with the data instead of freezing + # a threshold that was true the day somebody typed it. + # ⛔ NEVER BOTH. A leaf carrying `value` AND `ref` has two answers to one + # question and whichever the fold happened to read would be silent. Refuse + # the field. + if not allow_ref: + return None + if not isinstance(ref, dict) or condition.get('value') is not None: + return None + if op not in ROLLUP_REF_OPS: + return None + sigmas = ref.get('sigmas') + # `bool` is an `int` subclass, so `True` would otherwise validate as 1.0 + # sigma — a checkbox arriving in a numeric slot should be refused, not + # interpreted. + if isinstance(sigmas, bool) or not isinstance(sigmas, (int, float)): + return None + sigmas = float(sigmas) + if not (-ROLLUP_MAX_SIGMAS <= sigmas <= ROLLUP_MAX_SIGMAS): + return None # also catches NaN and +/-inf: every comparison is False + item['ref'] = {'sigmas': sigmas} + else: + value = condition.get('value') + if not isinstance(value, (str, int, float, bool)): + return None + item['value'] = str(value)[:1000] + conditions.append(item) + if conditions: + conj = str(raw.get(conj_key) or 'and').strip().lower() + if conj not in ROLLUP_CONDITION_CONJ: + return None + out[bag_key] = conditions + out[conj_key] = conj + return out + + +def _clean_field(raw, previous=None): + """One field dict → the stored shape, or None. The single validator for create AND patch, so + a column cannot be typed one way on the way in and another on the way back.""" + prev = previous or {} + raw = raw if isinstance(raw, dict) else {} + label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80] + key = re.sub(r'[^a-z0-9_]+', '_', + str(raw.get('key') or prev.get('key') or _slug(label)).strip().lower()) + key = key.strip('_')[:60] + ftype = str(raw.get('type') or prev.get('type') or 'text').strip().lower() + if not key or not label or ftype not in UT_FIELD_TYPES: + return None + out = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'} + # ⭐ 2026-08-10 — the DISPLAY format, on the add/patch door too. ⚠ `'format' in raw` rather than + # `raw.get('format')`, exactly like `description` below: an explicit `{}`/None CLEARS the + # format, while omitting the key keeps what is stored. A PATCH that sends only a label must not + # silently strip a formatting choice — that is how `agg` stopped totalling a money column. + fmt = _clean_format(raw.get('format') if 'format' in raw else prev.get('format')) + if fmt: + out['format'] = fmt + # Canonical descriptions are schema metadata, not the user's per-view note. Preserve them + # through every field write so product-owned presets can explain themselves in the header + # tooltip and schema drawer. An explicit blank clears a user-created field's description; + # omission keeps the previous declaration. + description_raw = (raw.get('description') if 'description' in raw + else prev.get('description')) + description = ' '.join(str(description_raw or '').split())[:300] + if description: + out['description'] = description + opts_raw = raw.get('options') if 'options' in raw else prev.get('options') + if ftype in ('select', 'multiselect'): + opts = [' '.join(str(o).split())[:60] for o in (opts_raw or []) if str(o).strip()][:50] + if opts: + out['options'] = opts + # ⭐⭐ WAVE-29 CONTRACT C1 (owner item 13) — **THE OPTION COLOURS, WHICH THIS DOOR DROPPED.** + # + # The client has carried `optionColors`/`colorCodeOptions` since wave 14 (`types.ts`, the + # `BRAND_SWATCHES` editor, `cells.optionTint`, the kanban card) and `aios_grid._clean_field` + # keeps both — but THIS validator, the one every `ut_*` create and patch goes through, had + # zero occurrences of either key. So the owner picked a colour on a user database, the pane + # saved, and the colour was gone on the next read: the same crossed-validator shape as item 2a, + # in a different pair of doors. `grid_events._FIELD_ECHO_PROPS` already carried both, so the + # echo wire was plumbed for a value nothing stored. + # + # ⛔ SIBLING KEYS, and `options` IS NOT RESHAPED. Every consumer treats options as `string[]` + # (this function flattens each to a bare `str` twelve lines up); folding colour into the option + # entries would be a wire change across the grid, the kanban, the editor and the filter panel. + # + # ⛔ OMIT-WHEN-ABSENT, exactly like `format`/`metric`/`link`, because `verify_api.py` and + # `verify_automation.py` assert `_clean_field(f) == f` over every engine-seeded field list — + # a branch that emitted `optionColors: {}` would redden both gates for every preset column in + # the product. ([[default-must-pass-its-own-guard]]) + # + # ⚠ INHERITED ACROSS A PATCH, for `agg`'s reason: a PATCH that sends only a label must not + # silently strip the colours somebody chose, which is precisely how a money column stopped + # totalling. + # + # ⚠ A HEX IS VALIDATED; BRAND MEMBERSHIP IS NOT, and that is deliberate. `_clean_option_colors` + # (`aios_grid.py:200`) — the sibling this mirrors — accepts any `#RRGGBB`, and the client + # renders a stored colour through `choiceColors.brandedTint`, which maps it to its NEAREST + # brand swatch at DISPLAY time while leaving what is stored byte-stable. Refusing a non-brand + # hex here would refuse values the other validator accepts on the same UI, and would break a + # column that moved between the two. + if ftype in ('select', 'multiselect'): + colors_raw = (raw.get('optionColors') if 'optionColors' in raw + else prev.get('optionColors')) + colors = _clean_option_colors(colors_raw, out.get('options') or []) + if colors: + out['optionColors'] = colors + color_code = (raw.get('colorCodeOptions') if 'colorCodeOptions' in raw + else prev.get('colorCodeOptions')) + if isinstance(color_code, bool): + out['colorCodeOptions'] = color_code + # ⭐ WAVE-29 T22's own repair: a `rating` column's star count. Grid-created columns on a `ut_*` + # database now come through this door rather than the per-user overlay, and this validator had + # no `max` — so a 3-star or 10-star column silently became 5-star on the way in. Bounded rather + # than refused, mirroring `aios_grid._clean_rating_max`: the field still holds its 1..max + # integers whatever the number was, so an unusable value is a default and never a lost column. + if ftype == 'rating': + max_raw = raw.get('max') if 'max' in raw else prev.get('max') + if max_raw is not None: + try: + out['max'] = max(2, min(int(max_raw), 10)) + except (TypeError, ValueError): + pass + role = str(raw.get('editRole') or prev.get('editRole') or 'admins').strip().lower() + out['editRole'] = role if role in FIELD_EDIT_ROLES else 'admins' + if prev.get('default') is True or raw.get('default') is True: + out['default'] = True + # ⭐ 2026-08-07 (owner ruling) — THE PRIMARY COLUMN, DECLARED RATHER THAN POSITIONAL. + # + # The grid's identity column is `fields.find(f => f.pinned)?.key ?? fields[0]?.key` + # (`useGridColumns.reconcileOrder` / `permsModel.identityKey`), forced to position 0 and + # force-shown. No `ut_*` field had ever declared `pinned`, so the fallback was the whole rule: + # whichever column happened to be created FIRST became the locked primary forever. On a + # database somebody made by hand and later pointed at an Instagram automation, that is the + # `name` column `create()` mints — which nothing ever writes, so the table's identity column + # was permanently blank while `handle` sat six columns to the right. + # + # ⚠ STICKY ACROSS A PATCH, like `default` one line up: the engine declares it (the C1 preset + # set pins `handle`), and a client PATCH that simply omits the key must not silently unpin the + # column it is editing. Clearing it is deliberate work, not a side effect of renaming a header. + if prev.get('pinned') is True or raw.get('pinned') is True: + out['pinned'] = True + # ⭐ 2026-08-09 (wave 28) — `agg` DECIDES WHETHER THE GRID'S TOTALS ROW SUMS THE COLUMN, and + # this door silently dropped it while `clean_fields` (:402) kept it. Two validators, one + # question, opposite answers — D-91's exact class, found from the other end: the register's row + # described `clean_fields` as the lax door, and by the time it was read that half had been + # fixed while THIS one still diverged. + # ⛔ THE CONSEQUENCE IS QUIET, WHICH IS WHY IT SURVIVED. Every currency and money-rollup column + # in the four Odoo databases declares `agg: "sum"`; the spawn writes definitions straight to + # the store, so the totals row worked — until anyone PATCHED one of those fields (a rename, a + # width, a description), at which point the column silently stopped totalling and nothing + # anywhere went red. Caught by a gate leg written to assert the LINK bag survives this door, + # which found `agg` instead. + # ⚠ Sticky across a patch for the same reason as `default` and `pinned`: a client PATCH that + # omits the key must not un-total a column as a side effect of renaming its header. + # ⭐ WAVE-29 C7 — the vocabulary widened from the single word `sum` to `FIELD_AGGS`. The + # STICKINESS is unchanged and now applies to every name: a PATCH that omits `agg` keeps + # whatever the column had, because dropping it is how a money column silently stopped + # totalling. ⛔ An unknown name is DROPPED rather than stored — a summary nothing can compute + # would render blank under a picker that says otherwise. + # ⚠ `'agg' in raw`, not `raw.get('agg') or …` — the same idiom `format` and `description` use + # two screens up, and it is what makes the difference between OMITTING the key (keep what is + # stored: the wave-28 stickiness that stops a rename un-totalling a money column) and sending + # it EMPTY (clear the summary, which the picker's "None" row has to be able to do). Written + # the loose way, "None" would have been a no-op with the menu cheerfully showing the change. + agg_name = str((raw.get('agg') if 'agg' in raw else prev.get('agg')) or '').strip() + if agg_name in FIELD_AGGS: + out['agg'] = agg_name + # An automation column carries the binding the engine reads; it is preserved across a patch + # rather than re-declared, because the automation editor owns it and this route does not. + auto = raw.get('automation') if 'automation' in raw else prev.get('automation') + if isinstance(auto, dict): + out['automation'] = auto + # C7 (wave 22): a metric bag is validated or the FIELD is refused — a mis-shaped metric + # column stored anyway would be a column of invented blanks wearing a number's name. + metric_raw = raw.get('metric') if 'metric' in raw else prev.get('metric') + if metric_raw is not None: + m = _clean_metric(metric_raw) + if m is None: + return None + out['metric'] = m + # C3 (wave 25, R7): the PROFILE FLAG. Same posture as the metric bag one line up — a + # mis-shaped flag refuses the FIELD, and an explicit `profile: None` CLEARS it, so the flag + # can be taken off a column without deleting the column. ⚠ `type: 'text'` is enforced here + # rather than left to the caller: R7 is *"a flag on an ordinary text field"*, and a flag on + # a select/int would promise a validated handle to a cell this module never validates. + profile_raw = raw.get('profile') if 'profile' in raw else prev.get('profile') + if profile_raw is not None: + p = _clean_profile(profile_raw) + if p is None or ftype != 'text': + return None + out['profile'] = p + # ⭐ 2026-08-07 — the relational pair. Same posture as the two bags above: a mis-shaped bag + # refuses the FIELD, and the bag is required for its own type. ⛔ THE TYPE/BAG PAIRING IS + # ENFORCED BOTH WAYS — a `link` with no `link` bag is a column that can never resolve a row, + # and a `link` bag on an `int` column is a declaration nothing reads. Either half alone is the + # silent kind of broken: the column exists, renders, and means nothing. + # ⚠ AN INHERITED BAG IS ONLY INHERITED WHILE THE TYPE STILL WANTS IT (2026-08-09, owner: + # *"everything is custom and changeable always"*). `prev`'s bag used to be picked up on ANY + # patch, so retyping a rollup column to text sent `{'type': 'text'}`, inherited the rollup bag + # from `prev`, hit the `ftype != 'rollup'` arm and returned None — i.e. the server refused to + # let a rollup become anything else, with a sentence about the name and type. Changing the + # type is the one act that MEANS "drop this bag", so a mismatched inherited bag is dropped. + # ⛔ AN EXPLICIT bag on a mismatched type STILL REFUSES: that is a caller declaring something + # nothing will ever read, which is the silent-broken half this pairing exists to catch. + prev_type = str(prev.get('type') or '').strip().lower() + link_raw = (raw.get('link') if 'link' in raw + else (prev.get('link') if prev_type == ftype else None)) + if link_raw is not None: + lk = _clean_link(link_raw) + if lk is None or ftype != 'link': + return None + out['link'] = lk + elif ftype == 'link': + return None + rollup_raw = (raw.get('rollup') if 'rollup' in raw + else (prev.get('rollup') if prev_type == ftype else None)) + if rollup_raw is not None: + rl = _clean_rollup(rollup_raw) + if rl is None or ftype != 'rollup': + return None + out['rollup'] = rl + elif ftype == 'rollup': + return None + # ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` column's language bag. + # + # ⚠ Deliberately the WEAKEST posture of the four bags above, and the asymmetry is the whole + # point rather than an inconsistency. `metric`/`link`/`rollup` bags DEFINE what their column + # is: without one the cell can never resolve a value, so a missing or mis-shaped bag refuses + # the field. A `language` chooses a HIGHLIGHTER over a string the column already holds + # perfectly well — so it is optional (an unconfigured code column renders plain), it survives + # a patch that omits it (the `automation` rule: the gear owns it, a header rename must not + # clear it), and an unknown language degrades to `plain` instead of destroying the column. + # R13's "NO execution engine" is why this can be so relaxed: the value selects a renderer and + # can never select an interpreter. + # ⚠ Function-local import, the convention `grid_events` already uses for the same module + # (`import aios_grid as _ag2`, five call sites): `core` must not import the layer above it at + # MODULE level. One vocabulary, one validator — declaring the language list a second time + # here is exactly the drift the fields contract exists to catch. + # ⚠ OMIT = KEEP, `plain` = CLEAR. `_clean_code` returns None for plain (see its docstring), + # so an explicit `{'language': 'plain'}` from the editor drops the key here while a patch + # that never mentions `code` inherits the previous bag. Without that split, switching a + # column back to Plain text would be a control that silently does not save. + code_raw = raw.get('code') if 'code' in raw else prev.get('code') + if code_raw is not None and ftype == 'code': + import aios_grid as _agc + c = _agc._clean_code(code_raw) + if c: + out['code'] = c + # ⭐⭐ 2026-08-10 — THE FORMULA EXPRESSION. Posture: `link`/`rollup`'s, not `code`'s — a + # formula column with no expression is a column that can never show a value, so the pairing is + # enforced BOTH WAYS and either half alone refuses the field. + # + # ⚠ `valid_keys` IS DELIBERATELY OMITTED, and it is the one real asymmetry with the overlay + # door. `grid_events` passes the workspace's own key set, so a `{ref}` naming no column is + # refused at write time there. This function is handed ONE field with no sibling context — + # `add_field`/`patch_field` could supply the table's keys, but `clean_fields` validates a list + # mid-construction where half the referenced columns do not exist yet, and `clean_machine_ + # fields` judges a list that has no table at all. A second, weaker copy of the ref rule in one + # of the three would be the divergence this module keeps paying for. STRUCTURE is checked here + # (charset, length, balanced parens and quotes, well-formed refs); an unresolvable ref blanks + # the CELL client-side, which is `_clean_formula`'s own documented read-time behaviour and the + # same answer a later-deleted column already produces. + # ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import. + formula_raw = (raw.get('formula') if 'formula' in raw + else (prev.get('formula') if prev_type == ftype else None)) + if formula_raw is not None: + fx = _ag_formula(formula_raw) + if fx is None or ftype != 'formula': + return None + out['formula'] = fx + elif ftype == 'formula': + return None + return out + + +def is_derived_link(field): + """Is this a link whose cell the ENGINE owns? (an `on` join, not a user-picked set) + + ⛔ ONE definition, read by the write wall, the compute pass and the wire. The alternative is + three places independently asking "does the bag have an `on`", which is how one of them + silently keeps taking writes after the other two stop. + """ + lk = (field or {}).get('link') + return isinstance(lk, dict) and bool(lk.get('on') or lk.get('inverse')) + + +def is_computed_cell(field): + """Is this field's cell computed by the SERVER, so a human write is an invented value? + + The `ut_*` twin of `aios_grid.READONLY_CUSTOM_TYPES`, and deliberately a PREDICATE rather + than a type set: a `link` is read-only when it is derived and editable when it is not, which + a set of type names cannot express. `grid_events` is the wall that reads it. + + ⭐⭐ 2026-08-10 — `formula` JOINS, and it is the one member here whose value the server does + NOT compute. Every other member is refused because the server owns the number; a formula cell + is refused because NOBODY owns a stored number — the browser recomputes the column on every + paint from the row's other cells, so a value written here is overwritten on screen before it + is ever read and sits in the store as a fossil no reader agrees with. The client already + declines to offer an editor (`READONLY_CELL_TYPES`) and declines the paste + (`coerceClipboardValue`); both are courtesy, and this is the wall + ([[schema-role-is-not-a-value-wall]] — a courtesy in front of no wall is the shape that + keeps shipping). + """ + f = field or {} + return (isinstance(f.get('metric'), dict) or f.get('type') in ('rollup', 'formula') + or is_derived_link(f)) + + +#: The numeric family and the date family, named once. Membership decides which cells the +#: refusal below can speak about at all — a `text` column takes any string by definition. +_NUMERIC_CELL_TYPES = ('int', 'currency', 'pct') +_DATE_CELL_TYPES = ('date',) + + +def cell_type_refusal(field, value): + """Why this value cannot go in this column, or None if it can. ⛔ REFUSES; never rewrites. + + ⭐⭐ W29-T81 — THE SERVER-SIDE HALF of "can this text be this field's value". Until now the + only evaluator was `coerceClipboardValue`, which lives in the BROWSER — so the import door + took `{"qa_count": "seventeen-ish"}` at an `int` column, answered `201 {"imported": 1}`, and + stored it verbatim. Measured against production on `bac40c2`. A rule enforced only by the + client is a rule that holds for exactly one client ([[one-evaluator-per-question]]). + + ⚠ DELIBERATELY NARROWER THAN THE CLIENT'S COERCER, and it must stay that way: this asks only + whether a NUMBER is a number and a DATE is a date, the two families where an unusable string + is later rendered as a fabricated figure. It accepts everything the client's coercer emits + (canonical ISO dates, plain decimals) plus the human spellings that coercer also takes + ("$1,234.50"), so no UI path can trip a wall the UI cannot see. Choice vocabularies stay + client-side for now: their options change under a stored row, and refusing an import against + a vocabulary edited yesterday would reject data that is merely out of date. + + ⚠ BLANK IS ALWAYS ACCEPTABLE. An empty cell is an empty cell in every column; the import + plan already omits them, and refusing one would make a ragged spreadsheet unimportable. + """ + f = field or {} + ftype = f.get('type') + text = '' if value is None else str(value).strip() + if not text: + return None + label = f.get('label') or f.get('key') or 'that column' + if ftype in _NUMERIC_CELL_TYPES: + try: + float(text.replace('$', '').replace(',', '').replace('%', '').replace(' ', '')) + except (TypeError, ValueError): + return f'{text[:40]!r} is not a number, and {label!r} holds numbers' + return None + if ftype == 'rating': + try: + n = int(text) + except (TypeError, ValueError): + return f'{text[:40]!r} is not a whole number, and {label!r} holds a rating' + top = f.get('max') if isinstance(f.get('max'), int) and f.get('max') else 5 + if not 1 <= n <= top: + return f'{label!r} takes a rating from 1 to {top}, and {text[:40]!r} is outside it' + return None + if ftype in _DATE_CELL_TYPES: + # ISO-prefixed, which is exactly what the client's coercer produces and what every + # reader of a stored date assumes. A month name is not refused as invalid English — it + # is refused because nothing downstream would parse it either. + if not re.match(r'^\d{4}-\d{2}-\d{2}', text): + return (f'{text[:40]!r} is not a date, and {label!r} holds dates ' + f'(write it as YYYY-MM-DD)') + return None + return None + + +#: ⭐⭐ 2026-08-09 (owner) — THE STAMP THAT SURVIVES A RECONCILER. +#: +#: Owner: *"No rollup field should be uneditable, everything is custom and changeable always."* +#: Unlocking `may_edit_field` alone would have been a lie with a delay on it: BOTH preset +#: reconcilers (`automation_engine._reconcile_ig_graph_fields` and `odoo_relational`'s forward +#: migration) OVERWRITE every contract key — `label`, `type`, `link`, `rollup` — on any field +#: still carrying `automation.preset`. So a reconfigured "Avg views · last 12 posts" would have +#: saved, rendered, recomputed, and then silently reverted to the shipped contract at the next +#: enrichment run or store resync. The edit would look like it worked, which is worse than a +#: refusal ([[flag-shipped-without-its-writer]]). +#: +#: ⛔ IT DOES NOT CLEAR `preset`, and that is deliberate. `preset` records PROVENANCE (this column +#: was spawned by the product, and `ut_ensure` re-stamps it on every run anyway — clearing it +#: would be undone within one tick). This stamp records CUSTODY: a human has taken this column +#: over, so the machine stops rewriting its definition while still owning its VALUES. +USER_EDITED_KEY = 'userEdited' + + +def user_edited(field): + """Has a human taken over this preset column's DEFINITION? (the reconcilers' skip signal)""" + auto = (field or {}).get('automation') + return isinstance(auto, dict) and auto.get(USER_EDITED_KEY) is True + + +def preset_editable(field): + """Does the pre-set lock let this column's DEFINITION be changed anyway? + + ⛔ ONE PREDICATE FOR ONE QUESTION, read by all three walls that enforce the lock + (`may_edit_field` here, `routes_tables._field_or_refuse`, and the client's `schemaLocked`). + Three independent copies of "is this preset?" is how one of them keeps refusing after the + other two stop — the shape D-107 wore, and the reason `is_derived_link` above is a function + rather than an expression repeated three times. + + Today it answers YES for exactly one thing: a ROLLUP. The owner's ruling names rollups, and + the argument is specific to them rather than a general softening of the lock — a rollup holds + no data of its own. It is a QUESTION asked of other rows ("average views over the last N + posts"), it can be re-asked at any time, and re-asking it costs nothing because the answer is + recomputed from the authoritative store on the next pass. Retyping a preset `followers` + column, by contrast, would strand real measurements in a column that can no longer read them. + """ + return str((field or {}).get('type') or '') == 'rollup' + + +def clean_machine_fields(fields): + """`(clean, refused)` — the SANCTIONED way for a machine writer to seed a table's schema + (wave 25, doc order step 2). `clean` is the list as this module would store it; `refused` + names every field that did not survive, so a caller can be LOUD rather than short. + + ⛔ WHY THIS EXISTS. `automation_engine.ut_ensure` writes field dicts STRAIGHT into the + `user_tables` bucket with `rt.update(UT_STORE_KEY, …)` — it never routes through this + module, so **the field layer's own validator has never judged a single column the engine has + ever spawned.** That is [[default-must-pass-its-own-guard]] with a live subject: the product + seeds fields against a law it does not run. Nothing has broken yet only because the drift is + one key wide and in the fail-closed direction (a field with no `editRole` reads as + not-`everyone`), which is luck, not design. + + ⚠ THIS IS THE "IMPOSSIBLE" HALF AND IT IS OPT-IN; the LOUD half is `verify_api`'s derived + section, which walks every field list the engine declares and refuses to let one through + that this validator would change in any way but adding `editRole`. A helper the caller may + ignore is not a control — the gate is the control, and it covers a list nobody remembered + to route through here. + """ + clean, refused = [], [] + for f in (fields or []): + got = _clean_field(f) + if got is None: + refused.append(str((f or {}).get('key') or f)) + else: + clean.append(got) + return clean, refused + + +def may_edit_field(table_key, fkey, viewer, is_admin=False, st=None): + """May `viewer` change THIS column's definition? Creator/admin always; others only when the + field itself says `editRole: 'everyone'`. Fail-closed on an unknown field.""" + table = get(table_key, st) or {} + field = next((f for f in (table.get('fields') or []) if f.get('key') == str(fkey)), None) + # Instagram's pre-set schema is product contract, not tenant configuration. Even an admin + # may sort/filter/hide it, but cannot rename, retype, duplicate or delete it. Ordinary fields + # on the same database remain user-owned, including user-created Links and Rollups. + if isinstance((field or {}).get('automation'), dict) \ + and field['automation'].get('preset') is True \ + and not preset_editable(field): + return False + if may_open(table_key, viewer, is_admin, st) and ( + bool(is_admin) or table.get('createdBy') == viewer): + return True + for f in (table.get('fields') or []): + if f.get('key') == str(fkey): + return f.get('editRole') == 'everyone' + return False + + +def flow_bound(bag, st=None): + """C8 (wave 22, owner item 5) — may this `automation` bag be STORED? Only when its + `flowId` names an automation definition that exists in this tenant's `automations` bucket. + Fail-closed both ways: no flowId is a column that silently never runs, and a flowId naming + a deleted flow is the same thing one delete later. (Read tolerance for pre-law fields + lives in `aios_grid._clean_automation` — this guards the WRITE doors only.)""" + if not isinstance(bag, dict): + return True # no bag, no law — an ordinary column + flow = str(bag.get('flowId') or '').strip() + if not flow: + return False + try: + return flow in (_st(st).get('automations') or {}) + except Exception: + return False + + +def add_field(table_key, raw, st=None): + """Append one column to the shared schema. Returns the stored field, or None if refused.""" + if not is_user_table(table_key, st): + return None + defn = get(table_key, st) or {} + have = [f for f in (defn.get('fields') or [])] + if len(have) >= MAX_FIELDS: + return None + field = _clean_field(raw) + if not field or any(f.get('key') == field['key'] for f in have): + return None + if not flow_bound(field.get('automation'), st): + return None # C8: no field without a flow (400 at the route) + # C3 (wave 25, R7's first mitigation): AT MOST ONE profile field per table. Without it the + # flag means nothing — "the profile column" stops being resolvable the moment there are two, + # and the enrich action would have to guess which handle it was pointed at. The route names + # the existing column in its refusal (`_refusal_sentence`); the LAW is here. + if field.get('profile') and _profile_of(have): + return None + + def _add(cur): + t = cur.get(str(table_key)) + if t is not None: + t.setdefault('fields', []).append(field) + return cur + + _st(st).update(STORE_KEY, _add, flush='sync') + return field + + +def _reciprocal_link_key(table_key, field_key): + digest = hashlib.sha1(f'{table_key}:{field_key}'.encode('utf-8')).hexdigest()[:10] + return f'linked_{digest}' + + +def sync_reciprocal_link(table_key, field_key, st=None): + """Create/repair Airtable's reciprocal link field for one ordinary link. + + The source cell stores picked target row ids. The reciprocal is a computed inverse link on + the target database; it lists source rows that include the current target id. Retargeting or + retyping the source removes the obsolete inverse in the same store update. + """ + table_key, field_key = str(table_key), str(field_key) + result = {'field': None, 'reciprocal': None} + + def _sync(cur): + cur = cur if isinstance(cur, dict) else {} + source = cur.get(table_key) or {} + source_field = next((f for f in (source.get('fields') or []) + if f.get('key') == field_key), None) + # Remove every old inverse for this source identity first. This makes retargeting and + # deleting deterministic instead of leaving a live-looking backlink on the old table. + for candidate in cur.values(): + if not isinstance(candidate, dict): + continue + candidate['fields'] = [f for f in (candidate.get('fields') or []) + if not (isinstance(f.get('link'), dict) + and f['link'].get('table') == table_key + and f['link'].get('inverse') == field_key)] + if not source_field or source_field.get('type') != 'link': + return cur + bag = dict(source_field.get('link') or {}) + if bag.get('on') or bag.get('inverse'): + return cur + target_key = str(bag.get('table') or '') + target = cur.get(target_key) + if target is None: + return cur + # ⛔ 2026-08-10 — THE COLUMN CAP APPLIES TO A COLUMN NOBODY ASKED FOR, TOO. + # + # `add_field` refuses at `MAX_FIELDS`; this function appended straight past it, and the + # field it appends lands on a table the user is not even looking at. The IG preset set is + # 41 columns against a cap of 60, so a Posts or Comments database is genuinely reachable — + # and the failure mode is the worst kind: the source link saves, the target silently grows + # its 61st column, and every LATER `add_field` on that table is refused with a sentence + # about a cap the user never crossed on purpose. + # ⚠ THE SOURCE LINK STILL SAVES. Refusing the whole relation because the OTHER database is + # full would be a refusal the user cannot act on from where they are standing; an ordinary + # link with no reciprocal is exactly what every link was before this function existed, and + # it still resolves its own cell. `reciprocal` is simply not stamped on the bag, so the + # next `sync` on a table with room creates it — this is a skip, not a tombstone. + if len(target.get('fields') or []) >= MAX_FIELDS: + return cur + reciprocal_key = _reciprocal_link_key(table_key, field_key) + reciprocal = _clean_field({ + 'key': reciprocal_key, + 'label': str(source.get('label') or table_key)[:80], + 'type': 'link', + 'link': {'table': table_key, 'inverse': field_key, + 'reciprocal': field_key}, + 'editRole': 'admins', + }) + if reciprocal is None: + return cur + target.setdefault('fields', []).append(reciprocal) + bag['reciprocal'] = reciprocal_key + source_field['link'] = bag + result['field'] = dict(source_field) + result['reciprocal'] = dict(reciprocal) + return cur + + _st(st).update(STORE_KEY, _sync, flush='sync') + return result + + +def patch_field(table_key, fkey, raw, st=None): + """Edit one column's definition IN PLACE. Returns the stored field, or None if refused. + + ⚠ THE KEY NEVER MOVES. A field's key is what every stored cell is filed under, so accepting + a new one here would orphan every value in the column while looking like a rename. The + LABEL is the renameable thing; the key is identity. + """ + if not is_user_table(table_key, st): + return None + fields = [dict(f) for f in ((get(table_key, st) or {}).get('fields') or [])] + idx = next((i for i, f in enumerate(fields) if f.get('key') == str(fkey)), -1) + if idx < 0: + return None + merged = dict(raw or {}) + merged['key'] = str(fkey) + field = _clean_field(merged, fields[idx]) + if not field: + return None + if 'automation' in merged and not flow_bound(field.get('automation'), st): + return None # C8: a patch cannot un-bind a column either + # C3: the one-per-table law on the PATCH door too — flagging a second column is the same act + # as adding one. `exclude=fkey` so re-saving the profile column does not collide with itself. + if field.get('profile') and _profile_of(fields, exclude=str(fkey)): + return None + # ⭐⭐ 2026-08-09 — THE CUSTODY STAMP IS WRITTEN HERE, at the ONE door a human reaches. + # + # `USER_EDITED_KEY`'s note has the why. It is written here rather than inside `_clean_field` + # because `_clean_field` is also the machine's validator (`clean_fields`, `clean_machine_ + # fields`): stamping there would mark the product's OWN spawn as user-edited and switch the + # reconcilers off for every preset column on the first tick, which is the inverse of the bug + # it exists to prevent. + # ⚠ Only when the definition ACTUALLY differs. A no-op save (open the editor, press Save) + # must not quietly take a column out of the product's contract. + if user_edited(fields[idx]) or (field != fields[idx] + and isinstance(fields[idx].get('automation'), dict) + and fields[idx]['automation'].get('preset') is True): + automation = dict(field.get('automation') or fields[idx].get('automation') or {}) + automation[USER_EDITED_KEY] = True + field['automation'] = automation + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + for i, f in enumerate(t.get('fields') or []): + if f.get('key') == str(fkey): + t['fields'][i] = field + break + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return field + + +def delete_field(table_key, fkey, st=None): + """Drop one column from the shared schema. Refuses to leave a table fieldless. + + ⚠ THE CELLS ARE LEFT IN THE ROWS ON PURPOSE. A deleted column whose values were also + scrubbed makes an accidental delete unrecoverable; the values are invisible without a field + declaring them, and re-adding the column with the same key brings them back. Same reasoning + as `routes_tables.delete_table` leaving the workspace bucket in place. Booked, not hidden. + """ + if not is_user_table(table_key, st): + return False + fields = (get(table_key, st) or {}).get('fields') or [] + if len(fields) <= 1 or not any(f.get('key') == str(fkey) for f in fields): + return False + doomed = next((f for f in fields if f.get('key') == str(fkey)), {}) + doomed_link = doomed.get('link') if isinstance(doomed.get('link'), dict) else {} + inverse_source = None + if doomed_link.get('inverse'): + source_table = str(doomed_link.get('table') or '') + source_field = str(doomed_link.get('inverse') or '') + source = get(source_table, st) or {} + if len(source.get('fields') or []) <= 1: + return False + inverse_source = (source_table, source_field) + + def _drop(cur): + t = cur.get(str(table_key)) + if t is not None: + t['fields'] = [f for f in (t.get('fields') or []) if f.get('key') != str(fkey)] + if inverse_source: + source_table, source_field = inverse_source + source = cur.get(source_table) + if isinstance(source, dict): + source['fields'] = [f for f in (source.get('fields') or []) + if f.get('key') != source_field] + # An ordinary link owns its reciprocal field. Delete the reciprocal in the same schema + # update so no target database can retain a live-looking backlink to a missing source. + for candidate in cur.values(): + if not isinstance(candidate, dict): + continue + candidate['fields'] = [f for f in (candidate.get('fields') or []) + if not (isinstance(f.get('link'), dict) + and ((f['link'].get('table') == str(table_key) + and f['link'].get('inverse') == str(fkey)) + or (inverse_source + and f['link'].get('table') == inverse_source[0] + and f['link'].get('inverse') == inverse_source[1])))] + return cur + + _st(st).update(STORE_KEY, _drop, flush='sync') + return True + + +def rename_choice_values(table_key, fkey, renames, st=None): + """Rename select/multiselect OPTIONS **and migrate every stored cell** (contract C-RENAME). + + `renames` = `[{'from': old, 'to': new}, …]` — an EXPLICIT MAPPING, never a diff. A diff + cannot tell "renamed Blue to Navy" from "deleted Blue and added Navy", and guessing wrong + empties a column silently. + + Returns `{'options': n, 'cells': n}`. This is the DEFINITION half — the base options list and + the base row values. A user's per-user overlay stratum and any view filter naming the old + value are `core.table_store`'s half of the same contract; the caller runs both. + """ + pairs = [] + for r in (renames or [])[:50]: + if not isinstance(r, dict): + continue + a = ' '.join(str(r.get('from') or '').split())[:60] + b = ' '.join(str(r.get('to') or '').split())[:60] + if a and b and a != b: + pairs.append((a, b)) + if not pairs or not is_user_table(table_key, st): + return {'options': 0, 'cells': 0} + mapping = dict(pairs) + counts = {'options': 0, 'cells': 0} + + def _apply(cur): + t = cur.get(str(table_key)) + if t is None: + return cur + multi = False + for f in (t.get('fields') or []): + if f.get('key') != str(fkey): + continue + multi = f.get('type') == 'multiselect' + opts = f.get('options') or [] + new_opts, seen = [], set() + for o in opts: + v = mapping.get(o, o) + if v not in seen: # a rename ONTO an existing option merges them + seen.add(v) + new_opts.append(v) + if o in mapping: + counts['options'] += 1 + if opts: + f['options'] = new_opts + for row in (t.get('rows') or {}).values(): + if not isinstance(row, dict) or str(fkey) not in row: + continue + cell = str(row.get(str(fkey)) or '') + if not cell: + continue + if multi: + # ⚠ MULTISELECT CELLS ARE COMMA-JOINED LABEL STRINGS, so a rename has to walk the + # parts. Rewriting the whole string would only ever hit a single-value cell. + parts = [p.strip() for p in cell.split(',')] + nxt = [mapping.get(p, p) for p in parts] + if nxt != parts: + row[str(fkey)] = ', '.join(dict.fromkeys(nxt)) + counts['cells'] += 1 + elif cell in mapping: + row[str(fkey)] = mapping[cell] + counts['cells'] += 1 + return cur + + _st(st).update(STORE_KEY, _apply, flush='sync') + return counts + + +def patch_cells(table_key, row_id, values, st=None): + """Write shared definition-row cells for trusted server-side writers.""" + if not is_user_table(table_key, st): + return False + rows = (get(table_key, st) or {}).get('rows') or {} + if str(row_id) not in rows: + return False + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + t.setdefault('rows', {}).setdefault(str(row_id), {}).update( + {str(k): str(v) for k, v in (values or {}).items()}) + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return True + + +def patch_link_cell(table_key, row_id, fkey, value, st=None): + """Persist one user-picked Link cell in the shared row and return its canonical id string. + + A Link is a relationship in the database schema, not one user's visual overlay. Persisting + it here makes the reciprocal field and every Rollup see the same source of truth. Derived + joins and inverse fields are engine-owned and are refused by this door. + """ + table_key, row_id, fkey = str(table_key), str(row_id), str(fkey) + if not is_user_table(table_key, st) or not records_mutable(table_key, st): + return None + table = get(table_key, st) or {} + if row_id not in (table.get('rows') or {}): + return None + field = next((f for f in (table.get('fields') or []) if f.get('key') == fkey), None) + bag = (field or {}).get('link') + if (field or {}).get('type') != 'link' or not isinstance(bag, dict) \ + or bag.get('on') or bag.get('inverse'): + return None + target = get(str(bag.get('table') or ''), st) or {} + valid = set((target.get('rows') or {}).keys()) + raw_ids = value if isinstance(value, (list, tuple, set)) else str(value or '').split(',') + picked, seen = [], set() + for raw_id in raw_ids: + rid = str(raw_id).strip() + if not rid or rid in seen: + continue + if rid not in valid or len(picked) >= LINK_MAX_IDS: + return None + seen.add(rid) + picked.append(rid) + if bag.get('single'): + break + canonical = ','.join(picked) + + def _set(cur): + current = cur.get(table_key) + if current is not None and row_id in (current.get('rows') or {}): + current['rows'][row_id][fkey] = canonical + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return canonical + + +def patch_profile_cell(table_key, row_id, fkey, value, st=None): + """Write a PROFILE cell — and, when it is blanked, clear that row's preset cells IN THE SAME + WRITE (wave 25, contract C3 + owner ruling R6). Returns + `{'handle', 'cleared': [key, …]}`, or None when the write is refused. + + ⭐ THIS IS THE `ut_*` WRITE DOOR R6 NAMES, and it is deliberately NOT the event seam. + HARD RULE 5: item 5a's clearing *looks* exactly like an event-trigger feature, and + implementing it by widening `grid_events`' emit guard is D-40 — deferred by ruling A-15 + because customer/product `EventCtx` carries no scoped handle and a nurilab user's edit would + fire against royal-imports' automations. Nothing here emits anything. + + ⚠ IT WRITES THE DEFINITION ROW, NOT AN OVERLAY (amendment C3-A1, MEASURED). An ordinary + `ut_*` cell edit lands in the typist's own overlay stratum, where the automation engine — + which reads `t['rows']` — can never see it. A handle nobody can enrich is the flag being + decorative, so a profile cell writes THROUGH to the shared row, exactly as the stage field + does for exactly the same reason. + + ⚠ ONE `update`, not two. "In the same write" is the contract, and it is also the only safe + shape: a blank committed without its clear leaves a row whose handle is gone and whose + follower count still reads 41,000 — stale numbers attributed to nobody, which is worse than + either the old row or the empty one. + + ⛔ IT DELETES NO FIELDS AND TOUCHES NO HISTORY. The columns stay (they are the table's + schema, and the next handle refills them); `ut_ig_snapshots` is a different table this + module never opens. R3's *"one store for one series"* is why that separation holds — the + cleared cells are the LATEST-value stamp, and the series itself was never in them. + """ + if not is_user_table(table_key, st): + return None + defn = get(table_key, st) or {} + rows = defn.get('rows') or {} + if str(row_id) not in rows: + return None + fields = defn.get('fields') or [] + fdef = next((f for f in fields if f.get('key') == str(fkey)), None) + if not isinstance(fdef, dict) or not isinstance(fdef.get('profile'), dict): + return None # not a profile column — the caller's wall failed open + handle, ok = normalize_profile(value, fdef['profile'].get('source')) + if not ok: + return None # refused with a sentence at the route + # Clear only the known Instagram presets that the automation declared, never every field with + # automation provenance. + cleared = [] + if not handle: + cleared = [f['key'] for f in fields + if f.get('key') in PROFILE_PRESET_KEYS + and isinstance(f.get('automation'), dict) + and str(rows.get(str(row_id), {}).get(f['key']) or '') != ''] + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + row = t.setdefault('rows', {}).setdefault(str(row_id), {}) + row[str(fkey)] = handle + for k in cleared: + row[k] = '' + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return {'handle': handle, 'cleared': cleared} + + +def delete_row(table_key, row_id, st=None): + """⛔ SYNC, AND IT WAS `async` — owner report 2026-08-10: *"it took me forever to delete a + record… it took me like 3 tries"*. + + `flush='async'` applies the drop to the in-process CACHE and defers the upload by + `store._FLUSH_DELAY` (2 s) so a burst coalesces into one commit. That trade was made for + `add_row`, and made there for a stated reason — the owner's *"adding a new record visually + takes too long, I need to be able to spam it"*. **This function inherited the flag and none of + the argument.** Nobody spams a delete: it is rare, it is destructive, and it is the one + mutation where "eventually consistent" is indistinguishable from "it did not work". Inside + that window the row is gone from this process and still present in the store, so anything that + re-reads the hub copy — a restart, an eviction, another reader — brings it back, and the user + deletes it again. + + ⚠ ONE COMMIT PER DELETE is the whole cost, and it is the correct one to pay: the alternative + is a destructive action whose durability depends on the container staying up for two seconds. + Adds keep their async path untouched — the spam case is real and is theirs. + ⚠ THIS IS NOT THE WHOLE OF D-118. A sync commit closes the lost-delete window; it does not make + two writers safe, which is D-4's job. + """ + if not is_user_table(table_key, st) or not records_mutable(table_key, st): + return False + + def _drop(cur): + t = cur.get(str(table_key)) + if t is not None: + (t.get('rows') or {}).pop(str(row_id), None) + return cur + + _st(st).update(STORE_KEY, _drop, flush='sync') + return True + + +#: Names the ENGINE uses for itself when no human is on the other end of a run. ⛔ A table +#: stamped with one of these has NO HUMAN OWNER, so `may_open` can only admit an admin — which +#: is why the automation engine must never mint one (see `ut_ensure`'s owner rule, wave 20). +#: Named here rather than inline so the two modules that care about it read the SAME list. +MACHINE_OWNERS = ('automation', 'scheduler') + + +def may_open(table_key, viewer, is_admin=False, st=None): + """FAIL-CLOSED visibility: the creator, or an admin. Nothing else. **THE one resolver.** + + A user table cannot be gated by `allowed_modules` — a module grant written last month cannot + describe a table created this morning — so it needs its own rule, and the safe rule is the + narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including + a BU-scoped sales agent) could open a table somebody else created just by knowing its key. + Sharing a user table with named colleagues is a follow-on, and it should reuse the shared- + VIEW vocabulary rather than inventing a second one. + + ⛔ WAVE 20 — THERE WAS A SECOND, WIDER RULE, AND THE TWO DISAGREED. `routes_automation`'s + table picker admitted `createdBy in (uname, 'automation', 'scheduler')`, so a non-admin could + SEE an automation-created database in the picker and then be refused when they opened it, + edited it, or tried to delete it. Two ideas of who owns a table is the same defect class as + the two ideas of an agent's book (D-30): both surfaces look right in isolation and only + disagree in front of a user. The picker now calls THIS function, and the engine stamps a + real owner (see `MACHINE_OWNERS`) so nothing legitimate is narrowed by the merge. + """ + t = get(table_key, st) + if not t: + return False + if bool(is_admin) or t.get('createdBy') == viewer: + return True + # ⭐ WAVE 20 (R10, closes DEBT D-32) — A GRANT ADMITS TOO. `core/shares.py` is the ONE registry + # for view/folder/database grants, and this is the line that makes a shared DATABASE actually + # open: without it the share was RECORDED and the receiver's nav never changed, which reads as + # "sharing is broken" rather than as a missing wall. + # + # ⚠ PURELY ADDITIVE, and deliberately last: it can only ever admit somebody a grant NAMES, and + # can never deny anyone the creator-or-admin rule already admits. `role_for` is fail-closed on + # a junk bucket or an unreadable store, so an unreachable registry degrades to today's rule + # rather than to an open door. + try: + import core.shares as shares + return shares.may_see('database', table_key, viewer, is_admin=is_admin, st=st) + except Exception: + return False + + +#: ⭐⭐ W31-T10 / CONTRACT C1 — THE TWO BUCKETS `may_open` READS, LENT FOR THE LENGTH OF ONE PASS. +#: +#: `core/store.py:Store.get` deep-copies the WHOLE tenant document on every call, hit or miss, +#: under `self._lock` — a documented 35.8 MB ceiling. So asking `may_open` about N databases cost +#: `1 + N` full copies of `user_tables`, plus up to another N of `object_shares` (its last leg), +#: and `GET /nav` measured a median of **9,548 ms live / 24,741 ms in-process** on tenant #0. +#: +#: ⛔ THE OBVIOUS FIX IS THE FORBIDDEN ONE, and `may_open`'s own docstring is the record of why: +#: wave 20 found TWO ideas of who owns a table (`routes_automation` admitted +#: `createdBy in (uname, 'automation', 'scheduler')`), so a non-admin saw a database in a picker +#: and was refused on opening it. Inlining creator-or-admin at a call site re-creates exactly +#: that. **`may_open` stays THE one resolver, unmodified and still asked about every table** — +#: it is simply no longer charged for documents the caller is already holding. +#: +#: ⚠ WHY THE SHARES BUCKET IS IN HERE TOO, when `routes_automation._LentTables` intercepts only +#: `user_tables`: `may_open`'s admit-by-grant leg calls `shares.may_see` → `role_for` → `grants`, +#: which is `st.get('object_shares')` PER TABLE. It is a different key, so a `user_tables`-only +#: lend passes it straight through — the exact residual C1 says needs its own answer. It is read +#: LAZILY (an admin, or a table's creator, is admitted before that leg runs and never pays for +#: it) and memoised for the pass. +#: +#: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through +#: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the +#: runtime through `__getattr__` — so this can serve a stale value for exactly two keys, for +#: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level +#: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS +#: `createdBy`). +_LENDABLE = (STORE_KEY, 'object_shares') + + +class _Lent: + """A read-only `st` that serves already-read documents and passes everything else through.""" + + def __init__(self, runtime, docs): + self._runtime, self._docs = runtime, dict(docs or {}) + + def get(self, name, fresh=False): + if fresh or name not in _LENDABLE: + return self._runtime.get(name, fresh) if fresh else self._runtime.get(name) + if name not in self._docs: + self._docs[name] = self._runtime.get(name) # lazily, once, for this pass + return self._docs[name] + + def get_projection(self, name, drop=()): + """⭐ W32-T01: serve the LENT document here too, or a projection re-reads what the pass + already holds — `__getattr__` would forward this straight to the runtime and the lend + would buy nothing for exactly the callers built to use it. + + ⚠ THE TWO MISMATCHES ARE BOTH SAFE, AND IN OPPOSITE WAYS — which is why this serves the + lent document as-is rather than re-projecting it: + • lent WHOLE, asked for a projection ⇒ the caller gets MORE than it asked for. Correct, + just not cheap — the one direction a shortcut here is allowed to be wrong in. + • lent PROJECTED, asked for the whole thing via `get` ⇒ the caller gets a `_Projected`, + and reading a dropped key RAISES with the reason. Loud, not silent. + A re-projection would cost a second copy to convert the first case and change nothing + about the second. + """ + if name not in _LENDABLE: + return self._runtime.get_projection(name, drop=drop) + if name not in self._docs: + self._docs[name] = self._runtime.get_projection(name, drop=drop) + return self._docs[name] + + def __getattr__(self, name): + return getattr(self._runtime, name) + + +def lend(st=None, **docs): + """A handle that serves `docs` for `_LENDABLE` keys and defers everything else to `st`. + + `lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and + hand the result to every predicate in that pass; see the note above for what makes it safe. + """ + return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None}) + + +def nav_entries(viewer=None, is_admin=False, st=None): + """Registry-SHAPED dicts so the nav can render user tables beside real modules with no + special-casing: the same keys the flat nav reads (`key`, `label`, `source`). + + Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse. + + ⭐ W31-T10 (C1): ONE document read for the whole listing. The filter is unchanged — the same + `may_open`, asked about the same tables, in the same order — it is handed the document this + function has already read instead of taking a fresh 35.8 MB copy per table. A caller that + already holds the document passes its own `lend(...)` as `st` and this pays nothing at all. + + ⭐⭐ W32-T02 (R8/D-175): and that ONE read is now a PROJECTED one. This function reads `label`, + `source` and `key`, and `may_open` reads `createdBy` (then the shares registry) — **no caller + of this listing has ever touched a row.** On tenant #0 that turns a 1,750 ms copy into 1.4 ms, + for both consumers: `nav()` and `_placeable_top_keys`, i.e. `GET /nav` and `GET /nav/prefs` — + the two that `Shell.tsx` fires CONCURRENTLY on the same `Store._lock`, which is why they added + up rather than overlapped and why the rail rows arrived seconds apart (owner item 5). + """ + tables = all_defs(st) + lent = lend(st, **{STORE_KEY: tables}) + out = [] + for key, t in sorted(tables.items(), + key=lambda kv: (kv[1].get('label') or '').lower()): + if viewer is not None and not may_open(key, viewer, is_admin, st=lent): + continue + out.append({'key': key, 'label': t.get('label') or key, + 'source': t.get('source') or BLANK_SOURCE, + 'user_table': True, 'nav': True, 'validate': False}) + return out diff --git a/platform/core/users.py b/platform/core/users.py index 0d37dfb63aea46b27d87a65720d0ce05c30f5463..a29435e3b3b9fe9b2c047f9f8d7bcda08d7375a5 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -445,20 +445,21 @@ def assignable_people(tenant=None): Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a - cross-tenant name leak. - - ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and - return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was - written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at - EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform - roster one forgotten kwarg away, with no error to notice - ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing). - ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass - `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate - changes — and a future caller that forgets gets an empty picker somebody notices instead of a - leak nobody does. + cross-tenant name leak. None = unscoped (the Streamlit host, tenant #0's process). """ - return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)}) + try: + reg = registry() or {} + except Exception: + return [] + want = str(tenant or '').strip().lower() + out = [] + for username, u in reg.items(): + if not isinstance(u, dict) or u.get('active') is False: + continue + if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue + out.append(str(u.get('name') or username)) + return sorted(set(out)) def set_avatar(username, data_url): @@ -485,41 +486,20 @@ def avatar_map(tenant=None): name, so the display name is the only join a renderer has. Two active accounts sharing a display name share one option; the first WITH a photo wins the key rather than a coin flip deciding whether the option has a face. `tenant` scopes it exactly as - `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed - blank, which both take from the ONE resolver below rather than each writing `if want and …`. - """ + `assignable_people(tenant)` does, and for the same leak.""" + try: + reg = registry() or {} + except Exception: + return {} + want = str(tenant or '').strip().lower() out = {} - for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]): + for username, u in sorted(reg.items()): + if not isinstance(u, dict) or u.get('active') is False: + continue + if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: + continue av = u.get('avatar') nm = str(u.get('name') or username) if av and nm not in out: out[nm] = str(av) return out - - -def _tenant_accounts(tenant): - """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED. - - ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice, - identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a - blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of - them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of - the same predicate and is lane C's to fold in. - - ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a - pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public` - normalises it the same way. What changed is the WANT side — a blank want is now a refusal - rather than a wildcard. - """ - want = str(tenant or '').strip().lower() - if not want: - # Not an error and not everything: nobody. See `assignable_people`'s note — the one - # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6. - return [] - try: - reg = registry() or {} - except Exception: - return [] - return [(n, u) for n, u in reg.items() - if isinstance(u, dict) and u.get('active') is not False - and str(u.get('tenant') or 'royal-imports').strip().lower() == want] diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index 2892c73329ad239d426a7a702f749413ad6a9fe9..c4be70047eeaea72f2055498b3a8159364893ce1 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -188,35 +188,14 @@ ENTITIES = { # backfill below has walked every partner (a partner's `write_date` did not move because # WE changed the schema). Every projection degrades a missing/NULL column to blank rather # than raising, so declaring early is safe — it is just not yet true. - # ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS - # ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors' - # etc."* He was right, and the census - # (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how - # much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76 - # on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here - # was measured non-empty on the live population first — none is added on the strength of - # "Odoo probably has that", which is how a schema grows columns nobody can fill. - # ⚠ THE MIRROR IS THE CEILING, NOT THE GRID. Every reader in `odoo_relational.py` reads - # THIS file, so a column added to a `*_fields()` contract with no entry here renders as a - # blank column forever — which is D-165's shape exactly, and the reason the widening starts - # on this line rather than on the grid. "fields": ["name", "team_id", "city", "state_id", "country_id", "active", "agent", "salesman_as_agent", "customer_rank", - "street", "street2", "zip", - "email", "phone", "mobile", "website", "vat", "ref", - "write_date"], + "street", "street2", "zip", "write_date"], # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next # sync_all() instead of needing a reseed. - # ⛔ D-165 IS ABOUT THIS LIST, AND THE CENSUS SETTLED WHICH HALF IS WRONG: `street`, - # `street2` and `zip` are `store=True` on `res.partner` and POPULATED in live Odoo — they - # are declared here and still NULL in the mirror purely because a backfill pass has not - # walked every partner (changing OUR schema does not move a partner's `write_date`, so an - # incremental sync never revisits them). Declaring is not filling; the fix is a run. "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28 "customer_rank", # added 2026-08-11 (W29-T54) - "street", "street2", "zip", # added 2026-08-12 (W30-T33) - "email", "phone", "mobile", # added 2026-08-14 (W33-T48) - "website", "vat", "ref"], + "street", "street2", "zip"], # added 2026-08-12 (W30-T33) # m2m fields land as _id (the FIRST linked id — the Customers-module convention: # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be diff --git a/platform/harness/meta_store.py b/platform/harness/meta_store.py index 049f6c7ff6a2cf21c03a15c1b65bb132e682f6f8..a58c0ab434f2705a9e12b2b6c961c03a2b9174a5 100644 --- a/platform/harness/meta_store.py +++ b/platform/harness/meta_store.py @@ -1,521 +1,521 @@ -"""meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48). - - python platform/harness/meta_store.py --sync pull into tenant #0's mirror - python platform/harness/meta_store.py --sync --tenant gtmlab - python platform/harness/meta_store.py --status what is in the mirror now - -⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is -hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of -spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the -per-tenant file (`path_for`), the process-singleton connection (`connect`), and the -delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one -store with two loaders, not two stores.** - -⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull -the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is -`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is -`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one -different first hop. - -⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json` -carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account. - Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57 -⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask -came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with -nothing going red — which is exactly the "do not drop any column" instruction, broken silently. -The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL. - -⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it -answers `403 (#200) Requires business_management permission`, which this token does not carry. One -column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause. - -⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is -larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id` -(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR. -""" -import argparse -import json -import os -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - -_HERE = Path(__file__).resolve().parent -if str(_HERE.parent) not in sys.path: - sys.path.insert(0, str(_HERE.parent)) - -from harness import datastore # noqa: E402 - -GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0" -GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}" - -#: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and -#: including anything Graph returns as a nested object, which is stored as compact JSON text. -_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks", - "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers", - "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number", - "min_daily_budget", "min_campaign_group_spend_cap"} -_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency", - "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap", - "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time", - "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target", - "lifetime_spend_cap", "lifetime_imps"} - -#: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account -#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by -#: the grid links in `meta_relational`. -ACCOUNT_FIELDS = ( - "account_id account_status age amount_spent balance business_city business_country_code " - "business_name business_state business_street business_street2 business_zip capabilities " - "created_time currency disable_reason end_advertiser end_advertiser_name funding_source " - "has_migrated_permissions id io_number is_attribution_spec_system_default " - "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account " - "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget " - "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type " - "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted" -).split() - -CAMPAIGN_FIELDS = ( - "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type " - "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status " - "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info " - "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution " - "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories " - "special_ad_category special_ad_category_country spend_cap start_time status stop_time " - "topline_id updated_time" -).split() - -ADSET_FIELDS = ( - "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount " - "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign " - "campaign_active_time campaign_attribution campaign_id configured_status created_time " - "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type " - "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative " - "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target " - "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal " - "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback " - "rf_prediction_id source_adset source_adset_id start_time status targeting " - "targeting_optimization_types time_based_ad_rotation_id_blocks " - "time_based_ad_rotation_intervals updated_time use_new_app_click" -).split() - -AD_FIELDS = ( - "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time " - "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status " - "conversion_domain created_time creative demolink_hash display_sequence effective_status " - "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name " - "preview_shareable_link priority recommendations source_ad source_ad_id status targeting " - "tracking_and_conversion_with_defaults tracking_specs updated_time" -).split() - -CREATIVE_FIELDS = ( - "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body " - "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria " - "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec " - "destination_set_id dynamic_ad_voice effective_authorization_category " - "effective_instagram_media_id effective_object_story_id enable_direct_install " - "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id " - "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec " - "link_deep_link_url link_destination_display_url link_og_id link_url " - "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec " - "object_type object_url place_page_set_id platform_customizations playable_asset_id " - "portrait_customizations product_set_id recommender_settings source_instagram_media_id status " - "template_url template_url_spec thumbnail_id thumbnail_url title url_tags " - "use_page_actor_override video_id" -).split() - -INSIGHT_FIELDS = ( - "account_currency account_id account_name action_values actions ad_id ad_name adset_id " - "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time " - "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type " - "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start " - "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions " - "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective " - "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach " - "social_spend spend unique_clicks unique_ctr unique_outbound_clicks " - "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions " - "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions " - "video_play_actions video_thruplay_watched_actions website_purchase_roas" -).split() - -#: table -> (graph edge on the account | None, measured fields, parent column) -SPECS = { - "meta_ad_accounts": (None, ACCOUNT_FIELDS, None), - "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"), - "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"), - "meta_ads": ("ads", AD_FIELDS, "adset_id"), - "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"), -} - -#: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights -#: has no id of its own, and it must be STABLE so a re-run updates instead of appending. -INSIGHTS_TABLE = "meta_insights_daily" -INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad" - -#: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields -#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"** -#: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the -#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at -#: page 25 answer 200. So the window is walked in slices with every column intact: -#: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows -#: 57 fields · account level · 7d -> 200, 7 rows -#: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over — -#: it drops columns R2 requires, and it does so invisibly. -INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90) -INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7) -INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25) - - -def _slices(days, size): - """[(since, until)] covering the last `days`, oldest first, in `size`-day windows.""" - from datetime import date, timedelta - end = date.today() - start = end - timedelta(days=max(1, days) - 1) - out, cur = [], start - while cur <= end: - stop = min(cur + timedelta(days=max(1, size) - 1), end) - out.append((cur.isoformat(), stop.isoformat())) - cur = stop + timedelta(days=1) - return out - -#: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is -#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is -#: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it -#: is reported as a problem, never absorbed. -MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200) -PAGE = int(os.environ.get("META_PAGE_SIZE") or 100) - - -class MetaError(RuntimeError): - """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it.""" - - -def token(): - """The Meta token from the environment or gitignored `platform/.env`; "" when absent. - - ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally: - `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps - (`core` never imports up). Fifteen lines is the price of that boundary. - """ - tok = os.environ.get("META_ADS_ACCESS_TOKEN") or "" - if tok: - return tok.strip() - env = _HERE.parent / ".env" - if env.exists(): - for line in env.read_text(encoding="utf-8", errors="replace").splitlines(): - if line.strip().startswith("META_ADS_ACCESS_TOKEN"): - _, _, v = line.partition("=") - return v.strip().strip('"').strip("'") - return "" - - -def _get(path, tok, **params): - params["access_token"] = tok - url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params) - req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"}) - try: - with urllib.request.urlopen(req, timeout=120) as r: - return json.loads(r.read().decode("utf-8", "replace")) - except urllib.error.HTTPError as e: - try: - msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {} - ).get("message") or "" - except Exception: - msg = "" - raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None - except Exception as e: - raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None - - -#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is -#: why it cannot be treated as "the server is broken, give up". -_TOO_MUCH = "reduce the amount of data" - -#: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a -#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account. -#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the -#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code. -#: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after -#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry -#: loop nobody can see. A partial mirror that says it is partial beats a hung sync. -_RATE_LIMITED = "too many calls" -_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3) -_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90) - - -def _walk(path, tok, log, **params): - """Every page of an edge, paged by CURSOR under our own parameters. - - ⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the - first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data - you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL - limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff. - Re-issuing each page ourselves with `after=` keeps the reduced limit for the whole walk. - - ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the - silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always - every column per row. - - -> (rows, hit_page_cap) - """ - out, pages = [], 0 - limit = int(params.pop("limit", None) or PAGE) - after, waited = None, 0 - while True: - call = dict(params, limit=limit) - if after: - call["after"] = after - try: - body = _get(path, tok, **call) - except MetaError as e: - if _TOO_MUCH in str(e) and limit > 5: - limit = max(5, limit // 4) - log(f" page too heavy for /{path} - retrying at limit={limit} " - f"(a payload ceiling, not a row cap; every column is still asked for)") - continue - if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES: - waited += 1 - log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting " - f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}") - time.sleep(_RATE_WAIT) - continue - if _RATE_LIMITED in str(e).lower(): - log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. " - f"{len(out)} row(s) collected so far are kept. Cause: the ads-management " - f"limit is per ad account and persists for minutes. Fix: re-run " - f"`--sync --only
` later; the upsert is idempotent.") - return out, True - raise - out.extend(body.get("data") or []) - pages += 1 - after = ((body.get("paging") or {}).get("cursors") or {}).get("after") - has_next = bool((body.get("paging") or {}).get("next")) and bool(after) - if not has_next: - return out, False - if pages >= MAX_PAGES: - log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. " - f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the " - f"window with META_INSIGHTS_DAYS.") - return out, True - - -def _cell(value): - """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than - being dropped: R2 says every field the API returns, and `targeting` is a field.""" - if value is None or isinstance(value, (str, int, float, bool)): - return value - return json.dumps(value, separators=(",", ":"), ensure_ascii=False) - - -def _coltype(name, rows=None): - """The DuckDB type for one column — decided by the DATA when there is data, by the name only - as a fallback. - - ⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`, - `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of - action-type objects** on the Insights edge: - [{"action_type":"outbound_click","value":"1.459854"}] - Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string - '[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked - like it worked and then blew up on the last table. - ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response, - do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because - that is what `_cell` stores; anything else falls back to the measured name hints. - """ - if rows: - seen, kinds = 0, set() - for r in rows: - v = r.get(name) - if v is None or v == "": - continue - kinds.add("json" if isinstance(v, (list, dict)) else - "bool" if isinstance(v, bool) else - "int" if isinstance(v, int) else - "float" if isinstance(v, float) else "str") - seen += 1 - if seen >= 200: - break - if kinds: - if "json" in kinds or "str" in kinds: - return "VARCHAR" # a JSON blob, or a numeric STRING - if kinds <= {"int", "bool"}: - return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT") - return "DOUBLE" - if name in _INT: - return "BIGINT" - if name in _DBL: - return "DOUBLE" - return "VARCHAR" # ids included — see the header - - -def _ensure(con, table, fields, rows=None): - """Create or widen the table, with every column typed from `rows` when they are available. - - ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a - column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type - disagreement drops and recreates rather than limping on with a column that refuses every - insert. The alternative is a mirror that is permanently unwritable for one bad guess. - """ - want = {f: _coltype(f, rows) for f in fields} - have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()} - if have: - clash = [f for f, ty in want.items() if f in have and have[f] != ty - and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")] - if clash: - con.execute(f"DROP TABLE {table}") - have = {} - if not have: - cols = ", ".join(f"{f} {want[f]}" for f in fields) - con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})" - if "id" not in fields else - f"CREATE TABLE IF NOT EXISTS {table} ({cols})") - return - for f in fields: - if f not in have: - con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}") - - -def _upsert(con, table, fields, rows): - """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so - a re-sync updates in place and can never append a second copy of the same object.""" - if not rows: - return 0 - cols = list(fields) - ids = [str(r.get("id") or "") for r in rows] - q = ",".join("?" for _ in ids) - con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids) - con.executemany( - f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})", - [[_cell(r.get(c)) for c in cols] for r in rows]) - return len(rows) - - -def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None): - """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token. - - Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop, - exactly as `odoo_relational.refresh` is. - """ - tok = tok or token() - # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a - # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s - # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already - # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path - # that trips the per-ad-account rate limit and never finishes. The comment beside that call - # claimed a short window the code could not deliver. - # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it - # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT. - days = int(insights_days or INSIGHTS_DAYS) - report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []} - if not tok: - report["problems"].append( - "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing " - "was pulled. This is a missing CREDENTIAL, not a missing capability.") - return report - - path = datastore.path_for(tenant_key) - if Path(datastore.DB_PATH) != Path(path): - datastore.use_path(path) - con = datastore.connect() - log(f" mirror: {Path(path).name}") - - accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE) - report["accounts"] = [a.get("id") for a in accts] - if not accts: - report["problems"].append("the token reaches no ad accounts") - return report - - # ── the five entity levels ──────────────────────────────────────────────────────────────── - for table, (edge, fields, _parent) in SPECS.items(): - rows, capped = [], False - for acct in accts: - aid = acct["id"] - if edge is None: - rows.append(_get(aid, tok, fields=",".join(fields))) - else: - got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE) - rows.extend(got) - capped = capped or hit - _ensure(con, table, fields, rows) - n = _upsert(con, table, fields, rows) - total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0] - report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped} - log(f" {table:<20} pulled {n:>6} mirror total {total:>6}" - + (" !! PAGE CAP HIT" if capped else "")) - if capped: - report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}") - - # ── the daily Insights grain ────────────────────────────────────────────────────────────── - if insights: - rows, capped = [], False - for acct in accts: - for since, until in _slices(days, INSIGHTS_SLICE): - got, hit = _walk(f"{acct['id']}/insights", tok, log, - fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL, - time_increment="1", limit=INSIGHTS_PAGE, - time_range=json.dumps({"since": since, "until": until})) - rows.extend(got) - capped = capped or hit - # ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync - # appends a second copy of the same day — so it is composed from the grain itself. - for r in rows: - r["id"] = ":".join(str(r.get(k) or "") for k in - (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop")) - fields = ["id"] + INSIGHT_FIELDS - _ensure(con, INSIGHTS_TABLE, fields, rows) - n = _upsert(con, INSIGHTS_TABLE, fields, rows) - total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0] - report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped} - log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}" - + (" !! PAGE CAP HIT" if capped else "")) - - con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", - ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}", - sum(t["in_mirror"] for t in report["tables"].values()), - time.strftime("%Y-%m-%d %H:%M:%S")]) - return report - - -def status(tenant_key="royal-imports"): - """What is in the mirror right now, per table. Never fetches.""" - path = datastore.path_for(tenant_key) - if Path(datastore.DB_PATH) != Path(path): - datastore.use_path(path) - out = {} - con = datastore.connect() - for table in list(SPECS) + [INSIGHTS_TABLE]: - try: - out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0] - except Exception: - out[table] = None # table absent = never synced - return out - - -def main(argv=None): - ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.") - ap.add_argument("--sync", action="store_true") - ap.add_argument("--status", action="store_true") - ap.add_argument("--tenant", default="royal-imports") - ap.add_argument("--no-insights", action="store_true") - ap.add_argument("--insights-days", type=int, default=None, - help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say") - a = ap.parse_args(argv) - if a.status: - for k, v in status(a.tenant).items(): - print(f" {k:<20} {'(never synced)' if v is None else v}") - return 0 - if not a.sync: - ap.print_help() - return 2 - rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days) - for p in rep["problems"]: - print(" PROBLEM:", p) - print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}") - return 1 if rep["problems"] else 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48). + + python platform/harness/meta_store.py --sync pull into tenant #0's mirror + python platform/harness/meta_store.py --sync --tenant gtmlab + python platform/harness/meta_store.py --status what is in the mirror now + +⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is +hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of +spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the +per-tenant file (`path_for`), the process-singleton connection (`connect`), and the +delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one +store with two loaders, not two stores.** + +⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull +the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is +`XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is +`Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one +different first hop. + +⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json` +carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account. + Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57 +⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask +came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with +nothing going red — which is exactly the "do not drop any column" instruction, broken silently. +The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL. + +⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it +answers `403 (#200) Requires business_management permission`, which this token does not carry. One +column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause. + +⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is +larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id` +(W26/R3), for the same reason, and it is why every `id` column here is VARCHAR. +""" +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +if str(_HERE.parent) not in sys.path: + sys.path.insert(0, str(_HERE.parent)) + +from harness import datastore # noqa: E402 + +GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0" +GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}" + +#: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and +#: including anything Graph returns as a nested object, which is stored as compact JSON text. +_INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks", + "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers", + "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number", + "min_daily_budget", "min_campaign_group_spend_cap"} +_DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency", + "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap", + "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time", + "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target", + "lifetime_spend_cap", "lifetime_imps"} + +#: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account +#: itself. `parent` names the column that ties a row to its parent, used by nothing here and by +#: the grid links in `meta_relational`. +ACCOUNT_FIELDS = ( + "account_id account_status age amount_spent balance business_city business_country_code " + "business_name business_state business_street business_street2 business_zip capabilities " + "created_time currency disable_reason end_advertiser end_advertiser_name funding_source " + "has_migrated_permissions id io_number is_attribution_spec_system_default " + "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account " + "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget " + "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type " + "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted" +).split() + +CAMPAIGN_FIELDS = ( + "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type " + "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status " + "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info " + "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution " + "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories " + "special_ad_category special_ad_category_country spend_cap start_time status stop_time " + "topline_id updated_time" +).split() + +ADSET_FIELDS = ( + "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount " + "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign " + "campaign_active_time campaign_attribution campaign_id configured_status created_time " + "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type " + "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative " + "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target " + "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal " + "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback " + "rf_prediction_id source_adset source_adset_id start_time status targeting " + "targeting_optimization_types time_based_ad_rotation_id_blocks " + "time_based_ad_rotation_intervals updated_time use_new_app_click" +).split() + +AD_FIELDS = ( + "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time " + "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status " + "conversion_domain created_time creative demolink_hash display_sequence effective_status " + "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name " + "preview_shareable_link priority recommendations source_ad source_ad_id status targeting " + "tracking_and_conversion_with_defaults tracking_specs updated_time" +).split() + +CREATIVE_FIELDS = ( + "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body " + "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria " + "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec " + "destination_set_id dynamic_ad_voice effective_authorization_category " + "effective_instagram_media_id effective_object_story_id enable_direct_install " + "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id " + "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec " + "link_deep_link_url link_destination_display_url link_og_id link_url " + "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec " + "object_type object_url place_page_set_id platform_customizations playable_asset_id " + "portrait_customizations product_set_id recommender_settings source_instagram_media_id status " + "template_url template_url_spec thumbnail_id thumbnail_url title url_tags " + "use_page_actor_override video_id" +).split() + +INSIGHT_FIELDS = ( + "account_currency account_id account_name action_values actions ad_id ad_name adset_id " + "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time " + "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type " + "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start " + "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions " + "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective " + "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach " + "social_spend spend unique_clicks unique_ctr unique_outbound_clicks " + "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions " + "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions " + "video_play_actions video_thruplay_watched_actions website_purchase_roas" +).split() + +#: table -> (graph edge on the account | None, measured fields, parent column) +SPECS = { + "meta_ad_accounts": (None, ACCOUNT_FIELDS, None), + "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"), + "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"), + "meta_ads": ("ads", AD_FIELDS, "adset_id"), + "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"), +} + +#: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights +#: has no id of its own, and it must be STABLE so a re-run updates instead of appending. +INSIGHTS_TABLE = "meta_insights_daily" +INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad" + +#: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields +#: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"** +#: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the +#: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at +#: page 25 answer 200. So the window is walked in slices with every column intact: +#: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows +#: 57 fields · account level · 7d -> 200, 7 rows +#: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over — +#: it drops columns R2 requires, and it does so invisibly. +INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90) +INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7) +INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25) + + +def _slices(days, size): + """[(since, until)] covering the last `days`, oldest first, in `size`-day windows.""" + from datetime import date, timedelta + end = date.today() + start = end - timedelta(days=max(1, days) - 1) + out, cur = [], start + while cur <= end: + stop = min(cur + timedelta(days=max(1, size) - 1), end) + out.append((cur.isoformat(), stop.isoformat())) + cur = stop + timedelta(days=1) + return out + +#: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is +#: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is +#: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it +#: is reported as a problem, never absorbed. +MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200) +PAGE = int(os.environ.get("META_PAGE_SIZE") or 100) + + +class MetaError(RuntimeError): + """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it.""" + + +def token(): + """The Meta token from the environment or gitignored `platform/.env`; "" when absent. + + ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally: + `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps + (`core` never imports up). Fifteen lines is the price of that boundary. + """ + tok = os.environ.get("META_ADS_ACCESS_TOKEN") or "" + if tok: + return tok.strip() + env = _HERE.parent / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.strip().startswith("META_ADS_ACCESS_TOKEN"): + _, _, v = line.partition("=") + return v.strip().strip('"').strip("'") + return "" + + +def _get(path, tok, **params): + params["access_token"] = tok + url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params) + req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"}) + try: + with urllib.request.urlopen(req, timeout=120) as r: + return json.loads(r.read().decode("utf-8", "replace")) + except urllib.error.HTTPError as e: + try: + msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {} + ).get("message") or "" + except Exception: + msg = "" + raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None + except Exception as e: + raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None + + +#: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is +#: why it cannot be treated as "the server is broken, give up". +_TOO_MUCH = "reduce the amount of data" + +#: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a +#: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account. +#: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the +#: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code. +#: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after +#: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry +#: loop nobody can see. A partial mirror that says it is partial beats a hung sync. +_RATE_LIMITED = "too many calls" +_RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3) +_RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90) + + +def _walk(path, tok, log, **params): + """Every page of an edge, paged by CURSOR under our own parameters. + + ⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the + first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data + you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL + limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff. + Re-issuing each page ourselves with `after=` keeps the reduced limit for the whole walk. + + ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the + silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always + every column per row. + + -> (rows, hit_page_cap) + """ + out, pages = [], 0 + limit = int(params.pop("limit", None) or PAGE) + after, waited = None, 0 + while True: + call = dict(params, limit=limit) + if after: + call["after"] = after + try: + body = _get(path, tok, **call) + except MetaError as e: + if _TOO_MUCH in str(e) and limit > 5: + limit = max(5, limit // 4) + log(f" page too heavy for /{path} - retrying at limit={limit} " + f"(a payload ceiling, not a row cap; every column is still asked for)") + continue + if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES: + waited += 1 + log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting " + f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}") + time.sleep(_RATE_WAIT) + continue + if _RATE_LIMITED in str(e).lower(): + log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. " + f"{len(out)} row(s) collected so far are kept. Cause: the ads-management " + f"limit is per ad account and persists for minutes. Fix: re-run " + f"`--sync --only
` later; the upsert is idempotent.") + return out, True + raise + out.extend(body.get("data") or []) + pages += 1 + after = ((body.get("paging") or {}).get("cursors") or {}).get("after") + has_next = bool((body.get("paging") or {}).get("next")) and bool(after) + if not has_next: + return out, False + if pages >= MAX_PAGES: + log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. " + f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the " + f"window with META_INSIGHTS_DAYS.") + return out, True + + +def _cell(value): + """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than + being dropped: R2 says every field the API returns, and `targeting` is a field.""" + if value is None or isinstance(value, (str, int, float, bool)): + return value + return json.dumps(value, separators=(",", ":"), ensure_ascii=False) + + +def _coltype(name, rows=None): + """The DuckDB type for one column — decided by the DATA when there is data, by the name only + as a fallback. + + ⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`, + `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of + action-type objects** on the Insights edge: + [{"action_type":"outbound_click","value":"1.459854"}] + Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string + '[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked + like it worked and then blew up on the last table. + ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response, + do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because + that is what `_cell` stores; anything else falls back to the measured name hints. + """ + if rows: + seen, kinds = 0, set() + for r in rows: + v = r.get(name) + if v is None or v == "": + continue + kinds.add("json" if isinstance(v, (list, dict)) else + "bool" if isinstance(v, bool) else + "int" if isinstance(v, int) else + "float" if isinstance(v, float) else "str") + seen += 1 + if seen >= 200: + break + if kinds: + if "json" in kinds or "str" in kinds: + return "VARCHAR" # a JSON blob, or a numeric STRING + if kinds <= {"int", "bool"}: + return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT") + return "DOUBLE" + if name in _INT: + return "BIGINT" + if name in _DBL: + return "DOUBLE" + return "VARCHAR" # ids included — see the header + + +def _ensure(con, table, fields, rows=None): + """Create or widen the table, with every column typed from `rows` when they are available. + + ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a + column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type + disagreement drops and recreates rather than limping on with a column that refuses every + insert. The alternative is a mirror that is permanently unwritable for one bad guess. + """ + want = {f: _coltype(f, rows) for f in fields} + have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()} + if have: + clash = [f for f, ty in want.items() if f in have and have[f] != ty + and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")] + if clash: + con.execute(f"DROP TABLE {table}") + have = {} + if not have: + cols = ", ".join(f"{f} {want[f]}" for f in fields) + con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})" + if "id" not in fields else + f"CREATE TABLE IF NOT EXISTS {table} ({cols})") + return + for f in fields: + if f not in have: + con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}") + + +def _upsert(con, table, fields, rows): + """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so + a re-sync updates in place and can never append a second copy of the same object.""" + if not rows: + return 0 + cols = list(fields) + ids = [str(r.get("id") or "") for r in rows] + q = ",".join("?" for _ in ids) + con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids) + con.executemany( + f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})", + [[_cell(r.get(c)) for c in cols] for r in rows]) + return len(rows) + + +def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None): + """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token. + + Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop, + exactly as `odoo_relational.refresh` is. + """ + tok = tok or token() + # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a + # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s + # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already + # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path + # that trips the per-ad-account rate limit and never finishes. The comment beside that call + # claimed a short window the code could not deliver. + # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it + # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT. + days = int(insights_days or INSIGHTS_DAYS) + report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []} + if not tok: + report["problems"].append( + "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing " + "was pulled. This is a missing CREDENTIAL, not a missing capability.") + return report + + path = datastore.path_for(tenant_key) + if Path(datastore.DB_PATH) != Path(path): + datastore.use_path(path) + con = datastore.connect() + log(f" mirror: {Path(path).name}") + + accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE) + report["accounts"] = [a.get("id") for a in accts] + if not accts: + report["problems"].append("the token reaches no ad accounts") + return report + + # ── the five entity levels ──────────────────────────────────────────────────────────────── + for table, (edge, fields, _parent) in SPECS.items(): + rows, capped = [], False + for acct in accts: + aid = acct["id"] + if edge is None: + rows.append(_get(aid, tok, fields=",".join(fields))) + else: + got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE) + rows.extend(got) + capped = capped or hit + _ensure(con, table, fields, rows) + n = _upsert(con, table, fields, rows) + total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0] + report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped} + log(f" {table:<20} pulled {n:>6} mirror total {total:>6}" + + (" !! PAGE CAP HIT" if capped else "")) + if capped: + report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}") + + # ── the daily Insights grain ────────────────────────────────────────────────────────────── + if insights: + rows, capped = [], False + for acct in accts: + for since, until in _slices(days, INSIGHTS_SLICE): + got, hit = _walk(f"{acct['id']}/insights", tok, log, + fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL, + time_increment="1", limit=INSIGHTS_PAGE, + time_range=json.dumps({"since": since, "until": until})) + rows.extend(got) + capped = capped or hit + # ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync + # appends a second copy of the same day — so it is composed from the grain itself. + for r in rows: + r["id"] = ":".join(str(r.get(k) or "") for k in + (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop")) + fields = ["id"] + INSIGHT_FIELDS + _ensure(con, INSIGHTS_TABLE, fields, rows) + n = _upsert(con, INSIGHTS_TABLE, fields, rows) + total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0] + report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped} + log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}" + + (" !! PAGE CAP HIT" if capped else "")) + + con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)", + ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}", + sum(t["in_mirror"] for t in report["tables"].values()), + time.strftime("%Y-%m-%d %H:%M:%S")]) + return report + + +def status(tenant_key="royal-imports"): + """What is in the mirror right now, per table. Never fetches.""" + path = datastore.path_for(tenant_key) + if Path(datastore.DB_PATH) != Path(path): + datastore.use_path(path) + out = {} + con = datastore.connect() + for table in list(SPECS) + [INSIGHTS_TABLE]: + try: + out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0] + except Exception: + out[table] = None # table absent = never synced + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.") + ap.add_argument("--sync", action="store_true") + ap.add_argument("--status", action="store_true") + ap.add_argument("--tenant", default="royal-imports") + ap.add_argument("--no-insights", action="store_true") + ap.add_argument("--insights-days", type=int, default=None, + help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say") + a = ap.parse_args(argv) + if a.status: + for k, v in status(a.tenant).items(): + print(f" {k:<20} {'(never synced)' if v is None else v}") + return 0 + if not a.sync: + ap.print_help() + return 2 + rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days) + for p in rep["problems"]: + print(" PROBLEM:", p) + print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}") + return 1 if rep["problems"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platform/harness/tools.py b/platform/harness/tools.py index 94e74480c49c1545e8c352b0eed06908e890e569..d046a442ba05436fe62116a22beae87809c2a74b 100644 --- a/platform/harness/tools.py +++ b/platform/harness/tools.py @@ -65,22 +65,13 @@ def _err(msg): # ------------------------------------------------------------------ schema tools def list_topics(): - """The 'what data exists' tool. - - W33-T49 adds `grid` — the store key of the database this topic describes, when it describes - one. It is what lets the agent connect "the thing the person is looking at" to "the topic I - can query", which is the whole point of the entity topics: before this, the semantic layer - could see `sale_order_line` and `account_move` and had never heard of `ut_odoo_customers`. - """ + """The 'what data exists' tool.""" out = [] for k, t in SEM.topics().items(): - row = {"topic": k, "label": t.get("label"), "entity": t.get("entity"), - "grain": t.get("grain"), - "dims": list((t.get("store") or {}).get("dims") or {}), - "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]} - if t.get("grid"): - row["grid"] = t["grid"] - out.append(row) + out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"), + "grain": t.get("grain"), + "dims": list((t.get("store") or {}).get("dims") or {}), + "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]}) return out @@ -92,30 +83,10 @@ def describe_topic(topic): mets = {k: {"label": m.get("label"), "description": m.get("description"), "format": m.get("format"), "ai_context": m.get("ai_context")} for k, m in SEM.metrics().items() if m["topic"] == topic} - out = {"topic": topic, "label": t.get("label"), "scope": t.get("scope"), - "grain": t.get("grain"), "ai_context": t.get("ai_context"), - "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()}, - "metrics": mets} - # ⭐⭐ W33-T49 — THE FIELD SEMANTICS, which this tool did not carry AT ALL. - # - # ⛔ THE GAP WAS NOT THE TOPIC FILES, IT WAS THIS RETURN. Before this, `describe_topic` - # answered with scope, grain, dims, metrics and `ai_context` — and **no vocabulary of - # COLUMNS**. So an agent told to describe a database could name what it MEASURES and never - # what it HOLDS: the owner's item 13 asks for "ALL populated fields … including its semantic - # layer", and adding eight topic files without this would have left `describe_topic` returning - # the same field-free shape it always did, one screen away from the fields. - # - # ⚠ `kind` is the part worth reading, and it is the difference between a column an agent can - # FILTER and one it cannot: `data` is stored on the row; `rollup` is computed from another - # topic (`from_topic` names it, and that is where a windowed or reconciled number must come - # from); `link` points at another database. An agent that treats a rollup as a stored column - # will ask this topic for a number this topic does not hold. - if t.get("fields"): - out["fields"] = [dict(f) for f in t["fields"]] - for k in ("grid", "subject", "entity"): - if t.get(k): - out[k] = t[k] - return out + return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"), + "grain": t.get("grain"), "ai_context": t.get("ai_context"), + "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()}, + "metrics": mets} # ------------------------------------------------------------------ query tools diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 0ff439a38cb7a3dc51951e01cfc34e14e911815a..eaf5ebac17decb7fc7029da5fe05c0cd5658c0f2 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -1,804 +1,797 @@ -"""modules/product_data.py — the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC). - -The second object on the table-page factory: same grid, same engine, same permission wall — the -only difference is the field schema and the identity. `modules/customer_data.pool()` is the -template and this deliberately mirrors its signature and its row shape. - -⛔ THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER. - -1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.** - Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an - integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the - same id on every pull and across processes — never an enumeration index, which would reshuffle - whenever the catalogue changes) and `code`, the real business key, as a visible column. - `_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would - silently merge in every downstream set operation, and a loud build failure is the only version - of that anyone would notice. - -2. **THE SCOPE RULE — inventory columns are CONSOLIDATED and are therefore OMITTED for a - BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory` - is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not - brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside - company-wide stock IN THE SAME ROW — the mixed-scope value defect wave 15's amendment 3 exists - for, arriving through a different door. The honest options were "omit" or "label the columns - company-wide"; omit is the fail-closed one, and a column that is absent asks a question, - whereas a column that is silently company-wide answers one wrongly. - -3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and - COUNTRY are not in Odoo at all — they live in the inventory WORKBOOK - ([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent. - Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost - allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it - does not pretend to cover columns that are absent. - -4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12, - 2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the - union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid - showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise: - - · ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date - window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics - that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because - ~2,550 active SKUs have never sold in wholesale scope at all. - · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the - revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and - not this one, and that zero is a measurement. A row that appears in NO revenue read - carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the - client renders as an empty cell. Blank is an admission; zero is a measurement. - · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where - that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW - UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product` - carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two - sources are JOINED rather than merged. -""" -import json -import zlib -from pathlib import Path - -import core.odoo as O -import core.periods as P -import core.shared_overlay as shared_overlay -import core.table_store as table_store -import modules.products as products - -#: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2. -#: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy -#: signal built on a stock number the caller cannot see would be a recommendation nobody could -#: check ([[no-unverifiable-aggregates]]). -#: -#: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers: -#: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products` -#: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field — -#: computed as `validate()`'s oracle, projected onto no wire — so it belongs here for the first -#: reader and is inert for the second. Removing it would stop the scope gate proving that a -#: BU-scoped pull withholds it. -#: -#: ⭐⭐ OWNER RULING 2026-08-11 — THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim: -#: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole -#: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether -#: anything was IN STOCK, which is the first question they ask. The ruling splits the block by -#: what a business unit can actually shape: -#: -#: * `on_hand` / `unit_cost` / `inv_value` — ONE physical warehouse, no Fisch shelf and no -#: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest") -#: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` — all SALES-derived, so a -#: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with -#: Sales from Fisch") -#: -#: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one": -#: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a -#: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer -#: asserts the columns are WITHHELD — it asserts the split above, which is a stronger claim and -#: cannot pass on an empty scoped payload (the shape the old rule and an outage share). -CONSOLIDATED_ONLY = () - -#: ⛔ POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD — they must never reach a browser. -#: -#: `rows_from_pool` projects strictly through the field contract, so "no Field" already means -#: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody -#: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts, -#: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one. -#: -#: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s -#: Field while keeping its computation, and the gate — which was asserting a FIELD property from -#: a ROW-key list — went red with no way to tell a deliberate divergence from a dropped column. -UNSHIPPED_ROW_KEYS = ("buy_now",) - -#: Wave 17 (owner item 13, ruling R3) — the SUPPLIER MASTER, from the curated mastersheet map -#: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo: -#: the owner confirmed this data came from the Inventory System Mastersheet, which is why -#: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than -#: quietly missing" and needed "a loader this module deliberately does not invent". This is -#: that loader. -#: -#: ⛔ THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT. -#: A user-created field and its values live in the PER-USER strata (`table_store.workspace` -#: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that -#: filtered on a user-created column would, for every OTHER account, name a column that does not -#: exist — and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES -#: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but -#: its author. Contract columns are identical for every reader, so the view means one thing. -#: -#: ⭐⭐ 2026-08-12 (W30-T36) — THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT -#: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."* -#: It was parked because a per-user column silently WIDENS a shared view — not because editing was -#: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum -#: that removes the objection: **one value per (row, column) for the whole tenant**, so the column -#: still means ONE thing to every reader and a shared view still filters honestly. -#: ⇒ The four columns below are now `source: "overlay"` + `shared: true` in the canonical -#: contract, their values live in `__shared`, and the master map is what SEEDS an -#: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`. -_SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json" -_SUPPLIER_CACHE = {} - - -def supplier_master(): - """`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process. - - Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table - that will not render because a master map is missing is a worse failure than one with blank - supplier columns. - """ - if _SUPPLIER_CACHE: - return _SUPPLIER_CACHE - try: - raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8")) - except Exception: - return {} - for code, meta in (raw or {}).items(): - if not isinstance(meta, dict): - continue - lead = meta.get("lead") - _SUPPLIER_CACHE[str(code)] = { - "supplier": (meta.get("vendor") or "") or None, - "lead_days": int(lead) if isinstance(lead, (int, float)) else None, - "origin_country": (meta.get("country") or "") or None, - "first_cost": meta.get("first_cost"), - } - return _SUPPLIER_CACHE - -#: Wave 16 C-TOPIC — the PRODUCT table's OWN workspace bucket. ⛔ Never the customer one: -#: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a -#: SHARED overlay bucket a hash collision would silently write a product note onto somebody's -#: customer (or the reverse). Separate store keys make that structurally impossible, which is -#: the whole reason the table-page factory exists ("a new table object gets its own -#: table_store.make('_table_workspace')"). -TABLE_KEY = 'product_table_workspace' - -_GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json' -_SHARED_KEYS = None - - -def SHARED_KEYS(): - """The product columns whose values are TENANT-WIDE — derived from the canonical contract's - own `shared: true`, never typed out a second time. - - ⛔ IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared - write into the per-user stratum with nothing going wrong anywhere — the widening defect - reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the - canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the - same file with no guard), so a raise here costs nothing that was still working. - """ - global _SHARED_KEYS - if _SHARED_KEYS is None: - doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8')) - _SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or [] - if f.get('shared')) - return _SHARED_KEYS - - -class _ProductTableStore(table_store.TableStore): - """The product workspace, with the SHARED columns routed to the tenant-wide stratum. - - ⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER - THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** — measured, zero - call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` → `grid_events. - handle_one` → `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which - `routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both - doors pass through; intercepting at either route would have left the other one writing a - per-user value that only its author could see. - - ⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store - handle as the per-user one it sits beside — `_tops`' own comment explains that a split, where - one side is tenant-scoped and the other is not, is worse than a stated residency error because - a user's value would vanish the moment they saved it. Reading `self.st` means both strata move - together the day that singleton gains a tenant handle. - """ - - def patch_overlay(self, username, pid, updates): - clean = dict(updates or {}) - if not clean: - return - keys = set(SHARED_KEYS()) - shared = {k: v for k, v in clean.items() if k in keys} - personal = {k: v for k, v in clean.items() if k not in keys} - if shared: - shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st) - if personal: - super().patch_overlay(username, pid, personal) - - -TABLE_OPS = _ProductTableStore(TABLE_KEY) - - -def shared_cells(pids, st=None): - """`{"": {key: value}}` for the SHARED columns of the rows named by `pids`. - - ⚠ `pids` is required and positional all the way down — `shared_overlay.cells` refuses to serve - "everything" by design, and the caller here always holds an already-scoped pool. - """ - return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st) - - -def sku_pid(code): - """A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and - always JSON-safe. Stable across processes and pulls, which an enumeration index is not.""" - return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF - - -def _assert_no_pid_collision(rows): - """Two SKUs sharing a pid would MERGE in every set operation downstream — cohort membership, - allowed_pids, the measure channel — and nothing would report it. Fail the build instead.""" - seen = {} - for r in rows: - prior = seen.get(r["pid"]) - if prior is not None and prior != r["code"]: - raise ValueError( - f"product_data: pid collision — {prior!r} and {r['code']!r} both hash to " - f"{r['pid']}. Downstream set operations would merge them silently; widen the id " - f"before shipping this catalogue.") - seen[r["pid"]] = r["code"] - - -def _inventory_by_code(t): - """`{code: {...}}` from the inventory module, or `{}` if it cannot be read. - - Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of - its own slow families: a product table that will not render because inventory is momentarily - unreachable is a worse failure than one with blank stock columns. - """ - try: - import modules.inventory as inventory - return inventory.sku_inventory(t=t) or {} - except Exception: - return {} - - -def _bu_ltm_share(t, team_id): - """`{code: 0.0..1.0}` — this unit's SHARE of the SKU's last-twelve-months units. - - ⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the - inventory block has to be re-shaped by a BU fact, and this is that fact. - - ⛔ A SHARE, NOT THE UNIT COUNT ITSELF — AND THE REASON IS A MEASURED IMPOSSIBILITY. My first - version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`, - leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one - question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live - check found **Fisch's LTM units EXCEEDING the company's** — a subset larger than its superset, - which no reader could explain and no reconciliation could survive. - - Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by - construction and the scoped figure can never exceed the consolidated one. It also leaves the - consolidated column exactly as it was — the Inventory page and the Product grid still agree - about company-wide units, which is what "leave the rest" asked for. - - `all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for - "this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover. - - Degrades to `{}` on any failure, matching `_inventory_by_code` — and a missing code then takes - the `_DEFAULT_SHARE` below rather than a silent 0. - """ - try: - f, to = P.ltm(t) - allq = {code: (r.get('qty') or 0.0) - for code, r in (products._sku_rev(f, to, None) or {}).items()} - buq = {code: (r.get('qty') or 0.0) - for code, r in (products._sku_rev(f, to, team_id) or {}).items()} - except Exception: - return {} - out = {} - for code, total in allq.items(): - out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0 - return out - - -#: What a SKU absent from the LTM sales read is worth to a business unit. ZERO — it did not sell -#: in anybody's book over the window, so no unit can claim its velocity. Named rather than -#: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print -#: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids. -_DEFAULT_SHARE = 0.0 - - -def _rescope_inventory(e, share): - """`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate. - - The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator — NOT a second - idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two - copies of a threshold table is how the Product grid and the Inventory page start disagreeing - about which SKUs are dead. - - ⚠ `on_hand` is whatever the warehouse holds, unscoped — so a Fisch reader's `dos` answers - "how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson - actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf, - and inventing one would put a number on screen that no Odoo query could reproduce. - """ - on_hand = e.get("on_hand") - if on_hand is None: - return None, None, None # no inventory row for this SKU: blank, never zero - qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0) - daily = qty / 365.0 - if daily > 0: - dos_raw = on_hand / daily - else: - dos_raw = float('inf') if on_hand > 0 else 0.0 - try: - import modules.inventory as inventory - bucket = inventory._bucket(dos_raw, on_hand, qty) - except Exception: - bucket = None - return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket - - -def _catalogue_by_code(): - """`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from. - - ⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point. - Inventory degrades to `{}` because a product table with blank stock columns beats one that - will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that - failed quietly would drop the grid straight back to the sold-only 2,717 with every gate - green and nothing on screen saying so, which is the defect this seam exists to end - ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise - into a 503 that names the cause, so the loud failure has somewhere honest to land. - - It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the - same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs - to build a pool without Odoo. - """ - return products.catalogue() - - -def _pricelist_by_code(): - """`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`. - - A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises - (these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a - pool without Odoo. ⛔ Unstubbed there, section H would reach live Odoo through the back door - and the whole file would stop being runnable offline. - """ - try: - return products.pricelist_by_code() - except Exception as e: - return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"} - - -def pool(team_id=None, t=None): - """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`. - - CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set - is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs - that sold in its window and contributes NO rows of its own. - - `team_id` shapes the revenue columns exactly as it does for customers (`products.directory` - passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep - driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a - catalogue has no team. - """ - t = t or P.today() - consolidated = team_id is None - # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock - # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`. - inv = _inventory_by_code(t) - bu_share = {} if consolidated else _bu_ltm_share(t, team_id) - sup = supplier_master() - prices, _price_report = _pricelist_by_code() - cat = _catalogue_by_code() - # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue - # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those - # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue - # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY). - rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} - - rows = [] - for code, meta in cat.items(): - r = rev.get(code) - row = { - "pid": sku_pid(code), - "code": code, - # ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's - # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active - # record's `display_name` is the current truth, and it is the same string for every - # SKU that is not re-SKUed. `directory()` derives the category identically. - "product": meta.get("product") or code, - "category": meta.get("category") or "(uncategorized)", - # ⭐⭐ W33-T43 (R2 / amendment A2) — ODOO'S OWN PRODUCT ID, beside the hashed `pid`. - # R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this - # is that column. ⛔ NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here, - # unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover - # it and it has to be carried from `products.catalogue()`. - # ⚠ `None`, never 0, when the catalogue somehow has no id — 0 is a real Odoo id. - "product_id": meta.get("id"), - # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev` - # keeps its measured numbers including a real 0.0 (it sold last year, not this one). - "rev_ytd": r.get("rev_ytd", 0.0) if r else None, - "rev_ly": r.get("rev_ly", 0.0) if r else None, - "yoy_pct": r.get("yoy_pct") if r else None, - "qty_ytd": r.get("qty_ytd", 0.0) if r else None, - "orders_ytd": r.get("orders_ytd", 0) if r else None, - } - # ⭐ WAVE 30 W30-T34 — the PRICELIST stratum, one column per declared list. Catalogue - # data like the supplier block, so it rides every pull, scoped or not: a price book is - # not a thing a business unit owns a slice of, and a Fisch reader who cannot see the - # Fisch price is exactly who this column is for. - # - # ⛔ `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is - # W29-T52's own negative control: the single `3_global` rule computes over `list_price`, - # which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer — - # it is a wrong one wearing a currency sign. - p = prices.get(code) or {} - for _col, _name in products.PRICELIST_COLUMNS: - row[_col] = p.get(_col) - # Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock). - s = sup.get(code) or {} - row.update({ - "supplier": s.get("supplier") or "(none)", - "lead_days": s.get("lead_days"), - "origin_country": s.get("origin_country") or "(none)", - "first_cost": s.get("first_cost"), - }) - e = inv.get(code) or {} - if consolidated: - qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket") - else: - qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE)) - row.update({ - # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf. - "on_hand": e.get("on_hand"), - "unit_cost": e.get("unit_cost"), - "inv_value": e.get("inv_value"), - # SCOPED: these three are functions of how fast THIS unit sells the SKU. - "qty_ltm": qty_ltm, - "dos": dos, - "stock_bucket": bucket, - }) - # THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule: - # buy when the shelf runs out before a reorder could land. - # - # ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400 - # no lead time; `dos` is null for anything that never sells through. "We don't know" - # and "you're fine" are different sentences, and only one of them is safe to print - # beside a purchasing decision. - # - # ⛔ `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it - # is not a preset field — it is a formula over the two columns beside it, and the - # platform has a formula field type for exactly that. Its Field is gone from - # `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that - # contract, so no Field means no cell on the wire. Nothing renders this key. - # - # It stays computed because deleting it would delete the PROOF. `validate()` below - # reconciles the formula's predicate against this one, row for row, which is the only - # thing that makes "the same exact figures" a claim rather than an assertion — and - # this repo has the scar already (wave 17: archiving `ar` "would have DELETED the - # proof"). Two comparisons per SKU is what that costs. - # - # ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer - # "does the shelf outlast a reorder AT FISCH'S RATE" — reading the consolidated figure - # here would print a buy signal computed from both units' velocity beside a days-of-supply - # computed from one, and the two columns would disagree on the same row. - lead = s.get("lead_days") - if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0: - row["cover_gap_d"] = int(round(dos - lead)) - row["buy_now"] = "Buy now" if dos < lead else "OK" - else: - row["cover_gap_d"] = None - row["buy_now"] = None - rows.append(row) - - # `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU - # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored - # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured - # nothing sold". - rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0)) - _assert_no_pid_collision(rows) - return rows - - -#: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03). -#: -#: The owner's ruling was that "Buy signal" is not a preset field — it is a formula over two -#: columns the product table already carries, and the platform has a formula field type for it. -#: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON -#: contract's `_product_removed_buy_now` note records: -#: -#: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "") -#: -#: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a -#: restatement of the business rule — that is the whole point, because the two could drift and -#: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly: -#: -#: · a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything -#: that is not a finite number or a string); -#: · `cmp` returns BLANK unless BOTH sides read as numbers — it never coerces a blank to 0, -#: which is the difference between this and a filter engine's `toNum(null) === 0`; -#: · `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison -#: propagates out as a blank cell rather than taking the false branch. -BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")' - - -def _buy_signal_formula(row): - """Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank).""" - def num(v): - # The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite - # number is null. `isinstance(True, int)` is True in Python, so bool is excluded first. - if isinstance(v, bool) or not isinstance(v, (int, float)): - return None - return v if v == v and v not in (float('inf'), float('-inf')) else None - - lead, dos = num(row.get("lead_days")), num(row.get("dos")) - if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank - return "" - if not lead > 0: - return "" # the formula's own else-branch - if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank - return "" - return "Buy now" if dos < lead else "OK" - - -def validate(team_id=None, t=None): - """Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number - which does not tie to Odoo does not ship. - - ⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING - (wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the - same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could - not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave. - No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either. - That is the leg below, and it is the control that would have caught it - ([[no-unverifiable-aggregates]]). - - ⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join - lose money?") and is still the right shape for that one. What changed is that it now has to - account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived - products out of the grid; the total is decomposed rather than compared loosely, so a growing - "outside the grid" figure shows up as a number rather than as slack in a tolerance. - - Still deliberately NOT covered, and said rather than implied: the inventory columns are a - different module's reconciliation, and they are absent on a scoped pull anyway. - """ - t = t or P.today() - rows = pool(team_id=team_id, t=t) - - # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active - # products, asked fresh — not a `len()` over anything this module or `products.catalogue()` - # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it. - n_active = products.catalogue_count() - checks = [{ - "check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT " - "Odoo search_count('product.product', active=True)", - "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active, - }] - - yf, yt = P.ytd(t) - sku_rev = products._sku_rev(yf, yt, team_id) or {} - in_pool = {r["code"] for r in rows} - outside = {c: v for c, v in sku_rev.items() if c not in in_pool} - ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2) - theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2) - dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2) - checks.append({ - "check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product " - "carries == modules.products._sku_rev over the same window", - "ours": round(ours + dropped, 2), "theirs": theirs, - "ok": abs((ours + dropped) - theirs) < 0.01, - "detail": {"in_the_grid": ours, "outside_the_grid": dropped, - "codes_outside": len(outside)}, - }) - # ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is - # trivially true when checked against the dict that did the dropping — that is the - # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE - # product carries means `products.catalogue()` missed a row that has revenue, which is the - # 2,717 defect returning in a smaller costume. - # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the - # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all. - coded_outside = sorted(c for c in outside if not str(c).startswith("pid:")) - still_active = (O.get_odoo().search_count( - 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)]) - if coded_outside else 0) - checks.append({ - "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived " - "stay out) — asked of Odoo, never of the catalogue that did the dropping", - "ours": still_active, "theirs": 0, "ok": still_active == 0, - "detail": {"codes": coded_outside[:10], "n_codes": len(outside)}, - }) - # ── ⭐⭐ WAVE 30 W30-T34 — THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ── - # - # Three legs, because the column can fail in three different ways and only one of them is a - # count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each - # rule over the products it names, IN PYTHON — that expansion is the fragile part, so every - # leg below re-asks ODOO instead of re-reading the map. - price_report = _pricelist_by_code()[1] - _pl_rows = O.search_read('product.pricelist', [], ['id', 'name']) - _pl_id = {} - for _p in _pl_rows: - _pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id']) - _today = P.today().isoformat() - - _targets_memo = {} - - def _rule_targets(list_name): - """(variant_ids, template_ids) a pricelist prices today — read FRESH from Odoo. - - Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and - without this `validate()` makes six identical round trips instead of three (measured: - ~48s of the run). Deliberately NOT an `lru_cache` — an oracle that survives the process - is an oracle reading yesterday's Odoo. - """ - if list_name in _targets_memo: - return _targets_memo[list_name] - var, tmpl = set(), set() - plid = _pl_id.get(list_name) - if plid is None: - return var, tmpl - for r in O.search_read( - 'product.pricelist.item', - [('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'), - ('applied_on', 'in', ['0_product_variant', '1_product'])], - ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price', - 'date_start', 'date_end']): - ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10] - if (ds and ds > _today) or (de and de < _today): - continue - fp = r.get('fixed_price') - if not isinstance(fp, (int, float)) or fp <= 0: - continue - if r.get('applied_on') == '0_product_variant' and r.get('product_id'): - var.add(O.m2o_id(r['product_id'])) - elif r.get('product_tmpl_id'): - tmpl.add(O.m2o_id(r['product_tmpl_id'])) - _targets_memo[list_name] = (var, tmpl) - return var, tmpl - - # LEG 1 — COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of - # ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS - # the data) but the EXPANSION is not, and the expansion is what breaks. - for _col, _name in products.PRICELIST_COLUMNS: - _var, _tmpl = _rule_targets(_name) - _dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)), - ('product_tmpl_id', 'in', sorted(_tmpl))] - theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0 - ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float))) - checks.append({ - "check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count " - f"of ACTIVE products its date-valid fixed rules reach", - "ours": ours, "theirs": theirs, "ok": ours == theirs, - "detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)}, - }) - - # LEG 2 — ⛔ THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must - # render BLANK. Asked of ODOO, never of the map that produced the blank — "everything I left - # empty was genuinely unpriced" is trivially true when checked against the dict that emptied - # it, which is the self-sealing shape the population leg above exists to end. - _unpriced = [r for r in rows - if not any(isinstance(r.get(c), (int, float)) - for c, _n in products.PRICELIST_COLUMNS)] - _codes = {r["code"] for r in _unpriced} - _stray = 0 - if _codes: - # Resolve those codes back to Odoo ids and ask whether ANY declared list prices them. - _ids, _tmpls = set(), set() - for _p in O.search_read('product.product', - [('active', '=', True), - ('default_code', 'in', sorted(c for c in _codes - if not c.startswith("pid:")))], - ['id', 'product_tmpl_id']): - _ids.add(_p['id']) - _tmpls.add(O.m2o_id(_p.get('product_tmpl_id'))) - for _col, _name in products.PRICELIST_COLUMNS: - _v, _t = _rule_targets(_name) - _stray += len((_v & _ids) | ({t for t in _tmpls if t in _t})) - checks.append({ - "check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared " - "lists (W29-T52's NC: no fallback dressed as a price) — asked of Odoo", - "ours": _stray, "theirs": 0, "ok": _stray == 0, - "detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)}, - }) - - # LEG 3 — ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you - # need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is - # counted here rather than dropped. The ASSERTION is the one thing that must never be true — - # a price cell that is present and not a positive number, i.e. a "free" SKU — while the - # rest rides `detail` so it is reported without reddening an honest day's data. - _bad = [r["code"] for r in rows - for c, _n in products.PRICELIST_COLUMNS - if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)] - # ⛔ MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends - # every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole - # dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this - # column ties to the cent HERE and ships rounded. Reported, never silently enforced. - _cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS - if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9) - _to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS - if isinstance(r.get(c), (int, float)) and round(r[c]) == 0) - checks.append({ - "check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; " - "an unset rule must be blank), and what the reader cannot see is COUNTED", - "ours": len(_bad), "theirs": 0, "ok": not _bad, - "detail": {"bad_cells": _bad[:10], "reader_report": price_report, - "wire_rounding_loses_cents_on": _cents, - "wire_rounding_to_zero_dollars": _to_zero}, - }) - - # WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every - # member of it must be re-derivable from the two columns beside it. A signal somebody buys - # stock on cannot be "mostly right": the failure that matters is a row that says OK because - # an input was missing, so the blank leg is checked as hard as the other two. - if team_id is None: # consolidated only — the inputs exist only there - buy = [r for r in rows if r.get("buy_now") == "Buy now"] - ok_rows = [r for r in rows if r.get("buy_now") == "OK"] - blank = [r for r in rows if r.get("buy_now") is None] - mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"])) - mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"])) - # A blank must be UNKNOWN — never a row we could have answered and quietly did not. - mis += sum(1 for r in blank - if isinstance(r.get("dos"), (int, float)) - and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0) - checks.append({ - "check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none " - "misclassified)", - "ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows), - "ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows), - "detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank), - "misclassified": mis}, - }) - # ⭐ OWNER 2026-08-03 — "use the formula fields to come up to the same EXACT figures". - # - # This is that sentence, as a check. The retired preset column and the formula that - # replaces it must agree on every SKU, in all three states, or the replacement is not a - # replacement. `_buy_signal_formula` is a line-by-line port of the client formula - # engine's evaluation of - # - # IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "") - # - # including the part that is easy to get wrong: a comparison against a BLANK is blank, - # never a coerced 0. (The client engine's `cmp` returns null unless both sides read as - # numbers, and `IF` refuses a non-boolean condition — so a missing `dos` yields "" and - # not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that - # never sells through; the formula engine does not, and this check is what holds it.) - # - # ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape. - disagree = sorted(r["code"] for r in rows - if (r.get("buy_now") or "") != _buy_signal_formula(r)) - checks.append({ - "check": 'Buy signal as a FORMULA field == the retired preset column, per SKU ' - '(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))', - "ours": len(rows) - len(disagree), "theirs": len(rows), - "ok": not disagree, - "detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)}, - }) - # The other half of "the same figures": the SHARED Buy list view can no longer filter on - # the retired column, and its replacement conditions must select the same SKUs. They are - # `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days` - # (_seed_wave17.views), so this reproduces exactly that conjunction. - # - # ⛔ THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the - # obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine - # gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on. - view_rows = {r["code"] for r in rows - if isinstance(r.get("dos"), (int, float)) - and isinstance(r.get("lead_days"), (int, float)) - and r["lead_days"] > 0 and r["dos"] < r["lead_days"]} - signal_rows = {r["code"] for r in buy} - rounding_would_miss = sorted( - c for c in signal_rows - if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0) - checks.append({ - "check": "Buy list view conditions (dos/lead_days, no retired column) select " - "exactly the Buy-now SKUs", - "ours": len(view_rows), "theirs": len(signal_rows), - "ok": view_rows == signal_rows, - "detail": {"only_in_view": sorted(view_rows - signal_rows)[:10], - "only_in_signal": sorted(signal_rows - view_rows)[:10], - # Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would - # have silently dropped. 0 today does not make that filter correct. - "cover_gap_rounds_to_zero": len(rounding_would_miss)}, - }) - return checks +"""modules/product_data.py — the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC). + +The second object on the table-page factory: same grid, same engine, same permission wall — the +only difference is the field schema and the identity. `modules/customer_data.pool()` is the +template and this deliberately mirrors its signature and its row shape. + +⛔ THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER. + +1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.** + Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an + integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the + same id on every pull and across processes — never an enumeration index, which would reshuffle + whenever the catalogue changes) and `code`, the real business key, as a visible column. + `_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would + silently merge in every downstream set operation, and a loud build failure is the only version + of that anyone would notice. + +2. **THE SCOPE RULE — inventory columns are CONSOLIDATED and are therefore OMITTED for a + BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory` + is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not + brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside + company-wide stock IN THE SAME ROW — the mixed-scope value defect wave 15's amendment 3 exists + for, arriving through a different door. The honest options were "omit" or "label the columns + company-wide"; omit is the fail-closed one, and a column that is absent asks a question, + whereas a column that is silently company-wide answers one wrongly. + +3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and + COUNTRY are not in Odoo at all — they live in the inventory WORKBOOK + ([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent. + Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost + allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it + does not pretend to cover columns that are absent. + +4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12, + 2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the + union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid + showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise: + + · ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date + window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics + that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because + ~2,550 active SKUs have never sold in wholesale scope at all. + · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the + revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and + not this one, and that zero is a measurement. A row that appears in NO revenue read + carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the + client renders as an empty cell. Blank is an admission; zero is a measurement. + · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where + that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW + UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product` + carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two + sources are JOINED rather than merged. +""" +import json +import zlib +from pathlib import Path + +import core.odoo as O +import core.periods as P +import core.shared_overlay as shared_overlay +import core.table_store as table_store +import modules.products as products + +#: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2. +#: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy +#: signal built on a stock number the caller cannot see would be a recommendation nobody could +#: check ([[no-unverifiable-aggregates]]). +#: +#: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers: +#: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products` +#: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field — +#: computed as `validate()`'s oracle, projected onto no wire — so it belongs here for the first +#: reader and is inert for the second. Removing it would stop the scope gate proving that a +#: BU-scoped pull withholds it. +#: +#: ⭐⭐ OWNER RULING 2026-08-11 — THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim: +#: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole +#: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether +#: anything was IN STOCK, which is the first question they ask. The ruling splits the block by +#: what a business unit can actually shape: +#: +#: * `on_hand` / `unit_cost` / `inv_value` — ONE physical warehouse, no Fisch shelf and no +#: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest") +#: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` — all SALES-derived, so a +#: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with +#: Sales from Fisch") +#: +#: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one": +#: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a +#: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer +#: asserts the columns are WITHHELD — it asserts the split above, which is a stronger claim and +#: cannot pass on an empty scoped payload (the shape the old rule and an outage share). +CONSOLIDATED_ONLY = () + +#: ⛔ POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD — they must never reach a browser. +#: +#: `rows_from_pool` projects strictly through the field contract, so "no Field" already means +#: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody +#: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts, +#: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one. +#: +#: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s +#: Field while keeping its computation, and the gate — which was asserting a FIELD property from +#: a ROW-key list — went red with no way to tell a deliberate divergence from a dropped column. +UNSHIPPED_ROW_KEYS = ("buy_now",) + +#: Wave 17 (owner item 13, ruling R3) — the SUPPLIER MASTER, from the curated mastersheet map +#: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo: +#: the owner confirmed this data came from the Inventory System Mastersheet, which is why +#: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than +#: quietly missing" and needed "a loader this module deliberately does not invent". This is +#: that loader. +#: +#: ⛔ THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT. +#: A user-created field and its values live in the PER-USER strata (`table_store.workspace` +#: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that +#: filtered on a user-created column would, for every OTHER account, name a column that does not +#: exist — and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES +#: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but +#: its author. Contract columns are identical for every reader, so the view means one thing. +#: +#: ⭐⭐ 2026-08-12 (W30-T36) — THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT +#: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."* +#: It was parked because a per-user column silently WIDENS a shared view — not because editing was +#: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum +#: that removes the objection: **one value per (row, column) for the whole tenant**, so the column +#: still means ONE thing to every reader and a shared view still filters honestly. +#: ⇒ The four columns below are now `source: "overlay"` + `shared: true` in the canonical +#: contract, their values live in `__shared`, and the master map is what SEEDS an +#: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`. +_SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json" +_SUPPLIER_CACHE = {} + + +def supplier_master(): + """`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process. + + Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table + that will not render because a master map is missing is a worse failure than one with blank + supplier columns. + """ + if _SUPPLIER_CACHE: + return _SUPPLIER_CACHE + try: + raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8")) + except Exception: + return {} + for code, meta in (raw or {}).items(): + if not isinstance(meta, dict): + continue + lead = meta.get("lead") + _SUPPLIER_CACHE[str(code)] = { + "supplier": (meta.get("vendor") or "") or None, + "lead_days": int(lead) if isinstance(lead, (int, float)) else None, + "origin_country": (meta.get("country") or "") or None, + "first_cost": meta.get("first_cost"), + } + return _SUPPLIER_CACHE + +#: Wave 16 C-TOPIC — the PRODUCT table's OWN workspace bucket. ⛔ Never the customer one: +#: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a +#: SHARED overlay bucket a hash collision would silently write a product note onto somebody's +#: customer (or the reverse). Separate store keys make that structurally impossible, which is +#: the whole reason the table-page factory exists ("a new table object gets its own +#: table_store.make('_table_workspace')"). +TABLE_KEY = 'product_table_workspace' + +_GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json' +_SHARED_KEYS = None + + +def SHARED_KEYS(): + """The product columns whose values are TENANT-WIDE — derived from the canonical contract's + own `shared: true`, never typed out a second time. + + ⛔ IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared + write into the per-user stratum with nothing going wrong anywhere — the widening defect + reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the + canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the + same file with no guard), so a raise here costs nothing that was still working. + """ + global _SHARED_KEYS + if _SHARED_KEYS is None: + doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8')) + _SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or [] + if f.get('shared')) + return _SHARED_KEYS + + +class _ProductTableStore(table_store.TableStore): + """The product workspace, with the SHARED columns routed to the tenant-wide stratum. + + ⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER + THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** — measured, zero + call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` → `grid_events. + handle_one` → `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which + `routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both + doors pass through; intercepting at either route would have left the other one writing a + per-user value that only its author could see. + + ⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store + handle as the per-user one it sits beside — `_tops`' own comment explains that a split, where + one side is tenant-scoped and the other is not, is worse than a stated residency error because + a user's value would vanish the moment they saved it. Reading `self.st` means both strata move + together the day that singleton gains a tenant handle. + """ + + def patch_overlay(self, username, pid, updates): + clean = dict(updates or {}) + if not clean: + return + keys = set(SHARED_KEYS()) + shared = {k: v for k, v in clean.items() if k in keys} + personal = {k: v for k, v in clean.items() if k not in keys} + if shared: + shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st) + if personal: + super().patch_overlay(username, pid, personal) + + +TABLE_OPS = _ProductTableStore(TABLE_KEY) + + +def shared_cells(pids, st=None): + """`{"": {key: value}}` for the SHARED columns of the rows named by `pids`. + + ⚠ `pids` is required and positional all the way down — `shared_overlay.cells` refuses to serve + "everything" by design, and the caller here always holds an already-scoped pool. + """ + return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st) + + +def sku_pid(code): + """A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and + always JSON-safe. Stable across processes and pulls, which an enumeration index is not.""" + return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF + + +def _assert_no_pid_collision(rows): + """Two SKUs sharing a pid would MERGE in every set operation downstream — cohort membership, + allowed_pids, the measure channel — and nothing would report it. Fail the build instead.""" + seen = {} + for r in rows: + prior = seen.get(r["pid"]) + if prior is not None and prior != r["code"]: + raise ValueError( + f"product_data: pid collision — {prior!r} and {r['code']!r} both hash to " + f"{r['pid']}. Downstream set operations would merge them silently; widen the id " + f"before shipping this catalogue.") + seen[r["pid"]] = r["code"] + + +def _inventory_by_code(t): + """`{code: {...}}` from the inventory module, or `{}` if it cannot be read. + + Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of + its own slow families: a product table that will not render because inventory is momentarily + unreachable is a worse failure than one with blank stock columns. + """ + try: + import modules.inventory as inventory + return inventory.sku_inventory(t=t) or {} + except Exception: + return {} + + +def _bu_ltm_share(t, team_id): + """`{code: 0.0..1.0}` — this unit's SHARE of the SKU's last-twelve-months units. + + ⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the + inventory block has to be re-shaped by a BU fact, and this is that fact. + + ⛔ A SHARE, NOT THE UNIT COUNT ITSELF — AND THE REASON IS A MEASURED IMPOSSIBILITY. My first + version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`, + leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one + question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live + check found **Fisch's LTM units EXCEEDING the company's** — a subset larger than its superset, + which no reader could explain and no reconciliation could survive. + + Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by + construction and the scoped figure can never exceed the consolidated one. It also leaves the + consolidated column exactly as it was — the Inventory page and the Product grid still agree + about company-wide units, which is what "leave the rest" asked for. + + `all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for + "this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover. + + Degrades to `{}` on any failure, matching `_inventory_by_code` — and a missing code then takes + the `_DEFAULT_SHARE` below rather than a silent 0. + """ + try: + f, to = P.ltm(t) + allq = {code: (r.get('qty') or 0.0) + for code, r in (products._sku_rev(f, to, None) or {}).items()} + buq = {code: (r.get('qty') or 0.0) + for code, r in (products._sku_rev(f, to, team_id) or {}).items()} + except Exception: + return {} + out = {} + for code, total in allq.items(): + out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0 + return out + + +#: What a SKU absent from the LTM sales read is worth to a business unit. ZERO — it did not sell +#: in anybody's book over the window, so no unit can claim its velocity. Named rather than +#: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print +#: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids. +_DEFAULT_SHARE = 0.0 + + +def _rescope_inventory(e, share): + """`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate. + + The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator — NOT a second + idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two + copies of a threshold table is how the Product grid and the Inventory page start disagreeing + about which SKUs are dead. + + ⚠ `on_hand` is whatever the warehouse holds, unscoped — so a Fisch reader's `dos` answers + "how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson + actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf, + and inventing one would put a number on screen that no Odoo query could reproduce. + """ + on_hand = e.get("on_hand") + if on_hand is None: + return None, None, None # no inventory row for this SKU: blank, never zero + qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0) + daily = qty / 365.0 + if daily > 0: + dos_raw = on_hand / daily + else: + dos_raw = float('inf') if on_hand > 0 else 0.0 + try: + import modules.inventory as inventory + bucket = inventory._bucket(dos_raw, on_hand, qty) + except Exception: + bucket = None + return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket + + +def _catalogue_by_code(): + """`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from. + + ⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point. + Inventory degrades to `{}` because a product table with blank stock columns beats one that + will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that + failed quietly would drop the grid straight back to the sold-only 2,717 with every gate + green and nothing on screen saying so, which is the defect this seam exists to end + ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise + into a 503 that names the cause, so the loud failure has somewhere honest to land. + + It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the + same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs + to build a pool without Odoo. + """ + return products.catalogue() + + +def _pricelist_by_code(): + """`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`. + + A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises + (these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a + pool without Odoo. ⛔ Unstubbed there, section H would reach live Odoo through the back door + and the whole file would stop being runnable offline. + """ + try: + return products.pricelist_by_code() + except Exception as e: + return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"} + + +def pool(team_id=None, t=None): + """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`. + + CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set + is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs + that sold in its window and contributes NO rows of its own. + + `team_id` shapes the revenue columns exactly as it does for customers (`products.directory` + passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep + driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a + catalogue has no team. + """ + t = t or P.today() + consolidated = team_id is None + # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock + # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`. + inv = _inventory_by_code(t) + bu_share = {} if consolidated else _bu_ltm_share(t, team_id) + sup = supplier_master() + prices, _price_report = _pricelist_by_code() + cat = _catalogue_by_code() + # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue + # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those + # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue + # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY). + rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} + + rows = [] + for code, meta in cat.items(): + r = rev.get(code) + row = { + "pid": sku_pid(code), + "code": code, + # ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's + # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active + # record's `display_name` is the current truth, and it is the same string for every + # SKU that is not re-SKUed. `directory()` derives the category identically. + "product": meta.get("product") or code, + "category": meta.get("category") or "(uncategorized)", + # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev` + # keeps its measured numbers including a real 0.0 (it sold last year, not this one). + "rev_ytd": r.get("rev_ytd", 0.0) if r else None, + "rev_ly": r.get("rev_ly", 0.0) if r else None, + "yoy_pct": r.get("yoy_pct") if r else None, + "qty_ytd": r.get("qty_ytd", 0.0) if r else None, + "orders_ytd": r.get("orders_ytd", 0) if r else None, + } + # ⭐ WAVE 30 W30-T34 — the PRICELIST stratum, one column per declared list. Catalogue + # data like the supplier block, so it rides every pull, scoped or not: a price book is + # not a thing a business unit owns a slice of, and a Fisch reader who cannot see the + # Fisch price is exactly who this column is for. + # + # ⛔ `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is + # W29-T52's own negative control: the single `3_global` rule computes over `list_price`, + # which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer — + # it is a wrong one wearing a currency sign. + p = prices.get(code) or {} + for _col, _name in products.PRICELIST_COLUMNS: + row[_col] = p.get(_col) + # Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock). + s = sup.get(code) or {} + row.update({ + "supplier": s.get("supplier") or "(none)", + "lead_days": s.get("lead_days"), + "origin_country": s.get("origin_country") or "(none)", + "first_cost": s.get("first_cost"), + }) + e = inv.get(code) or {} + if consolidated: + qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket") + else: + qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE)) + row.update({ + # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf. + "on_hand": e.get("on_hand"), + "unit_cost": e.get("unit_cost"), + "inv_value": e.get("inv_value"), + # SCOPED: these three are functions of how fast THIS unit sells the SKU. + "qty_ltm": qty_ltm, + "dos": dos, + "stock_bucket": bucket, + }) + # THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule: + # buy when the shelf runs out before a reorder could land. + # + # ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400 + # no lead time; `dos` is null for anything that never sells through. "We don't know" + # and "you're fine" are different sentences, and only one of them is safe to print + # beside a purchasing decision. + # + # ⛔ `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it + # is not a preset field — it is a formula over the two columns beside it, and the + # platform has a formula field type for exactly that. Its Field is gone from + # `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that + # contract, so no Field means no cell on the wire. Nothing renders this key. + # + # It stays computed because deleting it would delete the PROOF. `validate()` below + # reconciles the formula's predicate against this one, row for row, which is the only + # thing that makes "the same exact figures" a claim rather than an assertion — and + # this repo has the scar already (wave 17: archiving `ar` "would have DELETED the + # proof"). Two comparisons per SKU is what that costs. + # + # ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer + # "does the shelf outlast a reorder AT FISCH'S RATE" — reading the consolidated figure + # here would print a buy signal computed from both units' velocity beside a days-of-supply + # computed from one, and the two columns would disagree on the same row. + lead = s.get("lead_days") + if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0: + row["cover_gap_d"] = int(round(dos - lead)) + row["buy_now"] = "Buy now" if dos < lead else "OK" + else: + row["cover_gap_d"] = None + row["buy_now"] = None + rows.append(row) + + # `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU + # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored + # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured + # nothing sold". + rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0)) + _assert_no_pid_collision(rows) + return rows + + +#: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03). +#: +#: The owner's ruling was that "Buy signal" is not a preset field — it is a formula over two +#: columns the product table already carries, and the platform has a formula field type for it. +#: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON +#: contract's `_product_removed_buy_now` note records: +#: +#: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "") +#: +#: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a +#: restatement of the business rule — that is the whole point, because the two could drift and +#: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly: +#: +#: · a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything +#: that is not a finite number or a string); +#: · `cmp` returns BLANK unless BOTH sides read as numbers — it never coerces a blank to 0, +#: which is the difference between this and a filter engine's `toNum(null) === 0`; +#: · `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison +#: propagates out as a blank cell rather than taking the false branch. +BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")' + + +def _buy_signal_formula(row): + """Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank).""" + def num(v): + # The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite + # number is null. `isinstance(True, int)` is True in Python, so bool is excluded first. + if isinstance(v, bool) or not isinstance(v, (int, float)): + return None + return v if v == v and v not in (float('inf'), float('-inf')) else None + + lead, dos = num(row.get("lead_days")), num(row.get("dos")) + if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank + return "" + if not lead > 0: + return "" # the formula's own else-branch + if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank + return "" + return "Buy now" if dos < lead else "OK" + + +def validate(team_id=None, t=None): + """Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number + which does not tie to Odoo does not ship. + + ⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING + (wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the + same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could + not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave. + No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either. + That is the leg below, and it is the control that would have caught it + ([[no-unverifiable-aggregates]]). + + ⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join + lose money?") and is still the right shape for that one. What changed is that it now has to + account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived + products out of the grid; the total is decomposed rather than compared loosely, so a growing + "outside the grid" figure shows up as a number rather than as slack in a tolerance. + + Still deliberately NOT covered, and said rather than implied: the inventory columns are a + different module's reconciliation, and they are absent on a scoped pull anyway. + """ + t = t or P.today() + rows = pool(team_id=team_id, t=t) + + # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active + # products, asked fresh — not a `len()` over anything this module or `products.catalogue()` + # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it. + n_active = products.catalogue_count() + checks = [{ + "check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT " + "Odoo search_count('product.product', active=True)", + "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active, + }] + + yf, yt = P.ytd(t) + sku_rev = products._sku_rev(yf, yt, team_id) or {} + in_pool = {r["code"] for r in rows} + outside = {c: v for c, v in sku_rev.items() if c not in in_pool} + ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2) + theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2) + dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2) + checks.append({ + "check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product " + "carries == modules.products._sku_rev over the same window", + "ours": round(ours + dropped, 2), "theirs": theirs, + "ok": abs((ours + dropped) - theirs) < 0.01, + "detail": {"in_the_grid": ours, "outside_the_grid": dropped, + "codes_outside": len(outside)}, + }) + # ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is + # trivially true when checked against the dict that did the dropping — that is the + # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE + # product carries means `products.catalogue()` missed a row that has revenue, which is the + # 2,717 defect returning in a smaller costume. + # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the + # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all. + coded_outside = sorted(c for c in outside if not str(c).startswith("pid:")) + still_active = (O.get_odoo().search_count( + 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)]) + if coded_outside else 0) + checks.append({ + "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived " + "stay out) — asked of Odoo, never of the catalogue that did the dropping", + "ours": still_active, "theirs": 0, "ok": still_active == 0, + "detail": {"codes": coded_outside[:10], "n_codes": len(outside)}, + }) + # ── ⭐⭐ WAVE 30 W30-T34 — THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ── + # + # Three legs, because the column can fail in three different ways and only one of them is a + # count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each + # rule over the products it names, IN PYTHON — that expansion is the fragile part, so every + # leg below re-asks ODOO instead of re-reading the map. + price_report = _pricelist_by_code()[1] + _pl_rows = O.search_read('product.pricelist', [], ['id', 'name']) + _pl_id = {} + for _p in _pl_rows: + _pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id']) + _today = P.today().isoformat() + + _targets_memo = {} + + def _rule_targets(list_name): + """(variant_ids, template_ids) a pricelist prices today — read FRESH from Odoo. + + Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and + without this `validate()` makes six identical round trips instead of three (measured: + ~48s of the run). Deliberately NOT an `lru_cache` — an oracle that survives the process + is an oracle reading yesterday's Odoo. + """ + if list_name in _targets_memo: + return _targets_memo[list_name] + var, tmpl = set(), set() + plid = _pl_id.get(list_name) + if plid is None: + return var, tmpl + for r in O.search_read( + 'product.pricelist.item', + [('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'), + ('applied_on', 'in', ['0_product_variant', '1_product'])], + ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price', + 'date_start', 'date_end']): + ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10] + if (ds and ds > _today) or (de and de < _today): + continue + fp = r.get('fixed_price') + if not isinstance(fp, (int, float)) or fp <= 0: + continue + if r.get('applied_on') == '0_product_variant' and r.get('product_id'): + var.add(O.m2o_id(r['product_id'])) + elif r.get('product_tmpl_id'): + tmpl.add(O.m2o_id(r['product_tmpl_id'])) + _targets_memo[list_name] = (var, tmpl) + return var, tmpl + + # LEG 1 — COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of + # ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS + # the data) but the EXPANSION is not, and the expansion is what breaks. + for _col, _name in products.PRICELIST_COLUMNS: + _var, _tmpl = _rule_targets(_name) + _dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)), + ('product_tmpl_id', 'in', sorted(_tmpl))] + theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0 + ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float))) + checks.append({ + "check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count " + f"of ACTIVE products its date-valid fixed rules reach", + "ours": ours, "theirs": theirs, "ok": ours == theirs, + "detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)}, + }) + + # LEG 2 — ⛔ THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must + # render BLANK. Asked of ODOO, never of the map that produced the blank — "everything I left + # empty was genuinely unpriced" is trivially true when checked against the dict that emptied + # it, which is the self-sealing shape the population leg above exists to end. + _unpriced = [r for r in rows + if not any(isinstance(r.get(c), (int, float)) + for c, _n in products.PRICELIST_COLUMNS)] + _codes = {r["code"] for r in _unpriced} + _stray = 0 + if _codes: + # Resolve those codes back to Odoo ids and ask whether ANY declared list prices them. + _ids, _tmpls = set(), set() + for _p in O.search_read('product.product', + [('active', '=', True), + ('default_code', 'in', sorted(c for c in _codes + if not c.startswith("pid:")))], + ['id', 'product_tmpl_id']): + _ids.add(_p['id']) + _tmpls.add(O.m2o_id(_p.get('product_tmpl_id'))) + for _col, _name in products.PRICELIST_COLUMNS: + _v, _t = _rule_targets(_name) + _stray += len((_v & _ids) | ({t for t in _tmpls if t in _t})) + checks.append({ + "check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared " + "lists (W29-T52's NC: no fallback dressed as a price) — asked of Odoo", + "ours": _stray, "theirs": 0, "ok": _stray == 0, + "detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)}, + }) + + # LEG 3 — ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you + # need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is + # counted here rather than dropped. The ASSERTION is the one thing that must never be true — + # a price cell that is present and not a positive number, i.e. a "free" SKU — while the + # rest rides `detail` so it is reported without reddening an honest day's data. + _bad = [r["code"] for r in rows + for c, _n in products.PRICELIST_COLUMNS + if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)] + # ⛔ MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends + # every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole + # dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this + # column ties to the cent HERE and ships rounded. Reported, never silently enforced. + _cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS + if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9) + _to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS + if isinstance(r.get(c), (int, float)) and round(r[c]) == 0) + checks.append({ + "check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; " + "an unset rule must be blank), and what the reader cannot see is COUNTED", + "ours": len(_bad), "theirs": 0, "ok": not _bad, + "detail": {"bad_cells": _bad[:10], "reader_report": price_report, + "wire_rounding_loses_cents_on": _cents, + "wire_rounding_to_zero_dollars": _to_zero}, + }) + + # WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every + # member of it must be re-derivable from the two columns beside it. A signal somebody buys + # stock on cannot be "mostly right": the failure that matters is a row that says OK because + # an input was missing, so the blank leg is checked as hard as the other two. + if team_id is None: # consolidated only — the inputs exist only there + buy = [r for r in rows if r.get("buy_now") == "Buy now"] + ok_rows = [r for r in rows if r.get("buy_now") == "OK"] + blank = [r for r in rows if r.get("buy_now") is None] + mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"])) + mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"])) + # A blank must be UNKNOWN — never a row we could have answered and quietly did not. + mis += sum(1 for r in blank + if isinstance(r.get("dos"), (int, float)) + and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0) + checks.append({ + "check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none " + "misclassified)", + "ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows), + "ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows), + "detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank), + "misclassified": mis}, + }) + # ⭐ OWNER 2026-08-03 — "use the formula fields to come up to the same EXACT figures". + # + # This is that sentence, as a check. The retired preset column and the formula that + # replaces it must agree on every SKU, in all three states, or the replacement is not a + # replacement. `_buy_signal_formula` is a line-by-line port of the client formula + # engine's evaluation of + # + # IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "") + # + # including the part that is easy to get wrong: a comparison against a BLANK is blank, + # never a coerced 0. (The client engine's `cmp` returns null unless both sides read as + # numbers, and `IF` refuses a non-boolean condition — so a missing `dos` yields "" and + # not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that + # never sells through; the formula engine does not, and this check is what holds it.) + # + # ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape. + disagree = sorted(r["code"] for r in rows + if (r.get("buy_now") or "") != _buy_signal_formula(r)) + checks.append({ + "check": 'Buy signal as a FORMULA field == the retired preset column, per SKU ' + '(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))', + "ours": len(rows) - len(disagree), "theirs": len(rows), + "ok": not disagree, + "detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)}, + }) + # The other half of "the same figures": the SHARED Buy list view can no longer filter on + # the retired column, and its replacement conditions must select the same SKUs. They are + # `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days` + # (_seed_wave17.views), so this reproduces exactly that conjunction. + # + # ⛔ THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the + # obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine + # gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on. + view_rows = {r["code"] for r in rows + if isinstance(r.get("dos"), (int, float)) + and isinstance(r.get("lead_days"), (int, float)) + and r["lead_days"] > 0 and r["dos"] < r["lead_days"]} + signal_rows = {r["code"] for r in buy} + rounding_would_miss = sorted( + c for c in signal_rows + if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0) + checks.append({ + "check": "Buy list view conditions (dos/lead_days, no retired column) select " + "exactly the Buy-now SKUs", + "ours": len(view_rows), "theirs": len(signal_rows), + "ok": view_rows == signal_rows, + "detail": {"only_in_view": sorted(view_rows - signal_rows)[:10], + "only_in_signal": sorted(signal_rows - view_rows)[:10], + # Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would + # have silently dropped. 0 today does not make that filter correct. + "cover_gap_rounds_to_zero": len(rounding_would_miss)}, + }) + return checks diff --git a/platform/modules/products.py b/platform/modules/products.py index 459f07d937c81172a10eaeff383e1c98258744ea..f5c1ea9cfe3c52359185878c84b6d368bbc9eaac 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -1,737 +1,728 @@ -"""Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs -(catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse -(SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders. - -Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here. -Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it -runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the -project guardrails keep off the local PC. (See BACKLOG.) - -All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain. -Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/ -window), so it's its own function the app calls lazily and caches. -""" -import sys -import datetime as dt -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from functools import lru_cache -import core.odoo as O -import core.periods as P -import modules.sales as sales_mod - -# SKU-health views are about real products — exclude service/delivery pseudo-SKUs -# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby -# on the dot-path does not). Validation reconciles on this same non-service universe. -_NO_SVC = [('product_id.type', '!=', 'service')] - - -@lru_cache(maxsize=1) -def _code_map(): - """product_id → SKU code. Multiple product records can share a default_code (re-SKUing - /duplicates); keying by code merges them so a re-coded item doesn't read as a fake - decliner + fake riser. Products without a code fall back to a per-id key. Archived - products are INCLUDED — re-SKUing typically archives the old record and creates a new - one under the same code, and the old record still carries last-year sales.""" - prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'], - limit=50000) - return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") - for p in prods} - - -def _sku_rev(date_from, date_to, team_id=None): - """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded, - duplicate product records merged by SKU code).""" - g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC), - ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False) - codes = _code_map() - out = {} - for r in g: - pid = O.m2o_id(r.get('product_id')) - if not pid: - continue - key = codes.get(pid, f"pid:{pid}") - e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')), - 'rev': 0.0, 'qty': 0.0, 'orders': 0}) - e['rev'] += r.get('price_subtotal') or 0.0 - e['qty'] += r.get('product_uom_qty') or 0.0 - e['orders'] += r.get('__count') or 0 - return out - - -def yoy_movers(t=None, limit=20, team_id=None): - """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change.""" - t = t or P.today() - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this = _sku_rev(yf, yt, team_id) - last = _sku_rev(lf, lt, team_id) - rows = [] - for pid in set(this) | set(last): - tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')}) - lr = last.get(pid, {'rev': 0.0}) - name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or '' - rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'], - 'change': tr['rev'] - lr['rev']}) - risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit] - decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit] - return {'risers': risers, 'decliners': decliners} - - -def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None): - """Catalog rot: SKUs that sold materially last year but are ~dead this year.""" - t = t or P.today() - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this = _sku_rev(yf, yt, team_id) - last = _sku_rev(lf, lt, team_id) - rows = [] - for pid, lr in last.items(): - if lr['rev'] < min_prior: - continue - tr = this.get(pid, {'rev': 0.0}) - if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie - continue - rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'], - 'lost': lr['rev'] - tr['rev']}) - rows.sort(key=lambda x: -x['lost']) - return rows[:limit] - - -def new_winners(t=None, limit=20, min_this=1500.0, team_id=None): - """SKUs that barely sold last year but are selling well this year.""" - t = t or P.today() - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this = _sku_rev(yf, yt, team_id) - last = _sku_rev(lf, lt, team_id) - rows = [] - for pid, tr in this.items(): - if tr['rev'] < min_this: - continue - lr = last.get(pid, {'rev': 0.0}) - if lr['rev'] > 0.05 * tr['rev']: - continue - rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'], - 'gained': tr['rev'] - lr['rev']}) - rows.sort(key=lambda x: -x['gained']) - return rows[:limit] - - -def velocity_leaders(t=None, limit=25, team_id=None): - """Top SKUs by LTM unit velocity (units/month) and revenue.""" - t = t or P.today() - lf, lt = P.ltm(t) - sku = _sku_rev(lf, lt, team_id) - rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0, - 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()] - rows.sort(key=lambda x: -x['units_ltm']) - return rows[:limit] - - -def _code_category(): - """code -> category name. Built from _code_map (pid->code) + the sales category map - (pid->category); the first record carrying a code sets that code's category.""" - codes = _code_map() - cats = sales_mod._product_cat() - out = {} - for pid, code in codes.items(): - if code not in out and pid in cats: - out[code] = cats[pid] - return out - - -def directory(t=None, team_id=None): - """Every SKU that sold this YTD or the same period last year, with the fields the SKU - drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the - customer directory — the full list (filtering happens in the UI), sorted by YTD revenue.""" - t = t or P.today() - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this = _sku_rev(yf, yt, team_id) - last = _sku_rev(lf, lt, team_id) - code_cat = _code_category() - rows = [] - for code in set(this) | set(last): - tr = this.get(code, {}) - lr = last.get(code, {}) - rev_ytd = tr.get('rev', 0.0) - rev_ly = lr.get('rev', 0.0) - rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code, - 'category': code_cat.get(code, '(uncategorized)'), - 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly, - 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly), - 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)}) - rows.sort(key=lambda r: -r['rev_ytd']) - return rows - - -def catalogue(): - """`{sku_code: {'product', 'category', 'id'}}` — EVERY ACTIVE product, sold or not. - - ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s - row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU - that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`, - `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe — - and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of - 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in - the consumer, never a window removed from `directory()`: removing the window alone lands at - 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all. - - ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one - catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than - merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold. - - Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active - records that carry none — so the join against `directory()` is code-for-code with no - normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields - off a sale line, so a never-sold row wears the same format as a sold one. - - ⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull - would put the grid back at a plausible wrong number with every gate green — the exact failure - this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard - `modules/backorders.py:161` already uses). - """ - dom = [('active', '=', True)] - prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'], - limit=50000) - n = O.get_odoo().search_count('product.product', dom) - if len(prods) != n: - raise ValueError( - f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against " - f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with " - f"nothing reporting it; raise the limit before shipping this.") - code_cat = _code_category() - out = {} - for p in prods: - code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") - # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero - # active products share a code, so this branch is a guard, not a merge policy. - # ⭐⭐ W33-T43 (R2 / amendment A2) — `id` RIDES THE ROW. Odoo's `product.product` id is - # already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is - # the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the - # legacy key and keeps every data column, so retiring the twin without this would lose a - # real identifier rather than a navigational link. - # ⚠ A product's grid pid is `crc32(default_code)`, NOT the Odoo id — unlike a CUSTOMER row, - # whose pid IS the partner id. That asymmetry is exactly why this cannot be derived - # downstream in `aios_grid.py` and has to be carried from the source read. - out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code, - 'category': code_cat.get(code, '(uncategorized)'), - 'id': p['id']}) - return out - - -#: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) — THE PRICELIST STRATUM, PER LIST. -#: -#: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than -#: decorating it: -#: -#: · **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139, -#: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that -#: is Fisch 3,971 · Royal 1 1,920 · Royal 2 1,857 · Public 6 · Giftware 0. -#: · **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.** -#: · **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** — median relative -#: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a -#: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the -#: only question the column exists to answer. -#: · ⛔ **`list_price` is 1.00 on 5,817 of the 5,871** — so the single `3_global` rule -#: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls -#: through to it**, and that is the ticket's own negative control: a SKU with no specific -#: item renders BLANK, never a fallback dressed as a price. -#: -#: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go -#: blank and makes `report["lists_missing"]` name it — a hard-coded id would keep pointing at -#: whatever inherited it and mislabel every cell silently. -PRICELIST_COLUMNS = ( - ("price_fisch", "Fisch"), - ("price_royal_1", "Royal 1"), - ("price_royal_2", "Royal 2"), -) - - -def pricelist_by_code(): - """`({code: {column_key: price}}, report)` — the date-valid FIXED base-tier price for each - declared pricelist, keyed EXACTLY as `catalogue()` keys its rows. - - ⛔ THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a - `pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other - way would leave those SKUs permanently blank while an oracle counting active products - counted them — a red gate on a working build, or worse, a silent hole nobody counts. - - **The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at - or below an order's quantity, because it is pricing a LINE that has one. A catalogue column - has no quantity in hand, so it takes the LOWEST `min_quantity` — the price at qty 1. Rules - above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only - rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all). - - **Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant` - rule is the more specific statement about this exact SKU. - - ⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than - `catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is - momentarily unreachable is a worse failure than one with blank price columns. The blanks are - not silent — `product_data.validate()`'s coverage leg reconciles each column against a fresh - Odoo count and goes red at zero. - - ⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need - to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is - counted rather than dropped: rules on pricelists the contract does not declare, non-fixed - (`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a - quantity break. - """ - # ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a - # report and a misleading one. A rule is classified ONCE, by the first reason it is skipped: - # undeclared list → out of date → not fixed → global → zero price. So `rules_not_fixed: 0` - # means "no formula rule on a list we declare", NOT "this Odoo has no formula rules" - # (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the - # contract does not declare — so it lands in `rules_undeclared_list`). - report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0, - "rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0, - "rules_zero_price": 0, "qty_break_only": 0} - try: - pl_rows = O.search_read('product.pricelist', [], ['id', 'name']) - by_name = {} - for p in pl_rows: - by_name.setdefault(str(p.get('name') or '').strip(), p['id']) - wanted = {} - for col, name in PRICELIST_COLUMNS: - pid = by_name.get(name) - if pid is None: - report["lists_missing"].append(name) - else: - wanted[pid] = col - - # ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE. - # Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on - # this connection; the same rules under a server-side domain with 6 fields cost - # **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's - # first-ever build, so that 16.5s is 16.5s of somebody's page load. - # - # ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part - # that is easy to get wrong: an absent `date_start` is `False`, not a past date, so - # `('date_start','<=',today)` ALONE would drop every open-ended rule — which is almost - # all of them. - today = P.today().isoformat() - _declared = sorted(wanted) - _kinds = ['0_product_variant', '1_product'] - _live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'), - ('applied_on', 'in', _kinds), - '|', ('date_start', '=', False), ('date_start', '<=', today), - '|', ('date_end', '=', False), ('date_end', '>=', today), - ('fixed_price', '>', 0)] - rules = O.search_read( - 'product.pricelist.item', _live, - ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price', - 'min_quantity']) if _declared else [] - - # ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six - # counts cost ~1.2s together; the rules they count would cost 16s to read. What this - # reader cannot see is still REPORTED — it is just no longer transferred. - def _n(extra): - try: - return O.get_odoo().search_count('product.pricelist.item', extra) - except Exception: - return -1 # -1 reads as "not measured", never as zero - _dated = ['|', ('date_start', '=', False), ('date_start', '<=', today), - '|', ('date_end', '=', False), ('date_end', '>=', today)] - report["rules_total"] = _n([]) - report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \ - if _declared else report["rules_total"] - if _declared: - _on = [('pricelist_id', 'in', _declared)] - # `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot - # price is one the operator is TOLD about, never one that quietly becomes a blank. - report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')]) - report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'), - ('applied_on', 'not in', _kinds)]) - report["rules_out_of_date"] = ( - _n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]) - - _n(_on + [('compute_price', '=', 'fixed'), - ('applied_on', 'in', _kinds)] + _dated)) - # A zero price is not "free" — it is an unset rule. Blank says so; 0 does not. - report["rules_zero_price"] = _n( - _on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)] - + _dated + [('fixed_price', '<=', 0)]) - - by_var, by_tmpl = {}, {} - for r in rules: - col = wanted[O.m2o_id(r.get('pricelist_id'))] - if r.get('applied_on') == '0_product_variant' and r.get('product_id'): - by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r) - elif r.get('product_tmpl_id'): - by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r) - - dom = [('active', '=', True)] - prods = O.search_read('product.product', dom, - ['id', 'default_code', 'product_tmpl_id'], limit=50000) - n = O.get_odoo().search_count('product.product', dom) - if len(prods) != n: - # Same guard, same reason as `catalogue()`: a short pull renders as a plausible - # smaller set of priced SKUs with nothing reporting it. - raise ValueError( - f"products.pricelist_by_code: the product pull is TRUNCATED — read {len(prods)} " - f"rows against a search_count of {n}.") - except Exception as e: - report["error"] = f"{type(e).__name__}: {str(e)[:200]}" - return {}, report - - out = {} - for p in prods: - code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") - tmpl = O.m2o_id(p.get('product_tmpl_id')) - cells = {} - for col, _name in PRICELIST_COLUMNS: - cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl)) - if not cands: - continue - base = min(cands, key=lambda r: r.get('min_quantity') or 0.0) - if (base.get('min_quantity') or 0.0) > 1.0: - report["qty_break_only"] += 1 - cells[col] = base.get('fixed_price') - if cells: - out.setdefault(code, {}).update(cells) - return out, report - - -def catalogue_count(): - """The INDEPENDENT population oracle: Odoo's own count of active products. - - Deliberately a bare `search_count` and not a `len()` over anything this module built — the - 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides - of its reconciliation from `_sku_rev`, so the oracle could never see a missing row. - """ - return O.get_odoo().search_count('product.product', [('active', '=', True)]) - - -def categories(t=None, team_id=None): - """Sorted distinct category names present in the SKU directory (for the drill-down filter).""" - return sorted({r['category'] for r in directory(t, team_id)}) - - -def _coverage(date_from, date_to, team_id=None): - """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code. - (A customer buying two records sharing a code can count twice; duplicates are rare, and - code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)""" - g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC), - ['__count'], ['product_id', 'order_partner_id'], lazy=False) - codes = _code_map() - cov = {} - names = {} - for r in g: - pid = O.m2o_id(r.get('product_id')) - if not pid: - continue - key = codes.get(pid, f"pid:{pid}") - cov[key] = cov.get(key, 0) + 1 - names.setdefault(key, O.m2o_name(r.get('product_id'))) - return cov, names - - -def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None): - """SKUs that lost the most customer breadth YoY — early warning a SKU is dying even if - revenue hasn't fully cratered yet.""" - t = t or P.today() - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - cov_t, names_t = _coverage(yf, yt, team_id) - cov_l, names_l = _coverage(lf, lt, team_id) - rows = [] - for pid, lc in cov_l.items(): - if lc < min_prior_custs: - continue - tc = cov_t.get(pid, 0) - drop = lc - tc - if drop <= 0: - continue - rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '', - 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop, - 'pct_drop': drop / lc * 100}) - rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop'])) - return rows[:limit] - - -# ====================================================================== SKU DRAWER (mirror of customer drawer) -def _sku_product_ids(code): - """All product.product ids sharing this SKU code (merged variants / archived records).""" - return [pid for pid, c in _code_map().items() if c == code] - - -def _sku_dom(pids, date_from, date_to, team_id=None): - return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC) - - -def _sku_name_category(pids): - rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id']) - name = rows[0]['name'] if rows else '(unknown)' - catmap = sales_mod._product_cat() - main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)') - return name, main - - -def _sku_buyers(pids, date_from, date_to, team_id=None): - """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window.""" - g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id), - ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False) - out = {} - for r in g: - pid = O.m2o_id(r.get('order_partner_id')) - if pid: - out[pid] = {'name': O.m2o_name(r.get('order_partner_id')), - 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0} - return out - - -def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None): - """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code. - Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read.""" - t = t or P.today() - pids = _sku_product_ids(code) - if not pids: - return None - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - mf, mt = P.ltm(t) - name, category = _sku_name_category(pids) - - def s(df, dtt, field='price_subtotal'): - return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field) - rev_ytd, rev_ly = s(yf, yt), s(lf, lt) - qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty') - buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id)) - buyers_ly = len(_sku_buyers(pids, lf, lt, team_id)) - g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id), - ['price_subtotal:sum', 'margin:sum'], [], lazy=False) - line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0 - margin = (g[0].get('margin') if g else 0) or 0.0 - - # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected - # on sale.order.line), in 2 reads instead of 26 point queries. - range_start = dt.date(t.year - 2, t.month, 1).isoformat() - lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id), - ['price_subtotal', 'order_id']) - oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')}) - odate = {} - for i in range(0, len(oids), 1000): - for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']): - if o.get('date_order'): - odate[o['id']] = str(o['date_order'])[:7] - mrev = {} - for line in lines: - ym = odate.get(O.m2o_id(line.get('order_id'))) - if ym: - mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0) - monthly = [] - for ym, start, end in P.month_starts(n_months, t): - y, m = int(ym[:4]) - 1, int(ym[5:7]) - monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0), - 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)}) - - allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id) - total = sum(v['rev'] for v in allsku.values()) or 1.0 - rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev'])) - if c == code), None) - return { - 'code': code, 'name': name, 'category': category, - 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly), - 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly), - 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly, - 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin, - 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100, - 'monthly': monthly, - } - - -def sku_buyer_bridge(code, t=None, team_id=None, top=12): - """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list.""" - t = t or P.today() - pids = _sku_product_ids(code) - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id) - tset, lset = set(this), set(last) - retained, new, churned = tset & lset, tset - lset, lset - tset - churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']} - for p in churned], key=lambda x: -x['ly_rev'])[:top] - return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)}, - 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)}, - 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)}, - 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0, - 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset), - 'this_total': sum(v['rev'] for v in this.values())} - - -def sku_concentration(code, t=None, team_id=None): - """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM).""" - t = t or P.today() - mf, mt = P.ltm(t) - buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id) - revs = sorted([v['rev'] for v in buyers.values()], reverse=True) - total = sum(revs) or 1.0 - hhi = sum((r / total) ** 2 for r in revs) - return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0, - 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0, - 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0} - - -def sku_price_dispersion(code, t=None, team_id=None, cap=60): - """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP - accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever.""" - t = t or P.today() - mf, mt = P.ltm(t) - g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id), - ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False) - rows, tot_rev, tot_qty = [], 0.0, 0.0 - for r in g: - pid = O.m2o_id(r.get('order_partner_id')) - rev = r.get('price_subtotal') or 0.0 - qty = r.get('product_uom_qty') or 0.0 - if not pid or qty <= 0: - continue - rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')), - 'price': rev / qty, 'qty': qty, 'rev': rev}) - tot_rev += rev - tot_qty += qty - vwap = (tot_rev / tot_qty) if tot_qty else 0.0 - for r in rows: - r['recoverable'] = max(0.0, vwap - r['price']) * r['qty'] - rows.sort(key=lambda x: -x['recoverable']) - return {'vwap': vwap, 'n_buyers': len(rows), - 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]} - - -def sku_top_buyers(code, t=None, team_id=None, top=15): - """Ranked buyers of this SKU (YTD) with YoY — clickable to open the customer drawer.""" - t = t or P.today() - pids = _sku_product_ids(code) - yf, yt = P.ytd(t) - lf, lt = P.ytd_last_year(t) - this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id) - rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'], - 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()] - rows.sort(key=lambda x: -x['rev']) - return rows[:top] - - -_BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0), - ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)] - - -def sku_customer_analysis(code, t=None, team_id=None, top=15): - """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL - spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend — who gets hurt - most if it stocks out / who to protect).""" - t = t or P.today() - lf, lt = P.ltm(t) - pids = _sku_product_ids(code) - empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0} - if not pids: - return empty - buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}} - if not buyers: - return empty - bpids = list(buyers) - g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)], - ['amount_untaxed:sum'], ['partner_id'], lazy=False) - total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')} - - def tier(rev): - for nm, lo in _BUYER_TIERS: - if rev >= lo: - return nm - return _BUYER_TIERS[-1][0] - seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS} - dep = [] - for p, v in buyers.items(): - ct = total.get(p, v['rev']) or v['rev'] - s = seg[tier(ct)] - s['buyers'] += 1 - s['sku_rev'] += v['rev'] - dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct, - 'dependency_pct': (v['rev'] / ct * 100) if ct else None}) - tot = sum(s['sku_rev'] for s in seg.values()) or 1.0 - segments = [] - for nm, _ in _BUYER_TIERS: - s = seg[nm] - s['rev_share'] = s['sku_rev'] / tot * 100 - s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0 - segments.append(s) - dep.sort(key=lambda x: -x['sku_rev']) - deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None] - return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers), - 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0} - - -def sku_whitespace(code, t=None, team_id=None, top=15): - """Customers who buy this SKU's category but NOT this SKU — ranked prospect list.""" - import modules.customers as cust_mod - t = t or P.today() - pids = _sku_product_ids(code) - _name, category = _sku_name_category(pids) - cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set() - mf, mt = P.ltm(t) - prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id))) - if not prospects: - return {'category': category, 'rows': []} - g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)], - ['amount_untaxed:sum'], ['partner_id'], lazy=False) - rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']), - 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')] - rows.sort(key=lambda x: -x['total_spend']) - return {'category': category, 'rows': rows[:top]} - - -def sku_drawer_bundle(code, t=None, team_id=None, allsku=None): - """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate), - then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call).""" - detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku) - if detail is None: - return {'detail': None} - bridge, conc, price, buyers, white, ca = O.parallel([ - lambda: sku_buyer_bridge(code, t, team_id), - lambda: sku_concentration(code, t, team_id), - lambda: sku_price_dispersion(code, t, team_id), - lambda: sku_top_buyers(code, t, team_id), - lambda: sku_whitespace(code, t, team_id), - lambda: sku_customer_analysis(code, t, team_id), - ]) - return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price, - 'buyers': buyers, 'white': white, 'ca': ca} - - -def validate(t=None, team_id=None): - """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU - selected) every check runs SCOPED to that BU, so the validation panel never reconciles - against — or exposes — the other BU's numbers. All checks here are BU-scopeable (no cross-BU - mirror), so team_id threads straight through.""" - t = t or P.today() - yf, yt = P.ytd(t) - checks = [] - sku = _sku_rev(yf, yt, team_id) - sku_sum = sum(v['rev'] for v in sku.values()) - line_total = O.sum_field('sale.order.line', - O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal') - checks.append({'check': 'SKU rev: Σ(per-SKU) == total line revenue, ex-services (YTD)', - 'a': round(sku_sum, 2), 'b': round(line_total, 2), - 'gap': round(sku_sum - line_total, 2), - 'ok': abs(sku_sum - line_total) <= 1.0}) - - m = yoy_movers(t, limit=10**9, team_id=team_id) - movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners']) - lf, lt = P.ytd_last_year(t) - last_total = O.sum_field('sale.order.line', - O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal') - checks.append({'check': 'SKU movers: Σ(Δ) == (YTD − LY) total', - 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2), - 'gap': round(movers_sum - (line_total - last_total), 2), - 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0}) - - # SKU drawer: buyer-bridge reconciles — retained + new buyer revenue == this-period SKU revenue - top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None - if top_code: - bb = sku_buyer_bridge(top_code, t, team_id=team_id) - recon = bb['retained']['rev'] + bb['new']['rev'] - checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue', - 'a': round(recon, 2), 'b': round(bb['this_total'], 2), - 'gap': round(recon - bb['this_total'], 2), - 'ok': abs(recon - bb['this_total']) <= 1.0}) - return checks +"""Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs +(catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse +(SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders. + +Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here. +Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it +runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the +project guardrails keep off the local PC. (See BACKLOG.) + +All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain. +Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/ +window), so it's its own function the app calls lazily and caches. +""" +import sys +import datetime as dt +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from functools import lru_cache +import core.odoo as O +import core.periods as P +import modules.sales as sales_mod + +# SKU-health views are about real products — exclude service/delivery pseudo-SKUs +# (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby +# on the dot-path does not). Validation reconciles on this same non-service universe. +_NO_SVC = [('product_id.type', '!=', 'service')] + + +@lru_cache(maxsize=1) +def _code_map(): + """product_id → SKU code. Multiple product records can share a default_code (re-SKUing + /duplicates); keying by code merges them so a re-coded item doesn't read as a fake + decliner + fake riser. Products without a code fall back to a per-id key. Archived + products are INCLUDED — re-SKUing typically archives the old record and creates a new + one under the same code, and the old record still carries last-year sales.""" + prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'], + limit=50000) + return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + for p in prods} + + +def _sku_rev(date_from, date_to, team_id=None): + """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded, + duplicate product records merged by SKU code).""" + g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False) + codes = _code_map() + out = {} + for r in g: + pid = O.m2o_id(r.get('product_id')) + if not pid: + continue + key = codes.get(pid, f"pid:{pid}") + e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')), + 'rev': 0.0, 'qty': 0.0, 'orders': 0}) + e['rev'] += r.get('price_subtotal') or 0.0 + e['qty'] += r.get('product_uom_qty') or 0.0 + e['orders'] += r.get('__count') or 0 + return out + + +def yoy_movers(t=None, limit=20, team_id=None): + """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + rows = [] + for pid in set(this) | set(last): + tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')}) + lr = last.get(pid, {'rev': 0.0}) + name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or '' + rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'], + 'change': tr['rev'] - lr['rev']}) + risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit] + decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit] + return {'risers': risers, 'decliners': decliners} + + +def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None): + """Catalog rot: SKUs that sold materially last year but are ~dead this year.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + rows = [] + for pid, lr in last.items(): + if lr['rev'] < min_prior: + continue + tr = this.get(pid, {'rev': 0.0}) + if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie + continue + rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'], + 'lost': lr['rev'] - tr['rev']}) + rows.sort(key=lambda x: -x['lost']) + return rows[:limit] + + +def new_winners(t=None, limit=20, min_this=1500.0, team_id=None): + """SKUs that barely sold last year but are selling well this year.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + rows = [] + for pid, tr in this.items(): + if tr['rev'] < min_this: + continue + lr = last.get(pid, {'rev': 0.0}) + if lr['rev'] > 0.05 * tr['rev']: + continue + rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'], + 'gained': tr['rev'] - lr['rev']}) + rows.sort(key=lambda x: -x['gained']) + return rows[:limit] + + +def velocity_leaders(t=None, limit=25, team_id=None): + """Top SKUs by LTM unit velocity (units/month) and revenue.""" + t = t or P.today() + lf, lt = P.ltm(t) + sku = _sku_rev(lf, lt, team_id) + rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0, + 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()] + rows.sort(key=lambda x: -x['units_ltm']) + return rows[:limit] + + +def _code_category(): + """code -> category name. Built from _code_map (pid->code) + the sales category map + (pid->category); the first record carrying a code sets that code's category.""" + codes = _code_map() + cats = sales_mod._product_cat() + out = {} + for pid, code in codes.items(): + if code not in out and pid in cats: + out[code] = cats[pid] + return out + + +def directory(t=None, team_id=None): + """Every SKU that sold this YTD or the same period last year, with the fields the SKU + drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the + customer directory — the full list (filtering happens in the UI), sorted by YTD revenue.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this = _sku_rev(yf, yt, team_id) + last = _sku_rev(lf, lt, team_id) + code_cat = _code_category() + rows = [] + for code in set(this) | set(last): + tr = this.get(code, {}) + lr = last.get(code, {}) + rev_ytd = tr.get('rev', 0.0) + rev_ly = lr.get('rev', 0.0) + rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code, + 'category': code_cat.get(code, '(uncategorized)'), + 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly, + 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly), + 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)}) + rows.sort(key=lambda r: -r['rev_ytd']) + return rows + + +def catalogue(): + """`{sku_code: {'product', 'category'}}` — EVERY ACTIVE product, sold or not. + + ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s + row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU + that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`, + `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe — + and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of + 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in + the consumer, never a window removed from `directory()`: removing the window alone lands at + 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all. + + ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one + catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than + merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold. + + Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active + records that carry none — so the join against `directory()` is code-for-code with no + normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields + off a sale line, so a never-sold row wears the same format as a sold one. + + ⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull + would put the grid back at a plausible wrong number with every gate green — the exact failure + this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard + `modules/backorders.py:161` already uses). + """ + dom = [('active', '=', True)] + prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'], + limit=50000) + n = O.get_odoo().search_count('product.product', dom) + if len(prods) != n: + raise ValueError( + f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against " + f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with " + f"nothing reporting it; raise the limit before shipping this.") + code_cat = _code_category() + out = {} + for p in prods: + code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero + # active products share a code, so this branch is a guard, not a merge policy. + out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code, + 'category': code_cat.get(code, '(uncategorized)')}) + return out + + +#: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) — THE PRICELIST STRATUM, PER LIST. +#: +#: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than +#: decorating it: +#: +#: · **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139, +#: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that +#: is Fisch 3,971 · Royal 1 1,920 · Royal 2 1,857 · Public 6 · Giftware 0. +#: · **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.** +#: · **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** — median relative +#: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a +#: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the +#: only question the column exists to answer. +#: · ⛔ **`list_price` is 1.00 on 5,817 of the 5,871** — so the single `3_global` rule +#: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls +#: through to it**, and that is the ticket's own negative control: a SKU with no specific +#: item renders BLANK, never a fallback dressed as a price. +#: +#: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go +#: blank and makes `report["lists_missing"]` name it — a hard-coded id would keep pointing at +#: whatever inherited it and mislabel every cell silently. +PRICELIST_COLUMNS = ( + ("price_fisch", "Fisch"), + ("price_royal_1", "Royal 1"), + ("price_royal_2", "Royal 2"), +) + + +def pricelist_by_code(): + """`({code: {column_key: price}}, report)` — the date-valid FIXED base-tier price for each + declared pricelist, keyed EXACTLY as `catalogue()` keys its rows. + + ⛔ THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a + `pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other + way would leave those SKUs permanently blank while an oracle counting active products + counted them — a red gate on a working build, or worse, a silent hole nobody counts. + + **The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at + or below an order's quantity, because it is pricing a LINE that has one. A catalogue column + has no quantity in hand, so it takes the LOWEST `min_quantity` — the price at qty 1. Rules + above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only + rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all). + + **Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant` + rule is the more specific statement about this exact SKU. + + ⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than + `catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is + momentarily unreachable is a worse failure than one with blank price columns. The blanks are + not silent — `product_data.validate()`'s coverage leg reconciles each column against a fresh + Odoo count and goes red at zero. + + ⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need + to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is + counted rather than dropped: rules on pricelists the contract does not declare, non-fixed + (`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a + quantity break. + """ + # ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a + # report and a misleading one. A rule is classified ONCE, by the first reason it is skipped: + # undeclared list → out of date → not fixed → global → zero price. So `rules_not_fixed: 0` + # means "no formula rule on a list we declare", NOT "this Odoo has no formula rules" + # (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the + # contract does not declare — so it lands in `rules_undeclared_list`). + report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0, + "rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0, + "rules_zero_price": 0, "qty_break_only": 0} + try: + pl_rows = O.search_read('product.pricelist', [], ['id', 'name']) + by_name = {} + for p in pl_rows: + by_name.setdefault(str(p.get('name') or '').strip(), p['id']) + wanted = {} + for col, name in PRICELIST_COLUMNS: + pid = by_name.get(name) + if pid is None: + report["lists_missing"].append(name) + else: + wanted[pid] = col + + # ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE. + # Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on + # this connection; the same rules under a server-side domain with 6 fields cost + # **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's + # first-ever build, so that 16.5s is 16.5s of somebody's page load. + # + # ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part + # that is easy to get wrong: an absent `date_start` is `False`, not a past date, so + # `('date_start','<=',today)` ALONE would drop every open-ended rule — which is almost + # all of them. + today = P.today().isoformat() + _declared = sorted(wanted) + _kinds = ['0_product_variant', '1_product'] + _live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'), + ('applied_on', 'in', _kinds), + '|', ('date_start', '=', False), ('date_start', '<=', today), + '|', ('date_end', '=', False), ('date_end', '>=', today), + ('fixed_price', '>', 0)] + rules = O.search_read( + 'product.pricelist.item', _live, + ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price', + 'min_quantity']) if _declared else [] + + # ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six + # counts cost ~1.2s together; the rules they count would cost 16s to read. What this + # reader cannot see is still REPORTED — it is just no longer transferred. + def _n(extra): + try: + return O.get_odoo().search_count('product.pricelist.item', extra) + except Exception: + return -1 # -1 reads as "not measured", never as zero + _dated = ['|', ('date_start', '=', False), ('date_start', '<=', today), + '|', ('date_end', '=', False), ('date_end', '>=', today)] + report["rules_total"] = _n([]) + report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \ + if _declared else report["rules_total"] + if _declared: + _on = [('pricelist_id', 'in', _declared)] + # `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot + # price is one the operator is TOLD about, never one that quietly becomes a blank. + report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')]) + report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'), + ('applied_on', 'not in', _kinds)]) + report["rules_out_of_date"] = ( + _n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]) + - _n(_on + [('compute_price', '=', 'fixed'), + ('applied_on', 'in', _kinds)] + _dated)) + # A zero price is not "free" — it is an unset rule. Blank says so; 0 does not. + report["rules_zero_price"] = _n( + _on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)] + + _dated + [('fixed_price', '<=', 0)]) + + by_var, by_tmpl = {}, {} + for r in rules: + col = wanted[O.m2o_id(r.get('pricelist_id'))] + if r.get('applied_on') == '0_product_variant' and r.get('product_id'): + by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r) + elif r.get('product_tmpl_id'): + by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r) + + dom = [('active', '=', True)] + prods = O.search_read('product.product', dom, + ['id', 'default_code', 'product_tmpl_id'], limit=50000) + n = O.get_odoo().search_count('product.product', dom) + if len(prods) != n: + # Same guard, same reason as `catalogue()`: a short pull renders as a plausible + # smaller set of priced SKUs with nothing reporting it. + raise ValueError( + f"products.pricelist_by_code: the product pull is TRUNCATED — read {len(prods)} " + f"rows against a search_count of {n}.") + except Exception as e: + report["error"] = f"{type(e).__name__}: {str(e)[:200]}" + return {}, report + + out = {} + for p in prods: + code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + tmpl = O.m2o_id(p.get('product_tmpl_id')) + cells = {} + for col, _name in PRICELIST_COLUMNS: + cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl)) + if not cands: + continue + base = min(cands, key=lambda r: r.get('min_quantity') or 0.0) + if (base.get('min_quantity') or 0.0) > 1.0: + report["qty_break_only"] += 1 + cells[col] = base.get('fixed_price') + if cells: + out.setdefault(code, {}).update(cells) + return out, report + + +def catalogue_count(): + """The INDEPENDENT population oracle: Odoo's own count of active products. + + Deliberately a bare `search_count` and not a `len()` over anything this module built — the + 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides + of its reconciliation from `_sku_rev`, so the oracle could never see a missing row. + """ + return O.get_odoo().search_count('product.product', [('active', '=', True)]) + + +def categories(t=None, team_id=None): + """Sorted distinct category names present in the SKU directory (for the drill-down filter).""" + return sorted({r['category'] for r in directory(t, team_id)}) + + +def _coverage(date_from, date_to, team_id=None): + """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code. + (A customer buying two records sharing a code can count twice; duplicates are rare, and + code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)""" + g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC), + ['__count'], ['product_id', 'order_partner_id'], lazy=False) + codes = _code_map() + cov = {} + names = {} + for r in g: + pid = O.m2o_id(r.get('product_id')) + if not pid: + continue + key = codes.get(pid, f"pid:{pid}") + cov[key] = cov.get(key, 0) + 1 + names.setdefault(key, O.m2o_name(r.get('product_id'))) + return cov, names + + +def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None): + """SKUs that lost the most customer breadth YoY — early warning a SKU is dying even if + revenue hasn't fully cratered yet.""" + t = t or P.today() + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + cov_t, names_t = _coverage(yf, yt, team_id) + cov_l, names_l = _coverage(lf, lt, team_id) + rows = [] + for pid, lc in cov_l.items(): + if lc < min_prior_custs: + continue + tc = cov_t.get(pid, 0) + drop = lc - tc + if drop <= 0: + continue + rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '', + 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop, + 'pct_drop': drop / lc * 100}) + rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop'])) + return rows[:limit] + + +# ====================================================================== SKU DRAWER (mirror of customer drawer) +def _sku_product_ids(code): + """All product.product ids sharing this SKU code (merged variants / archived records).""" + return [pid for pid, c in _code_map().items() if c == code] + + +def _sku_dom(pids, date_from, date_to, team_id=None): + return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC) + + +def _sku_name_category(pids): + rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id']) + name = rows[0]['name'] if rows else '(unknown)' + catmap = sales_mod._product_cat() + main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)') + return name, main + + +def _sku_buyers(pids, date_from, date_to, team_id=None): + """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window.""" + g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False) + out = {} + for r in g: + pid = O.m2o_id(r.get('order_partner_id')) + if pid: + out[pid] = {'name': O.m2o_name(r.get('order_partner_id')), + 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0} + return out + + +def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None): + """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code. + Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read.""" + t = t or P.today() + pids = _sku_product_ids(code) + if not pids: + return None + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + mf, mt = P.ltm(t) + name, category = _sku_name_category(pids) + + def s(df, dtt, field='price_subtotal'): + return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field) + rev_ytd, rev_ly = s(yf, yt), s(lf, lt) + qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty') + buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id)) + buyers_ly = len(_sku_buyers(pids, lf, lt, team_id)) + g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id), + ['price_subtotal:sum', 'margin:sum'], [], lazy=False) + line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0 + margin = (g[0].get('margin') if g else 0) or 0.0 + + # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected + # on sale.order.line), in 2 reads instead of 26 point queries. + range_start = dt.date(t.year - 2, t.month, 1).isoformat() + lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id), + ['price_subtotal', 'order_id']) + oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')}) + odate = {} + for i in range(0, len(oids), 1000): + for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']): + if o.get('date_order'): + odate[o['id']] = str(o['date_order'])[:7] + mrev = {} + for line in lines: + ym = odate.get(O.m2o_id(line.get('order_id'))) + if ym: + mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0) + monthly = [] + for ym, start, end in P.month_starts(n_months, t): + y, m = int(ym[:4]) - 1, int(ym[5:7]) + monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0), + 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)}) + + allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id) + total = sum(v['rev'] for v in allsku.values()) or 1.0 + rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev'])) + if c == code), None) + return { + 'code': code, 'name': name, 'category': category, + 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly), + 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly), + 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly, + 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin, + 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100, + 'monthly': monthly, + } + + +def sku_buyer_bridge(code, t=None, team_id=None, top=12): + """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list.""" + t = t or P.today() + pids = _sku_product_ids(code) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id) + tset, lset = set(this), set(last) + retained, new, churned = tset & lset, tset - lset, lset - tset + churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']} + for p in churned], key=lambda x: -x['ly_rev'])[:top] + return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)}, + 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)}, + 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)}, + 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0, + 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset), + 'this_total': sum(v['rev'] for v in this.values())} + + +def sku_concentration(code, t=None, team_id=None): + """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM).""" + t = t or P.today() + mf, mt = P.ltm(t) + buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id) + revs = sorted([v['rev'] for v in buyers.values()], reverse=True) + total = sum(revs) or 1.0 + hhi = sum((r / total) ** 2 for r in revs) + return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0, + 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0, + 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0} + + +def sku_price_dispersion(code, t=None, team_id=None, cap=60): + """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP + accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever.""" + t = t or P.today() + mf, mt = P.ltm(t) + g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id), + ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False) + rows, tot_rev, tot_qty = [], 0.0, 0.0 + for r in g: + pid = O.m2o_id(r.get('order_partner_id')) + rev = r.get('price_subtotal') or 0.0 + qty = r.get('product_uom_qty') or 0.0 + if not pid or qty <= 0: + continue + rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')), + 'price': rev / qty, 'qty': qty, 'rev': rev}) + tot_rev += rev + tot_qty += qty + vwap = (tot_rev / tot_qty) if tot_qty else 0.0 + for r in rows: + r['recoverable'] = max(0.0, vwap - r['price']) * r['qty'] + rows.sort(key=lambda x: -x['recoverable']) + return {'vwap': vwap, 'n_buyers': len(rows), + 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]} + + +def sku_top_buyers(code, t=None, team_id=None, top=15): + """Ranked buyers of this SKU (YTD) with YoY — clickable to open the customer drawer.""" + t = t or P.today() + pids = _sku_product_ids(code) + yf, yt = P.ytd(t) + lf, lt = P.ytd_last_year(t) + this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id) + rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'], + 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()] + rows.sort(key=lambda x: -x['rev']) + return rows[:top] + + +_BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0), + ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)] + + +def sku_customer_analysis(code, t=None, team_id=None, top=15): + """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL + spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend — who gets hurt + most if it stocks out / who to protect).""" + t = t or P.today() + lf, lt = P.ltm(t) + pids = _sku_product_ids(code) + empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0} + if not pids: + return empty + buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}} + if not buyers: + return empty + bpids = list(buyers) + g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)], + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')} + + def tier(rev): + for nm, lo in _BUYER_TIERS: + if rev >= lo: + return nm + return _BUYER_TIERS[-1][0] + seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS} + dep = [] + for p, v in buyers.items(): + ct = total.get(p, v['rev']) or v['rev'] + s = seg[tier(ct)] + s['buyers'] += 1 + s['sku_rev'] += v['rev'] + dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct, + 'dependency_pct': (v['rev'] / ct * 100) if ct else None}) + tot = sum(s['sku_rev'] for s in seg.values()) or 1.0 + segments = [] + for nm, _ in _BUYER_TIERS: + s = seg[nm] + s['rev_share'] = s['sku_rev'] / tot * 100 + s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0 + segments.append(s) + dep.sort(key=lambda x: -x['sku_rev']) + deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None] + return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers), + 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0} + + +def sku_whitespace(code, t=None, team_id=None, top=15): + """Customers who buy this SKU's category but NOT this SKU — ranked prospect list.""" + import modules.customers as cust_mod + t = t or P.today() + pids = _sku_product_ids(code) + _name, category = _sku_name_category(pids) + cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set() + mf, mt = P.ltm(t) + prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id))) + if not prospects: + return {'category': category, 'rows': []} + g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)], + ['amount_untaxed:sum'], ['partner_id'], lazy=False) + rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']), + 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')] + rows.sort(key=lambda x: -x['total_spend']) + return {'category': category, 'rows': rows[:top]} + + +def sku_drawer_bundle(code, t=None, team_id=None, allsku=None): + """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate), + then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call).""" + detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku) + if detail is None: + return {'detail': None} + bridge, conc, price, buyers, white, ca = O.parallel([ + lambda: sku_buyer_bridge(code, t, team_id), + lambda: sku_concentration(code, t, team_id), + lambda: sku_price_dispersion(code, t, team_id), + lambda: sku_top_buyers(code, t, team_id), + lambda: sku_whitespace(code, t, team_id), + lambda: sku_customer_analysis(code, t, team_id), + ]) + return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price, + 'buyers': buyers, 'white': white, 'ca': ca} + + +def validate(t=None, team_id=None): + """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU + selected) every check runs SCOPED to that BU, so the validation panel never reconciles + against — or exposes — the other BU's numbers. All checks here are BU-scopeable (no cross-BU + mirror), so team_id threads straight through.""" + t = t or P.today() + yf, yt = P.ytd(t) + checks = [] + sku = _sku_rev(yf, yt, team_id) + sku_sum = sum(v['rev'] for v in sku.values()) + line_total = O.sum_field('sale.order.line', + O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal') + checks.append({'check': 'SKU rev: Σ(per-SKU) == total line revenue, ex-services (YTD)', + 'a': round(sku_sum, 2), 'b': round(line_total, 2), + 'gap': round(sku_sum - line_total, 2), + 'ok': abs(sku_sum - line_total) <= 1.0}) + + m = yoy_movers(t, limit=10**9, team_id=team_id) + movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners']) + lf, lt = P.ytd_last_year(t) + last_total = O.sum_field('sale.order.line', + O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal') + checks.append({'check': 'SKU movers: Σ(Δ) == (YTD − LY) total', + 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2), + 'gap': round(movers_sum - (line_total - last_total), 2), + 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0}) + + # SKU drawer: buyer-bridge reconciles — retained + new buyer revenue == this-period SKU revenue + top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None + if top_code: + bb = sku_buyer_bridge(top_code, t, team_id=team_id) + recon = bb['retained']['rev'] + bb['new']['rev'] + checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue', + 'a': round(recon, 2), 'b': round(bb['this_total'], 2), + 'gap': round(recon - bb['this_total'], 2), + 'ok': abs(recon - bb['this_total']) <= 1.0}) + return checks diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index a1df92945d76142f88d5e0f6a1696a666bebafe3..22262f3b280cd1d73fe8393d0fbce2d1be869b22 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -1,558 +1,545 @@ -{ - "fields": [ - { - "key": "customer", - "label": "Customer", - "type": "text", - "source": "odoo", - "pinned": true, - "default": true, - "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." - }, - { - "key": "partner_id", - "label": "Odoo ID", - "type": "int", - "source": "odoo", - "derived": true, - "default": false, - "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." - }, - { - "key": "odoo_status", - "label": "Odoo record", - "type": "status", - "source": "odoo", - "default": false, - "options": [ - "Active", - "Archived" - ], - "description": "Whether this customer still exists in Odoo. Archived means deleted there." - }, - { - "key": "agent", - "label": "Agent", - "type": "text", - "source": "odoo", - "default": true, - "description": "The sales agent who owns this account." - }, - { - "key": "dba", - "label": "DBA", - "type": "select", - "source": "odoo", - "default": false, - "options": [ - "Fisch", - "Royal", - "Both" - ], - "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." - }, - { - "key": "salesperson", - "label": "Salesperson", - "type": "text", - "source": "odoo", - "default": false, - "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." - }, - { - "key": "street", - "label": "Street", - "type": "text", - "source": "odoo", - "default": false, - "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." - }, - { - "key": "street2", - "label": "Street 2", - "type": "text", - "source": "odoo", - "default": false, - "description": "Second address line (suite, unit, floor) on the customer's Odoo address." - }, - { - "key": "city", - "label": "City", - "type": "text", - "source": "odoo", - "default": true, - "description": "City on the customer's Odoo address." - }, - { - "key": "state", - "label": "State", - "type": "text", - "source": "odoo", - "default": true, - "description": "State or province on the customer's Odoo address." - }, - { - "key": "country", - "label": "Country", - "type": "text", - "source": "odoo", - "default": false, - "description": "Country on the customer's Odoo address." - }, - { - "key": "zip", - "label": "ZIP", - "type": "text", - "source": "odoo", - "default": false, - "description": "Postal code on the customer's Odoo address." - }, - { - "key": "customer_since", - "label": "Customer since", - "type": "date", - "source": "odoo", - "default": false, - "description": "When this customer was first set up in Odoo." - }, - { - "key": "tags", - "label": "Tags", - "type": "text", - "source": "odoo", - "default": false, - "description": "Odoo labels on this customer, comma-separated." - }, - { - "key": "pricelist", - "label": "Price list", - "type": "text", - "source": "odoo", - "default": false, - "description": "The price list this customer buys on." - }, - { - "key": "payment_terms", - "label": "Payment terms", - "type": "text", - "source": "odoo", - "default": false, - "description": "Payment terms on this customer's account — Net 30, for example." - }, - { - "key": "last_order", - "label": "Last order", - "type": "date", - "source": "odoo", - "default": true, - "description": "Date of the most recent confirmed order." - }, - { - "key": "overdue_days", - "label": "Overdue days", - "type": "int", - "source": "odoo", - "default": true, - "description": "How many days late this customer is running against their own usual ordering rhythm." - }, - { - "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", - "key": "est_missed", - "label": "Est. missed $", - "type": "currency", - "source": "odoo", - "default": true, - "agg": "sum", - "filterable": false, - "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." - }, - { - "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", - "key": "ar_open", - "label": "AR current $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." - }, - { - "key": "ar_overdue", - "label": "AR overdue $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Invoiced money past due — same basis as the Collections page." - }, - { - "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", - "key": "ar_outstanding", - "label": "AR outstanding $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." - }, - { - "key": "ar_exposure", - "label": "Credit exposure $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." - }, - { - "key": "ar_aged_1_30", - "label": "1-30 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_31_60", - "label": "31-60 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_61_90", - "label": "61-90 days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "ar_aged_90_plus", - "label": "90+ days $", - "type": "currency", - "source": "odoo", - "default": false, - "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." - }, - { - "key": "days_to_pay", - "label": "Days to pay", - "type": "int", - "source": "odoo", - "default": false, - "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." - }, - { - "key": "top_category", - "label": "Top category", - "type": "text", - "source": "odoo", - "default": false, - "description": "The category this customer spent the most on in the last 12 months." - }, - { - "key": "top_category_pct", - "label": "Top category %", - "type": "pct", - "source": "odoo", - "default": false, - "description": "Share of last-12-months spend that went to the top category." - }, - { - "key": "sku_count", - "label": "SKUs bought", - "type": "int", - "source": "odoo", - "default": false, - "description": "Distinct products bought in the last 12 months." - }, - { - "key": "top_sku", - "label": "Top SKU", - "type": "text", - "source": "odoo", - "default": false, - "description": "The product this customer spent the most on in the last 12 months." - }, - { - "key": "days_since", - "label": "Days since order", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days since the last confirmed order." - }, - { - "key": "typical_gap_days", - "label": "Typical gap days", - "type": "int", - "source": "odoo", - "default": false, - "description": "Days this customer usually goes between orders, from their own history." - }, - { - "key": "notes", - "label": "Notes", - "type": "text", - "source": "overlay", - "default": false, - "description": "Your notes on this customer. Saved in this app only, visible only to you." - } - ], - "rows": [ - { - "pid": 101, - "customer": "Poppy Flowers", - "status": "New", - "agent": "Naomi Linnell Rivera", - "city": "Charlottesville", - "state": "Virginia (US)", - "last_order": "2026-07-21", - "est_missed": 0, - "notes": "", - "country": "United States", - "zip": "02720", - "payment_terms": "30 Days", - "pricelist": "Fisch 1 (USD)", - "tags": "Royal", - "customer_since": "2023-01-10", - "salesperson": "Jessica", - "ar_open": 0, - "ar_overdue": 0, - "ar_exposure": 0, - "top_category": "Styrofoam", - "top_category_pct": 0.47, - "sku_count": 93, - "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", - "days_to_pay": 34, - "_created": "2023-01-15 09:10:00", - "lat": 25.7617, - "lon": -80.1918, - "odoo_status": "Archived", - "dba": "Royal", - "ar_outstanding": 0 - }, - { - "pid": 102, - "customer": "Meadow & Vine Wholesale", - "status": "Growing", - "agent": "Carla Jimenez", - "city": "Portland", - "state": "Oregon (US)", - "last_order": "2026-07-19", - "est_missed": 0, - "notes": "Expanding to a second storefront.", - "country": "United States", - "zip": "77041", - "payment_terms": "Immediate Payment", - "pricelist": "Royal 1 (USD)", - "tags": "Royal, Key account", - "customer_since": "2024-02-11", - "salesperson": "Naomi", - "ar_open": 1240.5, - "ar_overdue": 0, - "ar_exposure": 1740.5, - "top_category": "Ribbon", - "top_category_pct": 0.95, - "sku_count": 4, - "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", - "days_to_pay": null, - "_created": "2023-02-15 09:11:00", - "lat": 27.9506, - "lon": -82.4572, - "odoo_status": "Active", - "dba": "Fisch", - "ar_outstanding": 1240.5 - }, - { - "pid": 103, - "customer": "Bluestem Floral Supply", - "status": "Growing", - "agent": "Naomi Linnell Rivera", - "city": "Kansas City", - "state": "Missouri (US)", - "last_order": "2026-07-17", - "est_missed": 0, - "notes": "", - "country": "United States", - "zip": "11219", - "payment_terms": "60 Days", - "pricelist": "Royal 1 (USD)", - "tags": "Fisch", - "customer_since": "2025-03-12", - "salesperson": "Karen", - "ar_open": 0, - "ar_overdue": 5120.25, - "ar_exposure": 5120.25, - "top_category": "Foams & Finishes", - "top_category_pct": 0.31, - "sku_count": 27, - "top_sku": "SATIN RIBBON 2IN", - "days_to_pay": 61, - "_created": "2023-03-15 09:12:00", - "lat": 28.5384, - "lon": -81.3789, - "odoo_status": "Active", - "dba": "Both", - "ar_outstanding": 5120.25 - }, - { - "pid": 104, - "customer": "Camellia Row Florist", - "status": "Declining", - "agent": "Devon Marsh", - "city": "Savannah", - "state": "Georgia (US)", - "last_order": "2026-05-30", - "est_missed": 41200, - "notes": "Switched some volume to a local grower.", - "country": "United States", - "zip": "07649", - "payment_terms": "30 Days", - "pricelist": "Fisch 1 (USD)", - "tags": "(none)", - "customer_since": "2026-04-13", - "salesperson": "(none)", - "ar_open": 8300, - "ar_overdue": 940, - "ar_exposure": 11440, - "top_category": "All", - "top_category_pct": 1.0, - "sku_count": 1, - "top_sku": "(none)", - "days_to_pay": 12, - "_created": "2023-04-15 09:13:00", - "lat": 30.3322, - "lon": -81.6557, - "odoo_status": "Active", - "dba": "Royal", - "ar_outstanding": 9240 - }, - { - "pid": 105, - "customer": "Harborlight Wholesale Blooms", - "status": "Growing", - "agent": "Carla Jimenez", - "city": "Seattle", - "state": "Washington (US)", - "last_order": "2026-07-22", - "est_missed": 0, - "notes": "Top-10 account.", - "country": "United States", - "zip": "33125", - "payment_terms": "Immediate Payment", - "pricelist": "Royal 1 (USD)", - "tags": "Royal", - "customer_since": "2023-05-14", - "salesperson": "Jessica", - "ar_open": 0, - "ar_overdue": 0, - "ar_exposure": 0, - "top_category": "Styrofoam", - "top_category_pct": 0.47, - "sku_count": 93, - "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", - "days_to_pay": 34, - "_created": "2023-05-15 09:14:00", - "lat": 26.1224, - "lon": -80.1373, - "odoo_status": "Active", - "dba": "", - "ar_outstanding": 0 - }, - { - "pid": 106, - "customer": "Dogwood & Fern Co.", - "status": "Dormant", - "agent": "Devon Marsh", - "city": "Asheville", - "state": "North Carolina (US)", - "last_order": "2026-02-11", - "est_missed": 52400, - "notes": "No spring order this year.", - "country": "United States", - "zip": "90210", - "payment_terms": "60 Days", - "pricelist": "Royal 1 (USD)", - "tags": "Royal, Key account", - "customer_since": "2024-06-15", - "salesperson": "Naomi", - "ar_open": 1240.5, - "ar_overdue": 0, - "ar_exposure": 1740.5, - "top_category": "Ribbon", - "top_category_pct": 0.95, - "sku_count": 4, - "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", - "days_to_pay": null, - "_created": "2023-06-15 09:15:00", - "lat": 27.3364, - "lon": -82.5307, - "odoo_status": "Active", - "dba": "Fisch", - "ar_outstanding": 1240.5 - }, - { - "pid": 107, - "customer": "Verbena Market Florals", - "status": "Lost", - "agent": "Naomi Linnell Rivera", - "city": "Austin", - "state": "Texas (US)", - "last_order": "2025-11-04", - "est_missed": 78300, - "notes": "Went with a competitor on freight terms.", - "country": "United States", - "zip": "08701", - "payment_terms": "30 Days", - "pricelist": "Fisch 1 (USD)", - "tags": "Fisch", - "customer_since": "2025-07-16", - "salesperson": "Karen", - "ar_open": 0, - "ar_overdue": 5120.25, - "ar_exposure": 5120.25, - "top_category": "Foams & Finishes", - "top_category_pct": 0.31, - "sku_count": 27, - "top_sku": "SATIN RIBBON 2IN", - "days_to_pay": 61, - "_created": "2023-07-15 09:16:00", - "lat": null, - "lon": null, - "odoo_status": "Active", - "dba": "Both", - "ar_outstanding": 5120.25 - }, - { - "pid": 108, - "customer": "Larkspur Lane Supply", - "status": "New", - "agent": "Carla Jimenez", - "city": "Denver", - "state": "Colorado (US)", - "last_order": "2026-07-14", - "est_missed": 0, - "notes": "First order in April.", - "country": "United States", - "zip": "60614", - "payment_terms": "Immediate Payment", - "pricelist": "Royal 1 (USD)", - "tags": "(none)", - "customer_since": "2026-08-17", - "salesperson": "(none)", - "ar_open": 8300, - "ar_overdue": 940, - "ar_exposure": 11440, - "top_category": "All", - "top_category_pct": 1.0, - "sku_count": 1, - "top_sku": "(none)", - "days_to_pay": 12, - "_created": "2023-08-15 09:17:00", - "lat": 33.749, - "lon": -84.388, - "odoo_status": "Active", - "dba": "Royal", - "ar_outstanding": 9240 - } - ] -} +{ + "fields": [ + { + "key": "customer", + "label": "Customer", + "type": "text", + "source": "odoo", + "pinned": true, + "default": true, + "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + }, + { + "key": "odoo_status", + "label": "Odoo record", + "type": "status", + "source": "odoo", + "default": false, + "description": "Whether this customer still exists in Odoo. Archived means deleted there." + }, + { + "key": "agent", + "label": "Agent", + "type": "text", + "source": "odoo", + "default": true, + "description": "The sales agent who owns this account." + }, + { + "key": "dba", + "label": "DBA", + "type": "select", + "source": "odoo", + "default": false, + "options": [ + "Fisch", + "Royal", + "Both" + ], + "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." + }, + { + "key": "salesperson", + "label": "Salesperson", + "type": "text", + "source": "odoo", + "default": false, + "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account." + }, + { + "key": "street", + "label": "Street", + "type": "text", + "source": "odoo", + "default": false, + "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." + }, + { + "key": "street2", + "label": "Street 2", + "type": "text", + "source": "odoo", + "default": false, + "description": "Second address line (suite, unit, floor) on the customer's Odoo address." + }, + { + "key": "city", + "label": "City", + "type": "text", + "source": "odoo", + "default": true, + "description": "City on the customer's Odoo address." + }, + { + "key": "state", + "label": "State", + "type": "text", + "source": "odoo", + "default": true, + "description": "State or province on the customer's Odoo address." + }, + { + "key": "country", + "label": "Country", + "type": "text", + "source": "odoo", + "default": false, + "description": "Country on the customer's Odoo address." + }, + { + "key": "zip", + "label": "ZIP", + "type": "text", + "source": "odoo", + "default": false, + "description": "Postal code on the customer's Odoo address." + }, + { + "key": "customer_since", + "label": "Customer since", + "type": "date", + "source": "odoo", + "default": false, + "description": "When this customer was first set up in Odoo." + }, + { + "key": "tags", + "label": "Tags", + "type": "text", + "source": "odoo", + "default": false, + "description": "Odoo labels on this customer, comma-separated." + }, + { + "key": "pricelist", + "label": "Price list", + "type": "text", + "source": "odoo", + "default": false, + "description": "The price list this customer buys on." + }, + { + "key": "payment_terms", + "label": "Payment terms", + "type": "text", + "source": "odoo", + "default": false, + "description": "Payment terms on this customer's account — Net 30, for example." + }, + { + "key": "last_order", + "label": "Last order", + "type": "date", + "source": "odoo", + "default": true, + "description": "Date of the most recent confirmed order." + }, + { + "key": "overdue_days", + "label": "Overdue days", + "type": "int", + "source": "odoo", + "default": true, + "description": "How many days late this customer is running against their own usual ordering rhythm." + }, + { + "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.", + "key": "est_missed", + "label": "Est. missed $", + "type": "currency", + "source": "odoo", + "default": true, + "agg": "sum", + "filterable": false, + "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + }, + { + "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.", + "key": "ar_open", + "label": "AR current $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + }, + { + "key": "ar_overdue", + "label": "AR overdue $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Invoiced money past due — same basis as the Collections page." + }, + { + "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.", + "key": "ar_outstanding", + "label": "AR outstanding $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Total invoiced money owed right now: AR current $ plus AR overdue $." + }, + { + "key": "ar_exposure", + "label": "Credit exposure $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + }, + { + "key": "ar_aged_1_30", + "label": "1-30 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_31_60", + "label": "31-60 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_61_90", + "label": "61-90 days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "ar_aged_90_plus", + "label": "90+ days $", + "type": "currency", + "source": "odoo", + "default": false, + "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + }, + { + "key": "days_to_pay", + "label": "Days to pay", + "type": "int", + "source": "odoo", + "default": false, + "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + }, + { + "key": "top_category", + "label": "Top category", + "type": "text", + "source": "odoo", + "default": false, + "description": "The category this customer spent the most on in the last 12 months." + }, + { + "key": "top_category_pct", + "label": "Top category %", + "type": "pct", + "source": "odoo", + "default": false, + "description": "Share of last-12-months spend that went to the top category." + }, + { + "key": "sku_count", + "label": "SKUs bought", + "type": "int", + "source": "odoo", + "default": false, + "description": "Distinct products bought in the last 12 months." + }, + { + "key": "top_sku", + "label": "Top SKU", + "type": "text", + "source": "odoo", + "default": false, + "description": "The product this customer spent the most on in the last 12 months." + }, + { + "key": "days_since", + "label": "Days since order", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days since the last confirmed order." + }, + { + "key": "typical_gap_days", + "label": "Typical gap days", + "type": "int", + "source": "odoo", + "default": false, + "description": "Days this customer usually goes between orders, from their own history." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "description": "Your notes on this customer. Saved in this app only, visible only to you." + } + ], + "rows": [ + { + "pid": 101, + "customer": "Poppy Flowers", + "status": "New", + "agent": "Naomi Linnell Rivera", + "city": "Charlottesville", + "state": "Virginia (US)", + "last_order": "2026-07-21", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "02720", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Royal", + "customer_since": "2023-01-10", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-01-15 09:10:00", + "lat": 25.7617, + "lon": -80.1918, + "odoo_status": "Archived", + "dba": "Royal", + "ar_outstanding": 0 + }, + { + "pid": 102, + "customer": "Meadow & Vine Wholesale", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Portland", + "state": "Oregon (US)", + "last_order": "2026-07-19", + "est_missed": 0, + "notes": "Expanding to a second storefront.", + "country": "United States", + "zip": "77041", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-02-11", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-02-15 09:11:00", + "lat": 27.9506, + "lon": -82.4572, + "odoo_status": "Active", + "dba": "Fisch", + "ar_outstanding": 1240.5 + }, + { + "pid": 103, + "customer": "Bluestem Floral Supply", + "status": "Growing", + "agent": "Naomi Linnell Rivera", + "city": "Kansas City", + "state": "Missouri (US)", + "last_order": "2026-07-17", + "est_missed": 0, + "notes": "", + "country": "United States", + "zip": "11219", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-03-12", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-03-15 09:12:00", + "lat": 28.5384, + "lon": -81.3789, + "odoo_status": "Active", + "dba": "Both", + "ar_outstanding": 5120.25 + }, + { + "pid": 104, + "customer": "Camellia Row Florist", + "status": "Declining", + "agent": "Devon Marsh", + "city": "Savannah", + "state": "Georgia (US)", + "last_order": "2026-05-30", + "est_missed": 41200, + "notes": "Switched some volume to a local grower.", + "country": "United States", + "zip": "07649", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "(none)", + "customer_since": "2026-04-13", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-04-15 09:13:00", + "lat": 30.3322, + "lon": -81.6557, + "odoo_status": "Active", + "dba": "Royal", + "ar_outstanding": 9240 + }, + { + "pid": 105, + "customer": "Harborlight Wholesale Blooms", + "status": "Growing", + "agent": "Carla Jimenez", + "city": "Seattle", + "state": "Washington (US)", + "last_order": "2026-07-22", + "est_missed": 0, + "notes": "Top-10 account.", + "country": "United States", + "zip": "33125", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "Royal", + "customer_since": "2023-05-14", + "salesperson": "Jessica", + "ar_open": 0, + "ar_overdue": 0, + "ar_exposure": 0, + "top_category": "Styrofoam", + "top_category_pct": 0.47, + "sku_count": 93, + "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack", + "days_to_pay": 34, + "_created": "2023-05-15 09:14:00", + "lat": 26.1224, + "lon": -80.1373, + "odoo_status": "Active", + "dba": "", + "ar_outstanding": 0 + }, + { + "pid": 106, + "customer": "Dogwood & Fern Co.", + "status": "Dormant", + "agent": "Devon Marsh", + "city": "Asheville", + "state": "North Carolina (US)", + "last_order": "2026-02-11", + "est_missed": 52400, + "notes": "No spring order this year.", + "country": "United States", + "zip": "90210", + "payment_terms": "60 Days", + "pricelist": "Royal 1 (USD)", + "tags": "Royal, Key account", + "customer_since": "2024-06-15", + "salesperson": "Naomi", + "ar_open": 1240.5, + "ar_overdue": 0, + "ar_exposure": 1740.5, + "top_category": "Ribbon", + "top_category_pct": 0.95, + "sku_count": 4, + "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD", + "days_to_pay": null, + "_created": "2023-06-15 09:15:00", + "lat": 27.3364, + "lon": -82.5307, + "odoo_status": "Active", + "dba": "Fisch", + "ar_outstanding": 1240.5 + }, + { + "pid": 107, + "customer": "Verbena Market Florals", + "status": "Lost", + "agent": "Naomi Linnell Rivera", + "city": "Austin", + "state": "Texas (US)", + "last_order": "2025-11-04", + "est_missed": 78300, + "notes": "Went with a competitor on freight terms.", + "country": "United States", + "zip": "08701", + "payment_terms": "30 Days", + "pricelist": "Fisch 1 (USD)", + "tags": "Fisch", + "customer_since": "2025-07-16", + "salesperson": "Karen", + "ar_open": 0, + "ar_overdue": 5120.25, + "ar_exposure": 5120.25, + "top_category": "Foams & Finishes", + "top_category_pct": 0.31, + "sku_count": 27, + "top_sku": "SATIN RIBBON 2IN", + "days_to_pay": 61, + "_created": "2023-07-15 09:16:00", + "lat": null, + "lon": null, + "odoo_status": "Active", + "dba": "Both", + "ar_outstanding": 5120.25 + }, + { + "pid": 108, + "customer": "Larkspur Lane Supply", + "status": "New", + "agent": "Carla Jimenez", + "city": "Denver", + "state": "Colorado (US)", + "last_order": "2026-07-14", + "est_missed": 0, + "notes": "First order in April.", + "country": "United States", + "zip": "60614", + "payment_terms": "Immediate Payment", + "pricelist": "Royal 1 (USD)", + "tags": "(none)", + "customer_since": "2026-08-17", + "salesperson": "(none)", + "ar_open": 8300, + "ar_overdue": 940, + "ar_exposure": 11440, + "top_category": "All", + "top_category_pct": 1.0, + "sku_count": 1, + "top_sku": "(none)", + "days_to_pay": 12, + "_created": "2023-08-15 09:17:00", + "lat": 33.749, + "lon": -84.388, + "odoo_status": "Active", + "dba": "Royal", + "ar_outstanding": 9240 + } + ] +} diff --git a/web/src/alerts/alertsModel.ts b/web/src/alerts/alertsModel.ts index 2537671c52718b6ead09a79463cbdb4b266f9726..e371859ee58dc93370ece81ffc1da313d3487742 100644 --- a/web/src/alerts/alertsModel.ts +++ b/web/src/alerts/alertsModel.ts @@ -1,418 +1,365 @@ -// --------------------------------------------------------------------------- -// alerts/alertsModel.ts — WAVE 20 item 25 (contract C-ALERT): the inbox's PURE -// half. React-free and fetch-free, so `verify_alerts.py` runs it under node. -// -// An alert says "tell me when a record ENTERS this view". The client half is -// small, and every part of it fails silently when it is wrong: -// -// · an unread count taken from `items.length` rather than from the server's -// own `unread` disagrees with the badge the moment a page is capped or a -// read lands in another tab — and a badge that says 3 when the list shows 9 -// teaches the reader to ignore the badge; -// · a notification whose `viewId` no longer resolves must open NOTHING rather -// than a wrong view — alerts outlive the views they were made from; -// · `topic` is the surface's scope key ("customer", "product", "ut_…"), and -// the ROUTE is a registry key ("customer_data") — mapping one to the other -// by guesswork sends every click to a page that does not exist. -// --------------------------------------------------------------------------- - -/** - * ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — WHERE AN INBOX ITEM OPENS. - * - * `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open - * in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's - * run log, or the VIEW ID to select on a database. One optional key, two destinations. - * - * ⛔ EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather - * than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew - * a module" into "the client silently drops the row". An unknown module is refused by the - * FRAME's dispatcher, out loud, which is a different thing from never arriving. - */ -export interface NotificationTarget { - module: string; - id: string; - tab?: string; -} - -/** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the - * Inbox's branch and the gate's fixtures cannot disagree about the word. */ -export const NOTIF_KIND_ALERT = "alert"; -export const NOTIF_KIND_AUTOMATION = "automation"; -export const NOTIF_KIND_SHARE = "share"; - -/** C3's `target.module` vocabulary, and the automation sub-selection. */ -export const TARGET_MODULE_DATABASE = "database"; -export const TARGET_MODULE_AUTOMATION = "automation"; -export const TARGET_TAB_RUNS = "runs"; - -/** One notification, as `GET /api/v1/notifications` sends it. */ -export interface Notification { - id: string; - alertId: string; - viewId: string; - topic: string; - rowId: string; - /** What entered — the record's own label. */ - label: string; - /** What the alert is called, so a row reads without opening anything. */ - alertLabel: string; - /** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here - * would re-introduce the browser-clock drift the offset exists to remove. */ - at: string; - read: boolean; - /** - * WAVE 23 (contract C6) — WHAT KIND of notification this is. - * - * Absent (and every notification written before this wave) means the original one: a record - * ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card - * arrived at a review stage and routes by `autoId` instead — a different destination reached - * from the same list. - * - * ⛔ A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary - * is the SERVER's, and a client union over it turns "the server grew a kind" into "the client - * silently drops the row". Unknown kinds fall through to the view route, which is exactly what - * they did before this field existed. - */ - kind?: string; - /** `automation_review` only: the automation whose review stage a card reached. Absent on - * every other kind — and an `automation_review` row that arrives WITHOUT one opens nothing - * rather than guessing, the same posture `viewId` gets. */ - autoId?: string; - /** Advisory. A surface that does not scroll to a stage simply selects the automation. */ - stageId?: string; - /** How many cards arrived in the batch. C6 queues ONE notification per run naming the count, - * never one per record — so this is the number the row's own text is built from. */ - count?: number; - /** - * ⭐ WAVE 32 · C3 — THE HEADER LINE, so the Inbox can be laid out like mail. - * - * `subject` is what the item is ABOUT (the alert's name, the automation's name, the database - * that was shared); `label` stays what HAPPENED (the record that entered, the run summary). - * They were one field, which is why the pane could only ever render a sentence with no sender. - * Absent on a server that predates this wave — {@link subjectOf} falls back rather than - * rendering a blank header. - */ - subject?: string; - /** - * ⭐⭐ WAVE 33 · W33-T28 — WHO IT IS FROM. Mail has a sender; this list did not. - * - * A `verifier` reading the finished wave-32 Inbox found the row's sender POSITION occupied by - * `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you", a CATEGORY - * standing where a who belongs — and no sender anywhere on the wire. Derived by the server - * (`routes_alerts.notification_view`), never here: only it knows that a share has a person - * behind it and an alert has a machine. - * ⚠ Optional, so a payload from a server that predates this wave keeps today's shape; - * {@link senderOf} falls back rather than rendering a blank From column. - */ - sender?: string; - /** ⭐ WAVE 32 · C3 — where clicking it goes. ABSENT when this product cannot resolve a - * destination (an alert on a table this account can no longer route to), and that absence is - * load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click - * that silently does nothing. */ - target?: NotificationTarget; -} - -/** One alert, as `GET /api/v1/alerts` sends it. */ -export interface Alert { - id: string; - viewId: string; - topic: string; - owner: string; - label: string; - createdAt: string; - /** How many records are in its remembered set right now. */ - matched: number; - seeded: boolean; - lastRunAt: string; - lastError: string; -} - -export interface Inbox { - unread: number; - items: Notification[]; - /** - * ⭐⭐ WAVE 33 · W33-T28 / D-208 — THE SERVER'S CLOCK, so a stamp can be mail-shaped. - * - * ⛔ It exists so {@link stampText} never reads the BROWSER's clock, which is D-208's exit - * condition word for word. `at` is UTC with its offset (D-18) precisely so every reader sees - * the same instant; asking "is this today?" of a local clock would put the drift back. - * ⚠ Optional: absent, `stampText` returns the old absolute string rather than guessing. - */ - now?: string; -} - -export const EMPTY_INBOX: Inbox = { unread: 0, items: [] }; - -/** - * ⭐⭐ WAVE 31 · T23 (owner item 1) — WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION. - * - * Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't - * remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own - * `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel - * and `GET /notifications` answering — measured at **3,280 ms live** — and for ever after a - * failed fetch: - * · `items` was `[]`, so the pane printed **"Nothing new."** — a claim, not a wait; - * · `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had - * already loaded could never be cleared. - * A confident sentence about somebody's inbox, and the one control that would fix it, both - * switched off by the same uninitialised state. - * - * ⛔ IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts` - * compiles and RUNS this module under node; markup it cannot reach is markup no control can - * mutate. Deciding here means the truth table is asserted and each branch is load-bearing. - */ -export type PaneView = "pending" | "rows" | "empty" | "error"; - -/** - * `pending` while the first read is in flight · `error` when it failed and we have nothing to - * show · `rows` when there is something · `empty` ONLY when a successful read returned nothing. - * - * ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while - * the pane already holds notifications should not blank them — the reader loses real information - * to a transient. The error still reaches them as the pane's message line. - */ -export function paneView( - phase: "pending" | "ready" | "error", - rowCount: number, - seededUnread = 0 -): PaneView { - if (rowCount > 0) return "rows"; - if (phase === "pending") return "pending"; - if (phase === "error") return "error"; - // ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says - // there is something; the page we were given does not contain it. Saying "Nothing new" there - // is the same false confidence in a different costume. - return seededUnread > 0 ? "error" : "empty"; -} - -/** - * May "Mark all read" be pressed? - * - * ⛔ NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state — zero - * until its fetch lands, zero for ever if the fetch fails — so the control was dead in exactly - * the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count - * the frame already holds, and a failed read does not take the verb away: `POST /notifications/ - * read` with `ids: null` clears the account's inbox whether or not we managed to list it. - */ -export function canMarkAll(unread: number, busy = false): boolean { - return !busy && Math.max(0, Math.floor(unread)) > 0; -} - -const str = (v: unknown): string => (typeof v === "string" ? v : ""); -const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0); - -/** - * ⭐ WAVE 32 · C3 — one wire `target` → a {@link NotificationTarget}, or `null`. - * - * ⛔ BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is - * for: a target with a module and no id dispatches an open request naming NOTHING — a click that - * appears to work and silently does not, which is this repo's most-repeated shape. `null` is - * rendered as an unclickable row, which a reader can SEE. - */ -export function parseTarget(raw: unknown): NotificationTarget | null { - if (!raw || typeof raw !== "object") return null; - const t = raw as Record; - const module = str(t.module).trim(); - const id = str(t.id).trim(); - if (!module || !id) return null; - const tab = str(t.tab).trim(); - return { module, id, ...(tab ? { tab } : {}) }; -} - -/** - * `GET /notifications` → the inbox, fail-closed. - * - * ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two - * can legitimately differ — the list is what this page holds, the count is what - * the account has — and recomputing it here would make the badge a function of - * whatever the last fetch happened to include. - */ -export function parseInbox(body: unknown): Inbox { - const b = (body && typeof body === "object" ? body : {}) as Record; - const raw = Array.isArray(b.items) ? b.items : []; - const items: Notification[] = []; - for (const item of raw) { - if (!item || typeof item !== "object") continue; - const n = item as Record; - const id = str(n.id); - // An id-less notification cannot be marked read, so it would sit unread for - // ever and hold the badge up. Dropped, not rendered. - if (!id) continue; - items.push({ - id, - alertId: str(n.alertId), - viewId: str(n.viewId), - topic: str(n.topic), - rowId: String(n.rowId ?? ""), - label: str(n.label) || String(n.rowId ?? ""), - alertLabel: str(n.alertLabel), - at: str(n.at), - read: n.read === true, - // WAVE 23 C6 — ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload - // that predates this wave keeps today's shape rather than gaining four `undefined` keys, - // and an `automation_review` row missing its `autoId` is left WITHOUT one rather than - // with an empty string that would render as a real destination. - ...(str(n.kind) ? { kind: str(n.kind) } : {}), - ...(str(n.autoId) ? { autoId: str(n.autoId) } : {}), - ...(str(n.stageId) ? { stageId: str(n.stageId) } : {}), - ...(num(n.count) > 0 ? { count: num(n.count) } : {}), - // ⭐ WAVE 32 · C3 — additive and spread-conditional, exactly like the four above. - ...(str(n.subject) ? { subject: str(n.subject) } : {}), - // ⭐ WAVE 33 · W33-T28 — additive and spread-conditional, like every flag above it. - ...(str(n.sender) ? { sender: str(n.sender) } : {}), - ...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}), - }); - } - // ⭐ W33-T28 / D-208 — the server's clock rides through, spread-conditionally like every other - // additive key here, so a server that predates this wave yields the same object it always did. - return { - unread: Math.max(0, num(b.unread)), - items, - ...(str(b.now) ? { now: str(b.now) } : {}), - }; -} - -/** `GET /alerts` → the alert list, fail-closed. */ -export function parseAlerts(body: unknown): Alert[] { - const b = (body && typeof body === "object" ? body : {}) as Record; - const raw = Array.isArray(b.alerts) ? b.alerts : []; - const out: Alert[] = []; - for (const item of raw) { - if (!item || typeof item !== "object") continue; - const a = item as Record; - const id = str(a.id); - if (!id) continue; - out.push({ - id, - viewId: str(a.viewId), - topic: str(a.topic), - owner: str(a.owner), - label: str(a.label) || "Untitled alert", - createdAt: str(a.createdAt), - matched: num(a.matched), - seeded: a.seeded === true, - lastRunAt: str(a.lastRunAt), - lastError: str(a.lastError), - }); - } - return out; -} - -/** - * A topic (the grid's scope key) → the hash route that renders it. - * - * The two built-ins are the only pair that differ, and they differ because the - * REGISTRY names the surface while the GRID names the scope; a user table is its - * own key in both. `null` for anything else: a notification for a topic this - * client cannot route to must do nothing, not navigate somewhere plausible. - */ -export function routeForTopic(topic: string): string | null { - const t = str(topic).trim(); - if (t === "customer") return "customer_data"; - if (t === "product") return "product_data"; - if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t; - return null; -} - - - - - - -/** - * The stamp, made readable WITHOUT touching a clock. - * - * ⛔ NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The - * server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the - * same instant; parsing it into a browser Date and formatting it back would - * re-introduce the drift the offset exists to remove — a tenant a day ahead - * being told an event happened tomorrow ([[date-window-vocabulary]]). This is - * STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`. - * Anything that does not look like an ISO stamp passes through untouched, so a - * format this function has never seen is shown as sent rather than mangled. - */ -const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; - -export function stampText(at: string, now?: string): string { - const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at)); - if (!m) return str(at); - const [, day, hhmm] = m; - // ⭐⭐ W33-T28 / D-208 — MAIL-SHAPED, AND STILL STRING SURGERY. - // - // ⛔ `now` COMES FROM THE SERVER (`inbox.now`), NEVER FROM `new Date()`. That is D-208's exit - // condition word for word — "without reading the browser clock" — and the reason is the same - // one this function has always carried: `at` is sent as UTC WITH its offset (D-18) so every - // reader sees the same instant, and deciding "is this today?" against a browser clock would - // re-introduce exactly the drift the offset removes. Both operands are the server's. - // - // ⚠ AND WITHOUT `now` IT DEGRADES TO THE OLD FORMAT RATHER THAN GUESSING. A caller that has no - // server clock gets `2026-08-13 09:41` — the pre-wave string, unambiguous and never wrong — - // instead of a relative stamp computed from something we do not trust. - const today = /^(\d{4}-\d{2}-\d{2})/.exec(str(now)); - if (!today) return `${day} ${hhmm}`; - if (today[1] === day) return hhmm; - const [y, mo, d] = day.split("-"); - const month = MONTHS[Number(mo) - 1] || mo; - // The year only when it differs — a mail client does not print "2026" on a message from March. - return today[1].slice(0, 4) === y - ? `${month} ${Number(d)}` - : `${month} ${Number(d)}, ${y}`; -} - -/** - * The badge's text. Never the raw number past 99: a nav row is 236px wide and a - * four-digit badge pushes the label out of it. - */ -export function badgeText(unread: number): string { - const n = Math.max(0, Math.floor(unread)); - if (n <= 0) return ""; - return n > 99 ? "99+" : String(n); -} - -/** - * Apply a read/unread change LOCALLY, mirroring what the server just stored, and - * return the new inbox with the count corrected. - * - * `ids === null` is "all of them" (the API's own convention for mark-all). The - * count is derived from the ITEMS here — deliberately, and it is the one place - * that is right to do so: the server's answer is in flight, and the alternative - * is a badge that keeps its old number until the refetch lands. - */ -export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox { - const wanted = ids === null ? null : new Set(ids); - const items = inbox.items.map((n) => - wanted === null || wanted.has(n.id) ? { ...n, read } : n - ); - const seenUnread = items.filter((n) => !n.read).length; - // A page can hold fewer notifications than the account has, so a partial read - // must SUBTRACT from the server's count rather than replace it with this - // page's tally — except when marking everything, where zero is the answer. - if (wanted === null) return { unread: read ? 0 : items.length, items }; - const changed = inbox.items.filter( - (n) => wanted.has(n.id) && n.read !== read - ).length; - const delta = read ? -changed : changed; - return { unread: Math.max(seenUnread, inbox.unread + delta), items }; -} - -/** - * The shell↔rail channel for "make an alert out of this view" — the view rail - * raises it, the frame (which knows the current route, and therefore the topic) - * answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot - * import the shell, and it does not know its own scope key. - */ -export const ALERT_CREATE_EVENT = "aios:alert-create"; - -export interface AlertCreateRequest { - viewId: string; - label: string; -} - -export function parseAlertCreate(detail: unknown): AlertCreateRequest | null { - if (!detail || typeof detail !== "object") return null; - const d = detail as Record; - const viewId = str(d.viewId).trim(); - if (!viewId) return null; - return { viewId, label: str(d.label).trim() || viewId }; -} +// --------------------------------------------------------------------------- +// alerts/alertsModel.ts — WAVE 20 item 25 (contract C-ALERT): the inbox's PURE +// half. React-free and fetch-free, so `verify_alerts.py` runs it under node. +// +// An alert says "tell me when a record ENTERS this view". The client half is +// small, and every part of it fails silently when it is wrong: +// +// · an unread count taken from `items.length` rather than from the server's +// own `unread` disagrees with the badge the moment a page is capped or a +// read lands in another tab — and a badge that says 3 when the list shows 9 +// teaches the reader to ignore the badge; +// · a notification whose `viewId` no longer resolves must open NOTHING rather +// than a wrong view — alerts outlive the views they were made from; +// · `topic` is the surface's scope key ("customer", "product", "ut_…"), and +// the ROUTE is a registry key ("customer_data") — mapping one to the other +// by guesswork sends every click to a page that does not exist. +// --------------------------------------------------------------------------- + +/** + * ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — WHERE AN INBOX ITEM OPENS. + * + * `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open + * in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's + * run log, or the VIEW ID to select on a database. One optional key, two destinations. + * + * ⛔ EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather + * than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew + * a module" into "the client silently drops the row". An unknown module is refused by the + * FRAME's dispatcher, out loud, which is a different thing from never arriving. + */ +export interface NotificationTarget { + module: string; + id: string; + tab?: string; +} + +/** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the + * Inbox's branch and the gate's fixtures cannot disagree about the word. */ +export const NOTIF_KIND_ALERT = "alert"; +export const NOTIF_KIND_AUTOMATION = "automation"; +export const NOTIF_KIND_SHARE = "share"; + +/** C3's `target.module` vocabulary, and the automation sub-selection. */ +export const TARGET_MODULE_DATABASE = "database"; +export const TARGET_MODULE_AUTOMATION = "automation"; +export const TARGET_TAB_RUNS = "runs"; + +/** One notification, as `GET /api/v1/notifications` sends it. */ +export interface Notification { + id: string; + alertId: string; + viewId: string; + topic: string; + rowId: string; + /** What entered — the record's own label. */ + label: string; + /** What the alert is called, so a row reads without opening anything. */ + alertLabel: string; + /** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here + * would re-introduce the browser-clock drift the offset exists to remove. */ + at: string; + read: boolean; + /** + * WAVE 23 (contract C6) — WHAT KIND of notification this is. + * + * Absent (and every notification written before this wave) means the original one: a record + * ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card + * arrived at a review stage and routes by `autoId` instead — a different destination reached + * from the same list. + * + * ⛔ A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary + * is the SERVER's, and a client union over it turns "the server grew a kind" into "the client + * silently drops the row". Unknown kinds fall through to the view route, which is exactly what + * they did before this field existed. + */ + kind?: string; + /** `automation_review` only: the automation whose review stage a card reached. Absent on + * every other kind — and an `automation_review` row that arrives WITHOUT one opens nothing + * rather than guessing, the same posture `viewId` gets. */ + autoId?: string; + /** Advisory. A surface that does not scroll to a stage simply selects the automation. */ + stageId?: string; + /** How many cards arrived in the batch. C6 queues ONE notification per run naming the count, + * never one per record — so this is the number the row's own text is built from. */ + count?: number; + /** + * ⭐ WAVE 32 · C3 — THE HEADER LINE, so the Inbox can be laid out like mail. + * + * `subject` is what the item is ABOUT (the alert's name, the automation's name, the database + * that was shared); `label` stays what HAPPENED (the record that entered, the run summary). + * They were one field, which is why the pane could only ever render a sentence with no sender. + * Absent on a server that predates this wave — {@link subjectOf} falls back rather than + * rendering a blank header. + */ + subject?: string; + /** ⭐ WAVE 32 · C3 — where clicking it goes. ABSENT when this product cannot resolve a + * destination (an alert on a table this account can no longer route to), and that absence is + * load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click + * that silently does nothing. */ + target?: NotificationTarget; +} + +/** One alert, as `GET /api/v1/alerts` sends it. */ +export interface Alert { + id: string; + viewId: string; + topic: string; + owner: string; + label: string; + createdAt: string; + /** How many records are in its remembered set right now. */ + matched: number; + seeded: boolean; + lastRunAt: string; + lastError: string; +} + +export interface Inbox { + unread: number; + items: Notification[]; +} + +export const EMPTY_INBOX: Inbox = { unread: 0, items: [] }; + +/** + * ⭐⭐ WAVE 31 · T23 (owner item 1) — WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION. + * + * Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't + * remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own + * `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel + * and `GET /notifications` answering — measured at **3,280 ms live** — and for ever after a + * failed fetch: + * · `items` was `[]`, so the pane printed **"Nothing new."** — a claim, not a wait; + * · `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had + * already loaded could never be cleared. + * A confident sentence about somebody's inbox, and the one control that would fix it, both + * switched off by the same uninitialised state. + * + * ⛔ IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts` + * compiles and RUNS this module under node; markup it cannot reach is markup no control can + * mutate. Deciding here means the truth table is asserted and each branch is load-bearing. + */ +export type PaneView = "pending" | "rows" | "empty" | "error"; + +/** + * `pending` while the first read is in flight · `error` when it failed and we have nothing to + * show · `rows` when there is something · `empty` ONLY when a successful read returned nothing. + * + * ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while + * the pane already holds notifications should not blank them — the reader loses real information + * to a transient. The error still reaches them as the pane's message line. + */ +export function paneView( + phase: "pending" | "ready" | "error", + rowCount: number, + seededUnread = 0 +): PaneView { + if (rowCount > 0) return "rows"; + if (phase === "pending") return "pending"; + if (phase === "error") return "error"; + // ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says + // there is something; the page we were given does not contain it. Saying "Nothing new" there + // is the same false confidence in a different costume. + return seededUnread > 0 ? "error" : "empty"; +} + +/** + * May "Mark all read" be pressed? + * + * ⛔ NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state — zero + * until its fetch lands, zero for ever if the fetch fails — so the control was dead in exactly + * the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count + * the frame already holds, and a failed read does not take the verb away: `POST /notifications/ + * read` with `ids: null` clears the account's inbox whether or not we managed to list it. + */ +export function canMarkAll(unread: number, busy = false): boolean { + return !busy && Math.max(0, Math.floor(unread)) > 0; +} + +const str = (v: unknown): string => (typeof v === "string" ? v : ""); +const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0); + +/** + * ⭐ WAVE 32 · C3 — one wire `target` → a {@link NotificationTarget}, or `null`. + * + * ⛔ BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is + * for: a target with a module and no id dispatches an open request naming NOTHING — a click that + * appears to work and silently does not, which is this repo's most-repeated shape. `null` is + * rendered as an unclickable row, which a reader can SEE. + */ +export function parseTarget(raw: unknown): NotificationTarget | null { + if (!raw || typeof raw !== "object") return null; + const t = raw as Record; + const module = str(t.module).trim(); + const id = str(t.id).trim(); + if (!module || !id) return null; + const tab = str(t.tab).trim(); + return { module, id, ...(tab ? { tab } : {}) }; +} + +/** + * `GET /notifications` → the inbox, fail-closed. + * + * ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two + * can legitimately differ — the list is what this page holds, the count is what + * the account has — and recomputing it here would make the badge a function of + * whatever the last fetch happened to include. + */ +export function parseInbox(body: unknown): Inbox { + const b = (body && typeof body === "object" ? body : {}) as Record; + const raw = Array.isArray(b.items) ? b.items : []; + const items: Notification[] = []; + for (const item of raw) { + if (!item || typeof item !== "object") continue; + const n = item as Record; + const id = str(n.id); + // An id-less notification cannot be marked read, so it would sit unread for + // ever and hold the badge up. Dropped, not rendered. + if (!id) continue; + items.push({ + id, + alertId: str(n.alertId), + viewId: str(n.viewId), + topic: str(n.topic), + rowId: String(n.rowId ?? ""), + label: str(n.label) || String(n.rowId ?? ""), + alertLabel: str(n.alertLabel), + at: str(n.at), + read: n.read === true, + // WAVE 23 C6 — ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload + // that predates this wave keeps today's shape rather than gaining four `undefined` keys, + // and an `automation_review` row missing its `autoId` is left WITHOUT one rather than + // with an empty string that would render as a real destination. + ...(str(n.kind) ? { kind: str(n.kind) } : {}), + ...(str(n.autoId) ? { autoId: str(n.autoId) } : {}), + ...(str(n.stageId) ? { stageId: str(n.stageId) } : {}), + ...(num(n.count) > 0 ? { count: num(n.count) } : {}), + // ⭐ WAVE 32 · C3 — additive and spread-conditional, exactly like the four above. + ...(str(n.subject) ? { subject: str(n.subject) } : {}), + ...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}), + }); + } + return { unread: Math.max(0, num(b.unread)), items }; +} + +/** `GET /alerts` → the alert list, fail-closed. */ +export function parseAlerts(body: unknown): Alert[] { + const b = (body && typeof body === "object" ? body : {}) as Record; + const raw = Array.isArray(b.alerts) ? b.alerts : []; + const out: Alert[] = []; + for (const item of raw) { + if (!item || typeof item !== "object") continue; + const a = item as Record; + const id = str(a.id); + if (!id) continue; + out.push({ + id, + viewId: str(a.viewId), + topic: str(a.topic), + owner: str(a.owner), + label: str(a.label) || "Untitled alert", + createdAt: str(a.createdAt), + matched: num(a.matched), + seeded: a.seeded === true, + lastRunAt: str(a.lastRunAt), + lastError: str(a.lastError), + }); + } + return out; +} + +/** + * A topic (the grid's scope key) → the hash route that renders it. + * + * The two built-ins are the only pair that differ, and they differ because the + * REGISTRY names the surface while the GRID names the scope; a user table is its + * own key in both. `null` for anything else: a notification for a topic this + * client cannot route to must do nothing, not navigate somewhere plausible. + */ +export function routeForTopic(topic: string): string | null { + const t = str(topic).trim(); + if (t === "customer") return "customer_data"; + if (t === "product") return "product_data"; + if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t; + return null; +} + + + + + + +/** + * The stamp, made readable WITHOUT touching a clock. + * + * ⛔ NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The + * server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the + * same instant; parsing it into a browser Date and formatting it back would + * re-introduce the drift the offset exists to remove — a tenant a day ahead + * being told an event happened tomorrow ([[date-window-vocabulary]]). This is + * STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`. + * Anything that does not look like an ISO stamp passes through untouched, so a + * format this function has never seen is shown as sent rather than mangled. + */ +export function stampText(at: string): string { + const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at)); + return m ? `${m[1]} ${m[2]}` : str(at); +} + +/** + * The badge's text. Never the raw number past 99: a nav row is 236px wide and a + * four-digit badge pushes the label out of it. + */ +export function badgeText(unread: number): string { + const n = Math.max(0, Math.floor(unread)); + if (n <= 0) return ""; + return n > 99 ? "99+" : String(n); +} + +/** + * Apply a read/unread change LOCALLY, mirroring what the server just stored, and + * return the new inbox with the count corrected. + * + * `ids === null` is "all of them" (the API's own convention for mark-all). The + * count is derived from the ITEMS here — deliberately, and it is the one place + * that is right to do so: the server's answer is in flight, and the alternative + * is a badge that keeps its old number until the refetch lands. + */ +export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox { + const wanted = ids === null ? null : new Set(ids); + const items = inbox.items.map((n) => + wanted === null || wanted.has(n.id) ? { ...n, read } : n + ); + const seenUnread = items.filter((n) => !n.read).length; + // A page can hold fewer notifications than the account has, so a partial read + // must SUBTRACT from the server's count rather than replace it with this + // page's tally — except when marking everything, where zero is the answer. + if (wanted === null) return { unread: read ? 0 : items.length, items }; + const changed = inbox.items.filter( + (n) => wanted.has(n.id) && n.read !== read + ).length; + const delta = read ? -changed : changed; + return { unread: Math.max(seenUnread, inbox.unread + delta), items }; +} + +/** + * The shell↔rail channel for "make an alert out of this view" — the view rail + * raises it, the frame (which knows the current route, and therefore the topic) + * answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot + * import the shell, and it does not know its own scope key. + */ +export const ALERT_CREATE_EVENT = "aios:alert-create"; + +export interface AlertCreateRequest { + viewId: string; + label: string; +} + +export function parseAlertCreate(detail: unknown): AlertCreateRequest | null { + if (!detail || typeof detail !== "object") return null; + const d = detail as Record; + const viewId = str(d.viewId).trim(); + if (!viewId) return null; + return { viewId, label: str(d.label).trim() || viewId }; +} diff --git a/web/src/automation/AutomationBuilder.tsx b/web/src/automation/AutomationBuilder.tsx index 7be785d1367c0125e4c8cd2d8a0bf8825ce73a10..e05f945e362f11c98842ddb2bca93ce615864af2 100644 --- a/web/src/automation/AutomationBuilder.tsx +++ b/web/src/automation/AutomationBuilder.tsx @@ -1,3533 +1,3101 @@ -// --------------------------------------------------------------------------- -// automation/AutomationBuilder.tsx — the BUILDER (owner item 3b, contracts -// C1/C2/C4, ruling R1: "exact layout, Loopable skin"). -// -// THE ANATOMY IS AIRTABLE'S, 1:1, and it is the owner's reference (images 1-9): -// a centre column of TRIGGER → ACTIONS with a spine down the left carrying each -// step's status chip, dashed add-boxes at the end of each section, and a -// Properties panel on the right whose sections are Trigger details / -// Configuration / Test step. Every colour, size and weight is OURS (R1, -// DESIGN.md) — no Airtable blue, no caps micro-labels (R6: "TRIGGER" is -// "Trigger" here, and de-capping takes the letter-spacing with it). -// -// ⛔ IT RENDERS SYNCHRONOUSLY FROM WHAT IT WAS HANDED (C14 leg 1). No fetch, no -// effect, no second frame: the trigger comes from `automation.trigger`, the -// actions from `automation.flow`, the machine steps from `automation.graph` — -// all three already on the list payload the rail used to draw the row you -// clicked. THIS IS THE GHOST FIX. The old detail rendered Steps, then swapped to -// the Board when `/board` answered, so switching automations painted the -// PREVIOUS one's shape inside the new one's frame for as long as a fetch takes. -// A view that cannot be in two shapes cannot show you the wrong one. -// -// ⛔ NO CLIENT UNION OVER A SERVER VOCABULARY, anywhere. Trigger keys, action -// kinds, comparison operators, ending modes, review deciders and every ceiling -// ride `GET /automations`. What this file owns is pixels and wording. -// -// WHAT PERSISTS WHEN. A discrete choice writes immediately — picking a trigger, -// adding an action, flipping a switch — because a control you have to remember -// to Save is not a control (R9). FREE TEXT does not: a value, a prompt or a name -// commits on BLUR, or every keystroke would PATCH a half-typed condition and the -// server would refuse most of them out loud (the cron-string precedent in -// AutomationTrigger). -// --------------------------------------------------------------------------- -import type { ReactNode } from "react"; -import { useEffect, useRef, useState } from "react"; - -// ⭐ WAVE 30 · ITEM 7 (R3 + R4) — connector identity, from the ONE module that owns it. Not a -// second copy of two SVGs: the directory, this menu and the trigger picker all answer "which -// company is this row about", and three answers is how a platform gets added in two of them. -import { brandForConnector, brandForKind, hasBrandKind } from "../connectors/brandMarks"; - -import type { - Action, - ActionCatalogRow, - Automation, - Cond, - FlowVocab, - GraphNode, - OAuthStatus, - TriggerOption, - UserTable, -} from "./automationApi"; -// ⭐ WAVE 26 · ITEM 22 / D-70 — the Run guard, imported rather than re-derived. This file and -// `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides. -// ITEM 9 / R11 — `createTable` + `AutomationError` for the in-place "+ New database". -import { AutomationError, createTable, runBlock } from "./automationApi"; -import CondBuilder, { condComplete } from "./CondBuilder"; -import { groupActions, groupBranches, groupByPanel, numberActions, reorderList } - from "./steps"; -// ⭐ WAVE 25 item 5b (C2) — the grouped picker, and `TriggerMark` with it. The icon vocabulary -// MOVED to that file rather than being copied into it: the whole of item 5b is that the compact -// control in Properties and the big "+ Add trigger" menu are one control, and two copies of the -// marks is how they drift back into looking like two features. -import PresetPlan from "./PresetPlan"; -import TriggerPicker, { TriggerMark } from "./TriggerPicker"; - -/** - * ⛔ A PRIVATE MIME, AND `text/plain` IS NEVER SET (owner item 11; the wave-13 C-LAYOUT scar). - * A drag carrying `text/plain` is a drag the whole operating system understands: drop it on any - * input, textarea or contenteditable — this builder is full of them — and the browser TYPES the - * payload in, so a mis-aimed reorder silently writes an action id into somebody's condition - * value. A private type is inert everywhere except the drop targets below, which is why the nav - * rail's own drag uses one (`Shell.tsx:90 NAV_DRAG_TYPE`). - * ⚠ The board's card drag still uses `text/plain` (`AutomationBoard.tsx:167`) — noted, not fixed - * here: that file belongs to another session this wave. - */ -const ACTION_DRAG_TYPE = "application/x-loopable-action"; - -interface Props { - automation: Automation; - /** - * THE AUTOMATION'S NAME (item 15b, C-DETAIL) — it left the page header and Properties is where - * it went, at the top, above whatever step is selected. REQUIRED, all three: an optional prop - * degrades to "the field does not exist", which looks exactly like never having built it. - * - * ⚠ TEXT AND COMMIT ARE SEPARATE ON PURPOSE. This is the cron field's shape - * (`AutomationTrigger`'s `onCronText` / `onSchedule`): free text stays controlled while typing - * and writes ONCE on blur, because per-keystroke would PATCH a half-typed name. Folding them - * into one callback would force a choice between a laggy controlled input and a save per - * keystroke, and this file has already paid for both. - */ - name: string; - onNameText: (v: string) => void; - onNameCommit: () => void; - /** The server's trigger vocabulary (C3). Absent is a state this file states, never fills in. */ - triggers?: TriggerOption[]; - catalog: ActionCatalogRow[]; - vocab?: FlowVocab; - tables: UserTable[]; - oauth: OAuthStatus | null; - /** True while a write is in flight — drives the "All changes saved" stamp and disables edits. */ - busy: boolean; - /** Write a partial definition. The caller owns the request and prints the refusal verbatim. */ - onPatch: (body: Record) => void; - onToggleNode: (nodeId: string) => void; - /** - * ⛔ CHOOSING A TRIGGER IS NOT A `{trigger:{key}}` PATCH, and getting that wrong is invisible. - * `schedule` is what today's engine actually READS, so a pick has to write BOTH halves or - * "At a scheduled time" would store a key and leave the cron switched off — a trigger that - * says it is scheduled and never fires. The detail already owns that two-write shape - * (`pickTrigger`); the builder calls it rather than composing a second copy of it. - */ - onPickTrigger: (key: string) => void; - onRunNow: () => void; - /** - * The MACHINE steps' config bodies (source / columns / capture / find / write). They live in - * `AutomationDetail`, so they are passed IN rather than moved: a builder that re-implemented - * them would be a second copy of five panels whose only job is to agree with the engine. - * - * ⚠ IT TAKES A PANEL KEY, NOT A NODE (item 5, C-CFG). It used to be - * `(node: GraphNode) => ReactNode`, and the caller's whole body was `panelBody(node.panel)` — - * so the node was only ever a wrapper around the key. Now that Properties renders the panels - * GROUPED BY that key, taking a node would invite the caller to pass one node of a group and - * quietly imply the body belongs to it alone. - */ - renderNodeBody: (panel: string) => ReactNode; - /** - * The SCHEDULE half of the trigger face (`AutomationTrigger` with its picker hidden). Passed - * in for the same reason the node panels are: the cron round-trip, the server's preset list - * and the tick-honesty note exist exactly once, and both this view and the board render the - * same element rather than two copies of three things that go wrong invisibly. - */ - scheduleFace: ReactNode; - /** - * ⭐ WAVE 27 · CONTRACT C11 — is the PROPERTIES panel the one on screen? - * - * ⛔ THE STATE LIVES IN `AutomationDetail`, NEVER HERE, and the contract says so for a - * structural reason: the two panels that share this column are children of DIFFERENT - * components — Properties is this file's, the Run log is `AutomationDetail`'s — so the only - * place that can know "exactly one is visible" is their common parent. A local boolean here - * could hide Properties while the log was also hidden, or show both. - * ⚠ REQUIRED. An optional flag defaulting to `true` would make an unmounted toggle look like - * a working panel, which is the shape this repo keeps paying for. - */ - showProperties: boolean; - /** The Properties/Run-history switch, OWNED by `AutomationDetail` and drawn in both panel - * heads so it is in the same place whichever one is showing. */ - panelTabs: ReactNode; - /** Announce that a step (or the trigger) was clicked, so the parent can bring Properties - * forward — C11's "clicking any step card switches to Properties". */ - onStepPicked: () => void; - /** - * Does the cron drive the CURRENT trigger (owner item 7)? REQUIRED, and decided by the caller - * for the reason `AutomationTrigger.schedules` gives: which keys the cron drives is a per-key - * fact, and no component below the detail is entitled to name trigger keys. - */ - schedules: boolean; - /** - * ⭐ THE DATABASE THE FLOW'S RECORDS WALK (owner item 12a) — the trigger's table, or the - * automation's own target, resolved by the caller the way `automation_engine._flow_table` does. - * REQUIRED: an absent one degrades to an empty field list, which is the exact defect this prop - * exists to fix and is indistinguishable from "this database has no columns". - */ - walkTable: string; - /** - * ⭐ WAVE 26 · ITEM 9 / R11 — RE-READ THE DATABASE LIST, after an action's picker created one. - * - * ⛔ REQUIRED, deliberately. The wave doc's rule is "prefer a REQUIRED prop; an optional one - * degrades silently to 'the feature does not exist'" — and here the silent degradation has a - * particularly bad shape: the database really would be created, on the server, and simply not - * appear in the picker that made it. The user then makes a second one with the same name - * (duplicate names are legal), and the automation still points at neither. - */ - onTablesChanged: () => Promise | void; - /** - * Is there an edit the HEADER's Save still owns? REQUIRED, and it exists so the panel's - * "All changes saved" stamp can stop being a claim about fields it does not cover — the - * machine steps' panels moved into Properties this wave and they commit with Save, not on - * change. - */ - dirty: boolean; -} - -/* - * ⚠ `"node"` LEFT THIS UNION WITH THE MACHINE CARDS (item 5). It existed so a click on one of - * those cards could open its panel; with the cards gone nothing can set it, and a variant no - * code path can reach is a branch that reads as supported and is not. - */ -type Sel = { kind: "trigger" | "action"; id: string }; - -/** Airtable phrases the empty state's shortcuts; ours come from the SERVER's own list (image 1). */ -const SUGGESTED = 6; - -/** - * ⭐⭐ WAVE 30 · W30-T23 / C3 — THE ENRICH KINDS, MIRRORING `automation_engine.ENRICH_KINDS`. - * - * ⛔ ONE PREDICATE, TWO CALL SITES, and both had to move or the ticket would have half-shipped: - * `seedFor` decides whether the menu row can be ADDED at all, and the properties panel decides - * whether the added step has any CONTROLS. Widening only the first yields a step a person can add - * and cannot configure; widening only the second yields a panel for a row that is permanently - * faded. D-79 was that pair coming apart on the Instagram side. - * ⚠ Named as a list rather than written twice, because the server's own validator branches on a - * TUPLE of the same two names — a third platform must be one edit here, not a grep. - */ -const ENRICH_KINDS = ["enrich_instagram", "enrich_tiktok"]; -const isEnrich = (kind: string): boolean => ENRICH_KINDS.includes(kind); - -/** - * ⭐⭐ WAVE 33 · W33-T52 (owner item 16: *"real configuration under Web Action"*) — THE FIVE WEB - * KINDS, and the SHAPE of each one's config, in ONE table read by BOTH halves. - * - * ⛔ THE PAIR THE COMMENT ABOVE WARNS ABOUT HAD ALREADY COME APART HERE, in the worse direction. - * `seedFor` gained all five branches at wave 31 QA, so every web action could be ADDED — and - * `ActionProps` never gained an arm, so none of them could be CONFIGURED. The step landed on the - * canvas with a blank config, the panel painted a heading and nothing else, and the runner blocked - * it at run time with a sentence naming keys no surface let anybody type. Five whole, correct, - * unusable actions: `web_*` appeared in this entire client exactly once, in a comment. - * ⚠ SO THE KEYS ARE DECLARED ONCE. The seed and the panel read the same object — the seed spreads - * it, the panel walks its keys IN ORDER to decide which controls to draw — because a second - * per-kind key list is precisely how the two halves diverged the first time - * ([[constant-two-features-share]]). - * ⛔ EVERY KEY HERE SURVIVES `_clean_action_config`, and that is not a coincidence to be kept by - * memory: the server's web arm is an ALLOWLIST, so a key this table invents is dropped on save — - * the panel would accept a value, the local state would keep it, and it would be gone on reload. - * `verify_automation_ui` pins this table against the server's own allowlist for that reason. - * ⚠ Which keys are REQUIRED is deliberately NOT here — it rides the wire from `ACTION_REQUIRED` - * (`FlowVocab.actionRequired`), so the red mark and the runner's refusal cannot disagree. - */ -const WEB_KINDS = ["web_goto", "web_read", "web_click", "web_fill", "web_repair"]; -const isWeb = (kind: string): boolean => WEB_KINDS.includes(kind); -const WEB_SEEDS: Record> = { - web_goto: { url: "", waitFor: "", timeoutMs: 20000 }, - web_read: { url: "", selector: "", field: "", attr: "text", all: false, waitFor: "", - timeoutMs: 20000 }, - web_click: { url: "", selector: "", waitFor: "", dryRun: false, timeoutMs: 20000 }, - web_fill: { url: "", selector: "", value: "", secret: false, waitFor: "", dryRun: false, - timeoutMs: 20000 }, - web_repair: { url: "", selector: "", hint: "", waitFor: "", timeoutMs: 20000 }, -}; -/** - * ⚠ `waitFor` AND `dryRun` WERE REACHABLE FROM THE RUNNER AND FROM NOWHERE ELSE, and they were - * found by the verifier that checked this ticket rather than by anything in it. The engine keeps - * both on every web kind and the job acts on them — `waitFor` on all five (`_after` waits for a - * selector once the action is done), `dryRun` on `web_fill` and `web_click` ONLY, where the job - * resolves the element and reports what it WOULD have typed or clicked instead of doing it. - * ⛔ SO `dryRun` IS SEEDED ON EXACTLY THOSE TWO. Offering it on `web_read` would paint a control - * that changes nothing, which is the same defect as a missing one wearing better clothes - * [[wrong-parent-not-broken-control]]. - */ - -/** The server's clamp on `timeoutMs`, mirrored so the BOX and the STORE cannot disagree. */ -const WEB_TIMEOUT_MIN = 1000; -const WEB_TIMEOUT_MAX = 120000; - -/** - * ⭐⭐ WAVE 33 · W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP'S OWN SHAPE. - * - * ⛔ IT IS NOT A `web_*` KIND AND MUST NOT BE ADDED TO `WEB_KINDS`. That list is held name-for-name - * against the server's own tuple by a gate, and against the RUNNER's three copies of it; widening - * it here to get a free seed would put `ai_agent` into a set the browser job's runner does not - * implement. It is a sibling, so it gets a sibling's entry. - * ⛔ AND IT NEEDS BOTH HALVES, TODAY, for the reason this file learned twice in one wave: a kind - * with a catalog row and no SEED is permanently unclickable (D-79), and a kind with a seed and no - * PANEL is addable and unconfigurable (W33-T52, the five web actions). Two out of three ships an - * invisible feature; three out of three is the only number that is a feature. - * ⚠ NO `selector`. Working the selector out is the whole step — a selector box here would be a - * control that contradicts the action's reason to exist. - */ -const AI_AGENT_SEED: Record = { - url: "", instruction: "", field: "", maxSteps: 6, dryRun: false, timeoutMs: 20000, -}; -/** The server's own ceiling on how many steps one fuzzy instruction may compose. */ -const AI_AGENT_MAX_STEPS = 12; - -/* ⛔ `TriggerMark` STOOD HERE AND MOVED TO `TriggerPicker.tsx` (wave 25 item 5b, C2). - It is IMPORTED back for the trigger card below. Moving rather than copying is the point of - the item: the compact control in Properties and this menu are one picker now, and the marks - are the thing the owner noticed missing from one of them (reference/ERROR 3.png). */ -function ActionMark({ kind }: { kind: string }) { - /* ⭐⭐ WAVE 30 · ITEM 7 / R4 — the same one-line resolution `TriggerMark` carries, and for the - same reason: this component is drawn in the add-action menu, on every step card and on every - machine node, so a connected action must wear its company's logo in all three or it wears two - identities on one screen. Unbranded kinds fall through to the house glyphs below, untouched. */ - const brand = brandForKind(kind); - if (brand) return brand; - const common = { - width: 15, height: 15, viewBox: "0 0 16 16", fill: "none", stroke: "currentColor", - strokeWidth: 1.4, strokeLinecap: "round" as const, strokeLinejoin: "round" as const, - "aria-hidden": true, - }; - if (kind === "group") - return ( - - - - - ); - if (kind === "create_record") - return ( - - - - - ); - if (kind === "find_records") - return ( - - - - - ); - /* - ⚠ THE MACHINE KINDS ARE HERE TOO, and leaving them out was a real defect for one render: - the engine's nodes are `source | capture | branch | write`, none of which matched above, so - "Write to the database" and "Fetch the page" both drew the PENCIL — an edit glyph on a step - that reads a web page and a step that writes a database. Caught by reading the screenshot, - not by a gate ([[ui-invisible-to-assertions]]): the icon was present, legible and wrong. - These are the shapes the retired `AutomationSteps.KindMark` used, kept with their meanings. - */ - if (kind === "source") - return ( - - - - - ); - if (kind === "write") - return ( - - - - - ); - if (kind === "capture") - return ( - - - - ); - if (kind === "branch") - return ( - - - - - ); - if (kind === "update_record") - return ( - - - - - ); - if (kind === "send_email") - return ( - - - - - ); - if (kind === "slack") - return ( - - - - ); - if (kind === "run_script") - return ( - - - - ); - if (kind === "generate_ai") - return ( - - - - ); - if (kind === "repeating_group") - return ( - - - - - ); - /* - ⛔ THE FALLBACK IS NEUTRAL, and it was not the first time round. The default used to BE the - pencil — a real member of the set (`update_record`) — so every kind with no arm silently - borrowed "edit": `send_email`, `run_script` and `repeating_group` all drew it in the action - menu, which reads as three actions that modify a record. A fallback that is a real member - cannot be told apart from a match; this one can. - */ - return ( - - - - - ); -} - -/** The label a table key reads as. A raw `ut_9f3a…` on a card is a key, not a subtitle. */ -function tableLabel(tables: UserTable[], key: string): string { - if (!key) return ""; - return tables.find((t) => t.key === key)?.label || key; -} - -export default function AutomationBuilder({ - automation, - name, - onNameText, - onNameCommit, - triggers, - catalog, - vocab, - tables, - oauth, - busy, - onPatch, - onToggleNode, - onPickTrigger, - onRunNow, - renderNodeBody, - scheduleFace, - showProperties, - panelTabs, - onStepPicked, - schedules, - walkTable, - onTablesChanged, - dirty, -}: Props) { - const [sel, setSel] = useState({ kind: "trigger", id: "" }); - const [picking, setPicking] = useState(false); - const [adding, setAdding] = useState(false); - /** The trigger key awaiting confirmation (image 6) — a change that can invalidate config. */ - const [confirmKey, setConfirmKey] = useState(""); - /** The card a reorder is hovering over (item 11). `""` = nothing is being dragged. */ - const [dragOver, setDragOver] = useState(""); - /** - * ⛔ ONE DRAG SUPPRESSES EXACTLY ONE CLICK (item 11; `AutomationBoard`'s trap 1, which this - * repo has already paid for once). Browsers can deliver a `click` after `dragend`, and the - * thing under the pointer here is the card's own selector — so a finished reorder would ALSO - * change which step the Properties panel is showing, which reads as the panel jumping on its - * own. A ref and not state, because the click arrives before a re-render would land. - */ - const draggedRef = useRef(false); - /** - * ⛔ THE TRIGGER MENU IS ANCHORED TO ITS BOX, AND THAT IS A LAYOUT FACT BEFORE IT IS A - * DISMISSAL ONE (owner's ERROR 4, 2026-08-06). `` was rendered at - * the BOTTOM of this component's fragment — a sibling of `.autox-flow` and the `.autox-props` - * aside, inside `.auto-work`, which is `display: flex`. So opening it added a THIRD COLUMN to - * the builder: the menu landed to the right of the Properties panel, off the edge of the - * window, and the flow column (`flex: 1 1 auto; min-width: 0`) collapsed under it until - * "Suggested triggers" wrapped one word per line and the canvas grew a horizontal scrollbar. - * Nothing was broken about the menu — it was in the wrong parent. - * - * It lives inside `.autox-main` beside its own button now, and this ref is what makes the pair - * ONE unit: an outside mousedown must not count the button that opens it as "outside", or the - * click would close and reopen it forever. `TriggerPicker`'s own doc says the caller owns - * open-ness for the `menu` variant precisely because the caller owns the anchor. - */ - const addWrap = useRef(null); - - // Escape and an outside click both close it — the `field` variant's rule, applied to the twin - // that had neither. Without this the empty state's menu was a one-way door: every row picks a - // trigger, so changing your mind meant picking one you did not want. - useEffect(() => { - if (!picking) return; - const onKey = (e: KeyboardEvent) => { - if (e.key === "Escape") setPicking(false); - }; - const onDown = (e: MouseEvent) => { - if (!addWrap.current?.contains(e.target as Node)) setPicking(false); - }; - window.addEventListener("keydown", onKey); - window.addEventListener("mousedown", onDown); - return () => { - window.removeEventListener("keydown", onKey); - window.removeEventListener("mousedown", onDown); - }; - }, [picking]); - - const trigger = automation.trigger || null; - const triggerKey = trigger?.key || (automation.schedule?.enabled ? "schedule" : "manual"); - const options = triggers || []; - const picked = options.find((t) => t.key === triggerKey) || null; - const actions = automation.flow?.actions || []; - const nodes = (automation.graph?.nodes || []).filter((n) => n.kind !== "trigger"); - const ops = vocab?.condOps || []; - const nullaryOps = vocab?.nullaryCondOps || []; - const table = tables.find((t) => t.key === (trigger?.table || "")) || null; - - /** - * ⛔ HAS ANYTHING BEEN CHOSEN? — and the first version of this line was wrong in a way only - * a screenshot showed. - * - * It read `!!trigger || key === "schedule" || key === "manual"`, and `triggerKey` DERIVES - * "manual" whenever nothing is stored — so it was true for every automation ever, and the - * Airtable empty state (image 1) was unreachable code that every gate was happy with. - * - * The honest test is what the DEFINITION holds, not what the picker displays: the engine - * deliberately stores no trigger for manual/schedule ("`schedule` already owns the cron"), so - * `trigger === null && !schedule.enabled` IS "nobody has decided yet". Manual is what that - * state DOES, not a thing you pick out of it — which is why the suggested list below omits - * it: an option whose selection stores nothing would bounce straight back to this state on - * the next reload, and a control that undoes itself is worse than no control. - */ - const chosen = !!trigger || !!automation.schedule?.enabled; - /** - * ⭐⭐ 2026-08-07 (owner report, the SECOND time on this feature) — *"I still can't do - * Enrichment, when the Trigger is Manual, it says that I need to bound it to a database, how?"* - * - * ⛔ THE DATABASE PICKER EXISTED AND WAS UNREACHABLE FOR THE ONE TRIGGER THAT NEEDS IT MOST, - * and the two decisions that combined to hide it were each locally right: - * 1. `clean_trigger` STORES NOTHING for `manual` and `schedule` (`:8389`) — "storing a no-op - * trigger would be a second copy of that fact". So a manual automation holds `trigger: - * null` forever, by design. - * 2. `chosen` is therefore FALSE for it (the note above says so explicitly), and W25/R14 hides - * the whole Configuration section until a trigger is `chosen`. - * ⇒ The picker `caee549` added for exactly this case sits INSIDE that section, so picking - * Manual could never reveal it. `run_plain` then refused with *"pick one on the trigger, or in - * Properties"* — a sentence naming two doors while rendering neither. MEASURED in the live - * nurilab store: `auto_2` "Enrichment test", `kind:'plain'`, `trigger:None`, - * `config.targetTable:''`, one `enrich_instagram` action, unrunnable. - * - * ⚠ AND THE SCREEN DISAGREED WITH THE STORE, which is why the owner reasonably believed they - * had picked something: `triggerKey` DERIVES `"manual"` when nothing is stored, so the panel - * displays "Manual" over a definition that holds no trigger at all. - * - * ⛔ THE FIX IS NOT TO START STORING A MANUAL TRIGGER. The note above already argued that out - * and it is still right — a control whose selection stores nothing bounces back on reload. What - * was wrong is treating "no trigger stored" as "nothing to configure": a `plain` automation - * whose trigger carries no table of its own ALWAYS has one thing to configure, the database its - * steps walk, and that is true in exactly the state where `chosen` is false. So Configuration - * opens for that too — the trigger-specific blocks inside are each keyed off `triggerKey` and - * stay hidden on their own. That widening lives in the Properties panel beside the picker it - * reveals (`needsOwnTable`), not here: `chosen` still means exactly what it says. - */ - const configured = !trigger || trigger.configured !== false; - - /** Write one key of the trigger, keeping the rest — the engine merges against `previous`. */ - const patchTrigger = (patch: Record) => - onPatch({ trigger: { key: triggerKey, ...patch } }); - - /** - * ⭐⭐ 2026-08-07 (owner report) — PATCH THE AUTOMATION'S OWN CONFIG. - * - * ⛔ IT SPREADS THE EXISTING CONFIG, and that is not defensive tidiness: `clean_definition` - * takes `raw["config"]` WHOLE when it is a dict (`cfg_raw = raw.get("config") if isinstance(...) - * else prev.get("config")`) — it does not merge. Sending `{targetTable: x}` alone would erase - * the discovery predicates, the lanes and the schedule face in one keystroke. - */ - const patchConfig = (patch: Record) => - onPatch({ config: { ...((automation.config as Record) || {}), ...patch } }); - - const patchFlow = (next: Action[]) => onPatch({ flow: { actions: next } }); - - /** - * ⭐ ONE TREE WALKER (wave 24, C-FORK). Rewrite or drop any action anywhere, branches included: - * `f` returns a replacement, or `null` to remove it. - * - * ⛔ THIS REPLACED FOUR HAND-ROLLED WALKS, and they were four places to forget the same thing. - * A group's children moved from `config.actions` to `config.branches[].actions` this wave, so - * every one of them would have quietly stopped descending into a fork — editing, deleting or - * counting nothing inside it, with `tsc` perfectly happy because `config` is - * `Record` and an absent key is just `undefined`. The failure would have been - * "the card is there and the delete button does nothing", per action, per fork. - */ - const mapTree = (list: Action[], f: (a: Action) => Action | null): Action[] => - (list || []) - .map((a) => { - const next = f(a); - if (next === null) return null; - if (next.kind !== "group") return next; - const brs = groupBranches(next); - if (!brs.length) return next; - return { - ...next, - config: { - ...next.config, - branches: brs.map((br) => ({ ...br, actions: mapTree(br.actions || [], f) })), - }, - }; - }) - .filter((a): a is Action => a !== null); - - /** Replace one action anywhere in the tree, forks included — ids are stable server-side. */ - const editAction = (id: string, change: (a: Action) => Action) => - patchFlow(mapTree(actions, (a) => (a.id === id ? change(a) : a))); - - const dropAction = (id: string) => { - // ⚠ REMOVING THE LAST ACTION IN A BRANCH IS REFUSED BY THE SERVER ("a branch with no actions - // inside it does nothing") and the refusal is printed verbatim, as every other one is. Not - // pre-empted here: a client that predicted it would be a second validator, and the honest - // repair — delete the branch, not its last child — is one the reader can act on. - patchFlow(mapTree(actions, (a) => (a.id === id ? null : a))); - if (sel.kind === "action" && sel.id === id) setSel({ kind: "trigger", id: "" }); - }; - - /** - * REORDER (owner item 11). The whole `flow.actions` list goes back through `patchFlow` — the - * same door every other edit uses — rather than a bespoke move endpoint, because the engine - * already validates a whole flow and a second write path would be a second thing to keep - * agreeing with `clean_actions`. - * - * ⚠ TOP-LEVEL SIBLINGS ONLY, deliberately, and it is a SCOPE rather than an oversight: a card - * inside a conditional group is not `draggable` at all, so there is no affordance that quietly - * does nothing. Group children are about to stop living under `config.actions` and start living - * under `config.branches[].actions` (C-FORK), and building a tree-walking reorder against a - * shape this same wave replaces would be work done twice with a window of wrongness in between. - */ - const moveAction = (dragId: string, dropId: string) => { - const next = reorderList(actions, dragId, dropId); - // `null` = same card, or one of them is not a top-level sibling. Nothing moved, so nothing - // is written — an aborted drag must not PATCH the flow back over itself. - if (next) patchFlow(next); - }; - - /** - * ⛔ A NEW ACTION'S CONFIG MUST BE ONE THE SERVER ACCEPTS — and the first version of this - * function got it wrong for FOUR of the five kinds, which is worth writing down because every - * gate was green and the harness could not see it (SSR renders no click). - * - * MEASURED against `clean_actions` itself (`python -c` over the engine module, no server): - * `{values:{"":""}}` -> "a value is written to a field with no name" - * `{table:"",values:…}` -> the same, twice - * `{table:""}` (find) -> "the find records action names no database" - * `{cond:…,actions:[]}` -> "a conditional group with no actions inside it does nothing" - * So clicking "Add → Update record" would have produced a red banner and NO CARD. This is the - * wave-22 discovery-predicate scar reproduced exactly ([[cg-condition-builder-items]] — every - * default must be one the server takes), in a file whose own comment cites it. - * - * The seeds below are built from data this component already holds, and `seedFor` returns - * NULL when it cannot build a legal one. A kind with no legal seed is offered DISABLED with - * the reason, rather than offered and refused: the two states look identical to a user and - * only one of them is honest. - */ - /** - * ⭐ THE WALKING RECORD'S COLUMNS (item 12a). Read off `walkTable`, NOT off `table`. - * - * `table` is the TRIGGER's database and it answers a different question — which columns the - * trigger may watch or test. This one answers what the ACTIONS write to and what a branch's - * condition compares, and on any automation without an event trigger (manual, schedule, - * `ig_profile_match`) the two are not the same table at all: the first is empty and the second - * is where all the records are. Reading actions' fields off the trigger is why every condition - * inside a fork offered an empty picker. - */ - const walkFields = (tables.find((t) => t.key === walkTable) || null)?.fields || []; - const firstField = walkFields[0]?.key || ""; - const firstTable = tables[0] || null; - const firstTableField = firstTable?.fields?.[0]?.key || ""; - - const seedFor = (kind: string): Record | null => { - if (kind === "update_record") - return firstField ? { values: { [firstField]: "" } } : null; - if (kind === "create_record") - return firstTable && firstTableField - ? { table: firstTable.key, values: { [firstTableField]: "" } } - : null; - if (kind === "find_records") - return firstTable ? { table: firstTable.key, cond: null, limit: 25 } : null; - /* - ⭐⭐ WAVE 31 · T38 (contract C5, owner ruling R10) — THE WEB READ, and this branch is here - because a gate demanded it rather than because anybody remembered. - - ⛔ FLIPPING `ready:True` ON THE SERVER IS NOT MOUNTING AN ACTION. `verify_automation_ui`'s - seed check DERIVES its subject from the server catalog, so the moment `web_read` became - ready it went red with `MISSING: ['web_read']` — the action would have rendered faded, - labelled "Needs a database" (a claim about databases, which is not the problem) and been - permanently unclickable. That is D-79's scar exactly, and the check written after it is what - caught the repeat. A capability is reachable when the SERVER offers it, the ENGINE dispatches - it and the CLIENT can seed it; two out of three ships an invisible feature. - - ⚠ THE SEED IS DELIBERATELY BLANK, and the server was changed to accept that. `url`, - `selector` and `field` are what a person types in the panel, so requiring them at validation - time would make a freshly-added step refuse itself — the wave-23 illegal-default-seed defect. - `_clean_action_config` stores an unconfigured step and the RUNNER fails closed with a - sentence naming what is missing, which is the same posture the enrich arm takes with - `profileField`. The values below mirror that validator's defaults exactly. - */ - if (kind === "web_read") - return { ...WEB_SEEDS.web_read, field: firstField || "" }; - /* - ⭐⭐ W31 QA — THE OTHER FOUR WEB KINDS, and the comment above predicted this exactly one - kind early. When the owner's revocation of D-51/R5 flipped `web_goto`/`web_fill`/`web_click`/ - `web_repair` to `ready:True`, the catalog-derived seed check went red with - `MISSING: ['web_click','web_fill','web_goto','web_repair']` — all four would have painted - faded and PERMANENTLY UNCLICKABLE while the server, the seam and the runner all offered them. - Three out of four is still an invisible feature. - - ⚠ BLANK FOR THE SAME REASON `web_read` IS BLANK: `url` / `selector` / `value` are what a - person types in the panel, so seeding them with anything real would either lie or refuse - itself at validation time. `_clean_action_config` stores the unconfigured step and the RUNNER - fails closed naming what is missing (`WEB_REQUIRED`, per kind) — which is why these seeds - carry no field the validator would reject and no field the runner would silently ignore. - ⚠ `web_fill` seeds `secret: false` EXPLICITLY rather than omitting it: the flag decides - whether the typed value is masked in the run log, and a masking control that only exists - once somebody discovers it is a control most people never turn on. - */ - // ⚠ W33-T52 — one branch over `WEB_SEEDS`, not four literals. The seed and the config panel - // now read the same table, so a sixth web kind is ONE edit and cannot ship addable-but- - // unconfigurable the way all five of these did. - if (isWeb(kind)) return { ...WEB_SEEDS[kind] }; - // W33-T56 — the fuzzy step. Its own seed, because it is not a `web_*` kind (see AI_AGENT_SEED). - if (kind === "ai_agent") return { ...AI_AGENT_SEED }; - /* - ⭐⭐ 2026-08-07 (owner report) — THE ENRICH ACTION HAD NO BRANCH HERE, SO IT WAS - PERMANENTLY UNCLICKABLE. `disabled={!c.ready || busy || !seedFor(c.kind)}` and this - function's `return null` fallback meant the row rendered faded, labelled "Needs a - database", with an EMPTY tooltip (`seedBlock` had no sentence for it either). The server - declared it `ready:true`, its config panel was fully built and its runner was correct — - one client-side list nobody updated made the whole capability unreachable. Owner: - *"The enrichment action, says that it needs a database. So I can't even click it and - assign a database? thats wicked."* - - ⛔ IT TAKES NO TABLE, AND THAT IS WHY THE OLD LABEL WAS ALSO WRONG. Enrichment acts on - the record WALKING THE FLOW — the flow's own target database — so there is nothing to - assign per action. `verify_automation`'s seed-coverage check now derives its subject from - the server catalog, so the NEXT ready action added without a seed goes red instead of - shipping invisible. - - The values mirror `clean_action_config`'s defaults exactly. A seed that disagreed with the - validator is the wave-23 illegal-seed defect (four kinds shipped a red banner and no card). - */ - /* - ⭐⭐ WAVE 30 · W30-T23 / CONTRACT C3 — ONE ARM FOR BOTH ENRICH KINDS. - The server's own validator branches on `ENRICH_KINDS = ("enrich_instagram","enrich_tiktok")` - — ONE branch, one set of keys, with `postMetrics`/`commentMetrics` answered `False` for - TikTok until W30-T10 wires post capture. So the seed is shared for the same reason: a seed - that disagreed with the validator is the wave-23 illegal-seed defect (four kinds shipped a - red banner and no card), and TWO seeds for one validator branch is that defect waiting. - ⚠ The two metric keys are seeded `false` for BOTH, which is what the server already answers - for TikTok — so nothing changes shape when T10 deletes its `and kind != "enrich_tiktok"` - clause. Seeding them only for Instagram would have made T10 a client change as well. - ⛔ THIS ARM AND ITS `seedBlock` SENTENCE HAD TO LAND TOGETHER. D-79's scar was exactly one - of the two missing: the row rendered faded, labelled "Needs a database" — a claim about - databases, which was not the problem — with an EMPTY tooltip. - */ - if (isEnrich(kind)) - return { - // ⭐ WAVE 28 · R5/R6 — `tier: "anonymous"` and `noFallback: false` are GONE from the seed. - // R5 retired the free ladder from enrichment, so a new action must not be born naming a - // source strategy the runner no longer consults. The server keeps ACCEPTING both keys on - // stored definitions and ignores them (C2, D-65's law) — but a SEED is what a new step - // starts with, and seeding a dead key is how a retired concept outlives its removal. - // ⚠ `postMetrics`/`commentMetrics` stay `false` here and that is R6's ruling verbatim - // ("always both off until toggled on"), matching `clean_action_config`'s reading of an - // absent key so the seed and the validator cannot disagree. - postMetrics: false, commentMetrics: false, - dryRun: false, maxPosts: 10, - fromView: "", sortField: "first_found", sortDir: "desc", - // ⚠ `skipRecent: true` MATCHES `_ensure_enrich_step`'s server seed, deliberately. One - // control must not have two defaults depending on whether the step was added by hand or - // seeded onto an Instagram search — that is the split-default class this repo has paid - // for twice ([[default-must-pass-its-own-guard]]). It also cannot surprise on first use: - // nothing has an `Enriched at` yet, so the cooldown changes nothing until run two, which - // is exactly when not re-paying is what you want. - // ⛔ NOT the validator's default. `clean_action_config` reads an ABSENT key as False, so - // every enrich action stored before today keeps the behaviour it already had — a seed is - // what a new step starts with, never a rule applied retroactively (D-65). - limit: 25, skipRecent: true, skipRecentDays: 30, - }; - if (kind === "group") - /* - ⭐ RE-SEEDED FOR C-FORK, and the old seed would now be REFUSED. It was - `{cond: null, actions: [child]}` — the pre-wave-24 shape, which `clean_actions` no longer - accepts as a config: the fork's legs live under `branches` and `cond`/`actions` are gone - from the group's own config entirely. Clicking "If / then" would have produced a red - banner and no card, which is precisely the illegal-default-seed defect this function's - header was written about, reproduced by the wave that quotes it. - - MEASURED against `clean_actions` rather than reasoned: - · a branch with no actions -> "a branch with no actions inside it does nothing" - · so the one branch is born with the cheapest legal child, exactly as before — not a - review, which would also mint a board stage nobody asked for by clicking "If / then". - ⛔ `label: ""` ON PURPOSE. The SERVER letters the branches by index and calls a null-cond - last leg "Otherwise", but it PRESERVES any label the client sends — so sending one would - freeze it, and a branch labelled "A" would still say "A" after the branch above it was - deleted. Sending nothing keeps the lettering correct by construction, and this fresh - single branch reads "Otherwise" until it is given a condition, which is what it is. - */ - return firstField - ? { - branches: [ - { - id: "b1", - label: "", - cond: null, - actions: [ - { id: "", kind: "update_record", enabled: true, when: null, - config: { values: { [firstField]: "" } } }, - ], - }, - ], - } - : null; - return null; - }; - - /** Why a kind cannot be added yet — the server's own precondition, said before the click. */ - const seedBlock = (kind: string): string => { - if (kind === "update_record" || kind === "group") - return "Name the trigger's database first — this needs a column to write."; - if (kind === "create_record" || kind === "find_records") - return "There is no blank database to point at yet."; - /* - ⛔ NOT `""`. An empty string here renders "Needs a database" with NO tooltip — a disabled - control that will not say why, which is exactly how the enrich action sat unreachable and - unexplained. A kind that reaches this line is one somebody forgot to seed, so the fallback - says the true thing rather than the plausible one: the label above claims a database is - missing, and for a forgotten seed that claim is a guess. - */ - return "This action cannot be added yet — it has no starting configuration."; - }; - - /** - * ⚠ `into` NAMES A BRANCH NOW, not just a group (C-FORK). A fork has several legs and "add an - * action to this group" stopped being a complete instruction the moment it did — putting it in - * the first branch by default would silently attach work to whichever leg happens to be first. - */ - const addAction = (kind: string, into?: { actionId: string; branchId: string }) => { - const seed = seedFor(kind); - if (!seed) return; - const fresh: Action = { id: "", kind, enabled: true, when: null, config: seed }; - if (!into) { - patchFlow([...actions, fresh]); - } else { - patchFlow( - mapTree(actions, (a) => { - if (a.id !== into.actionId) return a; - return { - ...a, - config: { - ...a.config, - branches: groupBranches(a).map((br) => - br.id === into.branchId - ? { ...br, actions: [...(br.actions || []), fresh] } - : br - ), - }, - }; - }) - ); - } - setAdding(false); - }; - - /** Add an empty-conditioned leg to a fork. The server letters it and enforces the ceiling. */ - const addBranch = (actionId: string) => { - const seedChild = seedFor("update_record"); - if (!seedChild) return; - patchFlow( - mapTree(actions, (a) => { - if (a.id !== actionId) return a; - const brs = groupBranches(a); - return { - ...a, - config: { - ...a.config, - // ⚠ APPENDED LAST, and the server refuses a null-cond branch that is not last — so a - // fork that already ends in an Otherwise leg refuses this with its own sentence - // rather than silently burying the catch-all in the middle, where every branch below - // it would be dead code. - branches: [ - ...brs, - { id: "", label: "", cond: null, - actions: [{ id: "", kind: "update_record", enabled: true, when: null, - config: seedChild }] }, - ], - }, - }; - }) - ); - }; - - const dropBranch = (actionId: string, branchId: string) => - patchFlow( - mapTree(actions, (a) => { - if (a.id !== actionId) return a; - const kept = groupBranches(a).filter((br) => br.id !== branchId); - // A fork with no legs at all "does nothing" — the server says so; do not send an empty - // list dressed up as an edit. - if (!kept.length) return a; - return { ...a, config: { ...a.config, branches: kept } }; - }) - ); - - /** - * R8's numbers for the whole flow, computed ONCE per render rather than per card — the map is - * built by walking the tree, so asking it per card would be quadratic in a deep fork, and more - * importantly a per-card computation could not see its siblings and would have to re-derive - * the position it is being told. - */ - const stepNos = numberActions(actions); - - /** - * ⭐ THE ACTION MENU'S GROUP ORDER, FROM THE SERVER (owner item 12b, contract C-ACT). - * - * ⛔ NEVER A LITERAL LIST OF GROUP NAMES HERE. The server ships `groupOrder` on every catalog - * row (1 Web action · 2 Database · 3 Connected · 4 Advanced logic, derived server-side from one - * table so two rows cannot disagree about their own group). A client - * `["Web action", "Database", …]` would be a second copy of the catalog's structure, and the - * day the engine adds or renames a group the client drops it off the end — or drops it - * entirely — without anything going red. - * - * ⚠ AN UNORDERED GROUP SORTS LAST, NOT FIRST, which the server's own gate asserts. `Infinity` - * and not `0`: a missing number meant "before everything" under a naive `|| 0`, so a server - * older than this client would have led its menu with whatever it had failed to classify. - * ⚠ The sort is STABLE over first appearance, so two groups sharing an order keep the payload's - * sequence rather than an arbitrary one. - */ - /* ⭐ WAVE 27 · ITEM 33 / C4 — the ordering above moved into `steps.ts` as `groupActions`, - which does the SAME arithmetic and additionally nests connector rows. Two reasons it moved - rather than gaining a clause here: `groupTriggers` already does exactly this for the trigger - picker and the two must not drift, and a pure function in `steps.ts` is one node can run — - `verify_steps` exercises the nest, which no amount of JSX here could be tested against. */ - const menuGroups = groupActions(catalog); - - /** ONE action row, so the nested and un-nested lists cannot render differently. */ - const actionRow = (c: ActionCatalogRow) => ( - - ); - - // ── the centre column ──────────────────────────────────────────────────────────────────── - const chip = () => { - if (!chosen) return null; - if (!configured) - return ( - - Finish configuration - - ); - if (trigger?.paused) - return Paused; - if (picked && picked.ready === false) - return Not set up; - const last = automation.runs?.[0]; - if (last) - return ( - - {last.ok ? "Last run succeeded" : "Last run failed"} - - ); - return null; - }; - - const actionCard = (a: Action, depth: number, hasNext = false): ReactNode => { - const row = catalog.find((c) => c.kind === a.kind); - const isGroup = a.kind === "group"; - const branches = isGroup ? groupBranches(a) : []; - /** - * ⭐ THE PERMANENT FIRST STEP (owner ruling 2026-08-06). An Instagram search produces rows - * and has nowhere to put them until something writes them, so Create record is step 1 and - * stays. DERIVED from the same rule the server enforces (`ig_action_pinned`) rather than - * from a stored flag — and the control is HIDDEN rather than disabled, because the server - * re-inserts the action on the next save, so a delete button here would appear to work and - * then silently undo itself. - */ - const pinned = - automation.kind === "discover_instagram" && depth === 0 && a.id === actions[0]?.id; - /** - * ⭐⭐ WAVE 32 · T45 (owner item 10) — CONFIGURED / UNCONFIGURED, and the SERVER decides. - * - * ⛔ NOT COMPUTED HERE. The same `engine.action_needs` that fills this list also refuses the - * run (`400 action_unconfigured`, and `run_now` for the tick and the webhook), so a card - * cannot claim Configured over an action the run will reject. A client-side "does it look - * filled in" test would be a second rule that agrees until one of them learns a new key. - * ⚠ Nested cards get it too: the list is keyed by ACTION ID over the whole flow, branches - * included, and a step inside an If / then is the one a person is least able to see. - */ - const needs = (automation.unconfigured || []).find((u) => u.id === a.id)?.needs || []; - /** Only TOP-LEVEL cards reorder (see `moveAction`) — depth 0, and never while a write is up. */ - const canDrag = depth === 0 && !busy && !pinned; - return ( -
-
{ - draggedRef.current = true; - e.dataTransfer.effectAllowed = "move"; - e.dataTransfer.setData(ACTION_DRAG_TYPE, a.id); - } - : undefined - } - onDragOver={ - canDrag - ? (e) => { - // Somebody else's drag — a file, a nav row, selected text — is not ours to - // accept. Without this test the card would light up for anything dragged over - // it and then swallow the drop. - if (!e.dataTransfer.types.includes(ACTION_DRAG_TYPE)) return; - e.preventDefault(); - e.dataTransfer.dropEffect = "move"; - setDragOver(a.id); - } - : undefined - } - onDragLeave={ - canDrag - ? (e) => { - // Moving onto a CHILD of this card is not leaving it; without this the - // highlight flickers off every time the pointer crosses the title. - if (e.currentTarget.contains(e.relatedTarget as Node)) return; - setDragOver((cur) => (cur === a.id ? "" : cur)); - } - : undefined - } - onDrop={ - canDrag - ? (e) => { - const dragId = e.dataTransfer.getData(ACTION_DRAG_TYPE); - setDragOver(""); - if (!dragId) return; - e.preventDefault(); - moveAction(dragId, a.id); - } - : undefined - } - onDragEnd={() => { - setDragOver(""); - // Cleared a tick later: the post-drag click arrives BEFORE this would run. - window.setTimeout(() => { - draggedRef.current = false; - }, 0); - }} - > - - {pinned ? ( - - Always first - - ) : ( - - )} -
- - {/* - ⭐ THE FORK (owner item 12a, ruling R8, contract C-FORK). A group used to hold ONE - nested column; it now holds a LETTERED LANE PER BRANCH, because R8 rules that a fork - occupies one step number and its legs are alternatives rather than later steps. - - THE LETTER AND THE "Otherwise" ARE THE SERVER'S WORDS — `br.label`, printed, never - composed. `clean_actions` letters by index and names a null-cond last leg "Otherwise", - so the lanes re-letter themselves correctly when one is deleted. - The nesting ceiling is still the SERVER's `maxGroupDepth`, so the add-here affordance - disappears at exactly the depth `clean_actions` refuses. - */} - {isGroup ? ( -
- {branches.map((br) => ( -
-
- {br.label} - - {/* ⚠ NOT A RENDERED CONDITION — the tree is edited in Properties, and a - second editable copy here would be two controls for one fact. This says - only WHICH leg you are looking at. A null cond is the catch-all, and the - server has already labelled it, so this says nothing twice. */} - {br.cond ? "when its conditions match" : "everything else"} - - {branches.length > 1 ? ( - - ) : null} -
-
- {(br.actions || []).map((k) => actionCard(k, depth + 1))} - {depth + 1 < (vocab?.maxGroupDepth ?? 2) || !(br.actions || []).length ? ( - - ) : null} -
-
- ))} - -
- ) : null} - - {/* - THE ARROW BACK INTO THE FLOW (item 12a). The lanes are alternatives and exactly one of - them runs — the engine takes the first matching branch and breaks — so they REJOIN, and - without a mark saying so a fork at the end of a column reads as several parallel endings. - Drawn only when there IS a next step to rejoin: an arrow into nothing is a promise the - flow does not keep. - */} - {isGroup && hasNext ? ( - - ) : null} -
- ); - }; - - return ( - <> -
- {/* ── TRIGGER ─────────────────────────────────────────────────────────────────── */} -
-
- Trigger - {chip()} -
-
- {chosen && options.length ? ( -
- -
- ) : ( - /* - THE EMPTY STATE (image 1): a dashed add-box and the server's own suggested - triggers. One line of chrome, no tour (R13) — the list IS the explanation, and - it is the server's list so it cannot describe a trigger we removed. - */ - <> - {/* THE BOX AND ITS MENU ARE ONE UNIT (see `addWrap`). The menu USED to render at - the bottom of this component, outside `.autox-flow` entirely, and became a - third column of the flex row that holds the canvas and Properties — the - owner's ERROR 4. */} -
- - {/* ⭐ WAVE 25 item 5b (C2) — `TriggerPicker`, the SAME component Properties → - Trigger details renders in its compact form. Category first (Time / - Database / Connector) with connector rows nested under their product. */} - {picking ? ( - setPicking(false)} - /> - ) : null} -
- {options.length ? ( -
-

Suggested triggers

- {options - // ⭐ 2026-08-07 (owner ruling) — MANUAL IS OFFERED AGAIN. It was excluded - // (`t.key !== "manual"`) because picking it stored NOTHING and bounced - // straight back to this empty state — "a control that undoes itself is - // worse than no control", correct while that was true. `clean_trigger` now - // STORES `{key:'manual'}`, so the pick sticks, `chosen` goes true and - // Configuration opens. Hiding it now would leave the one trigger a person - // most expects to find as the only one they cannot choose. - .filter((t) => t.ready !== false && !t.planned) - .slice(0, SUGGESTED) - .map((t) => ( - - ))} -
- ) : ( -

This server did not offer a trigger list.

- )} - {/* ONE LINE (R13), and it is what is TRUE of this state rather than a caption - for the box above it: with nothing chosen, Run now is the only thing that - starts this automation. Airtable's empty state means "it cannot run"; ours - does not, and saying so is the difference between a screen that is honest - and one that merely looks the same. */} -

Until then, only Run now starts it.

- - )} -
-
- - {/* ── ACTIONS ─────────────────────────────────────────────────────────────────── */} -
-
- Actions -
-
- {actions.map((a, i) => actionCard(a, 0, i < actions.length - 1))} - - {/* - ⛔ THE MACHINE STEPS ARE NOT CARDS ANY MORE (owner item 5, contract C-CFG). They - were engine-derived read-only cards sitting under the owner's own actions — "fetch - the page", "Bright Data", "Anonymous", "Write" — so the centre column mixed two - different things: what the AUTOMATION is, which nobody chose and nobody can reorder, - and what the OWNER added, which is the whole subject of this builder. Their - configuration and their switches moved to Properties → Configuration → - "How this fetches", one section per panel. - - ⚠ THE ENGINE STILL RUNS THEM. `graph()` emits the same nodes; this surface simply - stops DRAWING them, which is why the switches had to move rather than go — R7 names - Bright Data (money) and Write's dry run (reads and reports without writing a row) as - controls that must survive the cards that carried them. - */} - - - - {/* THE ACTION MENU (images 7/8): the server's catalog, grouped by its own `group` - key, with `ready:false` rows faded and carrying the server's reason. A shorter - menu would imply those actions do not exist — the owner asked to see all of - them, and `clean_actions` refuses the unready ones at the door, so faded is a - wall rather than a decoration. */} - {adding ? ( -
- {menuGroups.map((g) => ( -
-

{g.key}

- {/* ⭐ WAVE 27 · ITEM 33 / C4 — the group's OWN rows first, then one nest per - connector. `
` rather than a hand-built disclosure: it IS the - chevron-and-submenu the reference shows, it opens on Enter and Space - without a keydown handler, and it needs no open/closed state of its own - to get wrong. Closed by default — the reference shows collapsed rows, - and a connector nobody is using should cost one line, not six. */} - {[...g.rows, ...g.sub].map((entry) => ("rows" in entry ? ( -
- - {/* ⭐ WAVE 30 · ITEM 7 / R4 — THE COLLAPSED ROW GETS AN IDENTITY. - This is the row the owner clicks before choosing Instagram or - TikTok, and until now it was the only row in the menu with - nothing on its left: the chevron, then a bare word. The mark is - resolved from the connector KEY, so a connector the server adds - tomorrow gets a slot for free and simply renders no logo. */} - {brandForConnector(entry.key) ? ( - - {brandForConnector(entry.key)} - - ) : null} - - {entry.label} - - {entry.rows.length} action{entry.rows.length === 1 ? "" : "s"} - - - - {entry.rows.map((c) => actionRow(c))} -
- ) : actionRow(entry)))} -
- ))} -
- ) : null} -
-
-
- - {/* ── PROPERTIES ──────────────────────────────────────────────────────────────────── */} - - - {/* THE CONFIRM (image 6). A trigger change can invalidate the configuration under it, so - it ASKS — and it says what it will cost rather than "are you sure". */} - {confirmKey ? ( -
-

Change the trigger?

-

- Anything configured for {picked?.label || triggerKey} is dropped. -

-
- - -
-
- ) : null} - - {/* ⛔ THE PICKER USED TO RENDER HERE, and here is outside both columns. `.auto-work` is a - flex row of `.autox-flow` + `.autox-props`, so a third child was laid out as a third - COLUMN — the menu appeared past the right edge of the Properties panel and squeezed the - canvas until the suggested triggers wrapped one word per line (owner's ERROR 4). It is - rendered beside its own "+ Add trigger" box now; see `addWrap`. */} - - ); -} - -/** One action's subtitle — what it will DO, composed from its own config. */ -function actionSub(a: Action, tables: UserTable[]): string { - const cfg = a.config || {}; - if (a.kind === "update_record") { - const keys = Object.keys((cfg as { values?: Record }).values || {}) - .filter(Boolean); - return keys.length ? `Sets ${keys.join(", ")}` : "No values yet"; - } - if (a.kind === "create_record") { - const t = String((cfg as { table?: string }).table || ""); - return t ? `Into ${tableLabel(tables, t)}` : "No database yet"; - } - if (a.kind === "find_records") { - const t = String((cfg as { table?: string }).table || ""); - return t ? `In ${tableLabel(tables, t)}` : "No database yet"; - } - return ""; -} - -function findAction(list: Action[], id: string): Action | null { - for (const a of list) { - if (a.id === id) return a; - // ⚠ THROUGH THE BRANCHES (C-FORK). This read `config.actions`, which a fork no longer has — - // so selecting any action inside one would have found nothing and the Properties panel would - // have said "that action is no longer part of this flow" about a card visibly on screen. - if (a.kind === "group") - for (const br of groupBranches(a)) { - const hit = findAction(br.actions || [], id); - if (hit) return hit; - } - } - return null; -} - -/* - * ⛔ `nodePanel()` IS GONE (item 5). It rendered ONE machine node's panel, chosen by the card the - * reader had clicked, and included a "that step is no longer part of this automation" branch for - * a selection the graph had since dropped. Both facts died with the cards: the panels are now - * rendered together under "How this fetches", keyed by PANEL rather than by node id, so there is - * no selection left to go stale. - */ - -// ── the Properties panel's two faces ───────────────────────────────────────────────────────── - -function TriggerProps({ - automation, - triggerKey, - options, - picked, - table, - tables, - ops, - nullaryOps, - vocab, - oauth, - busy, - nodes, - onToggleNode, - renderNodeBody, - onAskChange, - onPatchTrigger, - onPatchConfig, - onPickTrigger, - onRunNow, - scheduleFace, - schedules, - chosen, -}: { - automation: Automation; - triggerKey: string; - options: TriggerOption[]; - picked: TriggerOption | null; - table: UserTable | null; - tables: UserTable[]; - ops: string[]; - nullaryOps: string[]; - vocab?: FlowVocab; - oauth: OAuthStatus | null; - busy: boolean; - /** The engine's machine steps (item 5) — their switches and bodies live under Configuration. */ - nodes: GraphNode[]; - onToggleNode: (nodeId: string) => void; - renderNodeBody: (panel: string) => ReactNode; - /** Ask before a trigger change that can invalidate the config under it (image 6). */ - onAskChange: (key: string) => void; - onPatchTrigger: (patch: Record) => void; - /** Writes the automation's OWN config — the database a table-less trigger has nowhere - * else to name (owner report 2026-08-07). Spreads on the way in; see `patchConfig`. */ - onPatchConfig: (patch: Record) => void; - onPickTrigger: (key: string) => void; - onRunNow: () => void; - scheduleFace: ReactNode; - schedules: boolean; - /** - * ⭐ WAVE 25 item 5b (ruling R14) — HAS A TRIGGER BEEN PICKED AT ALL? - * - * REQUIRED and passed IN, never re-derived: it is the same `chosen` the centre column gates its - * empty state on, and the honest test is what the DEFINITION holds rather than what the picker - * displays (`triggerKey` derives "manual" whenever nothing is stored, so anything computed from - * it is true for every automation ever — a mistake this file already made once and only a - * screenshot caught). - */ - chosen: boolean; -}) { - const trigger = automation.trigger || null; - const provider = picked?.connect?.provider || ""; - const connected = !!(provider && oauth?.[provider]?.connected); - const startUrl = picked?.connect?.startUrl || ""; - const needsTable = !!trigger && "table" in trigger; - /** - * ⭐⭐ 2026-08-07 — DOES THIS AUTOMATION HAVE TO NAME ITS OWN DATABASE? - * - * True for a `plain` automation whose trigger carries no table — which INCLUDES the state where - * no trigger is stored at all, and that inclusion is the whole fix. `clean_trigger` stores - * nothing for `manual`/`schedule` by design, so those automations hold `trigger: null`, `chosen` - * is false, and W25/R14's "Configuration does not render until a trigger is picked" hid the - * Database picker below from the exact automations that cannot get a database any other way. - * The owner hit it twice: *"when the Trigger is Manual, it says that I need to bound it to a - * database, how?"* — and the honest answer was that there was no how. - * - * ⚠ It is the SAME condition the picker itself renders on, named once and used twice, so the - * section cannot open without the control or the control appear without its section. - */ - const needsOwnTable = automation.kind === "plain" && !needsTable; - const last = automation.runs?.[0]; - /** ITEM 22 / D-70 / R12 — the shared Run guard (see `runBlock`). */ - const runState = runBlock(automation); - - return ( - <> -

Trigger details

-
- {/* - ⭐ WAVE 25 item 5b (C2) — THE NATIVE ` onPatchTrigger({ table: e.target.value })} - data-role="trigger-table" - > - - {trigger?.table && !tables.some((t) => t.key === trigger.table) ? ( - - ) : null} - {tables.map((t) => ( - - ))} - -
- ) : null} - - {/* - ⭐⭐ 2026-08-07 (owner report) — THE DATABASE A TABLE-LESS TRIGGER HAS NOWHERE ELSE TO NAME. - Owner: *"how come, enrich instagram when standalone 'manual' trigger, doesn't have a - database that it should point to? dont make this mistake again."* - - MEASURED across the trigger vocabulary: `manual`, `schedule` and `email` carry NO `table`, - so `needsTable` is false and the picker above never rendered — while `AutomationDetail`'s - Properties picker is gated to `scrape_db` / `field_instagram`, BOTH RETIRED KINDS. So - **nothing in the client wrote `config.targetTable` for a `plain` automation**, and - `run_plain` answered *"no database is bound yet — pick one on the trigger, or in - Properties"*: a refusal naming two doors, neither of which existed. Every scheduled or - manual flow — the owner's *"on a schedule, enrich these sets of influencer names"* — was - unbuildable. - - ⛔ `plain` ONLY, and that is what keeps it from being a second control for one fact. Every - other kind names its database somewhere of its own: an Instagram search through the pinned - Create record (W25/R2 made that action authoritative and `targetTable` follows it), and the - two retired kinds through their own Properties panels. Widening this would put two pickers - on one key and let them disagree, which is the exact defect R2 was written to end. - */} - {needsOwnTable ? ( -
- - -

- The records this automation’s steps walk. A trigger like Manual or a schedule does - not name one on its own. -

-
- ) : null} - - {triggerKey === "event_field" ? ( - <> - {/* - ⛔ "WATCHED COLUMN" IS GONE (owner item 7, C-TRIG law 4). "When a record matches - conditions" is a FILTER, and a watched column was a second, different question - answered in the same box — the trigger fired on a column changing AND on the - conditions holding, which is two triggers wearing one name. - ⚠ THE SERVER WENT FIRST, and that ordering was the point: `clean_trigger` stopped - reading `field` (C posted law 4 done) BEFORE this control came out. Deleting the - control while the validator still read the key would have left a stored watched - column that nothing displays and nothing can clear — and deleting the key while the - control still showed it would have made every Save silently drop what the user - picked. The CONDITION is what completes this trigger now: with none, it rides - `configured:false` and says so. - */} -

- * Conditions -

- onPatchTrigger({ when: next })} - fields={table?.fields || []} - ops={ops} - nullaryOps={nullaryOps} - maxDepth={vocab?.maxCondDepth ?? 3} - maxChildren={vocab?.maxCondChildren ?? 12} - disabled={busy} - /> - - ) : null} - - {triggerKey === "record_updated" ? ( -
- - -

Select none to fire on any column.

-
- ) : null} - - {triggerKey === "enters_view" ? ( -
- - {/* - ⭐ FOUR STATES, AND THREE OF THEM ARE SENTENCES (owner item 8, C-TYPES). - `views` now rides `GET /automations/tables`, so this control is live — but "drop the - note" would have collapsed it to two states and rebuilt the exact defect the contract - forbids. Absent and empty are DIFFERENT FACTS: - · no database chosen -> choose one first - · `views` ABSENT -> the server offered nothing (a server older than this - client). Saying "no views" here would be a claim - nobody measured. - · `views` present and [] -> this database genuinely has none. That IS measured, - and it is the one that tells the reader to go and - make a view rather than to go and find an admin. - · non-empty -> the picker. - */} - {!table ? ( -

Choose a database first.

- ) : !table.views ? ( -

This server did not offer a view list for that database.

- ) : !table.views.length ? ( -

- That database has no saved views yet — make one on its grid and it appears here. -

- ) : ( - whose `value` matches no - ) : null} - {table.views.map((v) => ( - - ))} - - )} -
- ) : null} - - {triggerKey === "email" ? ( -
- - onPatchTrigger({ query: e.target.value })} - /> -
- ) : null} - - {triggerKey === "webhook" && trigger?.token ? ( -
- - -

Minted once. It survives every other change to this trigger.

-
- ) : null} - - {/* THE SCHEDULE, rendered by the one component that owns the cron round-trip. - ⚠ NOT `triggerKey === "schedule"` ANY MORE (item 7, C-TRIG). `ig_profile_match` watches - nothing — it MAKES rows, on the cron — so its Configuration carries these controls - "always", and the caller says which keys those are. */} - {schedules ? scheduleFace : null} - {triggerKey === "manual" ? ( -

- It runs when you press Run once now, and nothing else starts it. -

- ) : null} - - )} - - {/* - ⭐ HOW THIS FETCHES (owner item 5, contract C-CFG) — the machine steps' switches and - config bodies, below the trigger's own fields, under ONE sub-heading. - - ⛔ ONE SECTION PER PANEL, NOT PER NODE (`groupByPanel`, and its note is the reason). Four - of `field_instagram`'s nodes share `panel: "capture"` and both of `discover_instagram`'s - share `panel: "find"`, so a section per node would print one body four times and mount the - discovery filter twice against a single `preds` array. - - ⛔ AND THE SWITCHES ARE THE POINT, not a leftover. R7: Bright Data buys exact counts with - MONEY and Write's switch is the difference between a run that writes rows and one that - reads and reports. They were on the cards this wave deleted, so deleting the cards without - moving them would have retired two shipped controls silently. Each still posts to the - node-toggle door — the server decides what a switch means, this only says which node. - - `plain` automations have no machine nodes at all, so the whole section is absent for them - rather than an empty heading (R6 makes that the common case from now on). - - ⭐ WAVE 26 · ITEM 17 / R15 — THE PROSE IS GONE AND THE SWITCHES STAYED, which is the whole - ruling. The owner quoted this section back verbatim — *"How this fetches / Find profiles / - Bio contains skincare… / Collect results / Takes about 20 minutes / Up to 10 profiles · - about $0.025"* — and asked for it gone. Every line of that quote is server-composed - (`automation_engine.graph`), and it maps EXACTLY onto three renderers, which is why the - deletion could be surgical rather than a section removal: - · `n.subtitle` -> "Bio contains skincare…" and "Takes about 20 minutes" (`:5030`, `:5036`) - · `g.nodes[0].detail` -> "Up to 10 profiles · about $0.025" (`:5032`) — the ESTIMATE LINE - · `n.title`/`

` -> "How this fetches", "Find profiles", "Collect results" — KEPT - ⛔ THE HEADING WAS A SYMPTOM, NOT THE SUBJECT. Deleting the section would have retired the - Bright Data MONEY switch and the write node's DRY RUN with it — the two controls that were - moved here precisely so deleting the cards would not retire them silently. They are the - `n.toggle` block below and they are untouched. - ⚠ `AutomationFind`'s "What it costs" estimate is NOT this line and is NOT in scope: it is - the ANSWER to a button a person pressed, and it is the only surface carrying D-23's - SPEC-basis caveat. The read-only paragraph went; the control did not. - */} - {nodes.length ? ( - <> -

How this fetches

- {groupByPanel(nodes).map((g) => ( -
- {g.nodes.map((n) => ( -
- - - - - {n.title} - - {/* ⭐ ITEM 18 / R14 — the per-node status dot stood here and is deleted. The - node already carries its own `ActionMark` above; a coloured dot beside it was - the second thing on one row claiming to say what this step is. */} - {n.toggle ? ( - - ) : null} -
- ))} - {/* ⭐ ITEM 17 / R15 — THE ESTIMATE LINE STOOD HERE. `g.nodes[0].detail` is where - "Up to 10 profiles · about $0.025" reached the screen (composed at - `automation_engine.py:5032`), and the owner named it. It was the engine's own - sentence rather than a client paraphrase, which is why it was defensible and - why it still had to go: an accurate paragraph nobody asked for is still the - architecture explaining itself (DESIGN.md §4). - ⚠ The COST question keeps a home — "What it costs" in the Find panel below, - behind a press, carrying its own SPEC caveat. This line stated a price nobody - had asked for beside a control that was not about price. */} - {renderNodeBody(g.panel)} -
- ))} - - ) : null} - -

Test step

- {/* - ⛔ HONEST TO OUR SEMANTICS, not to Airtable's. Airtable's "Test step" replays one step - against a chosen record. Ours has no step-replay: the engine runs a whole automation. So - the control says what it actually does — RUN IT — and the results below are the last - real run's, from the run log, rather than a rehearsal nobody performed. - */} - {/* ⭐ ITEM 22 / D-70 / R12 — THE SECOND DOOR ONTO THE SAME MONEY, guarded by the SAME - function. It was `!!automation.running`, which is process state and therefore false for - the entire 20-30 minute vendor wait; `runBlock` is the one definition both buttons read, - so a fix cannot land on one and miss the other. */} - - {runState.blocked && !automation.running ? ( -

{runState.why}

- ) : null} - {last ? ( - <> -

- {last.ok ? "Last run succeeded" : "Last run failed"} -

-

- {last.ts.replace("T", " ").slice(0, 16)} — {last.summary} -

- - ) : ( -

It has not run yet.

- )} - - ); -} - -/** - * ⭐ WAVE 26 · ITEM 9 / R11 — "+ New database", INSIDE the action's own database picker. - * - * Owner ruling R11, verbatim: *"name it and go. No dialog stack; columns follow what the action - * writes (how the IG presets already work)."* So this is a name box and a button, in place, and - * it selects what it creates — never a modal over a modal, and never a trip to another surface - * that loses the action being configured. - * - * ⛔ IT DOES NOT INVENT THE ROW IT JUST MADE. `POST /tables` answers `{key}` alone, so the - * component asks the caller to RE-LIST and only then selects the key. Splicing a client-built - * `UserTable` in would put a guessed `rowCount` and an empty `fields` on screen beside real ones, - * and the picker renders `rowCount` — a fabricated measurement, which is the one thing this - * codebase refuses everywhere else. - * - * ⚠ THE REFUSAL IS THE SERVER'S, PRINTED VERBATIM. `POST /tables` has real refusals a person can - * hit — an empty name, the `MAX_TABLES` ceiling, an unavailable store — and each answers with its - * own sentence. Re-deriving "you have too many databases" here would be a second copy of a rule - * that lives on the other side of the wall ([[schema-role-is-not-a-value-wall]]: find the line - * that ENFORCES it, and let that line do the talking). - */ -function NewDatabase({ - disabled, - onCreated, -}: { - disabled: boolean; - /** - * Called with the new key AFTER the caller's table list has been re-read. - * ⚠ IT MAY RETURN A PROMISE AND THE CALLER MUST AWAIT IT. Selecting the key before the list - * contains it makes the picker's own "stored value is always an option" guard paint - * "(not visible to you)" about the database the user just created — the guard doing exactly its - * job, on a race, and saying something alarming and false. - */ - onCreated: (key: string) => Promise | void; -}) { - const [open, setOpen] = useState(false); - const [name, setName] = useState(""); - const [busy, setBusy] = useState(false); - const [err, setErr] = useState(""); - - const create = () => { - const label = name.trim(); - if (!label || busy) return; - setBusy(true); - setErr(""); - createTable(label) - .then(async (r) => { - setOpen(false); - setName(""); - // AWAITED: the caller re-reads the table list, and only then may the key be selected. - await onCreated(r.key); - }) - .catch((e) => { - // The server's own words. `AutomationError` carries the composed sentence; anything else - // is a transport failure and says so rather than blaming the name. - setErr( - e instanceof AutomationError && e.message - ? e.message - : "could not create it — the workspace did not answer" - ); - }) - .finally(() => setBusy(false)); - }; - - if (!open) { - return ( - - ); - } - return ( -
- setName(e.target.value)} - // Enter creates, Escape backs out — the two keys a one-field form owes its user. - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); - create(); - } else if (e.key === "Escape") { - setOpen(false); - setErr(""); - } - }} - /> - - - {err ?

{err}

: null} -
- ); -} - -/** - * ⭐⭐ WAVE 33 · W33-T52 (owner item 16) — THE WEB ACTION'S CONFIGURATION, which did not exist. - * - * One component for all five kinds rather than five arms, because the kinds differ ONLY in which - * keys they carry — and that list already exists, once, in `WEB_SEEDS`. A per-kind arm would have - * been a fifth place the same key set is written down, and the previous four copies are why this - * ticket exists at all. - * - * ⛔ THE ORDER OF THE CONTROLS IS `WEB_SEEDS`' OWN KEY ORDER. Not a second list: an object literal - * preserves insertion order in every engine this ships to, so the table that says which keys a - * kind has also says what a person reads first. Adding a key to the seed adds its control. - * ⛔ THE RED MARK IS THE SERVER'S, not this file's. `vocab.actionRequired[kind]` is - * `ACTION_REQUIRED` on the wire — the same table `unconfigured_actions` and the run refusal read — - * so a control marked optional here is a control the runner will start on. With no vocabulary - * (a server older than this client) NOTHING is marked, which is the honest degradation: inventing - * a requirement the save door does not have would refuse a step that would have run. - * ⚠ EXPORTED for the same reason `ActionProps` is: a panel's defects (a control bound to no key, - * two controls bound to one key, a checkbox that paints unchecked over a stored `true`) are - * invisible to a source grep and need the real thing mounted. - */ -export function WebActionConfig({ - kind, - cfg, - required, - walkFields, - walkTable, - busy, - setCfg, -}: { - kind: string; - cfg: Record; - /** `ACTION_REQUIRED[kind]` off the wire — `[]` when the server did not send it. */ - required: { phrase: string; key: string }[]; - walkFields: { key: string; label: string; type: string }[]; - /** `""` when no record walks this flow — the R10 sentence, not an empty picker. */ - walkTable: string; - busy: boolean; - setCfg: (patch: Record) => void; -}) { - const shape = WEB_SEEDS[kind]; - if (!shape) return null; - const req = new Set(required.map((r) => r.key)); - const text = (v: unknown) => (v === undefined || v === null ? "" : String(v)); - const mark = (key: string) => (req.has(key) ? * : null); - - /* Every control is UNCONTROLLED (`defaultValue` + `onBlur`), which is this file's text idiom — - see the `find_records` limit box. It matters more here than there: `setCfg` merges into the - action and re-renders the whole builder, so a controlled input would round-trip the tree on - every keystroke of a CSS selector. */ - const line = (key: string, label: string, hint = "", mono = false) => ( -
- - setCfg({ [key]: e.target.value })} - /> - {hint ?

{hint}

: null} -
- ); - const flag = (key: string, label: string) => ( - - ); - - return ( - <> - {Object.keys(shape).map((key) => { - if (key === "url") - /* ⛔ THE HINT IS PER KIND, because the answer is. `web_goto` REQUIRES an address (the - server marks it), while the other four act on the page the journey is already on and - an address is how you START one — leaving it blank is the normal case for a step that - follows another. The first draft said "The page this step opens" on all five, which - reads as mandatory on the four where it is not, and invites a person to paste the same - URL into every step of a journey (found by this ticket's own verifier). */ - return line("url", "Web address", - (kind === "web_goto" - ? "The page this step opens." - : "Leave this blank to act on the page the flow is already on. Fill it in only to " - + "start somewhere new.") - + " Braces read a column off the record this flow is walking, so {{Website}} visits a " - + "different page for every row."); - if (key === "waitFor") - return line("waitFor", "Wait for this to appear first", - "A CSS selector. The step waits for it before reporting success — useful on a page " - + "that fills itself in after it loads. Leave blank to not wait.", true); - if (key === "selector") - return line("selector", "CSS selector", - "What on the page to act on — for example h1, .price, or #email.", true); - if (key === "value") - return line("value", "Value to type", - "Braces work here too, so {{Email}} types each row's own address."); - if (key === "hint") - return line("hint", "What to look for", - "Describe the thing in words. This step proposes a selector for it and writes the " - + "proposal to the run log; it changes nothing on the page."); - if (key === "attr") - return line("attr", "What to take", - "Leave this as `text` to take what a person would read. Name an attribute instead — " - + "href, src — to take that."); - if (key === "all") return flag("all", "Take every match, not just the first"); - if (key === "secret") - return flag("secret", "Hide this value in the run log (for a password or a token)"); - if (key === "dryRun") - return flag("dryRun", "Rehearse only — find the element and report what it would do"); - if (key === "field") { - /* ⛔ R10's RULE, and this key is the one place on the panel it applies. The value is a - column on the database the flow WALKS — `apply_actions` writes it back onto that row — - so with no walking record there is no column to choose and a picker with no options - would read as "this database has no columns". The sentence points at the surface that - can fix it, exactly as the condition editor below does. */ - if (!walkTable) - return ( -
- -

- This flow has no record to write to — give it a database on the trigger - first. -

-
- ); - const cur = text(cfg.field); - return ( -
- - ` whose `value` matches no - ` - ) : null} - {walkFields.map((f) => ( - - ))} - -
- ); - } - if (key === "timeoutMs") - return ( -
- - { - const n = Math.min(WEB_TIMEOUT_MAX, - Math.max(WEB_TIMEOUT_MIN, Number(e.target.value) || 20000)); - e.target.value = String(n); - setCfg({ timeoutMs: n }); - }} - /> -

- Between {WEB_TIMEOUT_MIN.toLocaleString()} and {WEB_TIMEOUT_MAX.toLocaleString()}. - Anything outside is pulled in. -

-
- ); - return null; - })} -

- This step runs in a browser job of its own, which takes about 10 to 30 seconds. A run stops - after 50 of them. -

- - ); -} - -/** - * ⭐ WAVE 28 — EXPORTED so the include-row render suite can mount the real panel. - * - * ⛔ THE EXPORT IS THE POINT, not a convenience. R6's three include rows are markup whose defect - * modes are invisible to a source grep: a row that renders with no key attached, a Profile - * checkbox that paints unchecked, two rows bound to the same key. `ReviewProps` below is already - * exported for the same reason, so this is the file's existing shape rather than a new one. - * `_test/` never ships (`deploy_web.py` excludes `_test/` and `_`-prefixed files both). - */ -export function ActionProps({ - action, - pinned, - catalog, - tables, - onTablesChanged, - walkFields, - walkTable, - ops, - nullaryOps, - vocab, - busy, - onEdit, -}: { - action: Action | null; - /** - * ⭐ THE PERMANENT STEP 1 OF AN INSTAGRAM SEARCH (owner report, 2026-08-06). It is the ONE - * action whose `values` and `uniqueOn` the engine never reads: `apply_actions` drops - * `actions[0]` for `discover_instagram` and the runner does the writing itself, upserting the - * profile columns on `(handle, created_by)`. - * - * The card stays — "this search puts what it finds in a database" is true and worth showing — - * but a `* Values` editor above `{{column_key}} reads that column off the record walking the - * flow` is a required-looking control over a step nothing walks, and the owner spent a session - * trying to make sense of it. Worse, it was ANSWERABLE: remapping it to `name: {{handle}}` - * saved cleanly, changed nothing, and left the panel describing a write the engine does not - * perform. A picture of the engine that disagrees with the engine is the defect this file - * refuses in five other places. - */ - pinned: boolean; - catalog: ActionCatalogRow[]; - tables: UserTable[]; - /** ITEM 9 / R11 — re-read the list after this panel's picker creates one. REQUIRED; see Props. */ - onTablesChanged: () => Promise | void; - /** - * ⛔ THE WALKING RECORD'S COLUMNS — the trigger's database, resolved by the builder. - * - * These three surfaces were built with `fields={[]}` and it made R3's headline feature - * unauthorable: a conditional group's condition, an action's `when`, and `update_record`'s - * column picker all offered "Choose a field…" and nothing else. An empty list is not a - * neutral default here — it is a picker with no options, which reads as "this database has - * no columns". - */ - walkFields: { key: string; label: string; type: string }[]; - /** - * ⭐ WAVE 26 · ITEM 5 / R10 — DOES A RECORD WALK THIS FLOW AT ALL? The key of the database it - * walks, or `""`. - * - * ⛔ REQUIRED, and it answers a question `walkFields` structurally cannot. Both a flow with no - * walking record and a flow walking an empty database hand this panel `[]`, and R10 wants - * OPPOSITE renderings for them: a sentence pointing at the trigger for the first, an ordinary - * (empty) picker for the second. - * ⚠ VERIFIED AGAINST THE RUNNER, not inferred from the trigger list — which C4 forbids by name - * ([[loopable-wave24]]: `CRON_DRIVEN_TRIGGERS` was exactly that mistake and became D-55). - * `run_flow` returns immediately on `if not table_key` and otherwise walks that table's rows - * evaluating `lane_match(act["when"], row)`, so a non-empty table key is EXACTLY the condition - * under which `when` is ever consulted. The builder derives it from `_flow_table`'s own - * precedence (`AutomationDetail.tsx`'s `walkTable`, whose comment carries that rule). - */ - walkTable: string; - ops: string[]; - nullaryOps: string[]; - vocab?: FlowVocab; - busy: boolean; - onEdit: (change: (a: Action) => Action) => void; -}) { - if (!action) return

That action is no longer part of this flow.

; - const row = catalog.find((c) => c.kind === action.kind); - const cfg = action.config || {}; - const setCfg = (patch: Record) => - onEdit((a) => ({ ...a, config: { ...a.config, ...patch } })); - const values = (cfg as { values?: Record }).values || {}; - const target = tables.find((t) => t.key === String((cfg as { table?: string }).table || "")); - const valueFields = action.kind === "create_record" ? target?.fields || [] : walkFields; - /** - * The saved views of the database the flow's records WALK — the enrich step's optional filter. - * - * ⚠ Read off `walkTable`, never off `target`: `target` is the Create-record action's own - * destination and answers a different question, and on an Instagram discovery flow the two are - * routinely different tables. Same trap `walkFields` carries its own note about. - * ⚠ `undefined` is a THIRD STATE and is preserved as one (`UserTable.views` is absent until the - * server sends it) — the panel says "no view list was offered" rather than drawing an empty - * picker that reads as "this database has no views". - */ - const walkViews = (tables.find((t) => t.key === walkTable) || null)?.views; - - return ( - <> -

{row?.label || action.kind}

- {row?.detail ?

{row.detail}

: null} - -

Configuration

- - {action.kind === "group" ? ( - <> - {/* - ⭐ ONE CONDITION EDITOR PER BRANCH (C-FORK). The group's own `config.cond` is GONE — - a fork has a condition per leg, not one for the whole thing — so an editor bound to - `cfg.cond` would now write a key `clean_actions` drops on the floor: the tree would - look saved, survive a reload from local state, and be absent the next time anyone - opened the automation. - THE HEADING IS THE SERVER'S LABEL. The last leg may carry no condition, and the - server has already named it "Otherwise" — giving it one here turns it into a lettered - branch, which is a real thing to want and needs no separate control. - */} - {groupBranches(action).map((br, i) => ( -
-

- {br.label} - {br.cond ? " — run these actions if…" : " — everything that reaches here"} -

- - onEdit((a) => ({ - ...a, - config: { - ...a.config, - branches: groupBranches(a).map((x) => - x.id === br.id ? { ...x, cond: next } : x - ), - }, - })) - } - fields={walkFields} - ops={ops} - nullaryOps={nullaryOps} - maxDepth={vocab?.maxCondDepth ?? 3} - maxChildren={vocab?.maxCondChildren ?? 12} - lead="" - disabled={busy} - /> -
- ))} -

- The first branch whose conditions match is the one that runs — the record does not go - down two of them. -

- - ) : null} - - {action.kind === "create_record" ? ( -
- - ` whose `value` matches no ` - ) : null} - {tables.map((t) => ( - - ))} - - {/* ITEM 9 / R11 — mint one without leaving the action being configured. */} - { - await onTablesChanged(); - setCfg({ table: key }); - }} - /> - {/* - ⭐ WAVE 25 item 4 (ruling R2b, contract C1, wiring W25-4) — WHAT POINTING IT HERE DOES. - The two lists are the SERVER's (`GET /automations/presets`), diffed against whatever - database this action names — R2 makes `config.table` authoritative, so a hand-made one - answers exactly like a spawned one. - */} - -
- ) : null} - - {/* WHAT THE SEARCH ACTUALLY WRITES, in place of the two controls below that do not apply to - it. ⚠ THE LIST IS `_candidate_row`'s, field for field: a shorter "the profile columns" - would be safe and useless, and a list that drifts from what the engine writes would be - the same lie the value map was. */} - {pinned ? ( -

- The search writes one row per profile it finds — handle, profile link, name, followers, - following, average engagement, bio, link in bio, verified and category — into that - database. It matches on the handle, so a profile found again updates its row instead of - adding another. There is nothing to map: the columns come from the search. -

- ) : null} - - {/* - ⭐ WAVE 25 item 2 (ruling R1a, contract C5, wiring W25-5) — KEEP RECORDS UNIQUE. - Today's Create record APPENDS FOREVER: `_commit_action_writes` mints `max(id)+1`, so any - scheduled flow using it duplicates a row per run — every night, silently, until somebody - looks at the table. `uniqueOn` names the column to upsert on instead. - - ⛔ `""` IS A REAL CHOICE AND IT IS THE DEFAULT, so no stored automation changes meaning the - day this ships. That is why the empty option is worded as a behaviour ("Add a new record - every time") rather than as an absence ("None"): it describes what will happen, which is - the thing the reader is choosing between. - - ⚠ THE COLUMN LIST IS THE TARGET DATABASE'S, not the walking record's — `valueFields` - already resolves that for `create_record`. And the server REFUSES a `uniqueOn` this action - does not write ("it writes: …"), so the honest client move is to offer the columns it - writes FIRST while still allowing the rest: pre-empting the refusal here would be a second - copy of a server rule, and this file's law is that the server owns legality. - */} - {action.kind === "create_record" && !pinned ? ( -
- - -

- {String((cfg as { uniqueOn?: string }).uniqueOn || "") - ? "A run that finds a matching record updates it instead of adding another." - : "Every run adds a row, even if the same one is already there."} -

-
- ) : null} - - {/* - ⭐ WAVE 25 item 1 (rulings R3/R4, contract C4) — THE ENRICH ACTION'S CONFIGURATION. - It replaces the whole `field_instagram` KIND: enriching a profile is something you do TO a - record, not a species of automation. The switches are that kind's, unchanged in meaning, - because they are the ones that decide what a run COSTS. - - ⛔ NO `profileField` PICKER HERE, DELIBERATELY, and it is not an omission. C3's profile - FLAG on the column is what binds this action (R7 — one flagged text field per table), and - the server resolves it at run time; an empty `profileField` is the A3 stored-inert state, - not an error. A second picker would let a reader choose a column the flag does not name, - which is two sources of truth for one binding — and the flag is the one the engine reads. - */} - {isEnrich(action.kind) ? ( - <> - {/* - ⭐⭐ 2026-08-07 (owner ruling) — WHICH RECORDS THIS STEP SPENDS ON. - Owner: *"how many records with what sort, based on a Filtered view or maybe top N - enrichment sorted by date"*. Every enrichment is a vendor call, so an enrich step - without this panel bills for EVERY record in the database on EVERY run — which is why - these controls shipped in the same change as the fix that made the action clickable at - all, rather than after it. - - ⛔ THE LIMIT IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS READ — the owner's own - clarification: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is - enriched last 30 days, then it goes to next list"*. The server walks PAST every skipped - record until the quota is filled, so the number in this box is a cost ceiling that - means the same thing every run. `enrich_selection` owns the rule; this panel never - re-implements it. - */} -

Records to enrich

-
- - ` whose - `value` matches no ` - ) : null} - {(walkViews || []).map((v) => ( - - ))} - - {/* ⛔ AND IT SAYS SO IN WORDS, not only as an option label. The option is what stops - the picker lying; the sentence is what tells somebody the automation is broken and - what to do — an option nobody opens the dropdown to read is a fix for the code and - not for the person. */} - {(cfg as { fromView?: string }).fromView && walkViews - && !walkViews.some((v) => v.id === (cfg as { fromView?: string }).fromView) ? ( -

- This step is bound to a view that no longer exists, so its runs walk no records. - Pick another view, or choose every record. -

- ) : null} - {/* ⚠ ABSENT vs EMPTY, the `UserTable.views` third state. "This server did not offer a - view list" is a different fact from "this database has no views", and rendering an - empty picker for the first one is the silent-empty defect wave 23 gated. */} - {!walkViews ? ( -

No view list was offered for that database.

- ) : null} -
-
- - -
-
- - -
-
- - setCfg({ limit: Number(e.target.value) || 25 })} - /> -

- Records enriched per run, at most 100. Skipped records do not use up the limit — - the run keeps going down the list until it has enriched this many. -

-
- - {/* ⛔ THE DAYS BOX ONLY EXISTS WHILE THE RULE IS ON. A number that configures nothing is - worse than a missing one ([[wrong-parent-not-broken-control]]) — and this one would - read as a cost guard that is running while it is switched off. */} - {(cfg as { skipRecent?: boolean }).skipRecent ? ( -
- - setCfg({ skipRecentDays: Number(e.target.value) || 30 })} - /> -

- Days. A profile enriched inside this window is passed over, and the run moves on to - the next one — so you never pay twice for the same profile, and the history - still gains a point once the window closes. -

-
- ) : null} - - {/* - ⭐⭐ WAVE 28 · R5 / R6 / R7 — THE SOURCE QUESTION IS GONE, AND THREE INCLUDE AXES - REPLACE IT. What stood here was a `Source` select writing `config.tier` - (Anonymous / Paid provider) plus a sibling `noFallback` checkbox — ONE control in two - pieces, asking the user to choose a VENDOR STRATEGY. - - ⛔ R5 RETIRED THE QUESTION, not just the control. Enrichment routes per capability to - the paid providers and reports blocked on a refusal; there is no thin anonymous row to - fall back to, so "which source" and "stop if it fails" no longer have answers a person - could give. `tier` and `noFallback` are accepted-and-ignored in stored configs (C2, - and D-65's law: never 400 a definition that was legal when it was written) — which is - why this panel simply stops writing them rather than migrating anything. - - ⛔ THE ROWS BELOW ARE A TRANSFORM, NOT AN ADDITION, and that distinction is the whole - defect risk in this change. `postMetrics` and `commentMetrics` ALREADY had checkboxes - 34 lines below this point ("Also capture per-post engagement" / "…per-comment…"). - Building three NEW rows and leaving those would have put TWO controls on each key — - which compiles, renders, and satisfies any check asking whether a Post-data switch - exists. The old pair is deleted; these carry their keys. - - ⚠ KEYS UNCHANGED ON PURPOSE (C2). Every stored enrich action round-trips untouched; - only the labels move. R7: no cost sentence anywhere in this panel — the run log keeps - honest spend reporting, and a warning printed permanently is chrome the eye stops - reading (DESIGN.md §4). - */} - {/* ⚠ A GROUP HEADING, NOT A `