diff --git a/RELEASES.json b/RELEASES.json index 79739322f0ad7638c9a9d6104bb0b07800124e13..89464650f708851a27e44b6348ae86f194a494f7 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,5 +1,5 @@ { - "current": "7269240", + "current": "bac40c2", "releases": [ { "version": "v24", diff --git a/VERSION b/VERSION index 0cef34bb6a7571bbcbb4e0f8ed0f94c5b3835538..690f16a6a301ce64adf22bceb4860225f7d09088 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7269240 +bac40c2 diff --git a/api/automation_engine.py b/api/automation_engine.py index 65ce5ae5f937ac71cdc0924897504053e8b309cc..80dccc87f59b1736cd62e97248c173f25e61339b 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -1,10483 +1,11194 @@ -"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. - -THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant -store and a RUN that lives in this process. The definition is durable, small and rarely written; -the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file -is built to avoid, and it has two halves: - - * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the - store outlives the process that wrote it, so a container restart mid-run leaves an automation - that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives - in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written - exactly ONCE per run, at the end. - * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s - floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape - by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes - its whole result first and commits it in a single `update` callback. - -WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not -deployed, so its extraction core is copied into this file (the wave doc's instruction), ported -httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with -it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: -the skill version takes a URL from a developer on a CLI; this takes one from a request body. -""" -from __future__ import annotations - -import datetime as _dt -import hashlib -import hmac -import ipaddress -import json -import os -import re -import socket -import threading -import time -from urllib.parse import urljoin, urlparse - -import requests - -# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability -# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — -# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the -# connector that routes, which is the shape the split was for: this file no longer knows that -# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore -# a signal, not a convenience — it means something in the automation runtime started making a -# vendor decision, and that belongs one layer down. - -try: # the parser the extraction half needs - from bs4 import BeautifulSoup -except Exception: # pragma: no cover — deploy lag; see mailbox ② - BeautifulSoup = None - -# --------------------------------------------------------------------------------------------- -# THE BUCKET (C4-AUTO) -# --------------------------------------------------------------------------------------------- - -#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. -STORE_KEY = "automations" -#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through -#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for -#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the -#: session-D mailbox as a finding against A's file rather than fixed from here. -UT_STORE_KEY = "user_tables" -UT_PREFIX = "ut_" - -MAX_AUTOMATIONS = 40 -MAX_RUNS = 20 # trimmed history per automation -MAX_NAME = 80 -MAX_UT_ROWS = 5000 # mirrors core.user_tables.MAX_ROWS -MAX_UT_TABLES = 40 - -#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. -#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the -#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull -#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's -#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour -#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still -#: report success. A time series that stops after five days without saying so is worse than one -#: that was never built. -#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE -#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: -#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day -#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts -#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had -#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the -#: silent version would have flatlined a chart inside a fortnight with a green dot over it. -#: -#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises -#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, -#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 -#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a -#: smaller number. -#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) -MAX_UT_IG_ROWS = 200_000 -IG_TABLE_PREFIX = "ut_ig_" - -#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was -#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — -#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY -#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would -#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket -#: serialisation cost, purely by an accident of naming. Naming the append tables is the version -#: that stays true when the next `ut_ig_*` table is not one. -IG_SNAPSHOTS_TABLE = "ut_ig_snapshots" -IG_POSTS_TABLE = "ut_ig_posts" -IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots" -IG_COMMENTS_TABLE = "ut_ig_comments" -AUTOMATION_RECORD_MODE = "automation" -APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE}) - -#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. -#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger -#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram -#: column, an Instagram search — and they survive on the automations that already use them -#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). -#: -#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: -#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind -#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was -#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole -#: description: three machine nodes it does not have, and a one-line summary about searching -#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. -KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram") -#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores -#: when the body names none, which after R6 is every create the client makes. -DEFAULT_KIND = "plain" -#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — -#: the ruling retires the door, not the two automations behind it (see `create`). -#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. -RETIRED_KINDS = ("scrape_db", "field_instagram") -#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were -#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name -#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence -#: per kind, in one place, because `create` and `clean_definition` both say it. -RETIRED_KIND_REPLACEMENT = { - "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", - "scrape_db": "Add a scrape step to any automation instead", -} -#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the -#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. -#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any -#: flow can take, so refusing the kind costs a person nothing but a different click. -#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared -#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a -#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch -#: door open on purpose, and there is a gate check that says so in those words. A kind may only be -#: walled off at a door once something else answers the same need. -#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) -KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database", - "field_instagram": "Instagram profile column", - "discover_instagram": "Find Instagram profiles"} -EXTRACTS = ("table", "jsonld") -STATES = ("idle", "running", "ok", "error", "partial") -#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). -#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a -#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). -TIERS = ("anonymous", "brightdata") -#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap -#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown -#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation -#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps -#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. -TIER_ALIASES = {"hiker": "brightdata"} - - -def clean_tier(raw): - """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" - t = str(raw or "").strip().lower() - t = TIER_ALIASES.get(t, t) - return t if t in TIERS else "" - - -def row_cap(table_key): - """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. - - An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything - else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. - """ - return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS - -#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI -#: shows and the vocabulary the parser accepts cannot drift. -CRON_PRESETS = [ - {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, - {"cron": "0 * * * *", "label": "Hourly"}, - {"cron": "0 6 * * *", "label": "Daily at 06:00"}, - {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, - {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, -] - -UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" - - -def _now(): - return _dt.datetime.now() - - -def _stamp(dt=None): - return (dt or _now()).strftime("%Y-%m-%d %H:%M") - - -def _iso(dt=None): - """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). - - ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables - and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A - naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC - minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is - not expressible at all. With the offset the same string is an instant, and relative rendering - becomes possible without migrating a single stored row. - - ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in - this module is against a naive `_now()`, and mixing aware and naive datetimes raises - `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. - """ - dt = dt or _now() - if dt.tzinfo is None: - dt = dt.astimezone() # a naive stamp from this process IS local time - return dt.isoformat(timespec="seconds") - - -#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns. -#: -#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one -#: function down: `first_found` cells exist as post-D-18 offset stamps -#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s -#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A -#: converter that understood only the newest shape would blank the oldest rows — and a migration -#: that empties cells is indistinguishable from one that moved them. -#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats "" -#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and -#: shows up as itself instead of disappearing. -def _day(s): - """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there.""" - raw = str(s or "").strip() - if not raw: - return "" - head = raw.replace("T", " ").split(" ")[0] - try: - _dt.date.fromisoformat(head) - except ValueError: - dt = _parse_iso(raw) - return dt.strftime("%Y-%m-%d") if dt else "" - return head - - -#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`. -#: -#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct` -#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every -#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse -#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1 -#: dialect and a 0–100 dialect, and they meet here). -#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null -#: — and `0.0` would claim we measured zero engagement. -def _pct100(v): - """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert.""" - if v is None or (isinstance(v, str) and not v.strip()): - return "" - try: - return _s(round(float(v) * 100.0, 4)) - except (TypeError, ValueError): - return "" - - -def _parse_iso(s): - """A stamp → a NAIVE LOCAL datetime, with or without an offset. - - Both shapes exist in the store simultaneously and always will: every run committed before - D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new - shape would silently return None for them — and `is_due` reads None as "never ran", which - would re-fire every schedule once. Reading both is what makes the change additive. - """ - raw = str(s or "").strip().replace("Z", "+00:00") - dt = None - try: - dt = _dt.datetime.fromisoformat(raw) - except Exception: # noqa: BLE001 - try: - dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") - except Exception: # noqa: BLE001 - return None - return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt - - -# --------------------------------------------------------------------------------------------- -# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher -# --------------------------------------------------------------------------------------------- -# The skill version guards the URL a developer typed. This one guards a URL that arrived in a -# request body, which changes the threat model in two ways the original does not cover: -# -# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` -# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` -# sails straight past a guard that only ever saw the first URL. The negative control ("a -# localhost URL is refused") would still pass while the rail was wide open. `fetch` below -# therefore takes the hops MANUALLY and re-guards every one. -# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the -# host and checks EVERY answer, not just the literal. -# -# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the -# record between the guard and the socket is not stopped by it. Closing that needs a pinned -# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather -# than implied away — the rail refuses the realistic cases and says what it does not cover. - -class Refused(ValueError): - """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" - - -def _ip_public(ip): - return not (ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_reserved or ip.is_unspecified or ip.is_multicast) - - -def guard(url, allowed=()): - """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers - checked too, plus the optional `--allowed` domain rail scrape.py carries.""" - p = urlparse(url or "") - if p.scheme not in ("http", "https"): - raise Refused(f"only http(s) allowed, not {p.scheme!r}") - host = (p.hostname or "").strip() - if not host: - raise Refused("no host in the URL") - low = host.lower() - if (low in ("localhost", "localhost.localdomain") - or low.endswith(".local") or low.endswith(".internal") - or low.endswith(".localhost")): - raise Refused(f"refused non-public host {host!r} (public web only)") - try: # a bare-IP host is decided on the literal - ip = ipaddress.ip_address(low) - except ValueError: - ip = None - if ip is not None and not _ip_public(ip): - raise Refused(f"refused non-public IP {host} (public web only)") - if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): - raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") - if ip is None: - try: - infos = socket.getaddrinfo(host, None) - except Exception as e: - raise Refused(f"could not resolve {host!r}: {type(e).__name__}") - for info in infos: - addr = info[4][0] - try: - resolved = ipaddress.ip_address(addr.split("%")[0]) - except ValueError: - continue - if not _ip_public(resolved): - raise Refused( - f"refused {host!r}: it resolves to the non-public address {resolved}") - return True - - -def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): - """GET `url`, taking redirects BY HAND so every hop passes `guard`. - - Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, - not a silent truncation — an endless redirect is indistinguishable from an attempt to walk - the fetcher somewhere it was told not to go. - """ - current = url - hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} - hdrs.update(headers or {}) - for _hop in range(max_hops + 1): - guard(current, allowed) - r = requests.get(current, timeout=timeout, headers=hdrs, - allow_redirects=False, stream=True) - if r.status_code in (301, 302, 303, 307, 308): - loc = r.headers.get("Location") - r.close() - if not loc: - raise Refused(f"redirect {r.status_code} with no Location header") - current = urljoin(current, loc) - continue - body = r.raw.read(max_kb * 1024, decode_content=True) or b"" - status, final = r.status_code, str(r.url) - r.close() - return status, final, body - raise Refused(f"more than {max_hops} redirects — refusing to follow further") - - -def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): - """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. - - ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by - hand because a redirected GET is ordinary. A redirected POST is not: re-sending a - credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail - exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever - re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. - (The vendor calls below are the only POSTs this module makes, and they carry the API key.) - """ - guard(url) - hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} - hdrs.update(headers or {}) - r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, - data=json.dumps(body if body is not None else {}), stream=True) - try: - if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, " - f"never re-sent to a location the server picked") - return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") - finally: - r.close() - - -# --------------------------------------------------------------------------------------------- -# EXTRACTION — vendored from scrape.py, unchanged in behaviour -# --------------------------------------------------------------------------------------------- - -def _soup(body): - if BeautifulSoup is None: - raise RuntimeError( - "beautifulsoup4 is not installed in this environment — the automation engine's " - "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " - "aios-web/requirements.txt).") - try: - return BeautifulSoup(body, "lxml") - except Exception: - return BeautifulSoup(body, "html.parser") - - -def tables(soup): - """Every HTML table as rows. Dict rows when the first row looks like a header.""" - out = [] - for t in soup.find_all("table"): - rows = [] - for tr in t.find_all("tr"): - cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] - if cells: - rows.append(cells) - if not rows: - continue - head, body = rows[0], rows[1:] - if body and len(head) == len(body[0]) and all(head): - out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) - else: - out.append(rows) - return out - - -def jsonld(soup): - out = [] - for s in soup.find_all("script", attrs={"type": "application/ld+json"}): - try: - out.append(json.loads(s.string or s.get_text())) - except Exception: - pass - return out - - -def meta(soup): - m = {} - if soup.title and soup.title.string: - m["title"] = soup.title.string.strip() - for tag in soup.find_all("meta"): - k = tag.get("name") or tag.get("property") - v = tag.get("content") - if k and v and (k in ("description", "keywords", "author") - or k.startswith(("og:", "twitter:"))): - m[k] = v.strip() - return m - - -def preview(url, extract="table", table_index=0, allowed=()): - """The field-map preview the editor calls BEFORE anything is created: what columns does this - page actually offer, and what do the first rows look like? Never writes.""" - status, final, body = fetch(url, allowed=allowed) - if not (200 <= status < 300): - return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status} — it may be bot-gated or gone", - "columns": [], "sample": [], "rowCount": 0} - soup = _soup(body) - rows = [] - if extract == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 - picked = found[idx] if found else [] - rows = [r for r in picked if isinstance(r, dict)] - cols, seen = [], set() - for r in rows[:50]: - for k in r: - if k not in seen: - seen.add(k) - cols.append(k) - return {"ok": True, "status": status, "url": final, "columns": cols, - "sample": rows[:8], "rowCount": len(rows), - "tableCount": len(tables(soup)) if extract != "jsonld" else 0, - "title": (meta(soup) or {}).get("title", "")} - - -def _scalar(v): - if isinstance(v, (str, int, float)) and not isinstance(v, bool): - return str(v) - if isinstance(v, bool): - return "1" if v else "" - if isinstance(v, dict): - return str(v.get("name") or v.get("@id") or "") - if isinstance(v, list): - return ", ".join(_scalar(x) for x in v[:8]) - return "" - - -# --------------------------------------------------------------------------------------------- -# CRON — a 5-field parser and a "what was the last fire time" walk -# --------------------------------------------------------------------------------------------- -# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward -# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute -# between the last run and now?"* — and answering it that way is what makes a missed tick -# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and -# not never. - -_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] - - -def _parse_field(spec, lo, hi): - out = set() - for part in str(spec).split(","): - part = part.strip() - if not part: - raise ValueError("empty cron field part") - step = 1 - if "/" in part: - part, _, s = part.partition("/") - step = int(s) - if step < 1: - raise ValueError("cron step must be >= 1") - if part in ("*", "?"): - a, b = lo, hi - elif "-" in part.lstrip("-"): - a_s, _, b_s = part.partition("-") - a, b = int(a_s), int(b_s) - else: - a = b = int(part) - if a < lo or b > hi or a > b: - raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") - out.update(range(a, b + 1, step)) - return out - - -def parse_cron(expr): - """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. - - Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at - WRITE time, because a cron string nobody can evaluate is an automation that silently never - runs and reports no error anywhere. - """ - parts = str(expr or "").split() - if len(parts) != 5: - raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") - sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] - dom_r = parts[2].strip() not in ("*", "?") - dow_r = parts[4].strip() not in ("*", "?") - return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), - dom_r, dow_r) - - -def _day_matches(day, doms, months, dows, dom_r, dow_r): - if day.month not in months: - return False - # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the - # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes - # a schedule that looks right fire almost never. - dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 - if dom_r and dow_r: - return day.day in doms or dow in dows - if dom_r: - return day.day in doms - if dow_r: - return dow in dows - return True - - -def prev_fire(expr, now=None, lookback_days=400): - """The most recent scheduled minute at or before `now`, or None inside the lookback. - - Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, - and only a matching day needs its hours/minutes searched. - """ - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - now = (now or _now()).replace(second=0, microsecond=0) - for back in range(lookback_days + 1): - day = (now - _dt.timedelta(days=back)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = back == 0 - for h in sorted(hours, reverse=True): - if same_day and h > now.hour: - continue - for m in sorted(minutes, reverse=True): - if same_day and h == now.hour and m > now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -def is_due(defn, now=None): - """Should the scheduler run this automation right now? - - Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the - moment the schedule was enabled, else creation) and at or before now. Anchoring on - `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at - 14:00" from running immediately and looking like a bug. - """ - if not isinstance(defn, dict): - return False - sched = defn.get("schedule") or {} - if not sched.get("enabled"): - return False - try: - fire = prev_fire(sched.get("cron"), now=now) - except ValueError: - return False - if fire is None: - return False - since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) - or _parse_iso(sched.get("enabledAt")) - or _parse_iso(defn.get("created"))) - return since is None or fire > since - - -def next_fire(expr, now=None, lookahead_days=400): - """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" - try: - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - except ValueError: - return None - now = (now or _now()).replace(second=0, microsecond=0) - for ahead in range(lookahead_days + 1): - day = (now + _dt.timedelta(days=ahead)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = ahead == 0 - for h in sorted(hours): - if same_day and h < now.hour: - continue - for m in sorted(minutes): - if same_day and h == now.hour and m <= now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -# --------------------------------------------------------------------------------------------- -# THE UPSERT — pure, so the arithmetic is testable without a store or a network -# --------------------------------------------------------------------------------------------- - -def upsert_rows(existing, incoming, key_field, cap=None): - """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. - - THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped - appearing on the source page has not necessarily stopped existing — the page changed its - filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream - hiccup into silent data loss, and the row may be carrying user-typed overlay values in - columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. - - Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. - - ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on - entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to - call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` - that same loop is hundreds of millions of dict copies and the automation simply never - finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) - - ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different - facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the - DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — - a property of the SYSTEM, and never benign. One number for both meant a table hitting its - ceiling was indistinguishable from a page with a few blank cells, which is how a time series - stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. - """ - cap = MAX_UT_ROWS if cap is None else int(cap) - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, - "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} - by_key, dupe_ids = {}, set() - for rid, row in rows.items(): - kv = str(row.get(key_field, "") or "").strip() - if not kv: - continue - if kv in by_key: - dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left - continue - by_key[kv] = rid - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 - seen_keys, incoming_dupes = set(), 0 - for src in incoming or []: - kv = str((src or {}).get(key_field, "") or "").strip() - if not kv: - counts["skipped"] += 1 # no key -> cannot be upserted; never guessed - continue - if kv in seen_keys: - incoming_dupes += 1 # the SOURCE listed it twice; first wins - continue - seen_keys.add(kv) - rid = by_key.get(kv) - if rid is None: - if len(rows) >= cap: - counts["capped"] += 1 # LOUD: the runner turns this into a partial run - continue - rid = str(next_id) - next_id += 1 - rows[rid] = dict(src) - by_key[kv] = rid - counts["inserted"] += 1 - continue - before = rows[rid] - changed = {k: v for k, v in src.items() if str(before.get(k, "")) != str(v)} - if changed: - before.update(src) - counts["updated"] += 1 - else: - counts["unchanged"] += 1 - counts["duplicates"] = incoming_dupes + len(dupe_ids) - counts["orphans"] = sum( - 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) - return rows, counts - - -def dedupe_canonical_rows(existing, key_field, newest_by=""): - """Collapse duplicate logical rows while preserving the lowest stable row id. - - Canonical entity tables use this before every upsert. Snapshot tables deliberately do not: - repeated shortcodes there are new timestamped observations, not duplicate posts. Values are - taken newest-first and then filled from older rows, so a sparse fresh projection does not - erase a field an earlier row knew. - """ - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - groups = {} - for rid, row in rows.items(): - identity = str(row.get(key_field) or "").strip() - if identity: - groups.setdefault(identity, []).append((rid, row)) - removed = 0 - for members in groups.values(): - if len(members) < 2: - continue - keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r)) - ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \ - if newest_by else members - merged = {} - for _rid, row in ordered: - for key, value in row.items(): - if key not in merged or str(merged.get(key) or "").strip() == "": - merged[key] = value - rows[keep] = merged - for rid, _row in members: - if rid != keep: - rows.pop(rid, None) - removed += 1 - return rows, removed - - -# --------------------------------------------------------------------------------------------- -# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) -# --------------------------------------------------------------------------------------------- - -def _ut_slug(label): - s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return (s or "table")[:40] - - -def ut_all(rt): - try: - return dict(rt.get(UT_STORE_KEY) or {}) - except Exception: - return {} - - -def disable_for_table(rt, table_key, note="target database deleted"): - """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. - - The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still - shows what existed and why it stopped — silently deleting a user's automation because its - target died would read as data loss. Returns the ids it touched.""" - key = str(table_key or "") - touched = [] - - def _up(cur): - for aid, d in (cur or {}).items(): - if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: - sch = d.get("schedule") - if not isinstance(sch, dict): - sch = d["schedule"] = {} - sch["enabled"] = False - d["statusNote"] = note - touched.append(str(aid)) - return cur - - if key: - rt.update(STORE_KEY, _up, flush="sync") - return touched - - -def ut_get(rt, key): - return ut_all(rt).get(str(key)) - - -def retire_automation_stage_fields(rt): - """Delete obsolete Board-only fields and their hidden cells, never user fields. - - The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField`` - metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old - generated timestamp/cycle cells are cleared too, including rows whose field definition was - removed by an interrupted earlier migration. Re-running this migration is a no-op. - """ - tables = ut_all(rt) - stage_keys = set() - for table in tables.values(): - if not isinstance(table, dict): - continue - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")): - stage_keys.add(str(field.get("key") or "")) - for row in (table.get("rows") or {}).values(): - for key in (row or {}): - text = str(key) - if text.startswith("stage_auto_"): - stage_keys.add(text.removesuffix("_at").removesuffix("_cycles")) - stage_keys.discard("") - if not stage_keys: - return {"tables": 0, "fields": 0, "cells": 0} - - changed = {"tables": set(), "fields": 0, "cells": 0} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for table_key, table in cur.items(): - if not isinstance(table, dict): - continue - kept, removed = [], set() - for field in table.get("fields") or []: - auto = field.get("automation") if isinstance(field, dict) else None - key = str(field.get("key") or "") if isinstance(field, dict) else "" - if key in stage_keys and isinstance(auto, dict) and \ - (auto.get("stageField") or auto.get("cyclesField")): - removed.add(key) - changed["fields"] += 1 - continue - kept.append(field) - if removed: - table["fields"] = kept - changed["tables"].add(str(table_key)) - for row in (table.get("rows") or {}).values(): - if not isinstance(row, dict): - continue - for stage_key in stage_keys: - for key in (stage_key, stage_key + "_at", stage_key + "_cycles"): - if key in row: - row.pop(key, None) - changed["cells"] += 1 - changed["tables"].add(str(table_key)) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return {"tables": len(changed["tables"]), "fields": changed["fields"], - "cells": changed["cells"]} - - -def bind_unbound_fields(rt): - """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` - are bound to the definition that already writes them — matched on the (targetTable, - fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no - definition references is DISABLED with the reason on it rather than deleted or guessed: - the column was already dead (nothing runs a column no flow names), and now it says so. - Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" - by_binding = {} - for aid, d in all_definitions(rt).items(): - cfg = d.get("config") or {} - if cfg.get("fieldKey") and cfg.get("targetTable"): - by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) - plan = {} - for tk, t in ut_all(rt).items(): - for f in (t.get("fields") or []): - a = f.get("automation") - # An already-DISABLED bag is a decision this migration made on a previous pass — - # re-planning it every call would turn the once-per-process sweep into a write - # per read. - if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ - and not a.get("disabled"): - plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) - if not plan: - return 0, 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for (tk, fk), aid in plan.items(): - for f in ((cur.get(tk) or {}).get("fields") or []): - if f.get("key") == fk and isinstance(f.get("automation"), dict): - if aid: - f["automation"]["flowId"] = aid - else: - f["automation"]["disabled"] = True - f["automation"]["statusNote"] = ( - "not bound to any flow — create an automation for this column " - "or delete it") - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - bound = sum(1 for v in plan.values() if v) - return bound, len(plan) - bound - - -def ut_key_for(label, key=None): - """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which - must not create anything) resolves the same key by construction rather than by copying the - derivation and drifting from it.""" - k = str(key or (UT_PREFIX + _ut_slug(label))) - return k if k.startswith(UT_PREFIX) else UT_PREFIX + k - - -#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same -#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging -#: `core` into the API's boot path — and the two are held in step by a gate check. -MACHINE_OWNERS = ("automation", "scheduler") - - -def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="", - record_mode="", lock_fields=False): - """Create the table if it is missing; return its key. Idempotent — a re-run of an automation - that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) - and an existing table with that key is adopted, not duplicated. - - Fields are MERGED, never replaced: a user who added a column to an automation's table keeps - it, and a new source column joins on the next run. - - `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped - `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their - provenance. Fields already on the table keep whatever tag they have (first flow wins; - shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not - ownership). - - ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever - or WHATEVER ran the automation, so the same table belonged to a person if its first run was - manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` - admits only the creator or an admin, so **whether you could open your own Instagram - snapshots depended on a race you never saw**. The owner is now the automation's creator, and - a table already stamped with a machine name is ADOPTED the next time a run knows a human - one. Adoption is a repair, not a widening: the automation's creator is the person who asked - for the table in the first place. - """ - key = ut_key_for(label, key) - # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point. - # - # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it - # would leave every table already in production on the old `text` schema forever while new - # ones got the honest types — the split schema R3's note warned about, arriving through the - # very function the note was written on. Migrating here means a table is brought forward - # immediately BEFORE anything appends to it, so no caller has to remember anything and no - # tenant is left behind by a script nobody ran. - # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is - # already current, which after the first run is every call. - if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS): - try: - migrate_ig_tables(rt, log=lambda *_a: None, only=key) - except Exception: # noqa: BLE001 - # A migration that cannot run must not stop the automation from writing its rows. - # The old schema still reads; a refused write loses the pull we just paid for. - pass - wanted = [] - for raw_field in (fields or []): - field = dict(raw_field) - if flow_tag or lock_fields: - automation = dict(field.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - if lock_fields: - automation["preset"] = True - field["automation"] = automation - wanted.append(field) - created = _iso() - human = username if username and username not in MACHINE_OWNERS else "" - - # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store - # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo - # budget, an idempotent helper that always writes is the same defect as a per-row insert. - have = ut_get(rt, key) - have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])} - missing = [f for f in wanted if f.get("key") not in have_fields] - missing_locks = [f for f in wanted - if lock_fields and f.get("key") in have_fields - and (not isinstance(have_fields[f.get("key")].get("automation"), dict) - or have_fields[f.get("key")]["automation"].get("preset") is not True)] - if (have is not None and not missing and not missing_locks - and not (record_mode and have.get("recordMode") != record_mode) - and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): - return key - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - if len(cur) >= MAX_UT_TABLES: - return cur - cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", - "createdBy": username, "created": created, - "fields": wanted, "rows": {}} - if record_mode: - cur[key]["recordMode"] = record_mode - return cur - have = {f.get("key") for f in (t.get("fields") or [])} - for f in wanted: - if f.get("key") not in have: - t.setdefault("fields", []).append(f) - have.add(f.get("key")) - elif lock_fields: - stored = next((g for g in (t.get("fields") or []) - if g.get("key") == f.get("key")), None) - if stored is not None: - automation = dict(stored.get("automation") or {}) - if flow_tag: - automation.setdefault("flowId", str(flow_tag)) - automation["preset"] = True - stored["automation"] = automation - # ADOPTION: a machine name is not an owner. It never overwrites a human one. - if human and (t.get("createdBy") or "") in MACHINE_OWNERS: - t["createdBy"] = human - # An automation's table SAYS an automation owns it — the nav badge reads from this. - t["source"] = t.get("source") or "Automation" - if record_mode: - t["recordMode"] = record_mode - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return key - - -def ut_write_rows(rt, key, rows): - """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t["rows"] = rows - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): - """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value - for k, rws in (extra_rows or {}).items(): - tt = cur.get(k) - if tt is not None: - tt["rows"] = rws - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -IG_FIELD_DESCRIPTIONS = { - "alt_text": "Accessibility text attached to the Instagram post.", - # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would - # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED - # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off - # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same - # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a - # description that named only the first would have quietly excluded the second. - "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus " - "rather than measured at the time shown.", - "avg_comments_12": "Average comments across the 12 most recent captured posts.", - "avg_engagement": "Average engagement rate reported for the profile.", - "avg_likes_12": "Average likes across the 12 most recent captured posts.", - "avg_plays_12": "Average plays across the 12 most recent captured posts.", - "avg_views_12": "Average views across the 12 most recent captured posts.", - "views": "View count for the post, as Instagram displays it.", - "video_duration": "Length of the video in seconds.", - "comments_disabled": "Checked when the creator has turned comments off for this post.", - "plays": "Times the video started playing, including replays.", - "bio": "Biography shown on the Instagram profile.", - "bio_hashtags": "Hashtags listed in the profile biography.", - "business_category": "Instagram's business category for the account.", - "caption": "Caption published with the Instagram post.", - "category": "Instagram's category for the profile.", - "comment_key": "Unique ID for the captured comment record.", - "commented_at": "Date the comment was posted.", - "comments": "Comment count reported for the post.", - "comments_captured": "Number of distinct comment records linked to this profile.", - "comments_link": "Comment records linked to this record.", - "country_code": "Country code reported for the profile.", - "created_by": "User whose automation first added this lead.", - "enriched_at": "Date the profile was last enriched.", - # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number - # beside it is the only thing that says how much of one. - "location_guess": "Most common city tagged across this profile's captured posts - a guess, " - "not a stated location.", - "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " - "city.", - "external_url": "Website linked from the Instagram biography.", - "external_url_title": "Title Instagram shows for the biography link.", - "fbid": "Facebook ID associated with the Instagram account.", - "first_found": "Date an automation first found this profile.", - "followers": "Latest reported follower count.", - "following": "Latest reported number of accounts followed.", - "found_count": "Number of times discovery automations found this profile.", - "full_name": "Display name shown on the Instagram profile.", - "handle": "Instagram username without the @ symbol.", - "has_channel": "Checked when the profile has an Instagram channel.", - "hashtags": "Hashtags extracted from the post caption.", - "highlights_count": "Total story highlight collections reported for the profile.", - "ig_id": "Instagram's internal ID for the account.", - "influencer_key": "Normalized handle linking this row to its profile.", - "is_business": "Checked when Instagram marks the account as a business.", - "is_joined_recently": "Checked when Instagram marks the account as recently joined.", - "is_private": "Checked when the Instagram profile is private.", - "is_professional": "Checked when Instagram marks the account as professional.", - "last_found": "Date an automation most recently found this profile.", - "likes": "Like count reported for this record.", - "measured_at": "Date engagement metrics on this post were last read.", - "measurements_captured": "Number of measurement rows stored for this post.", - "paid_partnership": "Checked when Instagram marks the post as a paid partnership.", - "partner": "Brand named in the post's paid partnership metadata.", - "partner_id": "Partner ID reported for the Instagram profile.", - "platform": "Social network for this profile.", - "plays": "Video plays, including repeat plays, reported for the post.", - "post_link": "Post record linked to this measurement or comment.", - "post_measurements_captured": "Number of distinct post measurements linked to this profile.", - "post_snapshot_key": "Unique ID for this post measurement.", - "post_snapshots_link": "Engagement measurement rows linked to this record.", - "posted_at": "Date the Instagram post was published.", - "posts_captured": "Number of distinct post records captured for this profile.", - "posts_count": "Total posts reported for the Instagram profile.", - "posts_link": "Post records linked to this profile.", - "profile_name": "Profile name returned by the data provider.", - "profile_reads": "Number of profile snapshots stored for this profile.", - "profile_snapshots_link": "Profile snapshot rows linked to this profile.", - "profile_url": "Direct URL to the Instagram profile.", - "pronouns": "Pronouns shown on the Instagram profile.", - "pulled_at": "Date this snapshot was collected.", - "related_accounts": "Accounts Instagram suggests alongside this profile.", - "replies": "Reply count reported for the comment.", - "shortcode": "Instagram shortcode that uniquely identifies the post.", - "source_payload": "Full source response for this record.", - "snapshot_key": "Unique ID for this profile snapshot.", - "source": "Method or provider used to read this profile.", - "tagged_location": "Location tagged on the post; it is not the creator's residence.", - "type": "Instagram media type: image, video, or carousel.", - "url": "Direct URL to the Instagram post.", - "verified": "Checked when Instagram marks the profile as verified.", -} - - -def field_def(text_key, label, ftype="text", **extra): - """One machine-spawned column definition. - - ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`, - `profile`), and it stays ONE constructor rather than growing a second for "special" fields. - ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s - W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole - preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]). - - ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a - feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so - `core.user_tables._clean_field` — the single validator every field created through the ordinary - door passes — has never judged a single field this engine spawned. The difference was exactly - one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f` - for every automation column in the product. - ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an - absent key was already not that, so the bypass was fail-closed and therefore silent. It is - stated now, and `verify_automation` asserts the equality field-by-field, so the next key - `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]). - """ - description = " ".join(str(extra.pop( - "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) - out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", - "editRole": "admins"} - if description: - out["description"] = description - out.update(extra) - return out - - -# --------------------------------------------------------------------------------------------- -# DEFINITIONS — validation and the bucket's read/write half -# --------------------------------------------------------------------------------------------- - -def _s(v, n=200): - return str(v if v is not None else "")[:n] - - -#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S. -#: -#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12 -#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config -#: disagree with what the run can possibly do. -#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect -#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve -#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is -#: worse than one that refuses it, because the refusal is the only place the ceiling can be -#: taught. So this REFUSES, and the sentence names the cap and who set it. -#: -#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the -#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the -#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask -#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30` -#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a -#: property of the call we happened to make. -#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of -#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE -#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to -#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it -#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12. -MAX_POSTS_PER_PULL = 30 -DEFAULT_POSTS_PER_PULL = 10 - -#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately -#: NOT `MAX_POSTS_PER_PULL`. -#: -#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap -#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so -#: raising the capture cap to 30 silently made every one of them average thirty posts under a -#: label that says twelve. Nobody would have looked at those columns again to check. -#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different -#: decisions, and only the first one was made. To widen the average, change this constant AND the -#: four keys and labels together — a column whose meaning changes underneath its own name is -#: worse than one that is merely narrow. -AVG_WINDOW_POSTS = 12 - - -#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a -#: filter cannot ask for a category that can never match a stored row. -POST_TYPES = ("video", "image", "carousel") - - -def clean_post_groups(raw, max_posts): - """`(groups, error)` for `config.postGroups` — "last N reels, last M images". - - ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape: - `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`. - - ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today - carries no such key, and inventing a filter for it would silently start DROPPING posts those - automations have always captured. Empty list and missing are the same answer. - - ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the - difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them - would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken - one, so the only way to see one is a person typing it, and a filter that silently ignores the - type you asked for is how you end up paying for reels and storing carousels. - """ - if raw in (None, "", [], {}): - return [], None - if not isinstance(raw, list): - return None, "the post groups have to be a list of {type, limit} entries" - out, seen = [], set() - for item in raw: - if not isinstance(item, dict): - return None, "each post group has to be a {type, limit} entry" - t = str(item.get("type") or "").strip().lower() - if t not in POST_TYPES: - return None, (f"'{t or 'blank'}' is not a post type — use one of " - f"{', '.join(POST_TYPES)}") - if t in seen: - return None, f"the post groups name '{t}' twice — one limit per type" - seen.add(t) - try: - lim = int(item.get("limit")) - except (TypeError, ValueError): - return None, f"how many {t} posts to keep has to be a whole number" - if lim < 1: - return None, f"a {t} group has to keep at least one post" - # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures - # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is - # refused HERE, where the number they typed is still on the screen in front of them. - if lim > max_posts: - return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " - f"group cannot keep {lim} — raise the post count first") - out.append({"type": t, "limit": lim}) - return out, None - - -def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): - """`(maxPosts, error)` — refuses out-of-range rather than clamping into range. - - ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this - takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old - clamp's default, which this cap now forbids. If an inherited value were refused the same way - a typed one is, every one of those automations would 400 on its next Save **forever**, for a - number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator - that refuses stored data does not protect the user from it, it locks them out of their own - automation. - So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely - INHERITED gets silently brought inside the cap it was already effectively subject to at the - vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]). - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default, None - try: - n = int(raw) - except (TypeError, ValueError): - if not submitted: - return default, None - return None, "the post limit must be a whole number" - if not submitted: - return max(1, min(n, MAX_POSTS_PER_PULL)), None - if n < 1: - return None, "keep at least one post per profile, or turn the post capture off" - if n > MAX_POSTS_PER_PULL: - return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts — that is the " - f"vendor's cap, not ours, so {n} would store the same " - f"{MAX_POSTS_PER_PULL} and read as more") - return n, None - - -def clean_config(kind, raw, previous=None): - """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. - - ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail - rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not - edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save" - from a panel that never knew about them would silently turn off the dry run, or turn ON a - per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep; - present ⇒ take it, including `false`. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - - def flag(name): - return bool(raw[name]) if name in raw else bool(prev.get(name)) - if kind == "plain": - # A plain automation has no machine step, so it only needs the database its records walk. - # - # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on - # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- - # trigger — so requiring a second copy of that fact here would be a mandatory field with - # exactly one legal answer, and a Save that refuses until you retype what you just picked. - # There is no `dryRun`: dry-run means "run the machine step but do not write", and there - # is no machine step to run. - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None - if kind == "scrape_db": - url = _s(raw.get("url"), 2000).strip() - if not url: - return None, "a source URL is required" - try: - guard(url) - except Refused as e: - return None, str(e) - extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" - fmap = {} - for k, v in list((raw.get("fieldMap") or {}).items())[:60]: - tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") - if tk: - fmap[_s(k, 120)] = tk[:60] - if not fmap: - return None, "map at least one source column to a field" - key_field = _s(raw.get("keyField"), 60).strip() - if key_field not in fmap.values(): - return None, "the key field must be one of the mapped fields" - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"url": url, "extract": extract, - "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), - "fieldMap": fmap, "keyField": key_field, - "targetTable": target, - "dryRun": flag("dryRun"), - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None - if kind == "field_instagram": - table = _s(raw.get("targetTable"), 60).strip() - if not table.startswith(UT_PREFIX): - return None, "an Instagram automation runs against a blank database (ut_*)" - fkey = _s(raw.get("fieldKey"), 80).strip() - if not fkey: - return None, "pick the automation column this run writes into" - # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2). - # They are read from nothing and written to nothing: a stored definition carrying either - # still SAVES — it simply loses them on the next write — and neither is ever a reason to - # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown - # key would 400 every automation a tenant stored before this wave, forever, on a screen - # that gives them no way to remove it. Dropping a retired key is a migration; refusing it - # is an outage. - # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary - # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired - # USER concept — renaming them would be a second, unrelated change wearing this one's - # justification. - # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel - # that does not edit the post count must not reset it to the default on Save. And an - # INHERITED value is clamped rather than refused; see `clean_max_posts`. - sent = "maxPosts" in raw - max_posts, perr = clean_max_posts( - raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent) - if perr: - return None, perr - return {"targetTable": table, "fieldKey": fkey, - "urlField": _s(raw.get("urlField"), 80).strip(), - # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked - # for: the Profiles row carries post IDENTITY for free but NO engagement - # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. - "postMetrics": flag("postMetrics"), - # The full Comments dataset can bill many rows per post. It is an explicit, - # independent opt-in and never follows the per-post switch automatically. - "commentMetrics": flag("commentMetrics"), - "dryRun": flag("dryRun"), - "maxPosts": max_posts}, None - if kind == "discover_instagram": - limit = int(raw.get("recordsLimit") or 0) - if limit < 1: - return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the " - "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") - if limit > BD_MAX_RECORDS: - return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what - # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading - # the operator afterwards is how the OR hole survived: the guard was handed the branches - # and never told they were a union. - join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" - preds, perr = clean_predicates(raw.get("predicates"), join) - if perr: - return None, perr - target = _s(raw.get("targetTable"), 60).strip() or DISCOVER_TABLE - if not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed")) - if serr: - return None, serr - return {"recordsLimit": limit, "predicates": preds, - "operator": join, - "targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or DISCOVER_LABEL, - # C6/R5: WHERE the conditions above came from, when they were derived. Stored so - # the surface can say "these were filled in from the 'Florists' view, over 12 - # records" instead of presenting them as if somebody typed them. - "seed": seed, - "dryRun": flag("dryRun")}, None - return None, f"unknown automation kind {kind!r}" - - -def clean_seed(raw): - """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter - somebody typed by hand has no seed, and that is the ordinary case. - - ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that - the derived conditions are "written into `config.predicates` as ordinary conditions — it is a - filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`, - like every other search. Keeping it means the surface can say where those rows came from, and - a user editing them freely is exactly what is supposed to happen. - """ - if raw in (None, "", {}): - return {}, None - if not isinstance(raw, dict): - return None, "the seed must be an object" - source = _s(raw.get("source"), 12).strip().lower() - if source not in SEED_SOURCES: - return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) - table = _s(raw.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, f"a seed reads a blank database (ut_*) — {table!r} is not one" - basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} - derived, _err = clean_predicates(raw.get("derived") or [], "and") - return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), - # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what - # was suggested; the conditions that matter are already in `predicates` and were - # validated there. Refusing a Save because a stored provenance note aged badly would - # block the user from editing the very filter it describes. - "derived": derived or [], - "basis": {"rows": max(0, int(basis.get("rows") or 0)), - "fields": [f for f in (basis.get("fields") or []) - if isinstance(f, dict)][:SEED_MAX_PREDICATES], - "related": basis.get("related") - if isinstance(basis.get("related"), dict) else {}, - "note": _s(basis.get("note"), 200)}}, None - - -def clean_schedule(raw, previous=None): - """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` - measures from — without it, enabling a daily job fires it immediately).""" - raw = raw if isinstance(raw, dict) else {} - cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" - parse_cron(cron) # raises -> the route answers 400 - enabled = bool(raw.get("enabled")) - prev = previous or {} - out = {"cron": cron, "enabled": enabled} - if enabled: - out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() - return out - - -# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── -# The owner's words: "we can make this automation a loop, so the ending should always be defined, -# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, -# or the user have to click a button to reset, or after a certain amount of time it can -# automatically reset to first cycle." -# -# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition -# that predates this field must not start moving records on its own the day the code ships. A -# loop is a thing somebody turns on. -def clean_flow(raw, previous=None): - """Validate the builder's ordered action list. `(flow, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions")) - if err: - return None, err - return {"actions": actions}, None - - -#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the -#: definition carries none. -#: -#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at -#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the -#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly -#: created automation. Two numbers for one fact, in the panel, before anybody had typed -#: anything. **Read off the screenshot; every assertion in the battery was green.** -#: -#: This is the same species as the default-versus-guard split fixed the same day: a value -#: decided in one file and a value decided in another, describing the same thing. Aligning the -#: constants closes the only window in which they can disagree — the seed — because every later -#: state comes from the stored config. -DISCOVER_SEED_RECORDS = 25 - - -def _ensure_ig_action(flow_raw, table): - """Create record is PERMANENT step 1 on `ig_profile_match` — it cannot be deleted or moved. - - ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, - on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a - comment warning that re-seeding on every clean would be "a control that will not take no for - an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be - deleted, because the nature of that automation is that it needs to first create a record in a - database somewhere from the list of profiles to fetch."* - - Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER - produces rows has nowhere to put them until something writes them, so an Instagram search - with no Create record is not a customised automation — it is a search whose results are - discarded. That is not a preference to respect. - - ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and - the values it maps are theirs. An existing `create_record` further down is MOVED to the front - rather than duplicated — re-seeding a second one would quietly double every run's writes. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if actions and actions[0].get("kind") == "create_record": - return _pin_unique(_ensure_enrich_step( - flow_raw if isinstance(flow_raw, dict) else flow)) - at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) - if at > 0: - actions.insert(0, actions.pop(at)) - else: - # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner - # and no card: the wave-23 header records four kinds that shipped with illegal seeds and - # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. - actions.insert(0, { - "id": "act_1", "kind": "create_record", "enabled": True, "when": None, - # `config.label` follows the `review` action's precedent in `_clean_action_config` — - # an action naming itself, inside the untyped config bag, so no shared TS type changes. - "config": {"table": str(table or DISCOVER_TABLE), - "label": "Save the profile", - # C5: the picture and the real write finally agree — see `_pin_unique`. - "uniqueOn": "handle", - "values": {"handle": "{{handle}}"}}, - }) - flow["actions"] = actions - return _pin_unique(_ensure_enrich_step(flow)) - - -def _ensure_enrich_step(flow_raw): - """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON AN INSTAGRAM SEARCH. - - Owner: *"make it default that this enrichment action is Step 2 always, and under Config of - Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for - the same reason: a search that finds profiles and never reads them has done half a job. The - difference is the control — step 1 is permanent because a flow without it discards its - results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and - disabling would be two ways to say one thing, and only one of them survives a re-save. - - ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains - this step by upgrading does not quietly start spending; switching the Source to the paid - provider is an explicit choice a person makes in front of the sentence that names the cost. - ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing — - a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow - re-buying the same profile nightly. Off-by-default would make the expensive behaviour the - accident. - - ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an - If/then branch has one; seeding a second at the top level would enrich twice and bill twice, - which is the duplication `apply_actions` already paid for once with the pinned step 1. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - if any(a.get("kind") == "enrich_instagram" for a in walk_actions(actions)): - return flow_raw if isinstance(flow_raw, dict) else flow - seeded = list(actions) - # Index 1 — after the pinned Create record, because there is nothing to enrich until the - # profiles have been written as records. `insert` past the end is a plain append, so a flow - # with only step 1 lands this at the end, which IS step 2. - seeded.insert(1, { - "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None, - "config": {"postMetrics": False, "commentMetrics": False, - "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL, - "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc", - "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True, - "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS}, - }) - return {**flow, "actions": seeded} - - -#: The key the discovery writer really upserts on. `run_discover_instagram` keys candidates on -#: `_ck(handle, created_by)`; `handle` is the half a person can see and the half this action's -#: values carry. -IG_PINNED_UNIQUE = "handle" - - -def _pin_unique(flow): - """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save. - - ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` — - the builder posts the whole action list on every save — so a client that omitted the key would - silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's - own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming - "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this - module refuses everywhere else. - - ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason - this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action - does not write, so the stamp is applied ONLY when `handle` is among the action's own values. - A user who remaps the card to write different columns keeps their edit and still saves; the - alternative — stamping unconditionally — is a product-seeded default that its own validator - would 400, which is precisely the post-W24 hotfix this rule exists because of. - - ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes - `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED - definition's own dict, so writing into it would edit the live store object in memory, before - (and regardless of) any commit. Every touched level is copied instead. - """ - if not isinstance(flow, dict): - return flow - acts = flow.get("actions") - if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict): - return flow - first = acts[0] - if first.get("kind") != "create_record": - return flow - cfg = first.get("config") - if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}): - return flow - if str(cfg.get("uniqueOn") or "").strip(): - return flow - return {**flow, - "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} - - -#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so -#: the seeder and the un-seeder cannot disagree about what "seeded" means. -_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich"} - - -def _is_untouched_ig_seed(action, table): - """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) - - ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and - that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build - the same dicts one screen above, so a step gaining a config key next wave gains it here too. - A remembered key list would quietly start calling every seeded action "edited". - - ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The - stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the - kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW - dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", - and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only - version where the two sides are the same kind of object. - - ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the - TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about - a step they still want, and an off seed is still a seed. - """ - if not isinstance(action, dict): - return False - kind = str(action.get("kind") or "") - if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): - return False - seeded, _seed_err = clean_actions( - (_ensure_ig_action({"actions": []}, table).get("actions") or [])) - fresh = {a.get("kind"): a for a in (seeded or [])} - seed = fresh.get(kind) - if not seed: - return False - cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) - if kind == "enrich_instagram": - cfg.pop("enabled", None) - seed_cfg.pop("enabled", None) - return cfg == seed_cfg and (action.get("when") or None) is None - - -def _drop_ig_seeds(flow_raw, table): - """Strip the trigger's own seeded steps, keeping any the person has since made theirs. - - ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT - THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it - does not delete the seeded actions … a create_record the person has since re-pointed at their - own table with their own values is THEIR action now"*. That reasoning is still correct and is - exactly what this function preserves — but it was applied to EVERY seeded action, including - the ones nobody had ever opened, and the result is what the owner reported: switch an - Instagram search to Manual and you are left holding a "Save the profile" step writing - `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not - somebody's work being protected; it is the machine's own leftover, pointed at a table the - automation has nothing to do with any more. - - ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as - `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a - branch that destroys something, so the conservative half is the load-bearing half. - """ - flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} - actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] - kept = [a for a in actions if not _is_untouched_ig_seed(a, table)] - if len(kept) == len(actions): - return flow_raw - return {**flow, "actions": kept} - - -def ig_action_pinned(defn, index): - """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` - flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" - return (defn or {}).get("kind") == "discover_instagram" and index == 0 - - -def clean_definition(raw, previous=None, username=""): - """Whole-definition validation. Returns `(defn, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous or {} - # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing - # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second - # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys - # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload - # invalid in BOTH its trigger and its config now answers with the trigger's sentence. - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr - # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this - # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a - # dead end the moment the wizard was deleted and the client's create body became {name}. - kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND - drop_ig_seeds = False - if (trigger or {}).get("key") == "ig_profile_match": - # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather - # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and - # whose kind says something else is not a preference to respect, it is two halves of one - # answer disagreeing, and the trigger is the half the person actually picked. - kind = "discover_instagram" - elif (kind == "discover_instagram" - and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match" - and (trigger or {}).get("key") != "ig_profile_match"): - # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG. - # - # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram - # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch - # the trigger to Manual and the automation stayed `discover_instagram`, which means - # - # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow - # the person had just made MANUAL fired a PAID Bright Data corpus search; - # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed - # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the - # instagram trigger, the 'always first' create record just stuck there"*. The server - # would have accepted the delete; the CLIENT hid the control, because both halves ask - # the kind and the kind was lying. - # - # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in - # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as - # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY - # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`. - # That turned five `section_bd` checks red and crashed the suite on a `StopIteration` - # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest - # question: did this automation stop being an Instagram search? - # - # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram - # trigger — a `discover_instagram` created with an explicit kind and some other trigger is - # somebody's deliberate state, not a mistake to correct. And only FROM - # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose - # triggers are their own business, and clobbering them would retire them by accident. - # - # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this - # replaces argued the other way, so here is why it was half right. It read: *"it does not - # delete the seeded actions … a create_record the person has since re-pointed at their own - # table with their own values is THEIR action now, and quietly destroying it is the silent - # data loss this module refuses everywhere else."* Every word of that is still true and is - # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the - # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to - # Manual still carrying a "Save the profile" step writing `{{handle}}` into - # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces - # handles, unpinned but still there, still runnable, and still pointed at a table that has - # nothing to do with this automation. - # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. - kind = DEFAULT_KIND - drop_ig_seeds = True - if kind not in KINDS: - return None, f"unknown automation kind {kind!r}" - # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. - # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` - # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. - # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way - # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two - # live automations save through this function every time their owner edits them. The gate's - # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved - # from `create` into `clean_definition` (the tempting simplification), this helper would go - # red across a dozen sections, which is the alarm that change deserves."* It did, and the - # alarm worked. - # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every - # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody - # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an - # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored - # ones are rewritten under their owner — and that is a ruling, not a refactor. - name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) - if not name: - return None, "name the automation" - cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - if kind == "discover_instagram" and prev.get("kind") != "discover_instagram": - # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very - # first Save after picking this trigger 400s — against the stored-inert-with- - # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). - # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard - # exists because an unbounded query is the one shape the vendor refuses outright, and - # coercing a blank into a number would be exactly the silent widening it protects against. - # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until - # a person presses Run now or arms the schedule. - cfg_raw = dict(cfg_raw or {}) - if not _ig_int(cfg_raw.get("recordsLimit")): - cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS - config, err = clean_config(kind, cfg_raw, prev.get("config")) - if err: - return None, err - if trigger: - # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own - # configuration IS the config, and `clean_trigger` could not see it. - trigger["configured"] = _trigger_configured(trigger, config) - try: - schedule = clean_schedule( - raw.get("schedule") if isinstance(raw.get("schedule"), dict) - else prev.get("schedule"), prev.get("schedule")) - except ValueError as e: - return None, str(e) - # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) - flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") - # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only - # call was what made the action deletable; running it on every save is what makes step 1 - # permanent, and it is deliberate rather than a widened condition nobody noticed. - if (trigger or {}).get("key") == "ig_profile_match": - flow_raw = _ensure_ig_action(flow_raw, config.get("targetTable") or DISCOVER_TABLE) - elif drop_ig_seeds: - # ITEM 15: the trigger just STOPPED being the Instagram one. Un-seed what the trigger - # planted and nobody has since made theirs — measured against the table the seeds were - # planted for, which is the PREVIOUS config's, not the one being saved. - flow_raw = _drop_ig_seeds( - flow_raw, (prev.get("config") or {}).get("targetTable") or DISCOVER_TABLE) - flow, ferr = clean_flow(flow_raw, prev.get("flow")) - if ferr: - return None, ferr - if (trigger or {}).get("key") == "ig_profile_match": - # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and - # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since - # wave 24: the pinned step 1 says where profiles are written, and the config says where - # the automation's records live — and for a discovery flow those are the same database by - # construction. When they disagreed, every surface picked a different winner (the canvas - # Write node read the config, the actual write read the action), which is a bug you can - # only see by comparing two panels. - # - # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the - # action's own configuration, so the table named there is the one they were looking at. - # - # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established: - # a stored definition carrying two different values is reconciled on its next clean, - # silently and exactly once, because nothing writes the losing value back. A refusal here - # would 400 every Save of a definition that was legal yesterday. - # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the - # TIE-BREAK rather than the whole rule. The owner's report: change the database on the - # TRIGGER, save, and the picker springs back to the old one with no message. The cause is - # the branch below read unconditionally — the pinned action still named yesterday's table, - # so it overwrote a change the person had just made in front of it, silently, every time. - # - # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled - # "when the two disagree, whose value is real?" — the action's, because R2b puts the - # preset list on the action's own panel, so that is the one they were looking at. That - # answers a definition at REST. A save is different: it carries an INTENT, and the honest - # question is which side this particular save changed. So: - # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to - # follow it (leaving the action behind would just re-revert on the next save); - # · only the action moved, or neither did and they were already inconsistent -> R2's - # migration, unchanged, reconciling a stored definition exactly once. - # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is - # also the only reading that cannot lose a value: the action's table is the one with - # the field mapping attached to it. - first = (flow.get("actions") or [{}])[0] - pinned = str((first.get("config") or {}).get("table") or "") \ - if first.get("kind") == "create_record" else "" - prev_target = str((prev.get("config") or {}).get("targetTable") or "") - prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] - ).get("config") or {}).get("table") or "") - target_moved = bool(prev_target) and config.get("targetTable") != prev_target - action_moved = bool(prev_pinned) and pinned != prev_pinned - if pinned and target_moved and not action_moved: - # The person changed the trigger's database. Carry it INTO the pinned step so the two - # agree, instead of throwing their edit away to keep a stale copy consistent. - cfg1 = dict(first.get("config") or {}) - cfg1["table"] = str(config.get("targetTable") or "") - flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] - elif pinned and pinned != config.get("targetTable"): - config["targetTable"] = pinned - # The stored label described a DIFFERENT database, so keeping it would caption the new - # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` - # never relabels a table that already exists — so an adopted hand-made database keeps - # the name its owner gave it. - config["targetLabel"] = "" - return { - "id": _s(prev.get("id") or raw.get("id"), 40), - "name": name, "kind": kind, "config": config, "schedule": schedule, - "trigger": trigger, - # The Builder's ordered actions. An absent flow is simply an automation with no actions. - "flow": flow, - "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, - "runs": list(prev.get("runs") or [])[:MAX_RUNS], - # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 - # minutes to build, so a run persists its id here and the NEXT run collects it. It is - # carried through `patch` untouched because a user pressing Save must not silently orphan - # a set the vendor is already building (and already counting against the funds gate). - "state": dict(prev.get("state") or {}), - "created": prev.get("created") or _iso(), - "createdBy": prev.get("createdBy") or username, - }, None - - -def set_state(rt, auto_id, patch_state): - """Merge into ONE automation's engine state. A key set to None is removed. - - Its own tiny store write rather than a field on `_commit_run`, because the handoff must - survive a run that ends in `error` — a snapshot the vendor is building does not stop existing - because the run that started it failed afterwards. - """ - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - st = dict(d.get("state") or {}) - for k, v in (patch_state or {}).items(): - st.pop(k, None) if v is None else st.__setitem__(k, v) - d["state"] = st - return cur - - rt.update(STORE_KEY, _up, flush="sync") - - -def _without_retired_board(definition): - """Return a definition with retired Board-only state removed, without mutating the store. - - This protects scheduled/manual execution before the next UI list request persists the - migration. It removes only the former Board configuration, actions, and audit trail; normal - actions and all user data remain intact. - """ - if not isinstance(definition, dict): - return definition, False - out, changed = dict(definition), False - cfg = out.get("config") - if isinstance(cfg, dict) and "lanes" in cfg: - cfg = dict(cfg) - cfg.pop("lanes", None) - out["config"] = cfg - changed = True - - def _actions(actions): - nonlocal changed - clean = [] - for action in actions if isinstance(actions, list) else []: - if not isinstance(action, dict): - clean.append(action) - continue - if action.get("kind") == "review": - changed = True - continue - item = dict(action) - if item.get("kind") == "group" and isinstance(item.get("config"), dict): - group_cfg = dict(item["config"]) - branches = group_cfg.get("branches") - if isinstance(branches, list): - kept = [] - for branch in branches: - if not isinstance(branch, dict): - changed = True - continue - next_actions = _actions(branch.get("actions")) - if not next_actions: - changed = True - continue - kept.append({**branch, "actions": next_actions}) - if not kept: - changed = True - continue - if kept != branches: - changed = True - group_cfg["branches"] = kept - item["config"] = group_cfg - elif isinstance(group_cfg.get("actions"), list): - nested = _actions(group_cfg["actions"]) - if not nested: - changed = True - continue - if nested != group_cfg["actions"]: - group_cfg["actions"] = nested - item["config"] = group_cfg - clean.append(item) - return clean - - flow = out.get("flow") - if isinstance(flow, dict): - next_flow = dict(flow) - actions = _actions(flow.get("actions")) - if actions != flow.get("actions"): - next_flow["actions"] = actions - if "ending" in next_flow: - next_flow.pop("ending", None) - changed = True - out["flow"] = next_flow - if "reviews" in out: - out.pop("reviews", None) - changed = True - return out, changed - - -def retire_automation_board_state(rt): - """Persist the idempotent Board retirement, then remove its generated table fields.""" - fields = retire_automation_stage_fields(rt) - try: - stored = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {"definitions": 0, **fields} - plan = {str(aid): cleaned for aid, raw in stored.items() - for cleaned, changed in [_without_retired_board(raw)] if changed} - if not plan: - return {"definitions": 0, **fields} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for aid, cleaned in plan.items(): - if aid in cur: - cur[aid] = cleaned - return cur - - rt.update(STORE_KEY, _up, flush="sync") - return {"definitions": len(plan), **fields} - - -def all_definitions(rt): - try: - raw = dict(rt.get(STORE_KEY) or {}) - except Exception: - return {} - return {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()} - - -def _new_id(existing): - n = 1 - while f"auto_{n}" in existing: - n += 1 - return f"auto_{n}" - - -#: C2's three answers to "which database does this automation work on?" — the FIRST question the -#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). -TARGET_MODES = ("existing", "new", "automated") - - -def resolve_target(rt, raw, username=""): - """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. - - - `existing` — bind a database that is already there. - - `new` — mint a blank one now, so the automation has somewhere to write before its first - run instead of conjuring a table the person never agreed to. - - `automated` — leave it to the runner: a scraping automation MINTS its target on first run - (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. - Nothing is created here, deliberately — an empty table created up front for a scrape that - never runs is litter nobody can explain. - """ - if not isinstance(raw, dict) or not raw: - return {}, None # no target block = the pre-wizard shape - mode = _s(raw.get("mode"), 20).strip() or "existing" - if mode not in TARGET_MODES: - return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) - if mode == "automated": - label = " ".join(_s(raw.get("label"), 60).split()) - return ({"targetLabel": label} if label else {}), None - if mode == "existing": - key = _s(raw.get("table"), 60).strip() - if not key: - return None, "choose the database this automation works on" - t = ut_get(rt, key) - if t is None: - return None, f"{key!r} is not a database in this workspace" - return {"targetTable": key, "targetLabel": t.get("label") or key}, None - label = " ".join(_s(raw.get("label"), 60).split()) - if not label: - return None, "name the new database" - import core.user_tables as _ut_new - key = _ut_new.create(label, username or "automation", st=rt) - if not key: - return None, ("the database could not be created — this workspace may be at its table " - "limit") - return {"targetTable": key, "targetLabel": label}, None - - -def _unmint(rt, key): - """Delete a database this call minted moments ago, because the call is refusing (D-50). - - ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user - needs to read; a rollback that raised would replace that sentence with a 500 and lose the - reason the create failed in the first place. The worst case of a swallowed failure here is - the orphan we had before — strictly no worse, and the refusal still reaches the person. - """ - if not key: - return - try: - import core.user_tables as _ut_del - _ut_del.delete(key, st=rt) - except Exception: # noqa: BLE001 - pass - - -def create(rt, raw, username=""): - existing = all_definitions(rt) - if len(existing) >= MAX_AUTOMATIONS: - return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" - raw = dict(raw or {}) - # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation - # happens after it, so a correct refusal ("a source URL is required") used to leave an - # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: - # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form - # twice silently accumulated empty databases in their nav. - # - # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It - # does not test for `mode == "new"`; it asks "did the table this call's target resolved to - # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and - # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target - # rather than "any table that appeared", so a concurrent create in another request is never - # collateral. - before = set(ut_all(rt) or {}) - patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) - if terr: - return None, terr - minted = str((patch_cfg or {}).get("targetTable") or "") - minted = minted if minted and minted not in before else "" - if patch_cfg: - raw["config"] = {**(raw.get("config") or {}), **patch_cfg} - # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY - # has, instead of letting the pure validator mint a second empty one. See - # `discover_default_table`; a target the caller named is never rewritten. - raw = _apply_discover_default(rt, raw) - defn, err = clean_definition(raw, None, username) - if err: - _unmint(rt, minted) - return None, err - # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations - # that already use them and NO NEW ONE CAN BE CREATED." - # - # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live - # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing - # the kind there would make both of them unsaveable — the ruling retires the door, not the - # automations behind it. Enforced on the server rather than left to the deleted wizard: a - # wall that holds only because the client stopped asking is a convention, not a wall. - if defn["kind"] in RETIRED_KINDS: - _unmint(rt, minted) - return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " - f"longer created — the ones already using it keep working. " - f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") - defn["id"] = _new_id(existing) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - rt.update(STORE_KEY, _up, flush="sync") - _presets_after_write(rt, defn, username) # R10: the columns exist before a run - if defn.get("trigger"): - _seed_event_state(rt, defn) - return defn, None - - -def patch(rt, auto_id, raw, username=""): - existing = all_definitions(rt) - prev = existing.get(str(auto_id)) - if prev is None: - return None, "no such automation" - merged = dict(prev) - # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a - # key missing here is accepted by the route, validated by `clean_definition`, and then - # DISCARDED — the client shows a saved flow that the store never received, and nothing goes - # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only - # ordering that never has a window where the bug exists. - for k in ("name", "config", "schedule", "kind", "trigger", "flow"): - if k in (raw or {}): - merged[k] = raw[k] - # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the - # Instagram trigger on an existing automation flips the kind, and the very next Save runs a - # config that has never carried a target through the pure validator, which mints the empty - # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. - merged = _apply_discover_default(rt, merged) - defn, err = clean_definition(merged, prev, username) - if err: - return None, err - defn["id"] = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - rt.update(STORE_KEY, _up, flush="sync") - _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS - # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching - # DISARMED — enabling never fires for records already in the state. - if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): - _seed_event_state(rt, defn) - return defn, None - - -def _presets_after_write(rt, defn, username): - """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run. - - Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset - columns immediately, "so R2b's list is real and the database is visible before anything is - spent". That last clause is the point — a person can look at the database, see the sixteen - columns, and decide whether to arm the schedule, instead of paying a vendor to find out what - they are agreeing to. - - ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs - nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first - run of a saved automation finds the table already correct and spends no commit. - - ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the - search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the - preset set, and spawning a subset here would mean the columns changed between Save and the - first run — two different answers to "what does this database look like", which is the exact - disagreement R10 exists to remove. - - ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure` - returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an - automation because the workspace is full would be a worse answer than saving one whose table - arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the - honest failure still has exactly one home. - """ - cfg = defn.get("config") or {} - target = str(_flow_table(defn) or cfg.get("targetTable") or "") - if not target: - return - try: - if (defn.get("trigger") or {}).get("key") == "ig_profile_match": - ut_ensure(rt, cfg.get("targetLabel") or DISCOVER_LABEL, CANDIDATE_FIELDS, username, - key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True) - return - enrich_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []), - "enrich_instagram") - table = ut_get(rt, target) or {} - named = next((str((a.get("config") or {}).get("profileField") or "").strip() - for a in enrich_actions - if str((a.get("config") or {}).get("profileField") or "").strip()), "") - bound = next((str(f.get("key") or "") for f in table.get("fields") or [] - if isinstance(f.get("profile"), dict)), "") or named - if enrich_actions and bound: - ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username, - key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True) - except Exception as e: # noqa: BLE001 - # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so - # raising here would answer 500 for an automation that IS stored — the caller would retry - # and create a second one. The columns then arrive on the first run, which is precisely - # the behaviour this function is an improvement on rather than a replacement for. - print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}") - - -def remove(rt, auto_id): - aid = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur.pop(aid, None) - return cur - rt.update(STORE_KEY, _up, flush="sync") - # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned - # silently: the column keeps its values and its config, and says why it stopped. Scanned - # first so a delete with no tagged fields costs no store commit. - hit = any(str((f.get("automation") or {}).get("flowId") or "") == aid - for t in ut_all(rt).values() for f in (t.get("fields") or [])) - if not hit: - return - - def _disable(cur): - cur = cur if isinstance(cur, dict) else {} - for t in cur.values(): - for f in (t.get("fields") or []): - a = f.get("automation") - if isinstance(a, dict) and str(a.get("flowId") or "") == aid: - a["disabled"] = True - a["statusNote"] = f"its automation was deleted {_stamp()}" - return cur - rt.update(UT_STORE_KEY, _disable, flush="sync") - - -# --------------------------------------------------------------------------------------------- -# RUN STATE — process memory only (see the module header for why this may never be persisted) -# --------------------------------------------------------------------------------------------- - -_RUN_LOCK = threading.RLock() -_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} - - -def running(tenant, auto_id): - with _RUN_LOCK: - return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None - - -def _claim(tenant, auto_id): - with _RUN_LOCK: - if (tenant, str(auto_id)) in _RUNNING: - return False - _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} - return True - - -def _step(tenant, auto_id, text): - with _RUN_LOCK: - cur = _RUNNING.get((tenant, str(auto_id))) - if cur is not None: - cur["step"] = text - - -def _no_step(_text): - """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner - directly). A runner must never REQUIRE the slot — the slot is process state, and the runner - is the part a test is allowed to call on its own.""" - - -def _release(tenant, auto_id): - with _RUN_LOCK: - _RUNNING.pop((tenant, str(auto_id)), None) - - -def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None): - """THE ONE store write a run performs on the `automations` bucket. - - `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here - because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to - invent a node status when this map is absent (every run stored before W19-C has no `steps`, - and painting those nodes green from the run's overall state would be a fabrication). - - ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the - comprehension below drops everything else — so the per-record sentence a vendor gave us had - literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the - product could say. MEASURED on nurilab: the same record blocked on three separate runs and - the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by - calling the vendors by hand. The note is the most valuable thing a run produces — a dead - handle, a vendor hiccup and an exhausted key are three different actions — and it was the one - thing thrown away. - ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it - did not ask for, and W25/R4 already retired writing status STRINGS into people's grids. - """ - entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), - "counts": {k: int(v) for k, v in (counts or {}).items() - if isinstance(v, (int, float))}, - "steps": {str(k): str(v) for k, v in (steps or {}).items()}, - # Bounded on both axes: a 100-profile run must not put 100 sentences into a store - # entry that is kept 20 deep per automation. - "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25], - "affected": list(affected or [])[:200]} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - d["status"] = {"state": state, "lastRunAt": entry["ts"], - "lastSummary": entry["summary"]} - d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] - # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with - # the reason on it — a dead credential must not burn quota (or a paid vendor's records) - # 96 times a day while a green toggle sits over a buried error log. Errors only: - # `partial` is the honest-progress state and pausing on it would punish honesty. - runs = d.get("runs") or [] - if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( - not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): - sch = d.get("schedule") - if isinstance(sch, dict) and sch.get("enabled"): - sch["enabled"] = False - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " - f"failures — fix the cause, then re-enable: " - f"{entry['summary'][:120]}") - return cur - - rt.update(STORE_KEY, _up, flush="sync") - _notify_outcome(rt, auto_id, state, entry) - return entry - - -def _notify_outcome(rt, auto_id, state, entry): - """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7). - - Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is - written, so it is the only place this can go without a second definition of "what happened". - - ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED. - 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the - MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a - background thread with no session, so an omitted `st` files nurilab's automation failures - in Royal Imports' inbox — D-16's class, silently, forever. - 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler - has no session at all, so the alternative is not "the wrong person" but "nobody", and - `notify` returns None on a blank owner — the notification would simply never exist, on - exactly the runs nobody was watching. - ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only - view-alert CREATION; `notify()` writes straight into the notifications bucket and - `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell. - ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that - SUCCEEDED into one that raised — the run entry is already committed one line above, and the - bell is a courtesy on top of it. - """ - try: - defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {} - owner = str(defn.get("createdBy") or "").strip() - if not owner: - return - from core import alerts as _alerts - _alerts.notify( - owner=owner, - label=_s(defn.get("name") or "Automation", 80), - topic="automation", - key=str(auto_id), - detail=f"{state}: {entry.get('summary') or ''}"[:300], - st=rt) - except Exception: # noqa: BLE001 - pass - - -# --------------------------------------------------------------------------------------------- -# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN -# --------------------------------------------------------------------------------------------- -# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and -# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) -# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: -# -# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, -# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the -# supplier takes the scraping risk; it is never an Instagram credential and this module must -# never be given one. -# -# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). -# -# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress -# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been -# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and -# the cell says which. A pull that quietly wrote zero posts and reported success would be worse -# than no automation at all: the table would look maintained and be empty. -# -# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, -# so a run history answers "what still works anonymously?" from data rather than from memory. - -PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) - -#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's -#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number -#: is a NOW value, so the series starts the day capture starts and a field we do not capture -#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows -#: has exactly the same cost as delaying capture — for that column — and is justified by exactly -#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to -#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. -#: Every returned provider field is retained in Source data. Promoted columns make commonly used -#: values filterable; the raw source document prevents a new or uncommon field from being lost. -#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose -#: almost none of them, which is what the `source` column is for. -SNAPSHOT_FIELDS = [ - field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"), - field_def("following", "Following", "int"), - field_def("posts_count", "Post count", "int"), - field_def("full_name", "Name"), field_def("bio", "Bio"), - field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"), - field_def("approx", "Counts are approximate", "checkbox"), - # The link in bio — the paid rung's field, and commercially the most useful single string an - # influencer row can carry (it is the shop/affiliate destination). - field_def("external_url", "Link in bio", "url"), - # --- the rest of the vendor's Profiles schema. - field_def("ig_id", "Instagram id"), - field_def("profile_url", "Profile", "url"), - field_def("avg_engagement", "Avg engagement", "pct"), - field_def("category", "Category"), - field_def("business_category", "Business category"), - field_def("is_business", "Business account", "checkbox"), - field_def("is_professional", "Professional account", "checkbox"), - field_def("is_private", "Private", "checkbox"), - field_def("highlights_count", "Story highlight count", "int"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("pronouns", "Pronouns"), - # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot - # row maps the WHOLE schema by design, so these belong here the moment the map reads them; - # leaving them out would be the "captured but unrecorded" half of the same loss the note at - # the top of this list is about. - field_def("profile_name", "Profile name"), - field_def("is_joined_recently", "Joined recently", "checkbox"), - field_def("has_channel", "Has channel", "checkbox"), - field_def("partner_id", "Partner id"), - field_def("external_url_title", "Link title"), - field_def("fbid", "Facebook id"), - field_def("related_accounts", "Related accounts"), - field_def("country_code", "Country"), - field_def("source_payload", "Source data", "json"), -] -POST_FIELDS = [ - field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), - field_def("posted_at", "Posted at", "date"), - field_def("type", "Type", "select", options=["image", "video", "carousel"]), - field_def("caption", "Caption"), field_def("url", "URL", "url"), - # A tagged place is useful content context and can be filtered as ordinary text. It is never - # presented as the creator's location: a creator can tag a holiday, venue or brand location. - field_def("tagged_location", "Tagged location"), - # The per-field map below gives operational fields first; this locked JSON document retains - # every other value Bright Data supplied, so a vendor schema addition is preserved - # instead of being silently discarded while the canonical field model catches up. - field_def("source_payload", "Source data", "json"), - # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous - # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). - field_def("paid_partnership", "Paid partnership", "checkbox"), - field_def("partner", "Partner brand"), - field_def("hashtags", "Hashtags"), - field_def("alt_text", "Alt text"), - # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT - # ONE HOP. - # - # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's - # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a - # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated - # transitively). So the post row carries its own LATEST value and the rollup reads it - # directly. - # - # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile - # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in - # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that - # was just appended — they are a projection of that store, never a second copy of it, and - # deleting them would cost a convenience rather than a measurement. - # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is - # off by default (it buys one vendor record per post), so on most pulls this stays empty, and - # empty means "not read", which is what makes an honest average possible at all. A 0 here - # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or - # COMMENT count is a real measurement and must survive. - # - # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08, - # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought). - # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct - # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in - # the very same rows. Mapping it here is what put one number on many unrelated posts. We - # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers; - # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays - # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained - # whole, so if the vendor ever populates it per-reel the history is re-derivable. - # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30 - # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the - # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named - # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10 - # (folded into a single "Views"), not because we are reading the wrong key. - # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED. - # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the - # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` — - # measured against a browser-read ground truth to the digit. Same column name, same type, same - # rollup; the engine underneath is swappable and the preset database never moved. That is the - # owner's schema ruling working exactly as intended. - # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung. - field_def("views", "Views", "int"), - field_def("plays", "Plays", "int"), - field_def("likes", "Likes", "int"), - field_def("comments", "Comments", "int"), - # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data - # further"*). Two fields the primary source does not return AT ALL, already paid for inside - # the same engagement response — so capturing them costs nothing extra. - # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on - # every write door, so a normaliser that emits `video_duration` without this line writes - # nothing and reports success — the wave-28 defect that swallowed nine preset cells. - field_def("video_duration", "Video length (s)", "int"), - field_def("comments_disabled", "Comments off", "checkbox"), - field_def("measured_at", "Engagement read at", "date"), - # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED": - # a post reaches its own comment rows the same way a profile reaches its posts. Derived from - # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance. - # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state - # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when - # `postMetrics` is off. - field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this post.", - link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}), - field_def("post_snapshots_link", "Measurement rows", "link", - description="Engagement measurements linked to this post.", - link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}), - field_def("measurements_captured", "Measurements captured", "rollup", - rollup={"link": "post_snapshots_link", "fn": "countall"}), -] - -#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE. -#: -#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to -#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default -#: is the capture. -#: -#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED. -#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE: -#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of -#: `instagram-capture.md`). They are the single most expensive thing this product can buy. -#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has -#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment -#: thread ingests identifiable people who never entered anybody's influencer list. -#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code -#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments` -#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on -#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is -#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md` -#: is written against. -#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as -#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs -#: nothing; the spend and the third-party ingest are a click somebody makes knowingly. -#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because -#: they are part of a record already paid for. The separate full Comments dataset remains OFF -#: unless `commentMetrics` is explicitly enabled on the enrichment action. -COMMENT_FIELDS = [ - field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"), - field_def("influencer_key", "Influencer"), - field_def("commented_at", "Commented at", "date"), - field_def("likes", "Likes", "int"), - field_def("replies", "Replies", "int"), - field_def("source_payload", "Source data", "json"), - field_def("post_link", "Post", "link", - description="Post record linked to this comment.", - link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), - # Author/text and every provider-specific value stay intact in Source data. The promoted - # columns intentionally keep the grid concise while Likes and Replies make comment engagement - # directly filterable and rollup-ready. -] -POST_SNAPSHOT_FIELDS = [ - field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), - field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"), - field_def("comments", "Comments", "int"), - # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits - # before any rollup argument is made. - # - # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with - # only the first can describe a measurement but not its AGE, so the single most useful thing - # this series can compute — views at N days old, i.e. velocity — is not expressible over it at - # all. `type` is the same shape one column over: "reels only" is the default lens on this data - # and without it every question has to go back through `ut_ig_posts` to ask what kind of post - # this was. - # - # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row - # from the same vendor record), so this costs one dict key each and no extra call. - # - # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24 - # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the - # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied - # from it deliberately rather than shared, because these are two different tables' columns that - # happen to agree today, and `verify_automation` asserts the agreement. - # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO - # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`, - # and `ut_ig_posts` already performs the first of them. See the verdict in - # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for - # moving the window here. - field_def("posted_at", "Posted at", "date"), - field_def("type", "Type", "select", options=["image", "video", "carousel"]), - # Plays move independently of likes on video, so it is its own series rather than a thing to - # derive. Paid rung only; blank means not read. - # The series regains Views alongside the latest-value projection on the Post row — one store - # for one series (R3) is untouched; this IS that store. - field_def("views", "Views", "int"), - field_def("plays", "Plays", "int"), - field_def("source_payload", "Source data", "json"), - field_def("post_link", "Post", "link", - description="Post record linked to this measurement.", - link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode", - "single": True}), -] - - -# --------------------------------------------------------------------------------------------- -# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`. -# --------------------------------------------------------------------------------------------- -# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE. -# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation -# runtime needs that, and the runtime is what this file is for. -# -# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation -# runtime still reaches for after the split — twenty, down from the seventy-one that used to be -# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor -# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and -# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that -# true without those files having to learn where the wire went. -# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked -# that would be green whether or not anything moved. `verify_automation`'s `section_split` -# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING -# system instead of trusting this import line ([[gate-answers-the-wrong-question]]). -# -# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back -# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily, -# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail -# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a -# module-level import, the names it wants already exist and the cycle still resolves. -from connectors_ig import ( # noqa: E402 - BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, - BD_RECORD_PRICE_SPEC, - _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_profile, _bd_rows, - _bd_tagged_location, - _first, _ig_int, - DEFERRED_MARK, - apify_profile, - bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready, - bd_snapshot_progress, depth_refusal, - ig_handle, public_source, pull_profile, select_post_groups, top_up_views, -) - - -# --------------------------------------------------------------------------------------------- -# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) -# --------------------------------------------------------------------------------------------- -# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's -# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns -# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND -# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower -# Georgia/Florida floral-design company — Fisch Floral's actual market. -# -# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: -# -# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` -# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is -# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the -# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. -# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, -# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, -# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still -# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a -# snapshot that is not ready when it looks. -# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery -# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` -# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is -# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the -# next run collects it. A `partial` that says "still building, the next run collects it" is -# the honest report of exactly what happened. -# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and -# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into -# a wider profile set would silently multiply the enrichment bill AND pollute the -# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich. -# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important -# test died on it. -# -# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE -# the API returns a number, `price: 0` means "not priced" rather than "free", and -# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to -# report. Every estimate below is derived from the PRICING PAGE and says so. - -#: The table a discovery run writes when the tenant has no Instagram profile database yet — the -#: FIRST one, never a second (`discover_default_table` elects an existing one before this is -#: reached). Upsert by handle, so re-finding a profile is free. -#: -#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG -#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said -#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`, -#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept -#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it -#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on -#: any tenant who renamed the table, which is exactly the mismatch the owner hit. -#: -#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three -#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the -#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key -#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant -#: would keep working; there simply is not one. -#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration -#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family. -DISCOVER_TABLE = "ut_ig_profile" -#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels, -#: so changing this can rename nothing that already exists. One constant because it was three -#: copies of the same string literal, and a default spelled three times is one edit from drifting. -DISCOVER_LABEL = "IG profile" -#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at -#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed -#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real -#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. -BD_MAX_RECORDS = 500 -#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). -BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) -BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) - -#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error -#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. -BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", - "array_includes", "not_array_includes", "is_null", "is_not_null") -#: Operators that take NO value (everything else requires one). -BD_NULLARY_OPS = ("is_null", "is_not_null") - -#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** -#: by the API and is therefore absent rather than offered-and-broken). -#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY -#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier -#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the -#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by -#: accident; adding them back is a decision, not a typo. -BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", - "category_name", "business_category_name", "is_business_account", - "is_professional_account", "is_verified", "account", "full_name", - "external_url", "bio_hashtags", "post_hashtags", "profile_url", - "related_accounts", "profile_name", "id", "fbid", "highlights_count") -#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: -#: these are the fields seen carrying values on real CORPUS rows. `category_name` and -#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter -#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". -BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") - -#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. -#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still -#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not -#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a -#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the -#: scanner still has to walk, so they refine a search and cannot BE one. -BD_NARROWING_FIELDS = frozenset({ - "biography", "account", "full_name", "profile_name", "category_name", - "business_category_name", "external_url", "bio_hashtags", "post_hashtags", - "related_accounts", "id", "fbid", "profile_url"}) -#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the -#: whole corpus, which is breadth wearing a condition's clothes. -BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) -#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed -#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% -#: populated: below it a filter misses more corpus than it matches, and the surface should -#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, -#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never -#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. -#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding -#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. -BD_POPULATED_FIELDS = frozenset({ - "account", "followers", "following", "posts_count", "avg_engagement", "biography", - "external_url", "is_business_account", "is_professional_account", "is_verified", - "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) -BD_MIN_NARROWING = 1 - -# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ────────────────── -# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY -# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what -# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner: -# *"look at each damn schema and only provide toggles operator that make sense"*. -# -# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx` -# defended raw names on the grounds that a refusal names the field exactly as the row does — and -# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI -# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio". -BD_FIELD_LABELS = { - "followers": "Followers", "following": "Following", "posts_count": "Post count", - "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category", - "business_category_name": "Business category", "is_business_account": "Business account", - "is_professional_account": "Professional account", "is_verified": "Verified", - "account": "Handle", "full_name": "Name", "profile_name": "Profile name", - "external_url": "Link in bio", "profile_url": "Profile link", - "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts", - "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID", - "highlights_count": "Story highlight count", -} -#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident; -#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson). -BD_FIELD_HINTS = { - "account": "The @username", - "avg_engagement": "Likes and comments as a share of followers", - "id": "Instagram's internal number for the account — for matching a list you already have", - "fbid": "The linked Facebook id — for matching a list you already have", - "profile_name": "The name shown above the bio, when it differs from the account name", - "related_accounts": "Accounts Instagram suggests alongside this one", -} -#: The KIND decides the comparisons offered and the control drawn for the value. -BD_FIELD_KINDS = { - **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement", - "highlights_count")}, - **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")}, - **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")}, - **{n: "choice" for n in ("category_name", "business_category_name")}, - **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url", - "profile_url", "id", "fbid")}, -} -#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of -#: each list is what `default_operator` must agree with. -BD_OPS_BY_KIND = { - # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second - # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one - # idea — and the second one is the one written in vendor grammar. - "text": ("includes", "not_includes", "=", "is_not_null", "is_null"), - "choice": ("=", "!=", "is_not_null", "is_null"), - "number": (">=", "<=", "=", ">", "<"), - # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean - # is `=` with the other value, written the confusing way. - "boolean": ("=",), - "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"), -} -#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire -#: (it is what the API accepts); nobody has to read it. -BD_OP_LABELS = { - "includes": "contains", "not_includes": "does not contain", - "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of", - ">=": "at least", "<=": "at most", ">": "more than", "<": "less than", - "array_includes": "has any of", "not_array_includes": "does not have", - "is_null": "is empty", "is_not_null": "is not empty", -} -#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into. -BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}] - -#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many -#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start` -#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing -#: the whole search to "match any" (which would also drag Followers into the union — the hole the -#: OR guard closed the same day). -#: -#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not -#: contain floral OR does not contain flower" matches nearly every account alive — a negative over -#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a -#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs -#: them enough to write the AND branch. -BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"}) -MAX_PREDICATE_VALUES = 12 - - -def field_label(name): - """The human name for a searchable field — used by the SURFACE and by every REFUSAL.""" - return BD_FIELD_LABELS.get(name) or str(name or "that field") - - -def field_kind(name): - return BD_FIELD_KINDS.get(name, "text") - - -def ops_for(name): - """The comparisons this field may be asked. A tuple, in offer order.""" - return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"]) - -#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing -#: law, because the two drifted apart and the drift cost a user their first automation. -#: -#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no -#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its -#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS` -#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door -#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in — -#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one -#: sentence of drift, and nothing red anywhere at any point. -#: -#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field -#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks -#: one itself — a client-side default is a second copy of this table, free to disagree with the -#: guard that judges it, which is exactly what happened. -BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"}) -BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"}) -#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a -#: worse question than an equality, and both narrow. -BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"}) - - -def default_operator(name): - """The operator a NEW condition on `name` starts with. - - Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from - saveable rather than two (change the comparison, then type a value) — with the second step - being one the surface never told anyone to take. - """ - kind = field_kind(name) - if kind == "tags": - return "array_includes" - if kind in ("boolean", "choice"): - # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through - # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so - # the default would have been an operator the same module refuses to offer — the exact - # default-versus-guard split this function was written to close, reintroduced one commit - # later by adding a kind. Asserted per field in `verify_automation.py`. - return "=" - if kind == "number": - # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person - # reaching for Followers means, and the guard's sentence is the honest answer when it is - # the ONLY condition present. - return ">=" - return "=" if name in BD_EXACT_FIELDS else "includes" - - -def predicate_narrows(p): - """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" - return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS - and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) - - -def narrowing_refusal(preds, operator="and"): - """C4's server law: the sentence when a predicate set does not narrow, else ''. One function - so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law) - refuse in the same words. - - ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY - predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a - UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR - followers >= 10000` saved clean and asked the vendor for every account over ten thousand - followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop. - A money wall that a dropdown three rows above it can walk through is not a wall. - - So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows - either way — `all([])` is True, which would have made "no conditions at all" the widest legal - query of the lot. - """ - preds = list(preds or []) - if str(operator or "").lower() == "or": - if preds and all(predicate_narrows(p) for p in preds): - return "" - return ("with Match set to any, every condition has to describe the account itself " - "(Bio, Handle, Name, a hashtag). Matching any means the results are added " - "together, so one follower or yes/no condition widens the whole search. Narrow " - "every condition, or set Match to all") - if any(predicate_narrows(p) for p in preds): - return "" - return ("add a condition that describes the account itself — Bio contains, Handle is, a " - "hashtag. Follower counts and yes/no conditions alone match too much of Instagram " - "to search") - - -def filter_meta(): - """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]` - over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, - R3). - - `defaultOperator` rides here rather than being a client constant for the reason written over - `default_operator`: the client's own default was `is_not_null` for every field, which the - narrowing guard refuses on every field. - """ - return [{"name": n, - "label": field_label(n), - "hint": BD_FIELD_HINTS.get(n, ""), - "kind": field_kind(n), - "populated": n in BD_POPULATED_FIELDS, - "narrowing": n in BD_NARROWING_FIELDS, - "defaultOperator": default_operator(n), - # The comparisons THIS field may be asked, already in plain English and already in - # offer order. A client that filtered a global list by field kind would be a second - # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale. - "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o), - "nullary": o in BD_NULLARY_OPS, - # `in`/`is any of` and the tag operators take a LIST — the control - # has to know that, and deriving it from the token in the client is - # the same second-copy mistake one level down. - "multi": o in BD_MULTI_VALUE_OPS} - for o in ops_for(n)], - "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []} - for n in BD_FILTER_FIELDS] - -#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET. -#: -#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what -#: "unified across tenant" means and what makes the pooled master series joinable at all — two -#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query. -#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET -#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat. -#: -#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in -#: `ut_ig_snapshots`, which the metric fields already read. ONE STORE FOR ONE SERIES: no per-record -#: `json` history column, no second copy of a number that already has a home. -#: -#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION, -#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields -#: available to us from Bright Data to be pre-set Fields for us and populated."* -#: -#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule -#: and the next reader will otherwise re-derive it and prune these columns back. It said: a -#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on -#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles, -#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is -#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were -#: captured into the snapshots and were NOT preset columns. -#: -#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be -#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the -#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus; -#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does -#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and -#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a -#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make. -#: Source data retains every field the provider returns. The only separate decision is whether to -#: request the paid full Comments dataset (`commentMetrics`, default OFF). -#: -#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is -#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`). -#: -#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person -#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and -#: written down here rather than inferred from a runner's module name, because the day a second -#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the -#: profile simply appears twice. -#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it. -#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals -#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as -#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names -#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things, -#: which is exactly why both are written down instead of inferred. -PROFILE_SOURCE_IG = "instagram" - -PLATFORM_INSTAGRAM = "Instagram" -PLATFORM_TIKTOK = "TikTok" # D-9's runner will stamp this; nothing writes it yet -PLATFORM_FACEBOOK = "Facebook" -PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK) - -#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT. -#: -#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as -#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring -#: `followers` as `int` gives NEW tables a schema every table already in production does not have -#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument -#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason -#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format? -#: change this."* -#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without -#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it -#: would do it silently, because a merged-by-key field list produces no error when two tables -#: disagree about what a column IS. -#: -#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor -#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this -#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction -#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the -#: mapping and in the migration both. See `_pct100`. -PRESET_PROFILE_FIELDS = [ - # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now. - # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different - # people as often as not, so the dedup key is (platform, handle) and never handle alone - # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung - # answered* (`field_def("source", "Read via")`), and one word meaning two things across two - # tables in one product is the drift this whole list exists to prevent. - field_def("platform", "Platform"), - # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING. - # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be - # REPLACED or made as the Handle. in any case the user can add their own record right and edit - # those records."* Two declarations, one field, and they are the same sentence read twice: - # - # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT - # this the primary is an accident of creation order. It is a DECLARATION, not a - # reorder: the stored field order is left alone (saved view orders keep working) - # and the client pins the column to position 0 wherever it sits. That is why this - # fixes tables that already exist without moving a single stored field. - # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to - # (`profile_field_key` step 2), which is what turns "I added a profile to my - # database, how do I enrich it?" from an unanswerable question into a step. It - # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle - # all normalise to the bare handle) and routes a typed value to the SHARED - # definition row rather than one user's overlay (C3-A1) — which is precisely what - # makes the owner's second clause true: a record you add BY HAND is a record the - # engine can read and enrich. - # - # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag - # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on - # anything but `text`. - field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}), - field_def("profile_url", "Profile", "url"), - field_def("full_name", "Name"), field_def("followers", "Followers", "int"), - field_def("following", "Following", "int"), - field_def("avg_engagement", "Avg engagement", "pct"), - field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), - field_def("verified", "Verified", "checkbox"), field_def("category", "Category"), - # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields - # MEASURED ≥60% populated (see the note above). - field_def("posts_count", "Post count", "int"), - field_def("highlights_count", "Story highlight count", "int"), - field_def("is_business", "Business account", "checkbox"), - field_def("is_professional", "Professional account", "checkbox"), - # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would - # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it. - field_def("ig_id", "Instagram id"), - # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction - # (see the block comment above for what that reverses). Types are HONEST from birth, which - # costs nothing here: no table has ever carried these columns, so there is no stored `text` - # definition for the migration to convert — R3's conversion rule applies to the columns that - # already exist, and these are new everywhere. - field_def("business_category", "Business category"), - field_def("is_private", "Private account", "checkbox"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("pronouns", "Pronouns"), - field_def("profile_name", "Profile name"), - field_def("is_joined_recently", "Joined recently", "checkbox"), - field_def("has_channel", "Has channel", "checkbox"), - field_def("partner_id", "Partner id"), - field_def("external_url_title", "Link title"), - # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity. - field_def("fbid", "Facebook id"), - field_def("related_accounts", "Related accounts"), - field_def("country_code", "Country"), - field_def("source_payload", "Source data", "json"), - # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT. - # - # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is - # linked to the profile automatically… and this rollup needs to have formula that we can use - # to calculate things like average Views over last N posts."* Both halves are these five - # columns, and they are PRESETS rather than something a person assembles, because "linked - # automatically" is the instruction — a relation you have to wire up by hand is the feature - # not existing. - # - # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged - # column and then the pinned one, both of which are `handle` on every preset table — so this - # links correctly on a database where the handle column was renamed, and on one where the - # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject - # [[gate-answers-the-wrong-question]] warns about, one layer down. - # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total. - # Two count-like columns with the same label would be unreadable. Caught by the label-collision - # gate rather than on screen, which is what that gate is for. - field_def("posts_link", "Post rows", "link", - description="Post records linked to this profile.", - link={"table": IG_POSTS_TABLE, "on": "influencer_key"}), - field_def("profile_snapshots_link", "Profile history", "link", - description="Profile snapshots linked to this profile.", - link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}), - field_def("post_snapshots_link", "Post measurement rows", "link", - description="Post engagement measurements linked to this profile.", - link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}), - field_def("comments_link", "Comment rows", "link", - description="Comment records linked to this profile.", - link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}), - # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup - # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further - # out in `ut_ig_post_snapshots`. - # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be - # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30 - # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant - # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a - # `limit`, and this is why — "the last 12" has to name what makes one post later than another. - # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours - # because its input was an account-grain constant; with the input now a real per-reel - # measurement the average means what its label says again. - field_def("avg_views_12", "Avg views · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "views", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "plays", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "likes", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup", - rollup={"link": "posts_link", "field": "comments", "fn": "average", - "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc", - "distinctBy": "shortcode"}), - # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO - # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after - # a paid enrich. This counts the post rows actually captured — a different number and a true - # one, which is why it gets its own column and its own label rather than quietly filling - # `posts_count` with something that is not what that field means. - field_def("posts_captured", "Posts captured", "rollup", - rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}), - field_def("profile_reads", "Profile reads", "rollup", - rollup={"link": "profile_snapshots_link", "fn": "countall", - "distinctBy": "snapshot_key"}), - field_def("post_measurements_captured", "Post measurements captured", "rollup", - rollup={"link": "post_snapshots_link", "fn": "countall", - "distinctBy": "post_snapshot_key"}), - field_def("comments_captured", "Comments captured", "rollup", - rollup={"link": "comments_link", "fn": "countall", - "distinctBy": "comment_key"}), - # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING. - # - # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we - # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one - # thing a buyer most wants and the one thing the profile row cannot answer. The competitor - # teardown found the same gap solved by GUESSING from bio words, two buckets deep - # ([[janney-ai-teardown]]). - # - # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place - # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the - # post row, plus the vendor's fuller `location`/`location_details` inside the paid - # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a - # creator's own posts is a far better residency signal than a word in a bio, and it costs a - # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1. - # - # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator - # posting from twelve cities gets a low confidence rather than a confident wrong answer, and - # `location_confidence` is the number a person filters on before trusting the guess. A single - # geotagged post produces NO guess at all — n=1 dressed as a pattern is what - # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain. - field_def("location_guess", "Location (guess)"), - field_def("location_confidence", "Location confidence", "pct"), - # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never - # enriched" and a stale one means "enriched in March", and no cell on the row can tell them - # apart. It is the single field that turns the other fifteen from numbers into measurements. - field_def("enriched_at", "Enriched at", "date"), - # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE. - # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the - # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the - # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying - # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two, - # rewritten each run, so a person reading the profile row can see the recent posts without a - # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2. -] -#: The keys the preset set owns — derived, so a field added above cannot be forgotten here. -PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS) - -#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile -#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what -#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the -#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now -#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to -#: enforce a rule nobody declares any more. -#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" — -#: never as "field number zero". -PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "") -PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "") - -#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the -#: alternative is two lists that describe the same columns and drift one label at a time, which is -#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found -#: them, who for, and a human's decision) rather than about the profile, so they are discovery's -#: and not part of the cross-tenant set. -CANDIDATE_FIELDS = [ - *PRESET_PROFILE_FIELDS, - # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell. - # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner - # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a - # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its - # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what - # changed is that "when did we first see this account" is a DAY, and a day is all anybody - # asks it for. `migrate_ig_field_types()` converts the stored strings. - field_def("found_count", "Times found", "int"), - field_def("first_found", "First found", "date"), - field_def("last_found", "Last found", "date"), - # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it - # describes, which is the first place anybody looks up what this column means. It read: *"the - # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two - # people discovering the same profile each get their own row, their own found_count and their - # own review card."* Wave 26 · R4/R5 retired every clause of that. - # - # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two - # people in one workspace who discover the same profile share ONE row, one `found_count` and - # one history — because they are looking at one company's leads, not two private lists. - # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it - # first. It is no part of the dedup key, and a run must never branch on it. - # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged, - # and the one clause of the old note that was still true. - field_def("created_by", "Found by"), - # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have - # a Field called stage to track the progress of the Automation per Record. We don't need - # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage - # column: every candidate carried BOTH a `stage_` select saying where it was AND a - # boolean saying whether it was kept — two progress fields that could disagree, with no rule - # about which one won. The stage field is the one progress column. Nothing replaces this. -] - - -def _profile_backlink_field(table_key, table_label, profile_key): - """A deterministic reciprocal link from one canonical IG table to one profile database. - - The key includes the target table identity, so ten separate Profile databases can all point - into the same canonical Posts/Comments/history tables without one relation overwriting the - next. The join is derived in both directions and therefore needs no cross-table fan-out write. - """ - digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10] - return field_def( - f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link", - description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.", - link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"}, - ) - - -def _profile_schema_for(bound_key): - """Preset fields for an existing Profile table without inventing a second identity column.""" - out = [] - for field in PRESET_PROFILE_FIELDS: - item = dict(field) - if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY: - item.pop("profile", None) - out.append(item) - return out - - -def _locked_ig_field(field): - """One canonical Instagram field with the immutable preset declaration attached.""" - item = dict(field) - automation = dict(item.get("automation") or {}) - automation["preset"] = True - item["automation"] = automation - return item - - -def _profile_binding(table): - """The declared Profile identity field, falling back only to the canonical handle.""" - fields = list((table or {}).get("fields") or []) - flagged = next((str(f.get("key") or "") for f in fields - if isinstance(f.get("profile"), dict)), "") - if flagged: - return flagged - return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else "" - - -def _ig_schema_contract(rt, profile_tables=None): - """The complete per-tenant Instagram graph contract, including every Profile backlink. - - The fixed child tables are shared. Profile tables are discovered structurally, so an older - user-named target and the built-in candidate database receive the same preset columns, Links, - Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted. - """ - tables = ut_all(rt) - profile_keys = sorted(set(profile_tables if profile_tables is not None - else _ig_profile_tables(rt))) - backlinks, wanted, drops = [], {}, { - IG_SNAPSHOTS_TABLE: {"post_hashtags"}, - # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today - # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS - # and fed by the `ig_post_views` capability. Leaving it here would have made the migration - # delete, on every authenticated read, the column the same release just added — the - # drop set and the field list are two halves of ONE contract and must move together. - } - for table_key in profile_keys: - table = tables.get(table_key) or {} - bound = _profile_binding(table) - if not bound: - continue - label = str(table.get("label") or table_key) - backlinks.append(_profile_backlink_field(table_key, label, bound)) - schema = _profile_schema_for(bound) - # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram - # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT - # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION. - # - # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every - # contract key it owns — `rollup` included, by its own note — so a column converted beside - # the contract is re-typed back to `int` on the next reconcile, silently, hours later. - # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the - # schema instead of two that disagree. - # - # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a - # column only where this tenant's own rows show the fold already equals the stored value — - # measured, because a named list would have blanked 221 live cells on nurilab alone - # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts - # fewer columns and loses nothing; as its history fills in, later passes convert more. - derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {})) - wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f - for f in schema] - # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states - # to disagree, so it is retired by key just like the old nested Posts JSON. - drops[table_key] = {"posts", "post_hashtags", "tracked"} - wanted.update({ - IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks], - IG_POSTS_TABLE: [*POST_FIELDS, *backlinks], - IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks], - IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks], - }) - return wanted, drops - - -# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use -# the scalar strings their renderers expect; engagement is the exception because Bright Data's -# 0-1 fraction becomes this product's 0-100 percentage. -_IG_RETYPE_CONVERTERS = { - (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100, -} - - -def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): - """Repair machine-owned IG field declarations in one coalesced store update. - - `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not - sufficient for a canonical relation: a stale `link.table`, rollup function, or type would - survive forever. This pass overwrites the contract keys for the fields we own, preserves - unrelated/user fields, and removes only explicitly retired preset columns plus their cells. - """ - drop_by_table = drop_by_table or {} - wanted_by_table = { - key: [_locked_ig_field(f) for f in fields] - for key, fields in wanted_by_table.items() - } - changed = False - - def _up(cur): - nonlocal changed - cur = cur if isinstance(cur, dict) else {} - for table_key, wanted_fields in wanted_by_table.items(): - table = cur.get(table_key) - if table is None: - continue - wanted = {str(f.get("key")): dict(f) for f in wanted_fields} - drops = set(drop_by_table.get(table_key) or ()) - fields, seen = [], set() - drops = _guarded_drops(table, drops) - for stored in table.get("fields") or []: - key = str(stored.get("key") or "") - if key in drops: - changed = True - continue - desired = wanted.get(key) - if desired is None: - fields.append(stored) - continue - # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A - # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten - # from the shipped contract, `rollup` included, so without this the newly - # unlocked "edit a preset rollup" would save, render, recompute and then revert - # at the next enrichment run — an edit that looks like it worked and undoes - # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp. - if _ut().user_edited(stored): - fields.append(stored) - seen.add(key) - continue - repaired = dict(stored) - for dkey, value in desired.items(): - if dkey != "automation": - repaired[dkey] = value - automation = dict(stored.get("automation") or {}) - automation.update(desired.get("automation") or {}) - repaired["automation"] = automation - # These bags define behaviour, not decoration. If the desired field does not use - # one, a stale bag from an earlier type must not remain attached to it. - for bag in ("link", "rollup", "profile", "pinned"): - if bag not in desired: - repaired.pop(bag, None) - if desired.get("type") not in ("select", "multiselect"): - repaired.pop("options", None) - converter = _IG_RETYPE_CONVERTERS.get((table_key, key)) - if (converter and stored.get("type") in ("text", "", None) - and repaired.get("type") != stored.get("type")): - for row in (table.get("rows") or {}).values(): - if str(row.get(key) or "").strip(): - converted = converter(row.get(key)) - if converted: - row[key] = converted - changed = True - if repaired != stored: - changed = True - fields.append(repaired) - seen.add(key) - for key, desired in wanted.items(): - if key not in seen and not any(str(f.get("key") or "") == key for f in fields): - fields.append(desired) - changed = True - if drops: - for row in (table.get("rows") or {}).values(): - for key in drops: - if key in row: - row.pop(key, None) - changed = True - table["fields"] = fields - return cur - - # Avoid a commit when the declarations are already exact. - snapshot = ut_all(rt) - needs = False - for table_key, wanted_fields in wanted_by_table.items(): - if table_key not in snapshot: - continue - table = snapshot.get(table_key) or {} - by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []} - # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip - # the commit when nothing would change; a spared `tracked` column left in the drop set - # answers "yes, work to do" on every single call, forever, and the schema pass would - # rewrite the table on every authenticated read without ever changing a byte. - drops = _guarded_drops(table, drop_by_table.get(table_key) or ()) - if drops & set(by_key): - needs = True - break - for desired in wanted_fields: - stored = by_key.get(str(desired.get("key") or "")) - mismatch = stored is None - if stored is not None: - for key, value in desired.items(): - if key == "automation": - if any((stored.get("automation") or {}).get(k) != v - for k, v in value.items()): - mismatch = True - break - elif stored.get(key) != value: - mismatch = True - break - if not mismatch: - stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired) - mismatch = any(k in stored for k in stale_bags) - if (not mismatch and desired.get("type") not in ("select", "multiselect") - and "options" in stored): - mismatch = True - if mismatch: - needs = True - break - if needs: - break - if needs: - rt.update(UT_STORE_KEY, _up, flush="sync") - return changed - - -def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""): - """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys. - - Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new - Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect - each Profile database to the same Posts, Comments, profile-history, and post-history stores. - """ - profile = ut_get(rt, profile_table) if profile_table else None - bound = str(profile_field or "").strip() - if profile is not None and not bound: - bound = next((str(f.get("key") or "") for f in profile.get("fields") or [] - if isinstance(f.get("profile"), dict)), "") - profile_label = str((profile or {}).get("label") or profile_table) - backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \ - if profile_table and profile is not None and bound else [] - - graph = { - IG_SNAPSHOTS_TABLE: ("IG snapshots", [*SNAPSHOT_FIELDS, *backlink]), - IG_POSTS_TABLE: ("IG posts", [*POST_FIELDS, *backlink]), - IG_POST_SNAPSHOTS_TABLE: ("IG post snapshots", [*POST_SNAPSHOT_FIELDS, *backlink]), - IG_COMMENTS_TABLE: ("IG comments", [*COMMENT_FIELDS, *backlink]), - } - keys = {} - for key, (label, fields) in graph.items(): - keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag, - record_mode=AUTOMATION_RECORD_MODE, lock_fields=True) - - if profile_table and profile is not None and bound: - profile_fields = _profile_schema_for(bound) - ut_ensure(rt, profile_label or profile_table, profile_fields, username, - key=profile_table, flow_tag=flow_tag, lock_fields=True) - # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older - # Profile database and IG candidates on the same universal schema while all of them point to - # the same four children. Existing flow provenance is merged, never replaced. - wanted, drops = _ig_schema_contract(rt) - _reconcile_ig_graph_fields(rt, wanted, drops) - return keys - - -# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ─────────────── -# -# Two changes land on data that already exists, and neither is optional once the field defs move: -# -# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a -# `date` column and a "20872" string in an `int` column are the split-schema the old -# "types are text on purpose" note correctly feared. Re-typing without converting is worse -# than not re-typing. -# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as -# `(handle, created_by)` DUPLICATES must be merged rather than left to collide. -# -# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a -# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would -# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a -# column is converted only while its STORED definition still says `text`; the moment it says `pct` -# the work is done and re-running is a no-op. That makes the migration safe to call on every boot, -# which is what it is for. -# -# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice. -# `observed_categories` named its two source tables and went silently empty the day wave 25 let a -# user point an automation somewhere else. Naming tables here would leave exactly those user-named -# databases on the old schema, which is the same bug wearing a migration's clothes. - -#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset -#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told. -MIGRATE_MIN_PRESET_FIELDS = 3 -#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/''). -_MIGRATE_CONVERT = { - "first_found": _day, "last_found": _day, "enriched_at": _day, - "avg_engagement": _pct100, -} - - -def _ig_profile_tables(rt): - """Every `ut_*` table in this tenant that looks like an Instagram profile table.""" - out = [] - for key, tbl in sorted((ut_all(rt) or {}).items()): - keys = {f.get("key") for f in (tbl or {}).get("fields") or []} - if "handle" in keys and len(keys & set(PRESET_PROFILE_KEYS)) >= MIGRATE_MIN_PRESET_FIELDS: - out.append(key) - return out - - -def discover_default_table(rt): - """Which database should a discovery automation write to when nobody has said? (2026-08-10) - - Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there. - We only need ONE IG profile so it's not confusing."* - - ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_ - leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the - label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own - note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is - a constant rather than a question, a tenant that already had an Instagram profile database got - a SECOND, empty one the first time a discovery automation was saved without a target. - MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside - `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them - pure confusion. - - So the default becomes a question asked of the tenant: - - exactly one profile database (excluding the fallback) -> that one - several, exactly one of which holds rows -> the one in use - several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE) - - ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and - picking one silently would write a paid discovery run into a database the user did not name — - worse than the empty table this exists to prevent. "" means "no opinion", and the caller's - existing fallback stands, which is exactly today's behaviour for that case. - ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset - schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the - empty table would keep re-electing it and nothing would ever change. - """ - try: - tables = ut_all(rt) or {} - found = [k for k in _ig_profile_tables(rt) if k != DISCOVER_TABLE] - except Exception: # noqa: BLE001 - return "" - if len(found) == 1: - return found[0] - if len(found) > 1: - used = [k for k in found if (tables.get(k) or {}).get("rows")] - if len(used) == 1: - return used[0] - return "" - - -def _apply_discover_default(rt, raw): - """Fill in a discovery automation's target BEFORE the pure validator invents one. - - ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that - turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure - validator with a two-argument contract every gate and route depends on). Resolving only at the - RUN sites would be cosmetic: the literal is written into the STORED config at save time, so - `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted. - `create`/`patch` are the one pair that both know the tenant and sit above that validator. - ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save - stored — is the user's choice and is never rewritten. - """ - cfg = raw.get("config") - if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip(): - return raw - default = discover_default_table(rt) - if not default: - return raw - return {**raw, "config": {**cfg, "targetTable": default}} - - -def _vestigial_name_field(fields, rows): - """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half. - - `user_tables.create()` mints EXACTLY ONE column on a hand-made database — - `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the - automation's columns after it, never reordering. So on every Instagram database somebody - created before pointing an automation at it, column 1 is a `Name` nothing will ever write. - - ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative - because this is the one branch in the migration that DESTROYS something. A column carrying any - declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with - a value in any row is somebody's data. Either way it survives as an ordinary column and merely - stops being the locked primary, which the `pinned` half achieves on its own. - - ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted - `custom__` (`buildOverlayField`) and every machine one comes from a `field_def` - list, so nothing but `create()`'s default can hold this key. The label and type are checked - anyway — a renamed or retyped column is a column someone touched on purpose. - """ - f = next((f for f in fields if f.get("key") == "name"), None) - if f is None or f.get("label") != "Name" or f.get("type") != "text": - return None - if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")): - return None - if any(str(r.get("name") or "").strip() for r in rows.values()): - return None - return f - - -def _merge_candidates(rows): - """C3's merge rule, applied to rows now colliding on `(platform, handle)`. - - Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell - takes the first non-blank, preferring the most recently enriched row. - - ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and - board cards; minting a new one would orphan all of them, so a merge picks a survivor rather - than creating one. - """ - groups, keepers = {}, {} - for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))): - r = rows[rid] - h = str(r.get("handle") or "").strip() - if not h: - # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a - # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a - # manual handle in my 'Beauty influencer leads' database, after an automation run, it - # gets deleted?"* - # - # A row you add by hand is born blank and stays blank until you type into it, so the - # window is not narrow — it is every row, from creation until the cell is committed. - # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S - # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was - # dropped even after it looked filled in on screen. - # - # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is - # not a candidate — it is somebody's row. It cannot collide with anything (there is - # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A - # merge pass that deletes what it cannot classify is not a merge, it is a filter. - keepers[rid] = r - continue - groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h), - []).append((rid, r)) - out, merged = dict(keepers), 0 - for _key, members in groups.items(): - if len(members) == 1: - out[members[0][0]] = members[0][1] - continue - merged += len(members) - 1 - # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement. - ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True) - keep_id = sorted(rid for rid, _r in members)[0] - acc = {} - for _rid, r in ordered: - for k, v in r.items(): - if k not in acc and str(v or "").strip(): - acc[k] = v - firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")] - lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")] - if firsts: - acc["first_found"] = min(firsts) - if lasts: - acc["last_found"] = max(lasts) - total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members) - if total: - acc["found_count"] = str(total) - out[keep_id] = acc - return out, merged - - -#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER -#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites -#: *"we already have a Field called stage… we don't need another checkbox for this"* — and -#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still -#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing -#: writes, nothing reads, and anybody can still click. -#: -#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the -#: record of who approved what before the wave, or leave them and let the next hand-made `tracked` -#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving -#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT -#: as a column nobody declared is the worse half. -TRACKED_KEY = "tracked" - - -def _retired_tracked_field(fields): - """The retired `tracked` column when it is still the MACHINE's own, else None. - - ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that - DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a - column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED, - or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as - an ordinary column and merely stops being fed by anything, which was already true. - - ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are - dropped unguarded: both are machine-derived cells with no history of anyone editing them. - `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test. - """ - f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None) - if f is None or f.get("type") not in ("checkbox", "", None): - return None - if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")): - return None - return f - - -def _guarded_drops(table, drops): - """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81). - - ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check. - `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks - `_retired_tracked_field` whether the column is still the machine's own, and the schema - reconciliation, which drops by KEY with no question asked. Guarding only the first was - decorative — the schema pass runs immediately afterwards on the same table and deleted the - retyped column the guard had just spared. Two paths to one deletion with one of them checked - is the shape where a control looks present and enforces nothing - ([[defects-that-mask-each-other]]). - """ - drops = set(drops or ()) - if TRACKED_KEY in drops: - stored = [f for f in (table or {}).get("fields") or [] - if str(f.get("key") or "") == TRACKED_KEY] - if stored and _retired_tracked_field(stored) is None: - drops.discard(TRACKED_KEY) - return drops - - -def _orphan_tracked_rows(fields, rows): - """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject. - - ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE - BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose - declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every - surface that renders from `fields` — and a migration that only looks at `fields` finds nothing - to do and `continue`s past the table. Storage, export and any future re-declaration of the key - all still see them. - """ - if any(str(f.get("key") or "") == TRACKED_KEY for f in fields): - return [] # the column above owns its own cells - return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})] - - -def backfill_corpus_snapshots(rt, log=print): - """The BACKFILL half: every profile row holding a measurement with no series behind it gets - its corpus observation. Returns how many were written. - - ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows - that arrived before the forward fix existed. A second row-shape here would be two ideas of - what a corpus observation is, and they would disagree on the next promoted column. - - ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is - deliberate. A profile that HAS been enriched already owns real observations; synthesising a - corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and - replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind - it, which is the only case where a corpus observation can only improve matters. - - ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on - every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has - a series, so it stops matching and the scan costs a dict comparison. - """ - snaps = ut_get(rt, IG_SNAPSHOTS_TABLE) - if snaps is None: - return 0 # no series store — see `append_ig_snapshots` on why we never mint one - known = {str((r or {}).get("influencer_key") or "").strip().lower() - for r in (snaps.get("rows") or {}).values()} - known.discard("") - #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived - #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a - #: named list goes silently empty the day the schema moves). - measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS - if fd["key"] not in _SNAPSHOT_OWN_KEYS - and fd.get("type") in ("int", "pct")) - incoming = [] - for table_key in _ig_profile_tables(rt): - for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): - handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if not handle or handle in known: - continue - if not any(str((row or {}).get(k) or "").strip() for k in measured): - continue # nothing was ever read about this row — there is no observation to record - built = corpus_snapshot_row(row) - if built: - incoming.append(built) - known.add(handle) # two profile tables holding one handle write ONE observation - return append_ig_snapshots(rt, incoming, log=log) - - -def backfill_post_snapshot_grain(rt, log=print): - """Copy `posted_at`/`type` off the POST row onto every measurement missing them. - - ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's - is not, so a pass that has already run matches nothing and a post that genuinely has no date - is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration - around it. - - ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN - THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own - record of the moment is the one to keep — a fact table that lets a dimension rewrite its - history is not a fact table. - """ - psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE) - posts = ut_get(rt, IG_POSTS_TABLE) - if psnaps is None or posts is None: - return 0 - have = {f.get("key") for f in (psnaps.get("fields") or [])} - carried = [k for k in ("posted_at", "type") if k in have] - if not carried: - return 0 # the columns are not declared yet — `ut_ensure` adds them first - by_shortcode = {} - for row in (posts.get("rows") or {}).values(): - code = str((row or {}).get("shortcode") or "").strip() - if code: - by_shortcode[code] = row or {} - changes = {} - for rid, row in (psnaps.get("rows") or {}).items(): - post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip()) - if not post: - continue - for key in carried: - want = str(post.get(key) or "").strip() - if want and not str((row or {}).get(key) or "").strip(): - changes.setdefault(str(rid), {})[key] = _s(want, 200) - if not changes: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - table = cur.get(IG_POST_SNAPSHOTS_TABLE) - if table is not None: - for rid, values in changes.items(): - table.setdefault("rows", {}).setdefault(rid, {}).update(values) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind") - return len(changes) - - -#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work -#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS. -#: -#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same -#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset -#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already -#: relational (the 12 links and rollups), or have no observation behind them at all -#: (`location_guess`/`location_confidence`, which WE infer rather than read). -#: -#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST. -#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells — -#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those -#: values were never recorded as observations (the hole this same wave closed going forward, in its -#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three -#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a -#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author -#: assumed. -#: -#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that -#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`. -#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not -#: safe today is not refused forever, it is simply not converted yet. -IG_DERIVABLE_KEYS = tuple( - fd["key"] for fd in PRESET_PROFILE_FIELDS - if fd.get("type") not in ("link", "rollup") - and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found", - "found_count", "created_by") - and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS}) - - -def _derived_profile_bag(key): - """The rollup that replaces the mapper's write of ONE profile column. - - ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's - value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an - expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the - scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest - observation that ACTUALLY READ this field", which is what the materialised cell meant. - MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the - observation itself, which is why the guard below is a proof and not a hope. - """ - return {"link": "profile_snapshots_link", "field": key, "fn": "latest", - "sortBy": "pulled_at", "sortDir": "desc", - "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"} - - -def _derived_profile_columns(table, snapshots): - """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table. - - = the ones already converted, PLUS the ones whose derived value equals the stored value on - every row. Returns keys. - - ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with - a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns — - `rollup` included, by design — so a converted column that this function stopped naming would be - re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written - by the mapper. The edit would look applied and undo itself later, which is exactly - [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore - included unconditionally rather than re-proved: its cells ARE the derived values, so a proof - would be comparing the fold against itself. - - ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody - measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere - else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest - authority is the data in front of the migration. - ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an - oversight: there is nothing to lose, and leaving it materialised would mean the schema differs - between two tenants for no reason a reader could discover. - """ - fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []} - rows = (table or {}).get("rows") or {} - by_subject = {} - for snap in ((snapshots or {}).get("rows") or {}).values(): - key = str((snap or {}).get("influencer_key") or "").strip().lower() - if key: - by_subject.setdefault(key, []).append(snap or {}) - # The engine's own ordering, once per subject — newest first, unrankable rows last. - ordered = {} - for subject, observations in by_subject.items(): - keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations] - rankable = [(o, k) for o, k in keyed if k is not None] - rankable.sort(key=lambda ok: ok[1], reverse=True) - ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None] - # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph - # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose - # link resolves to nothing — a blank cell wearing a configured column's clothes. - if "profile_snapshots_link" not in fields: - return [] - out = [] - for key in IG_DERIVABLE_KEYS: - field = fields.get(key) - if not field: - continue - if field.get("type") == "rollup": - out.append(key) # already converted — see the note above on why - continue - declared = str(field.get("type") or "text") - safe = True - for row in rows.values(): - stored = str((row or {}).get(key) or "").strip() - window = [o for o in ordered.get( - str((row or {}).get("handle") or "").strip().lower(), []) - if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here - derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared) - or "").strip() - if stored != derived: - safe = False - break - if safe: - out.append(key) - return out - - -def _as_derived_field(field): - """One preset scalar declaration → the same column declared as a rollup over the series.""" - key = str(field.get("key") or "") - return {**{k: v for k, v in field.items() if k not in ("agg",)}, - "type": "rollup", "rollup": _derived_profile_bag(key)} - - -def migrate_ig_tables(rt, log=print, only=""): - """Bring every Instagram Profile and canonical child table onto one current contract. - - Returns counted changes rather than a reassuring line. Profile cells are converted before - their type declarations change; then the graph reconciliation adds any missing preset - Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes - only retired machine fields. - - `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides - the WRITE PATH, so a table is brought forward immediately before anything appends to it and no - caller has to remember to run anything. - """ - stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0, - # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from - # `stamped` because they answer different questions: that one is "how many ROWS got a - # platform", these are "how many TABLES changed shape". - "pinned": 0, "flagged": 0, "droppedName": 0, - "droppedRecentPosts": 0, "droppedPostHashtags": 0, - "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0} - want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS} - # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red - # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a - # first run the table does not exist yet: with the detection bypassed it fell through to the - # writer and stamped a stub table with empty fields, clobbering the creation that was the whole - # point of the call (the owner stamp and the flow tag went with it). A table that does not - # exist, or that is not an Instagram profile table, has nothing to migrate. - detected = set(_ig_profile_tables(rt)) - targets = ([only] if only in detected else []) if only else sorted(detected) - for table_key in targets: - tbl = ut_get(rt, table_key) or {} - fields = [dict(f) for f in tbl.get("fields") or []] - rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()} - # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched, - # so the cell conversions below can never run twice on the same value. - stale = [f for f in fields - if f.get("key") in want and f.get("type") != want[f["key"]] - and f.get("type") in ("text", "", None)] - stale_keys = {f["key"] for f in stale} - has_platform = any(f.get("key") == "platform" for f in fields) - needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()] - # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING. - # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the - # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in - # production needs them stamped here, and that is the whole reason this branch exists. - # - # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule, - # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which - # is exactly why these are safe to run on every write where a value conversion would not be. - pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \ - if PRESET_PINNED_KEY else None - flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \ - if PRESET_FLAG_KEY else None - # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write - # doors, and a migration is not exempt from it. If somebody has already flagged another - # column on this table, stamping the handle too would leave the table in a state its own - # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would - # silently bind to whichever came first in the field list. Leave it alone and pin only. - other_profile = next((f for f in fields if f is not flag_f - and isinstance(f.get("profile"), dict)), None) - want_pin = pin_f is not None and pin_f.get("pinned") is not True - want_flag = (flag_f is not None and other_profile is None - and not isinstance(flag_f.get("profile"), dict)) - vestigial = _vestigial_name_field(fields, rows) - retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} - if _retired_tracked_field(fields) is not None: - retired.add(TRACKED_KEY) - orphan_tracked = _orphan_tracked_rows(fields, rows) - if (not stale_keys and has_platform and not needs_stamp - and not want_pin and not want_flag and vestigial is None and not retired - and not orphan_tracked): - continue - stats["tables"] += 1 - if want_pin: - pin_f["pinned"] = True - stats["pinned"] += 1 - if want_flag: - flag_f["profile"] = {"source": PROFILE_SOURCE_IG} - stats["flagged"] += 1 - if vestigial is not None: - # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer - # exists is invisible everywhere except the next export, where it reappears as a column - # nobody declared. `_vestigial_name_field` has already proven every one of them blank. - fields = [f for f in fields if f is not vestigial] - for r in rows.values(): - r.pop("name", None) - stats["droppedName"] += 1 - if retired: - fields = [f for f in fields if f.get("key") not in retired] - for r in rows.values(): - for key in retired: - r.pop(key, None) - stats["droppedRecentPosts"] += int("posts" in retired) - stats["droppedPostHashtags"] += int("post_hashtags" in retired) - stats["droppedTracked"] += int(TRACKED_KEY in retired) - for rid in orphan_tracked: - # D-74: a tick whose column was already gone. Counted apart from the column drop - # because they answer different questions — "how many tables still declared it" and - # "how many rows were still carrying it after the declaration went". - rows[rid].pop(TRACKED_KEY, None) - stats["droppedTrackedCells"] += 1 - - for key in stale_keys: - conv = _MIGRATE_CONVERT.get(key) - if not conv: - continue # int/checkbox already store the right text shape - for r in rows.values(): - if str(r.get(key) or "").strip(): - new = conv(r[key]) - # "" from a converter means UNREADABLE, and the cell keeps its original text. - # A migration that empties a cell is indistinguishable from one that moved it. - if new: - r[key], stats["converted"] = new, stats["converted"] + 1 - for f in stale: - f["type"] = want[f["key"]] - stats["retyped"] += 1 - for rid in needs_stamp: - rows[rid]["platform"] = PLATFORM_INSTAGRAM - stats["stamped"] += 1 - if not has_platform: - fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields - rows, merged = _merge_candidates(rows) - stats["merged"] += merged - - def _up(cur, _f=fields, _r=rows): - cur = cur or {} - t = dict(cur.get(table_key) or {}) - t["fields"], t["rows"] = _f, _r - cur[table_key] = t - return cur - - rt.update(UT_STORE_KEY, _up) - # The profile loop owns value conversions and deduplication. The schema pass owns the - # declarations across ALL related databases and is safe only after those conversions, because - # the stored old type is the idempotency key for unit-changing values such as engagement. - wanted, drops = _ig_schema_contract(rt) - stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops)) - # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT - # IDEMPOTENT, which is the one property everything else here is built on. - # - # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the - # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs - # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our - # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired - # profile cell against an UNREPAIRED observation, finds them different, and declines to convert - # a column that is in fact perfectly derivable. The next call then converts it — and - # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that - # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1. - # - # ⚠ The module already knew this shape one level up — the profile loop's own note says the - # schema pass "is safe only after those conversions, because the stored old type is the - # idempotency key". Same argument, one table further down. - # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same - # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is - # what moves the data, and it has already run. - wanted2, drops2 = _ig_schema_contract(rt) - stats["schema"] = int(bool(stats["schema"] - | int(_reconcile_ig_graph_fields(rt, wanted2, drops2)))) - # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose: - # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running - # it first would find nothing to fill and report a truthful zero about the wrong moment. - # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which - # is what makes them safe on a function that rides every `ut_ensure`. - stats["series"] = backfill_corpus_snapshots(rt, log=log) - stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log) - if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]: - log(f"[ig-migrate] {stats}") - return stats - - -# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ────────────────── -# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their -# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never -# hidden (R5). -# -# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN: -# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the -# corpus barely populates produces a filter that returns nothing and looks exactly like -# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about). -# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The -# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`, -# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them -# narrows under its own `default_operator`, asserted in the gate rather than assumed. -# -# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`, -# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and -# nothing else — a search that returns what you gave it, at full price. So the ranker reads only -# the three that describe a KIND of account: shared words in the bio, a shared domain in the link, -# and shared words in the name. -SEED_SOURCES = ("view", "cohort") -SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N] -SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter -SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared -SEED_MIN_ROWS = 2 # one row has nothing to share WITH -#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two -#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every -#: bio word in a two-row seed qualified as "shared", and the surface would have offered a -#: characteristic derived from a single record as the thing those records have in common. A -#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is -#: the fabrication this module refuses everywhere else. -SEED_MIN_ROWS_SHARING = 2 -#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor -#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down. -SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"), - ("full_name", "full_name")) -#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT — -#: an aggressive list would silently drop a real signal, and the coverage bar already removes most -#: noise. Anything ≤2 characters is dropped by length instead of by enumeration. -SEED_STOPWORDS = frozenset({ - "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from", - "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more", - "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"}) - - -def _seed_tokens(text): - """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars.""" - words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower()) - return {w for w in words if w not in SEED_STOPWORDS} - - -def _seed_domain(url): - """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a - domain and that IS the shared characteristic; the path is the thing that differs.""" - raw = str(url or "").strip() - if not raw: - return "" - host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower() - return host[4:] if host.startswith("www.") else host - - -def seed_predicates(rows): - """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network. - - `basis` is what makes the offer honest rather than magic: how many rows were read, which - characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads - as weak instead of being presented with the same confidence as a strong one. - - ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED - (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24 - default-versus-guard defect rebuilt — and it would be worse here, because the user did not - type it: they would be told their own product's suggestion is invalid. - - ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage - bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks - plausible, returns zero rows, and reads as "no such accounts exist". - """ - rows = [r for r in (rows or []) if isinstance(r, dict)] - total = len(rows) - basis = {"rows": total, "fields": [], - # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature. - # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1 - # preset set, so on our own seed rows it is usually simply absent — which the surface - # must SAY, or a lane that adds nothing reads as a lane that failed. - "related": {"tried": total, - "found": sum(1 for r in rows - if str(r.get("related_accounts") or "").strip())}, - "note": ""} - if total < SEED_MIN_ROWS: - basis["note"] = ("a seed needs at least two records — one record has nothing to share " - "with anything") - return [], basis - ranked = [] - for row_key, vendor_field in SEED_FROM_ROW: - counts = {} - for r in rows: - vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url" - else _seed_tokens(r.get(row_key))) - for v in vals: - if v: - counts[v] = counts.get(v, 0) + 1 - for value, n in counts.items(): - cov = n / total - if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING: - ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3), - "rows": n}) - # Strongest first; ties broken by the LONGER value, which is the more specific one. - ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"])) - derived, seen_fields = [], set() - for f in ranked: - if len(derived) >= SEED_MAX_PREDICATES: - break - # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the - # accounts carrying both words, which is a much smaller search than the seed implies. - if f["name"] in seen_fields: - continue - p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]} - if not predicate_narrows(p): - continue # cannot be offered; it would be refused at the save door - seen_fields.add(f["name"]) - derived.append(p) - basis["fields"].append({"name": f["name"], "label": field_label(f["name"]), - "value": f["value"], "coverage": f["coverage"]}) - if derived and narrowing_refusal(derived, "and"): - # Belt AND braces: the per-predicate check above should make this unreachable, and it is - # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest - # empty rather than offering something the save door will reject. - basis["note"] = ("the shared characteristics found are not specific enough to search on " - "— add a condition of your own") - return [], basis - if not derived: - basis["note"] = basis["note"] or ( - "these records share no bio words, link domain or name in common — nothing could be " - "derived, so the conditions are yours to write") - return derived, basis - - -def seed_rows(rt, table_key, view_id=""): - """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by - `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]]).""" - t = ut_get(rt, str(table_key or "")) - if t is None: - return [], f"{table_key!r} is not a database in this workspace" - rows = list((t.get("rows") or {}).values()) - if str(view_id or "").strip(): - # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a - # different row set from the view it names would be describing a view nobody has. - tree, _fields, problem = view_filter(rt, table_key, view_id) - if problem: - return [], problem - rows = [r for r in rows if lane_match(tree, r)] - return rows[:SEED_MAX_ROWS], "" - - -def automation_tables(defn): - """Every `ut_*` table THIS automation writes into — its config target plus every - `create_record` action's table, deduped, order preserved. - - ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition. - Split out rather than parameterised because the two questions are different: that one asks - "where could a Category value be, anywhere in this tenant" and is deliberately generous; this - one asks "which rows has THIS automation already found" and must not reach into a table it - does not write, or a nightly search would start excluding another automation's finds. - """ - keys, seen = [], set() - cfg = (defn or {}).get("config") or {} - for raw in [cfg.get("targetTable"), - *(((a or {}).get("config") or {}).get("table") - for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]: - k = str(raw or "").strip() - if k.startswith(UT_PREFIX) and k not in seen: - seen.add(k) - keys.append(k) - return keys - - -def already_found_handles(rt, defn, cap=None): - """The handles this automation has already written, for the vendor-side exclusion (item 7). - - ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a - profile has always been harmless — and never free: the vendor bills for every record it - returns, so a nightly search over stable keywords pays again, in full, for the same accounts - every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty - scout re-found its whole result set on every run. This is the list that stops it, sent as one - `not_in` the vendor evaluates BEFORE billing. - - ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would - hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords - are two questions, and one must not answer with "somebody already looked at that". - """ - cap = BD_EXCLUDE_MAX if cap is None else int(cap) - out, seen = [], set() - for table_key in automation_tables(defn): - for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): - h = str((row or {}).get("handle") or "").strip().lstrip("@").lower() - if h and h not in seen: - seen.add(h) - out.append(h) - if len(out) >= cap: - return out - return out - - -def observed_category_tables(rt): - """Every table a Category value could have been written into, DERIVED from the automations - this tenant actually has — the default discovery table and the snapshot table are the FLOOR, - not the list. - - ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared: - `config.targetTable` (the discovery runner's own write) and any `create_record` action's - `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker - writes). Both are read, deduped, order preserved so the defaults come first. - """ - keys, seen = [], set() - - def _add(k): - k = str(k or "").strip() - if k and k.startswith("ut_") and k not in seen: - seen.add(k) - keys.append(k) - - _add(DISCOVER_TABLE) - _add("ut_ig_snapshots") - try: - for defn in (all_definitions(rt) or {}).values(): - cfg = (defn or {}).get("config") or {} - _add(cfg.get("targetTable")) - for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")): - _add(((act or {}).get("config") or {}).get("table")) - except Exception: # noqa: BLE001 - # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults — - # the same posture as the master lane below: degrade to less vocabulary, never to an error. - pass - return keys - - -def observed_categories(rt, limit=60): - """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts - free text. - - D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen - (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that - still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**: - transcribe a published taxonomy and hope it lines up."* This is the first branch. - - ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the - corpus does not use returns zero rows and looks exactly like an honest "no such accounts - exist" — so a picker built from Instagram's own category list would mislead precisely when it - looked most authoritative. Every option here has been observed on a real row, and each carries - its COUNT so a value seen once reads differently from one seen forty times. - - ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a - control that refused anything else would be a second, quieter version of the same lie. - """ - seen = {} - - def _eat(rows, key): - for r in rows or []: - v = " ".join(str((r or {}).get(key) or "").split())[:60] - if v: - seen[v] = seen.get(v, 0) + 1 - - # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped - # green. The two constants below were the whole list, which was correct until **wave 25 R2 - # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a - # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely - # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows - # while the tenant's two real target tables held 20 each, so this function honestly reported - # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that - # names its sources goes quietly empty the moment the product lets a user choose one. - for key in observed_category_tables(rt): - _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category") - try: - # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three - # candidates still gets a vocabulary drawn from every profile the platform has captured. - import ig_master - if ig_master.configured(): - # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which - # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers - # None, so the master lane would have degraded to nothing INSIDE the except below and - # this would have shipped as a silent no-op with the gate green. Verified against - # `ig_master.series_for`, which reads the same bucket the same way. - _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category") - except Exception: # noqa: BLE001 - # A master that cannot be read costs the tenant its own values and nothing else. - pass - return [{"value": v, "count": n} - for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]] - - -def preset_plan(rt, table_key): - """C1 / R2b: the preset set diffed against ONE database's CURRENT fields. - - `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}` - - ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns - an automation will USE and which it will CREATE — and the only thing that can answer it is - whatever holds both lists. A client that diffed the preset set against a table's fields would - be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that - actually runs; it would be wrong precisely when the two lists differ, which is the only case - anybody is asking about. - - ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save - moment, and the honest answer is "all of them will be created". `present` is a fact about the - table, so it reads False for every field rather than the call refusing. - - ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG - (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the - relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY. - That is CORRECT for this payload — it answers "which columns will be used vs created", where - a name and a type are the whole question — and it is safe today because nothing RENDERS a - column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes - `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from - `PRESET_PROFILE_FIELDS`, which carries the bag. - ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would - render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or - widen this projection deliberately; do not assume a bag is here because the field has one. - """ - have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields") - or [])} - fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text", - "present": f["key"] in have} - for f in PRESET_PROFILE_FIELDS] - return {"table": str(table_key or ""), - "exists": ut_get(rt, str(table_key or "")) is not None, - "fields": fields, - "willUse": [f for f in fields if f["present"]], - "willCreate": [f for f in fields if not f["present"]]} - - -def clean_predicates(raw, operator="and"): - """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. - - Refuses rather than coerces: a predicate naming a field the API rejects comes back as a - 400 with the field named, because the alternative — dropping it — turns "find me verified - accounts in Georgia" into "find me any account" and bills for the difference. - """ - out = [] - for p in (raw or [])[:12]: - if not isinstance(p, dict): - continue - name = _s(p.get("name") or p.get("field"), 60).strip() - op = _s(p.get("operator") or p.get("op"), 20).strip() - if name not in BD_FILTER_FIELDS: - return None, ("that field cannot be searched — the searchable ones are: " - + ", ".join(field_label(f) for f in BD_FILTER_FIELDS)) - # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly - # because every operator was legal on every field; a yes/no column offering "at least" - # is a question with no meaning that the vendor is nevertheless asked. - allowed = ops_for(name) - if op not in allowed: - return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r} — " - "it takes: " - + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) - entry = {"name": name, "operator": op} - if op not in BD_NULLARY_OPS: - raw_v = p.get("value") - # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and - # it is stored as a list rather than as N sibling predicates so the row a person sees - # and the row that is stored are the same thing. - vals = raw_v if isinstance(raw_v, list) else [raw_v] - vals = [v.strip() if isinstance(v, str) else v for v in vals] - vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))] - if not vals: - return None, f"give {field_label(name)} something to compare against" - if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: - return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value — " - f"{field_label(name)} can only be given a list with " - + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) - if o in BD_MULTI_VALUE_OPS)) - if len(vals) > MAX_PREDICATE_VALUES: - return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values " - "in one condition") - if field_kind(name) == "boolean": - ok = {o["value"] for o in BD_BOOLEAN_OPTIONS} - bad = [v for v in vals if str(v).lower() not in ok] - if bad: - return None, f"{field_label(name)} is answered Yes or No" - # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers - # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and - # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field - # was a free-text box until this wave, so the Yes/No dropdown that makes it easy - # to ask is also the thing that would have made every boolean condition fail. - # The dropdown's option VALUES stay "true"/"false" (a carries strings); + # the conversion belongs here, at the edge that talks to the vendor. + vals = [str(v).lower() == "true" for v in vals] + if field_kind(name) == "number": + try: + vals = [float(v) if "." in str(v) else int(v) for v in vals] + except (TypeError, ValueError): + return None, f"{field_label(name)} takes a number" + # A single value stays a SCALAR on the wire. The vendor has only ever been sent + # scalars for these operators; the list form is expanded at send time and the + # one-value case must not quietly start exercising an untested shape. + entry["value"] = vals[0] if len(vals) == 1 else vals + out.append(entry) + if not out: + # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES. + # This used to refuse, which was right while a wizard collected the filters before the + # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger + # CREATES the automation, and the filters are typed afterwards — so a save-time refusal + # here meant the trigger could never be picked at all. It is the A3 + # stored-inert-with-`configured:false` pattern this module already runs on. + # + # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient: + # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a + # search, deliberately, because "a stored config can predate the law, and the vendor + # bills for breadth whether the filter was saved yesterday or last month". The RUN is the + # wall. Every MALFORMED predicate above is still refused here, and the narrowing law + # below still refuses a non-empty filter that narrows nothing — only EMPTY changed. + return [], None + # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the + # measured hang this guards against. Same sentence at create/patch and at run. + guard_err = narrowing_refusal(out, operator) or depth_refusal(out, operator) + if guard_err: + return None, guard_err + return out, None + + +def discover_estimate(records_limit): + """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the + section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" + n = max(0, int(records_limit or 0)) + return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), + "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", + "note": "estimated from the published rate — an exact price is only known after a run" + "a price before a run, and this account's token cannot read a balance"} + + +def _candidate_row(row, stamp): + """One corpus row → a `ut_ig_candidates` row. None when it has no handle. + + ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already + stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted + `tracked` either; R6 deleted that column in wave 26.) + """ + if not isinstance(row, dict): + return None + handle = str(_first(row, "account", "username", default="") or "").strip() + if not handle: + return None + return { + # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads + # Instagram, so it is a constant here rather than something derived from the row; the + # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart. + "platform": PLATFORM_INSTAGRAM, + "handle": handle, + "profile_url": str(_first(row, "profile_url", "url", + default=f"https://www.instagram.com/{handle}/")), + "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), + "followers": _s(_ig_int(_first(row, "followers"))), + "following": _s(_ig_int(_first(row, "following"))), + # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is + # the path where it carries values (measured on all five of the 2026-08-04 result set). + # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`. + "avg_engagement": _pct100(_first(row, "avg_engagement", default="")), + "bio": _s(_first(row, "biography", "bio", default=""), 500), + "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), + "verified": "1" if _first(row, "is_verified", default=False) else "", + "category": _s(_first(row, "category_name", "business_category_name", default="")), + # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp + # still exists on the snapshot series, which is where a time axis belongs. + "last_found": _day(stamp), + } + + +# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ────────────────────────────────────────── +# +# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a +# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a +# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via +# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers, +# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at +# all, so discovery was the one rung in this module that read numbers and recorded no observation. +# +# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold +# real corpus numbers that are already on screen and already filterable. The gap is that the +# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law +# exists to forbid one level up. +# +# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make +# it honest already existed: `source` says **Discovery** (this was read off the vendor's +# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not +# read this as an exact count taken at `pulled_at`" — which is the whole difference between +# recording what we know and inventing what we do not. +# +# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it +# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as +# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over +# the series therefore prefers the exact read over the corpus read whenever both happened on one +# day — the ordering you would have to hand-write, falling out of the grain. + +#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell. +IG_VIA_DISCOVERY = "brightdata:discovery" + +#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement +#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather +#: than typed out — a hand-list is what silently stops carrying the next promoted column. +_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx") + + +def corpus_snapshot_row(row, day=""): + """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable. + + `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's + only date is `first_found`. + + ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is + `@`, so re-running this over the same rows on the same day rewrites the same key + and `upsert_rows` merges it — the migration is safe to call on every write, which is the only + way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys + idempotency on the stored TYPE). + + ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty + ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed. + Here the merge is the hazard instead: a second discovery on the same day returning a thinner + corpus record would otherwise ERASE what the first one learned. An absent key renders exactly + as an empty cell, so nothing is lost by leaving it out. + """ + handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if not handle: + return None + when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "") + if not when: + return None + out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when, + "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"} + for fd in SNAPSHOT_FIELDS: + key = fd["key"] + if key in _SNAPSHOT_OWN_KEYS: + continue + value = (row or {}).get(key) + if str(value or "").strip(): + out[key] = str(value) if key == "source_payload" else _s(value, 400) + return out + + +def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print): + """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written. + + ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only + honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls + `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the + creator from there is unbounded recursion. A tenant with no series store has nothing to + back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this + function then has somewhere to write. + """ + rows_in = [r for r in (incoming or []) if r] + if not rows_in: + return 0 + existing = ut_get(rt, snap_key) + if existing is None: + return 0 + merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key", + cap=row_cap(snap_key)) + if counts["capped"]: + # D-11's law: a full append table means the SERIES has stopped growing, which is the one + # failure a chart cannot show you. + log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by " + f"{snap_key}'s row cap") + written = counts["inserted"] + counts["updated"] + if not written: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if cur.get(snap_key) is not None: + cur[snap_key]["rows"] = merged + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return written + + +# --------------------------------------------------------------------------------------------- +# RUNNERS +# --------------------------------------------------------------------------------------------- + +def cap_note(capped, missing=()): + """The sentence(s) a cap breach adds to a run summary. + + `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. + `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not + be created at all. + + ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the + ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that + does not say WHERE is only marginally better than a silent one. + + ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused + SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the + runners' `if tgt is not None` write guard then skipped that bucket, and the run reported + success over a database that does not exist. Exactly the silent-stop failure the row cap + already had, one level up — and harder to spot, because the table is not there to look at. + """ + out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} " + f"from this run were NOT written" for k, n in (capped or []) if n] + out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its " + f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" + for k in (missing or [])] + return "; ".join(out) + + +def ut_missing(rt, *keys): + """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" + have = ut_all(rt) + return [k for k in keys if k and k not in have] + + +def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" + cfg = defn.get("config") or {} + fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") + dry = bool(cfg.get("dryRun")) + # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from + # the rollup — inferring is exactly how a green dot ended up over an empty table (see the + # rollup comment in `run_field_instagram`). + steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} + step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}") + status, final, body = fetch(cfg["url"]) + if not (200 <= status < 300): + steps["fetch"] = "error" + return ("error", f"{cfg['url']} answered {status} — nothing was written", + {"status": status}, [], steps) + steps["fetch"] = "ok" + soup = _soup(body) + if cfg.get("extract") == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 + raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] + if not raw_rows: + steps["extract"] = "error" + return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], + steps) + steps["extract"] = "ok" + step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}") + + incoming = [] + for r in raw_rows: + mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} + if any(v for v in mapped.values()): + incoming.append(mapped) + + fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) + for src, tgt in fmap.items()] + # the key column leads, so the table opens on the identity it is upserted by + fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) + label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" + if dry: + # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, + # which would create the table: a dry run that leaves a new empty database behind is not + # a dry run. + table_key = ut_key_for(label, cfg.get("targetTable") or None) + else: + table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) + t = ut_get(rt, table_key) or {} + rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, + cap=row_cap(table_key)) + touched = [rid for rid, row in rows.items() + if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] + if not dry: + ut_write_rows(rt, table_key, rows) + counts["source_rows"] = len(incoming) + affected = touched[:200] + summary = (f"{counts['inserted']} new, {counts['updated']} updated, " + f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " + f"(kept)") + capped = [(table_key, counts.get("capped", 0))] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + note = cap_note(capped, missing) + if note: + summary += f" — {note}" + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + steps["write"] = "skipped" + elif not note: + steps["write"] = "ok" + state = "partial" if (counts.get("capped") or counts.get("skipped") or missing + or dry) else "ok" + log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def capture_rows(res, pulled): + """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`. + + ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT + RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path — + do not fork them." A second copy of this mapping is the shape that goes wrong invisibly, + because the two copies would each be *plausible* and would disagree only on the rows a + particular rung happened to return. + + ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE: + * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every + time and a blank is unambiguous — it means THIS PULL did not read it. + * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are + readable on some rungs and not others (the Bright Data PROFILE row carries post identity + with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and + `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run + ERASE what an expensive one learned. + * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is + the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement + rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k + ceiling and drawing a chart of nothing. A row in a time series should mean "this was true + then"; a row meaning "nobody looked" belongs nowhere. + """ + prof = (res or {}).get("profile") or {} + snap_row = { + "snapshot_key": f"{prof.get('username')}@{pulled}", + "influencer_key": prof.get("username"), "pulled_at": pulled, + # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called + # "Read via". `res["via"]` keeps the real key for the call graph and the server log. + "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")), + "bio": _s(prof.get("bio"), 500), + "external_url": _s(prof.get("external_url"), 300), + } + for fd in SNAPSHOT_FIELDS: + k = fd["key"] + if k not in snap_row: + value = prof.get(k) + # Profile history declares the same percentage dialect as the current Profile row. + # Converting only the latest projection left the append series at 0-1 while its field + # now said pct (0-100), so the two views of one measurement disagreed by 100x. + if k == "source_payload": + snap_row[k] = str(value or "") + else: + snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400) + idents, metrics_rows, comment_rows = [], [], [] + for p in (res or {}).get("posts") or []: + ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"), + "url": _s(p.get("url"), 300)} + for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), + ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), + ("alt_text", 400), ("tagged_location", 400)): + if p.get(k): + ident[k] = _s(p.get(k), n) + if p.get("source_payload"): + ident["source_payload"] = str(p["source_payload"]) + metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")} + # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF. + # + # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average + # views over the last 12 posts" would have to walk profile → posts → each post's most + # recent snapshot, and a two-hop rollup is a much larger feature with a much worse + # invalidation story. The post row now carries its own latest, exactly as the profile row + # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3). + # + # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the + # authoritative engagement series and still gets its appended row. These cells are a + # projection of the row being appended in the same breath, never a second source. + # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent + # key leaves the previous pull's value standing — which is the correct behaviour for a + # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing + # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the + # same failure the snapshot-append rule above is written against. + measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""} + if measured: + ident.update({k: _s(v, 40) for k, v in measured.items()}) + # The stamp is what makes the numbers readable: a blank `plays` beside a + # `measured_at` of last week means "we looked and the vendor had nothing", and with + # no stamp it means that AND "we never looked", indistinguishably. + ident["measured_at"] = _day(pulled) + idents.append(ident) + if any(v is not None for v in metrics.values()): + metrics_rows.append({ + "post_snapshot_key": f"{p['shortcode']}@{pulled}", + "shortcode": p["shortcode"], "influencer_key": prof.get("username"), + "pulled_at": pulled, + # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the + # SAME vendor record the identity row above is built from, so they cannot disagree + # with it on this pull. + # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split` + # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull + # that learns a post's engagement often does not learn its date in the same + # breath. `backfill_post_snapshot_grain` fills those from the post row, which by + # then may have learned it from a different rung. + "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200), + "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), + "views": _s(metrics["views"]), "plays": _s(metrics["plays"]), + "source_payload": str(p.get("source_payload") or "")}) + for comment in p.get("embedded_comments") or []: + # The embedded shape often names only the commenter. Its parent Post is the + # authoritative owner, so bind every preview to this profile and post explicitly. + comment_rows.append({**comment, "shortcode": p["shortcode"], + "influencer_key": prof.get("username") or ""}) + comment_rows.extend((res or {}).get("comments") or []) + # The same comment can be returned as both `latest_comments` and `top_comments`, or by the + # optional full endpoint after an embedded preview. The canonical key is the one row truth. + unique_comments = {} + for row in comment_rows: + key = str((row or {}).get("comment_key") or "") + if key: + unique_comments[key] = {**(unique_comments.get(key) or {}), **row} + return snap_row, idents, metrics_rows, list(unique_comments.values()) + + +#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched. +#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of +#: `row[...] = prof.get(...)` that a future field addition can silently miss. +#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did +#: not read it", and `enriched_at` is what distinguishes that from "never enriched"). +PRESET_FROM_PROFILE = { + "username": "handle", "profile_url": "profile_url", "full_name": "full_name", + "followers": "followers", "following": "following", "avg_engagement": "avg_engagement", + "bio": "bio", "external_url": "external_url", "verified": "verified", + "category": "category", "posts_count": "posts_count", + "highlights_count": "highlights_count", "is_business": "is_business", + "is_professional": "is_professional", "ig_id": "ig_id", + # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column + # that can only ever be blank, so "make the preset fields populated" is THIS half of the + # owner's instruction and the field list is only the other half. + # ⚠ The two lists are held in step by a derived gate rather than by care: every + # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared + # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`). + "business_category": "business_category", "is_private": "is_private", + "bio_hashtags": "bio_hashtags", + "pronouns": "pronouns", "profile_name": "profile_name", + "is_joined_recently": "is_joined_recently", "has_channel": "has_channel", + "partner_id": "partner_id", "external_url_title": "external_url_title", + "fbid": "fbid", "related_accounts": "related_accounts", + "country_code": "country_code", "source_payload": "source_payload", +} + +#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a +#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate +#: could only be written as "these three are fine" — a hard-coded exception list, which is the +#: shape [[gate-answers-the-wrong-question]] warns about. +#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an +#: enrichment run — which is the whole point of them: they stay true when the LINKED table +#: changes, and a pull that touched no profile still updates a profile's post count. +PRESET_WRITTEN_ELSEWHERE = ( + "platform", "handle", "enriched_at", + "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link", + "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12", + "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured", +) + + +#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a +#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING` +#: refuses on the discovery side for exactly the same reason. +LOCATION_MIN_POSTS = 2 + + +def _post_place(post): + """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload + second. + + ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the + vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the + `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details` + shape. The complete vendor row is retained in `source_payload` on the same record, already + paid for, so a post whose normalised field came back blank can still be read here. + ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region, + country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to + "where is this person"; "Indonesia" mostly is not. + """ + if not isinstance(post, dict): + return "" + raw = str(post.get("tagged_location") or "").strip() + if not raw: + try: + payload = json.loads(post.get("source_payload") or "{}") + except Exception: # noqa: BLE001 + payload = {} + raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else "" + return raw.split(",")[0].strip() + + +def location_guess(posts): + """`(city, confidence_pct)` from the places a creator's own posts were tagged in. + + `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not + the share of ALL posts, because a creator who geotags three of twelve is telling us about + three, and dividing by twelve would report a real signal as weak. How thin the evidence is + stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two. + """ + places = [p for p in (_post_place(x) for x in posts or []) if p] + if len(places) < LOCATION_MIN_POSTS: + return "", None + counts = {} + for p in places: + counts[p] = counts.get(p, 0) + 1 + # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be + # resolved alphabetically, which is an arbitrary answer wearing a confident number. + best = max(places, key=lambda p: (counts[p], -places.index(p))) + return best, round(100.0 * counts[best] / len(places), 1) + + +def preset_cells(res, pulled): + """C4/R3: one pull → the LATEST-value cells written onto the enriched record. + + ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the + other fifteen readable: without it a blank `followers` means both "never enriched" and + "enriched in March and the vendor had nothing", and no cell on the row can tell them apart. + """ + prof = (res or {}).get("profile") or {} + cells = {} + for src, dest in PRESET_FROM_PROFILE.items(): + v = prof.get(src) + if v is not None and v != "": + # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The + # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a + # straight-through copy here would write the same wrong number the discovery path + # used to write. Both writers convert; neither is the exception. + cells[dest] = (_pct100(v) if dest == "avg_engagement" else + (str(v) if dest == "source_payload" else _s(v, 500))) + # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an + # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have + # arrived with no platform at all — and a blank half of the dedup key is how one account + # becomes two rows. + cells["platform"] = PLATFORM_INSTAGRAM + # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when + # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose + # twelve posts happened to carry no geotag would make the column worse the more it ran. + place, confidence = location_guess((res or {}).get("posts")) + if place: + cells["location_guess"] = _s(place, 120) + cells["location_confidence"] = str(confidence) + # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a + # question in days; the full stamp keeps its precision on the snapshot series. + cells["enriched_at"] = _day(pulled) + return cells + + +def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, + rows=None): + """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public + profile through the paid capability chain, write a status string into the automation column, + and append a timestamped row to each of the three IG tables. + + ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per + CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two + "which rung answered" counters this function used to keep have nothing left to distinguish.""" + cfg = defn.get("config") or {} + table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") + post_metrics = bool(cfg.get("postMetrics")) + comment_metrics = bool(cfg.get("commentMetrics")) + dry = bool(cfg.get("dryRun")) + # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a + # status written under a node id `graph()` no longer emits is a dot nothing renders, which is + # indistinguishable from a step that never ran. + steps = {"trigger": "ok", "source": "idle", + "capture_posts": "idle" if post_metrics else "skipped", + "capture_comments": "idle" if comment_metrics else "skipped", + "write": "idle"} + t = ut_get(rt, table_key) + if not t: + steps["source"] = "error" + return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) + url_field = cfg.get("urlField") or _auto_url_field(t, fkey) + if not url_field: + steps["source"] = "error" + return ("error", "the automation column has no URL field bound to it", {}, [], steps) + steps["source"] = "ok" + rows = dict(t.get("rows") or {}) + # the relational tables the pull lands in (R7): every row timestamped for time-range filters + if dry: + # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) + snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE, + IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE) + else: + graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=table_key) + snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], + graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) + missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key) + snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) + posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) + psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) + comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {}) + + counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, + "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0, + "missing_tables": len(missing)} + cells, affected, notes = {}, [], [] + pending_metrics = [] + # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` + # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. + # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation + # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) + in_snaps, in_posts, in_psnaps, in_comments = [], [], [], [] + targets = [(rid, str(r.get(url_field, "") or "").strip()) + for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] + for i, (rid, url) in enumerate(targets): + if i: + time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) + # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile + # column automation blocks for minutes by design. A counting step is the difference + # between "working through them" and "stuck". + step(f"Capturing profile {i + 1} of {len(targets)}") + counts["profiles"] += 1 + res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log, + post_metrics=post_metrics, comment_metrics=comment_metrics, + pending_metrics=(pending_metrics if not dry else None)) + pulled = _iso() + via = res.get("via") or "" + read_ok = res["state"] in ("ok", "partial") + # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID + # vendor actually answered, which is the run's spend report — the thing the owner reads to + # reconcile a bill. What died is the free rung it used to be contrasted with, so the test + # is now simply "did a vendor answer" rather than "did the vendor we were told to try". + if read_ok and via in ("brightdata", "apify"): + counts["paid"] += 1 + if read_ok: + counts["ok" if res["state"] == "ok" else "partial"] += 1 + prof = res["profile"] + snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled) + in_snaps.append(snap_row) + in_posts.extend(ident_rows) + in_psnaps.extend(metric_rows) + in_comments.extend(captured_comments) + counts["metrics"] += len(metric_rows) + counts["comment_rows"] += len(captured_comments) + counts["posts"] += len(res.get("posts") or []) + detail = f"{prof.get('followers') or '?'} followers" + if prof.get("approx"): + detail += " (approx)" + if res.get("posts"): + detail += f", {len(res['posts'])} posts" + elif res["state"] == "partial": + detail += ", posts not readable on the rung that answered" + cells[rid] = f"{res['state']} · {_stamp()} · {detail}" + elif res["state"] == "blocked": + counts["blocked"] += 1 + cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "blocked") + else: + counts["error"] += 1 + cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "error") + affected.append(rid) + + # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. + snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) + posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) + comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key") + comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + counts["new_posts"] = c_post["inserted"] + capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), + (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])] + counts["capped"] = sum(n for _k, n in capped) + + if not dry: + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, val in cells.items(): + tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), + (comment_key, comments)): + tgt = cur.get(k) + if tgt is not None: + tgt["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics) + if queued: + counts["metric_batches_pending"] = queued + + # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: + # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), + # `error` a LOUD partial — a pooled history with silent holes is worse than none. + master_note = "" + if not dry and (in_snaps or in_posts or in_psnaps): + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), + in_snaps, in_posts, in_psnaps) + if m_status == "error": + master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \ + f"is complete and the next run re-appends" + counts["master_failed"] = 1 + elif m_status == "ok": + counts["master"] = len(in_snaps) + len(in_psnaps) + + # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while + # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail + # showed a green dot over a table with no posts in it. A summary that disagrees with the + # cells it summarises is the failure this whole module's honest-status rule exists to + # prevent, so `partial` now propagates. Measured 2026-08-04. + read = counts["ok"] + counts["partial"] + if not read: + state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" + elif counts["blocked"] or counts["error"] or counts["partial"]: + state = "partial" + else: + state = "ok" + if counts.get("metric_batches_pending"): + state = "partial" + # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) + if not targets: + cap_state = "idle" + elif not read: + cap_state = "error" if counts["error"] else "blocked" + else: + cap_state = "partial" if (counts["blocked"] or counts["error"] + or counts["partial"]) else "ok" + # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to + # read a profile now, and the node that names the profile set is the honest owner of "did + # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch + # that no longer exists in behaviour OR on screen. + steps["source"] = cap_state if targets else steps["source"] + # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually + # appended. "It was switched on" is not the same fact as "it answered", and painting the + # second from the first is the fabrication `node_status` refuses to make. + # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A + # run that found no posts to enrich did not have a rung refuse it — nothing was requested — + # so that reads `skipped`, the same word an off switch earns. + steps["capture_posts"] = ("ok" if counts["metrics"] + else "partial" if counts.get("metric_batches_pending") + else "blocked" if (post_metrics and counts["posts"]) + else "skipped") + steps["capture_comments"] = ("ok" if counts.get("comments") + else "blocked" if (comment_metrics and counts["posts"]) + else "skipped") + + summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " + f"({counts['new_posts']} new)") + if counts.get("metric_batches_pending"): + summary += (f", {counts['metric_batches_pending']} post-engagement batch" + f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building " + "(collected automatically)") + if counts["paid"]: + summary += f", {counts['paid']} with exact counts" + if counts["metrics"]: + summary += f", {counts['metrics']} post engagement snapshots" + elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"): + # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there + # was no engagement", which is a claim about Instagram rather than about our run. + summary += ", no post engagement was readable" + if counts["comment_rows"]: + summary += f", {counts['comment_rows']} comment rows captured" + elif comment_metrics and counts["posts"]: + summary += ", no comment engagement was readable" + if counts["partial"]: + summary += f", {counts['partial']} profile-only (posts not readable)" + if counts["blocked"]: + summary += f", {counts['blocked']} blocked" + note = cap_note(capped, missing) + if note: + summary += f" — {note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + elif not dry: + steps["write"] = "ok" + if master_note: + summary += f" — {master_note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + steps["write"] = "skipped" + state = "partial" if state != "error" else state + if notes: + summary += f" — {notes[0][:120]}" + if not dry: + # C7: the fresh master rows are exactly what the table's metric cells summarise. + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + return (state, summary, counts, affected, steps) + + +def _auto_url_field(table, fkey): + """The URL field an automation column is bound to: its own `automation.urlField`, else the + first url-typed column on the table. Never guessed silently — the caller reports which.""" + for f in table.get("fields") or []: + if f.get("key") == fkey: + bound = ((f.get("automation") or {}).get("urlField") or "").strip() + if bound: + return bound + for f in table.get("fields") or []: + if f.get("type") == "url": + return f.get("key") + return "" + + +def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_step, + rows=None): + """Automation #3 (owner ruling R7 / D-23): FIND handles nobody here has typed in. + + Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus + query and hands its snapshot id to the next run, or COLLECTS one a previous run started. + Every outcome is a sentence about what actually happened; none of them is a green zero. + """ + cfg = defn.get("config") or {} + dry = bool(cfg.get("dryRun")) + limit = int(cfg.get("recordsLimit") or 5) + preds = list(cfg.get("predicates") or []) + auto_id = str(defn.get("id") or "") + steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} + counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, + "missing_tables": 0} + + if not bd_ready(): + # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is + # the vendor's — so this is where the run stops, saying which env var is missing. + steps["find"] = "blocked" + return ("partial", "Profile search is not set up yet — nothing was searched and nothing " + "was charged", counts, [], steps) + + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + if not pending: + # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and + # the vendor bills for breadth whether the filter was saved yesterday or last month. + # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config + # stored before this check existed must not reach the vendor either. + guard_err = (narrowing_refusal(preds, cfg.get("operator")) + or depth_refusal(preds, cfg.get("operator"))) + if guard_err: + steps["find"] = "blocked" + return ("error", f"the search was not started — {guard_err}", counts, [], steps) + if dry: + est = discover_estimate(limit) + steps["find"] = steps["write"] = "skipped" + return ("partial", + f"Test run — nothing saved. Would look for up to {limit} " + f"profile{'' if limit == 1 else 's'} matching " + f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " + f"${est['usd']}", counts, [], steps) + step("Starting the search") + # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never + # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the + # vendor refuses. Reported either way, because "why did this run find fewer" and "why did + # this run cost the same as last night" are both questions this line answers. + excl = {} + sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, + exclude_handles=already_found_handles(rt, defn), + applied=excl) + if excl.get("excluded"): + step(f"excluded {excl['excluded']} already-found " + f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") + elif excl.get("dropped"): + step(f"searching without the already-found list — {excl['dropped']}") + if note: + steps["find"] = "blocked" + return ("error", f"the search was not started — {note}", counts, [], steps) + pending = sid + # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes + # through `rt.update`, which can RAISE on an unavailable store — and that exception + # becomes an error run, so a `log()` placed after it never executes. The snapshot would + # then exist, be billed, and have its id in neither the store nor the logs. Logging first + # means the worst case is still recoverable by a human reading the Space output. + log(f"[aios-auto] discover {auto_id}: started {sid}") + # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop + # existing because this process dies thirty seconds later, and a lost id is a set we + # paid attention to and can never collect. + set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) + steps["find"] = "ok" + + waited, status, size, note = 0.0, "", 0, "" + while True: + # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to + # BD_FILTER_WAIT (120 s) at the vendor, and D measured that this wait IS the whole of + # "Run once now is laggy / looks stuck". A counter here is what makes a legitimate + # two-minute wait distinguishable from a hung thread; without it the rendered step reads + # the same word for both, which is a progress indicator that cannot indicate progress. + step(f"Searching — {int(waited)}s of " + f"{int(BD_FILTER_WAIT)}s") + status, size, note = bd_filter_status(pending) + if note: + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", note, counts, [], steps) + if status != "building" or waited >= BD_FILTER_WAIT: + break + time.sleep(BD_FILTER_POLL) + waited += BD_FILTER_POLL + step("Reading the profiles the search found") + + if status == "building": + # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 + # minutes; holding a worker thread that long on a free-tier container to poll would be + # the wrong shape. Say plainly that it is still running and who collects it. + steps["collect"] = "partial" + # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made + # this read as "stuck": *"The next run picks up the results"* is a promise about a run + # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick + # finish it, so the promise is now kept by something rather than by the reader. + mins = 0 + try: + since = (defn.get("state") or {}).get("pendingSince") + if since: + mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) + - _dt.datetime.fromisoformat(str(since))).total_seconds() + // 60)) + except Exception: # noqa: BLE001 + mins = 0 + been = f" — {mins} min so far" if mins else "" + # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by + # naming the corpus size: *"the wait is the SEARCH over 620 million profiles, not the 10 + # rows"*. It was TRUE, and it was answering a question the person had not asked. Somebody + # watching a run that has taken eighteen minutes wants to know whether it is broken and + # when it ends; how big the index is, is our fact about our supplier. Keeping it also + # meant this line was the one place a vendor's catalogue size was quoted to a customer, so + # it would have gone stale the day the vendor grew — a number nobody would think to + # re-measure because nothing depends on it. + # ⚠ THE SHAPE OF THE ANSWER SURVIVES and is the part that mattered: the wait does not + # shrink when you ask for fewer records, and a person who does not know that reads + # "20 minutes for 10 rows" as a fault. + return ("partial", + f"Still searching at the provider{been}. A corpus search takes about 20 minutes " + f"however few records you asked for. Nothing is lost, you are not charged twice, " + f"and the results are collected automatically as soon as they are ready", + counts, [], steps) + if status == "empty": + # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword + # that is too specific, so it says what to do about it instead of reporting a fault. + steps["collect"] = "ok" + steps["write"] = "skipped" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("partial", "No profiles matched. Try a shorter or more common keyword — " + "'floral' finds more accounts than 'floral design studio'", + counts, [], steps) + if status != "ready": + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", + counts, [], steps) + + rows, dnote = bd_filter_rows(pending) + if dnote: + # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. + steps["collect"] = "partial" + return ("partial", f"the search finished with {size} match" + f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + steps["collect"] = "ok" + + stamp = _iso() + incoming = [c for c in (_candidate_row(r, stamp) for r in rows) if c] + counts["found"] = len(incoming) + # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and + # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to + # "the vendor delivered 3". It also catches the shape where an error document comes back + # down the rows path and parses as one unusable row instead of an error. + counts["dropped"] = max(0, len(rows) - len(incoming)) + label = cfg.get("targetLabel") or DISCOVER_LABEL + table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry + else ut_ensure(rt, label, CANDIDATE_FIELDS, username, + key=cfg.get("targetTable") or DISCOVER_TABLE, + flow_tag=str(defn.get("id") or ""), lock_fields=True)) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`. + # + # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning + # that built it is worth stating before it is discarded rather than after: per-user rows gave + # each person their own `found_count` and their own review card, so two colleagues scouting the + # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The + # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no + # longer part of the identity. + # + # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN + # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on + # TikTok are routinely different people, so keying on the handle ALONE would have silently + # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with + # no error, discovered months later by someone wondering why a creator's followers halved. + # Hence `platform`, and hence it being part of the key rather than a label beside it. + _ck = candidate_key + + # ⚠ A PRE-WAVE-26 ROW HAS NO `platform` CELL, and `_ck` defaults it to Instagram rather than to + # blank on purpose: every row that exists today was written by the Instagram runner, so that + # default is a FACT about the stored data, not an assumption. Blank-keyed rows would fail to + # match their own re-find and duplicate the whole table on the next run. + for r in existing.values(): + if not str(r.get("platform") or "").strip(): + r["platform"] = PLATFORM_INSTAGRAM + existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))} + for rid, r in existing.items()} + # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding + # a profile is the signal that it keeps matching, so the counter grows instead of + # resetting. `first_found` is written only when the pair is new, so it never moves. + seen = {r["_ckey"]: r for r in existing2.values()} + for c in incoming: + # "Found by" — the first finder is recorded and later finders do not overwrite them, which + # is what the column means now that it is no longer part of the identity. + c["created_by"] = username + c["_ckey"] = _ck(c.get("platform"), c["handle"]) + prev_row = seen.get(c["_ckey"]) + if prev_row and str(prev_row.get("created_by") or "").strip(): + c["created_by"] = prev_row["created_by"] + if prev_row: + counts["seen_again"] += 1 + c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) + else: + counts["new"] += 1 + c["found_count"] = "1" + c["first_found"] = stamp + # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF + # "is this row new to the TABLE". + # + # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's + # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true, + # and then drew nothing, because this stamp lived inside the `else` above. `skey` is + # `stage_`; the branch it sat in asks whether some OTHER automation had + # already created the row. So the first automation to reach a handle claimed it, and every + # later automation sharing that database silently had no cards — with a summary saying it + # had found them. Nothing errored, and both boards are green surfaces. + # + # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an + # unconditional write: a re-find must never pull a card somebody has already moved back + # into Review. So the question is "has THIS automation placed this record yet?" — not "is + # this record new?" A blank stage cell for this automation means unplaced, which is + # exactly the state a new card is in. + merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) + merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} + for rid, r in merged.items()} + counts["capped"] = mc["capped"] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + if not dry and not missing: + ut_write_rows(rt, table_key, merged) + steps["write"] = "ok" + # ⭐⭐ THE FORWARD HALF (2026-08-10). Every candidate this run wrote now also leaves an + # OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the + # next discovery run re-opens exactly the gap the backfill just closed, which is how a + # data defect becomes a recurring one. + # + # ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change + # worth owning: a tenant that has never enriched now gets the four canonical child + # databases the first time discovery runs (four of `MAX_UT_TABLES = 40`). The alternative + # writes a series into a store the profile table has no LINK to — the numbers would be + # recorded and unreachable, which is half a feature wearing a whole one's clothes. A + # tenant running discovery is an Instagram tenant; this moves the spawn earlier, it does + # not add one. + # ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not + # to every row the table happens to hold. + try: + graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key) + counts["series"] = append_ig_snapshots( + rt, [corpus_snapshot_row(c, day=stamp) for c in incoming], + graph[IG_SNAPSHOTS_TABLE], log=log) + except Exception as e: # noqa: BLE001 + # The candidates already landed. A series write that fails must not throw away the + # rows we just paid the vendor for — the same posture the preset top-up takes. + log(f"[aios-auto] discover {auto_id}: corpus series write failed: " + f"{type(e).__name__}: {e}") + elif dry: + steps["write"] = "skipped" + + affected = [rid for rid, r in merged.items() + if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] + est = discover_estimate(counts["found"]) + summary = (f"{counts['found']} profile{'' if counts['found'] == 1 else 's'} found — " + f"{counts['new']} new, {counts['seen_again']} seen before. About " + f"${est['usd']}") + if counts["dropped"]: + summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " + f"username and could not be saved") + cnote = cap_note([(table_key, counts["capped"])], missing) + if cnote: + summary += f" — {cnote}" + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" + if not dry and not missing: + try: + compute_metric_cells(rt, table_key) # C7 — see the field runner + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + log(f"[aios-auto] discover {auto_id} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge +#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a +#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own +#: cheaper approximation. +TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count", + "first_found", "last_found", "source", "source_payload"}) + + +def _tt_candidate_row(row, stamp): + """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle. + + ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so + writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as + `_candidate_row` on the Instagram side. + ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what + the enrich path will use too, so a corpus row and a scrape row cannot disagree about which + vendor key becomes which column. + """ + import connectors_tt as _tt + cells = _tt.normalize_profile(row if isinstance(row, dict) else {}) + if not str(cells.get("handle") or "").strip(): + return None + return {**cells, "last_found": stamp, "source": "corpus"} + + +def run_discover_tiktok(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """⭐⭐ DEBT D-9, open since wave 19 — FIND TIKTOK HANDLES NOBODY HERE HAS TYPED IN. + + Two-phase by construction, exactly like the Instagram runner: a run either STARTS a corpus + query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every + outcome is a sentence about what actually happened; none of them is a green zero. + + ⛔ IT IS A SIBLING OF `run_discover_instagram`, NOT A REFACTOR OF IT, AND THAT IS A DECISION. + The two-phase scaffolding below is duplicated. Extracting it would be architecturally right — + the `SOURCES` registry says so in its own comment — and it would rewrite the one code path this + product bills money through, which `section_bd`, `section_w27`, NC33/NC34 and four derived + sweeps all pin. R2 already chose "parallel, zero risk to the live Instagram rows" for the data; + the same reasoning applies to the runner. The extraction is booked as a PENDING line with the + Bright Data transport, which is the second duplication pointing at the same seam — two of them + is a better argument for the next wave than a half-finished refactor is for this one. + + ⛔ THE DISCOVERY HALVES BRIGHT DATA SELLS FOR TIKTOK BY KEYWORD AND BY PROFILE **404 FOR OUR + KEY** (probed: `gd_lj71gn6l68bz7y9hc`, `gd_lilwhto81z415d9mdl`). Discovery therefore routes + through the Profiles dataset's corpus filter, which is the same door Instagram uses. + """ + import connectors_tt as _tt + cfg = defn.get("config") or {} + dry = bool(cfg.get("dryRun")) + limit = int(cfg.get("recordsLimit") or 5) + preds = list(cfg.get("predicates") or []) + auto_id = str(defn.get("id") or "") + steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} + counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, + "missing_tables": 0} + + if not bd_ready(): + # Fail-closed, and it says which credential is missing. Discovery has no free rung to drop + # to — the corpus is the vendor's. + steps["find"] = "blocked" + return ("partial", "Profile search is not set up yet — nothing was searched and nothing " + "was charged", counts, [], steps) + + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + if not pending: + guard_err = (narrowing_refusal(preds, cfg.get("operator")) + or depth_refusal(preds, cfg.get("operator"))) + if guard_err: + steps["find"] = "blocked" + return ("error", f"the search was not started — {guard_err}", counts, [], steps) + if dry: + est = discover_estimate(limit) + steps["find"] = steps["write"] = "skipped" + return ("partial", + f"Test run — nothing saved. Would look for up to {limit} TikTok " + f"profile{'' if limit == 1 else 's'} matching " + f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " + f"${est['usd']}", counts, [], steps) + step("Starting the search") + excl = {} + # ⚠ `already_found_handles` NEEDS NO TIKTOK BRANCH — it is scoped to THIS definition's own + # tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` excludes + # exactly the handles it has itself found. Verified rather than assumed: the function reads + # `config.targetTable` plus each `create_record` action's table, and names no IG constant. + sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, + dataset_id=_tt.TT_DS_PROFILES, + exclude_handles=already_found_handles(rt, defn), + applied=excl) + if excl.get("excluded"): + step(f"excluded {excl['excluded']} already-found " + f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") + elif excl.get("dropped"): + step(f"searching without the already-found list — {excl['dropped']}") + if note: + steps["find"] = "blocked" + return ("error", f"the search was not started — {note}", counts, [], steps) + pending = sid + # LOG BEFORE PERSISTING: `set_state` can raise on an unavailable store, and a snapshot that + # exists, is billed and has its id in neither the store nor the logs is unrecoverable. + log(f"[aios-auto] discover-tt {auto_id}: started {sid}") + set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) + steps["find"] = "ok" + + waited, status, size, note = 0.0, "", 0, "" + while True: + step(f"Searching — {int(waited)}s of {int(BD_FILTER_WAIT)}s") + status, size, note = bd_filter_status(pending) + if note: + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", note, counts, [], steps) + if status != "building" or waited >= BD_FILTER_WAIT: + break + time.sleep(BD_FILTER_POLL) + waited += BD_FILTER_POLL + step("Reading the profiles the search found") + + if status == "building": + mins = 0 + try: + since = (defn.get("state") or {}).get("pendingSince") + if since: + mins = max(0, int((_dt.datetime.now(_dt.timezone.utc) + - _dt.datetime.fromisoformat(str(since))).total_seconds() + // 60)) + except Exception: # noqa: BLE001 + mins = 0 + been = f" — {mins} min so far" if mins else "" + steps["collect"] = "partial" + return ("partial", + f"Still searching at the provider{been}. A corpus search takes about 20 minutes " + f"however few records you asked for. Nothing is lost, you are not charged twice, " + f"and the results are collected automatically as soon as they are ready", + counts, [], steps) + if status == "empty": + steps["collect"] = "ok" + steps["write"] = "skipped" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("partial", "No profiles matched. Try a shorter or more common keyword — " + "'floral' finds more accounts than 'floral design studio'", + counts, [], steps) + if status != "ready": + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", + counts, [], steps) + + rows, dnote = bd_filter_rows(pending) + if dnote: + steps["collect"] = "partial" + return ("partial", f"the search finished with {size} match" + f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + steps["collect"] = "ok" + + stamp = _iso() + incoming = [c for c in (_tt_candidate_row(r, stamp) for r in rows) if c] + counts["found"] = len(incoming) + # Count what we could NOT use: "the vendor delivered 50 and we kept 3" must not read the same + # as "the vendor delivered 3". + counts["dropped"] = max(0, len(rows) - len(incoming)) + label = cfg.get("targetLabel") or TT_TABLE_LABELS[TT_PROFILE_TABLE] + table_key = (ut_key_for(label, cfg.get("targetTable") or TT_PROFILE_TABLE) if dry + else ut_ensure(rt, label, TT_PROFILE_FIELDS, username, + key=cfg.get("targetTable") or TT_PROFILE_TABLE, + flow_tag=auto_id, lock_fields=True)) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + # ⚠ A ROW WITH NO `platform` CELL IN A `ut_tt_*` TABLE IS STAMPED TIKTOK, not Instagram. The + # shared `candidate_key` defaults a blank to Instagram — correct for the IG table, where every + # legacy row came from that runner, and wrong here, where every row came from this one. + for r in existing.values(): + if not str(r.get("platform") or "").strip(): + r["platform"] = PLATFORM_TIKTOK + existing2 = {rid: {**r, "_ckey": candidate_key(r.get("platform"), r.get("handle"))} + for rid, r in existing.items()} + seen = {r["_ckey"]: r for r in existing2.values()} + for c in incoming: + c["created_by"] = username + c["_ckey"] = candidate_key(c.get("platform"), c["handle"]) + prev_row = seen.get(c["_ckey"]) + if prev_row and str(prev_row.get("created_by") or "").strip(): + c["created_by"] = prev_row["created_by"] + if prev_row: + counts["seen_again"] += 1 + c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) + # ⛔⛔ DEBT D-116, AND THIS RUNNER DELIBERATELY DOES NOT INHERIT IT. On the Instagram + # side a discovery re-find OVERWRITES an enriched `followers` with the corpus number — + # the corpus is cheaper, rounder and older than a scrape, so the row silently gets + # WORSE every night while the automation reports success. The fix is a precedence rule, + # not a blank check: a row that has been ENRICHED (`enriched_at` is stamped only by the + # enrich path) keeps its exact measurements, and discovery may only fill what is + # genuinely empty and update its own bookkeeping. + # ⚠ `TT_DISCOVERY_KEYS` is what discovery OWNS; everything else on a corpus row is a + # measurement, and an approximation must never replace an exact one. + if str(prev_row.get("enriched_at") or "").strip(): + for k in list(c): + if (k not in TT_DISCOVERY_KEYS and k != "_ckey" + and str(prev_row.get(k) or "").strip()): + c.pop(k) + else: + counts["new"] += 1 + c["found_count"] = "1" + c["first_found"] = stamp + merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) + merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} + for rid, r in merged.items()} + counts["capped"] = mc["capped"] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + if not dry and not missing: + ut_write_rows(rt, table_key, merged) + steps["write"] = "ok" + elif dry: + steps["write"] = "skipped" + + affected = [rid for rid, r in merged.items() + if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] + est = discover_estimate(counts["found"]) + summary = (f"{counts['found']} TikTok profile{'' if counts['found'] == 1 else 's'} found — " + f"{counts['new']} new, {counts['seen_again']} seen before. About " + f"${est['usd']}") + if counts["dropped"]: + summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no " + f"username and could not be saved") + cnote = cap_note([(table_key, counts["capped"])], missing) + if cnote: + summary += f" — {cnote}" + steps["write"] = "partial" + if dry: + summary = f"Test run — nothing saved. Would have written: {summary}" + state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" + log(f"[aios-auto] discover-tt {auto_id} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def _predicate_sentence(preds, operator="and"): + """A predicate list as something a person can read back. Used in summaries and the canvas. + + Human words on BOTH halves — it used to render `biography includes floral`, the vendor's + column name beside the vendor's operator token, in a sentence shown to a customer. + """ + joiner = " or " if str(operator or "").lower() == "or" else " and " + parts = [] + for p in preds or []: + v = p.get("value") + shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v + parts.append(f"{field_label(p.get('name'))} " + f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}" + + ("" if shown is None else f" {shown}")) + return joiner.join(parts) or "no conditions" + + +#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather +#: than a second, smaller number nobody could explain: a plain automation's records ARE its +#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a +#: flow could be pointed at and which would not finish. +MAX_FLOW_RECORDS = MAX_UT_ROWS + + +def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole + automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special + case and `apply_actions` still runs at the ONE call site that already exists. + + Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed + `error: unknown automation kind 'plain'` — on the kind every new automation now has. + + ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here. + `apply_actions` returns immediately when there is no table, so a flow whose only action is + `create_record` into some other database does NOTHING — and a green dot over nothing is the + exact defect this module names in three other places. It reports the honest state and says + which fact is missing. + """ + table = _flow_table(defn) + if not table: + return ("partial", + "no database is bound yet — a plain automation walks the records of its target " + "database, and this one names none (pick one on the trigger, or in Properties)", + {}, [], {}) + all_rows = (ut_get(rt, table) or {}).get("rows") or {} + # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug + # on the kind every new automation now has. `plain` has no machine step, so it has nothing of + # its own to call "the records this run touched" — and walking the WHOLE table would mean + # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the + # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them + # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what + # those genuinely mean. + ids = ([r for r in (rows or []) if str(r) in all_rows] if rows + else sorted(all_rows, key=_rid_num)) + + # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its + # saved View and quota are not merely how the action decides whether to spend; they ARE the + # set of records the automation was asked to process. Walking every row first meant a run + # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows. + # Besides being misleading, that shape made the runtime scale with unrelated historical rows. + # + # Keep this deliberately narrow: any flow with another action must still hand that action the + # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record + # semantics to preserve, so it can begin at its selected records safely. + flow_actions = list((defn.get("flow") or {}).get("actions") or []) + enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)] + enrich_scope_note = "" + if rows is None and len(enabled_direct) == 1 \ + and enabled_direct[0].get("kind") == "enrich_instagram": + enrich_cfg = enabled_direct[0].get("config") or {} + profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField")) + if profile_key: + # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES. + # `enrich_selection` is called TWICE per run — here, to decide which records the + # runner walks and announces, and again inside `_walk` to decide which the action + # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so + # the run announced *"(1 of the 30 asked for)"* in its summary while its own note + # from the second call said *"0 of the 30"* — one run, two answers, both printed. + # ⚠ `gone` is read from the definition here rather than passed in, because this + # function has the definition and `_walk` does the same read; a third source of the + # same fact is how the two would drift again ([[one-evaluator-per-question]]). + selected, enrich_scope_note = enrich_selection( + rt, table, enrich_cfg, profile_key, + gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})) + if enrich_scope_note.startswith("the enrich step names a view it cannot read"): + return ("partial", f"nothing walked in {table} — {enrich_scope_note}", {}, [], {}) + ids = selected + step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") + counts = {"records": len(ids)} + if len(ids) > MAX_FLOW_RECORDS: + # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names + # both numbers so "it only processed some of them" is readable rather than deducible. + total = len(ids) + ids = ids[:MAX_FLOW_RECORDS] + return ("partial", + f"{total} records in {table} — this run walked the first " + f"{MAX_FLOW_RECORDS}", counts, ids, {}) + n = len(ids) + # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK. + # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)? + # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours: + # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product + # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key. + # · `(all)` meant "every record, because a manual run has no trigger records to narrow to", + # which is unguessable from the word. It reads as a filter nobody chose. + # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole + # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that + # this sentence never claimed to report. What was wrong was that nothing said so. + label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table + scope = ("the records that fired the trigger" if rows + else ("the records selected for Instagram enrichment" + if len(enabled_direct) == 1 + and enabled_direct[0].get("kind") == "enrich_instagram" + else "every record — this run was started by hand, so nothing narrowed it")) + if enrich_scope_note: + scope += f" ({enrich_scope_note})" + # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400 + # characters by `_commit_run`, and this note is the longest thing a run says — it names + # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING + # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy + # of this sentence is never produced. Without this line the explanation exists only in a + # string that is about to be truncated, on exactly the runs that look like the automation + # has stopped working. + counts[RUN_NOTES_KEY] = [enrich_scope_note] + return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}", + counts, ids, {}) + + +RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, + "field_instagram": run_field_instagram, + "discover_instagram": run_discover_instagram, + "discover_tiktok": run_discover_tiktok} + + +# --------------------------------------------------------------------------------------------- +# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) +# --------------------------------------------------------------------------------------------- +# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → +# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness +# boolean on the wire, four client strings and two gate sections. That is what a hard-coded +# vendor costs, and TikTok (D-9) would have paid it again from scratch. +# +# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the +# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: +# +# probe "is this configured?" -> the honest readiness bit the UI shows +# capture enrich a handle we know -> `pull_profile`-shaped +# discover find handles we do not -> the corpus query +# +# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in +# later; it is the honest statement that this source has no discovery route, and a surface that +# offers one anyway is offering a button that must refuse. +SOURCES = { + "instagram": { + # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing + # copy, and which provider serves a capability is a routing decision the product owns — + # `providers.py` still holds the real keys, chains and costs. + "key": "instagram", "label": "Instagram", "vendor": "Scraper", + "probe": bd_ready, + "capture": pull_profile, + "discover": bd_filter_start, + "captureKind": "field_instagram", + "discoverKind": "discover_instagram", + "tiers": TIERS, + }, + # ⭐⭐ WAVE 29 — D-9 LANDS, AND IT LANDED AS THE ONE ENTRY THIS REGISTRY PREDICTED IT WOULD. + # The canvas, the append tables, the caps and the honest-status contract were already in place; + # what TikTok added was a field map, a discovery runner and three dataset ids. + "tiktok": { + # `vendor` NAMES NO COMPANY (owner instruction 2026-08-09) — it reaches operator-facing + # copy, and which provider serves a capability is a routing decision `providers.py` owns. + "key": "tiktok", "label": "TikTok", "vendor": "Scraper", + "probe": bd_ready, + # ⚠ `capture` IS None UNTIL W29-T05 WIRES THE ENRICHMENT ACTION, and `source_status` does + # not read it — but a surface that offered a capture this source cannot do would be + # offering a button that must refuse, which is exactly what this registry's own note + # forbids. Declared as absent rather than pointed at the Instagram function. + "capture": None, + "discover": bd_filter_start, + "captureKind": None, + "discoverKind": "discover_tiktok", + # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never + # had one. An empty tuple is the honest statement, not a gap to fill in later. + "tiers": (), + }, +} + + +def source_status(): + """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. + + ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface + needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the + credential to a browser would put a billable secret in every user's devtools. + """ + # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which + # supplier this rung actually uses — that history is worth keeping in the code — but a + # customer has no use for it, and a server-supplied string rendered by a generic component + # is precisely the half a client-file scan cannot see. + return [{"key": s["key"], "label": s["label"], + "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} + for s in SOURCES.values()] + + +# --------------------------------------------------------------------------------------------- +# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client +# --------------------------------------------------------------------------------------------- +# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are +# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — +# a canvas that still shows a step the engine stopped running looks completely fine. So the node +# list is derived from the DEFINITION by the same module that executes it, rides on the payload +# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. +# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. +# +# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is +# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine +# walks it linearly. Nothing here should be read as a general branching runtime. + +#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be +#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw +#: away the only information the dot is there to carry. +NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") + +#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a +#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is +#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones +#: here are all REAL — each changes what the next run does: +#: schedule the trigger fires on its cron, or only by hand +#: postMetrics likes/comments per post are bought, or the engagement series does not grow — +#: it costs a vendor record PER POST rather than per profile (measured: the +#: profile row carries post identity and no engagement) +#: commentMetrics the Comments dataset is bought, which can bill many rows per post +#: write DRY RUN — read everything, compute the counts, write nothing anywhere +#: +#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is +#: the point of the ruling: the money switches used to be "which rung do we try" (a question about +#: our plumbing), and they are now "what do you want captured" (a question about the user's data). +#: Profile is always captured and has no switch — an automation that fetches nothing is not an +#: automation with a step turned off, it is a broken one with a lie on it. +#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and +#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the +#: three toggles" names the CAPTURE ladder it is reshaping. +NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics", + "capture_comments": "commentMetrics", "write": "write"} + + +def _cron_label(cron): + return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") + + +def node_status(steps, nid): + """A node's dot — from the last run's MEASURED step outcome, or `idle`. + + ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely + so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, + and painting those nodes green because the run said `ok` would manufacture a measurement — the + same defect as the green dot over the empty posts table recorded in `run_field_instagram`. + No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. + """ + v = str((steps or {}).get(nid) or "") + return v if v in NODE_STATES else "idle" + + +def graph(defn): + """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" + defn = defn if isinstance(defn, dict) else {} + cfg = defn.get("config") or {} + sched = defn.get("schedule") or {} + runs = defn.get("runs") or [] + last = runs[0] if runs and isinstance(runs[0], dict) else {} + steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} + + def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): + return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, + "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), + "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} + + on = bool(sched.get("enabled")) + trg = defn.get("trigger") or {} + if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): + # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail + # read the same node, so an event-triggered flow must not read "Schedule". The membership + # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old + # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so + # the else-branch was reachable) — green, compiling, and wrong on screen. + t_on = bool(trg.get("enabled", True)) and not trg.get("paused") + sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) + watched = ", ".join(trg.get("fields") or []) or "any field" + # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the + # whole of it) and `record_updated` no longer names a condition (its watched fields are). + # These two lines are the migration's visible half: a node still describing a `.field` + # the validator has stopped storing would be the surface and the store disagreeing. + det = {"event_field": (f"{trg.get('table', '')}" + + (f" — {_lane_sentence(trg.get('when'))}" + if trg.get("when") else "")), + "record_updated": f"{trg.get('table', '')} — {watched}", + "record_created": str(trg.get("table") or ""), + "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}", + "form_submitted": str(trg.get("table") or ""), + "webhook": "POST the hook URL to fire it", + "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet — nothing to search for"), + "email": str(trg.get("query") or "")}.get(trg["key"], "") + if not trg.get("configured", True): + det = "Finish setting this trigger up before it can fire" + if trg.get("paused"): + det = str(defn.get("statusNote") or "paused") + nodes = [node("trigger", "trigger", sub, + "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] + else: + # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off + # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with + # the user's own pick — the same class of defect as the Database picker that could not be + # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the + # disabled-schedule gate asserts, and it was never the wrong half. + # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires + # TODAY, not that it may never be scheduled — the cron has to stay one click away, and + # this is the only node that offers it. + nodes = [node("trigger", "trigger", "Schedule" if on else "Manual", + _cron_label(sched.get("cron")) if on else "Manual only", 0, + panel="schedule", on=on, + detail="" if on else "It still runs when you press Run now")] + edges = [] + + if defn.get("kind") == "field_instagram": + # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED. + # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and + # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5 + # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the + # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners + # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas. + # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an + # implementation detail: the profile is always captured (R6), Posts and Comments are the + # opt-ins, and both are OFF until asked for. + metrics_on = bool(cfg.get("postMetrics")) + comments_on = bool(cfg.get("commentMetrics")) + max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL + nodes += [ + node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, + panel="source", + detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") + else "URL column: the one the field is bound to"), + # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY + # and no engagement (measured), so likes/comments are a SECOND vendor call per post. + # A switch that multiplies a run's cost by the post count deserves to be visible on + # the canvas rather than buried in a config panel. + node("capture_posts", "capture", "Post data", + "Likes and comments per post" if metrics_on else "Off", + 2, panel="capture", on=metrics_on, + detail=(f"Up to {max_posts} posts per profile" if metrics_on else + "ut_ig_post_snapshots only grows while this is on")), + node("capture_comments", "capture", "Comment data", + "Comments on those posts" if comments_on else "Off", + 3, panel="capture", on=comments_on, + detail=("The full comments dataset — many rows per post" if comments_on else + "Comments already embedded in a paid post row are kept either way")), + node("write", "write", "Write", + cfg.get("targetTable") or "No database", 4, panel="write", + on=not cfg.get("dryRun"), + detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), + ] + edges = [{"from": "trigger", "to": "source", "label": ""}, + {"from": "source", "to": "capture_posts", "label": ""}, + {"from": "capture_posts", "to": "capture_comments", "label": ""}, + {"from": "capture_comments", "to": "write", "label": ""}] + elif defn.get("kind") == "discover_instagram": + limit = int(cfg.get("recordsLimit") or 0) + est = discover_estimate(limit) + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + nodes += [ + node("find", "source", "Find profiles", + _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, + panel="find", + detail=(f"Up to {limit} profiles · about ${est['usd']}" + if bd_ready() + else "Profile search is not set up yet")), + node("collect", "capture", "Collect results", + "Searching…" if pending else "Takes about 20 minutes", 2, + panel="find", + detail=""), + node("write", "write", "Save results", + cfg.get("targetTable") or DISCOVER_TABLE, 3, panel="write", + on=not cfg.get("dryRun"), + detail=""), + ] + edges = [{"from": "trigger", "to": "find", "label": ""}, + {"from": "find", "to": "collect", "label": ""}, + {"from": "collect", "to": "write", "label": ""}] + elif defn.get("kind") == "scrape_db": + url = str(cfg.get("url") or "") + host = (urlparse(url).hostname or url or "No URL") if url else "No URL" + which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" + else f"HTML table #{int(cfg.get('tableIndex') or 0)}") + mapped = len(cfg.get("fieldMap") or {}) + nodes += [ + node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), + node("extract", "capture", "Extract", which, 2, panel="columns", + detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " + f"{cfg.get('keyField') or '—'}"), + node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, + panel="write", on=not cfg.get("dryRun"), + detail=cfg.get("targetTable") or "a new database"), + ] + edges = [{"from": "trigger", "to": "fetch", "label": ""}, + {"from": "fetch", "to": "extract", "label": ""}, + {"from": "extract", "to": "write", "label": ""}] + # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point. + # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught + # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML + # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A + # surface that draws nothing is obviously incomplete; one that draws a fetch step that does + # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at + # once because all three read this. + # + # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE + # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node, + # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on + # that promise, so it is stated rather than left to be inferred from the control flow. + return {"nodes": nodes, "edges": edges} + + +def toggle_node(rt, auto_id, node_id): + """Flip ONE node's switch. Returns `(defn, error)`. + + The node → config mapping lives on the SERVER for the reason the whole graph does: a client + that knew which field each node writes would be a second copy of that knowledge, free to + drift. The canvas just says "this node was clicked". + """ + prev = all_definitions(rt).get(str(auto_id)) + if prev is None: + return None, "no such automation" + target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) + if target is None: + return None, f"there is no {node_id!r} step in this automation" + if not target.get("toggle"): + return None, (f"the {target['title']} step cannot be turned off — it is what this " + f"automation is") + cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) + which = target["toggle"] + trg_key = str((prev.get("trigger") or {}).get("key") or "") + if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS: + # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping + # the cron under a node labelled "When a field changes" would be a switch that lies. + # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that + # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so + # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so + # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half. + trg = dict(prev.get("trigger") or {}) + trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) + trg["paused"] = False # a deliberate flip clears an auto-pause: human wins + return patch(rt, auto_id, {"trigger": trg}) + if which == "schedule": + sched["enabled"] = not sched.get("enabled") + elif which == "postMetrics": + cfg["postMetrics"] = not cfg.get("postMetrics") + elif which == "commentMetrics": + cfg["commentMetrics"] = not cfg.get("commentMetrics") + elif which == "write": + cfg["dryRun"] = not cfg.get("dryRun") + return patch(rt, auto_id, {"config": cfg, "schedule": sched}) + + +# --------------------------------------------------------------------------------------------- +# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed +# --------------------------------------------------------------------------------------------- +# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were +# records of the target database, and each record's position lived in a single-select STAGE FIELD +# the engine created on the customer's own database. Wave 26 deleted the board's built-in +# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`, +# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review +# branch of the action walk, and the endings that sent a terminal card back round. +# +# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote +# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles` +# counter per automation — to render a view of state the RUN LOG already holds. The customer paid +# for that in columns they did not ask for, in a store commit on every scheduled run, and in a +# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we +# had added. `migrate_ig_tables` now drops those columns wherever an automation created them. +# +# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE: +# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the +# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was +# one of four callers. The name is the last thing the board left behind here. +# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board` +# — the MIGRATION. It must outlive the thing it retires, or a definition written before the +# retirement walks into a runtime that no longer understands it. +# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are +# parked with no caller and say so at their own definitions. + +LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", + "is_empty", "is_not_empty") +LANE_NULLARY_OPS = ("is_empty", "is_not_empty") + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` +# +# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why +# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches +# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc +# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the +# same function applied one level up, so the cheaper honest version is to evaluate the leaf where +# it lies and never touch the bytes.) +# +# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An +# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and +# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. +# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an +# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that +# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine +# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. +MAX_COND_DEPTH = 3 +MAX_COND_CHILDREN = 12 +COND_GROUP_KEYS = ("all", "any") + + +def clean_cond(raw, depth=0, where=""): + """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no + condition", which is legal everywhere a condition is optional (the catch-all lane, an + unconditioned trigger, an action group that always runs).""" + if raw in (None, "", {}): + return None, None + at = f" on {where}" if where else "" + if not isinstance(raw, dict): + return None, f"the condition{at} must be an object" + conj = [k for k in COND_GROUP_KEYS if k in raw] + if len(conj) > 1: + return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the " + f"other") + if conj: + key = conj[0] + if depth + 1 > MAX_COND_DEPTH: + return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — " + f"the group{at} is deeper") + kids_raw = raw.get(key) + if not isinstance(kids_raw, list) or not kids_raw: + return None, f"the '{key}' group{at} needs at least one condition inside it" + if len(kids_raw) > MAX_COND_CHILDREN: + return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — " + f"the '{key}' group{at} has {len(kids_raw)}") + kids = [] + for child in kids_raw: + c, err = clean_cond(child, depth + 1, where) + if err: + return None, err + if c is None: + return None, (f"an empty condition sits inside the '{key}' group{at} — " + f"finish it or remove it") + kids.append(c) + return {key: kids}, None + field = _s(raw.get("field"), 80).strip() + op = _s(raw.get("op") or raw.get("operator"), 20).strip() + if not field: + # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could + # act on. It states a fact about the stored tree; it never said what to DO, and the usual + # cause is not a typo but an empty field picker (the flow had no walking record to offer + # columns from). The fix belongs in the message. + return None, (f"the condition{at} names no field — pick a column, or remove the " + f"condition") + if op not in LANE_OPS: + return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) + cond = {"field": field, "op": op} + if op not in LANE_NULLARY_OPS: + v = raw.get("value") + if v is None or (isinstance(v, str) and not v.strip()): + return None, f"give a value to compare {field!r} against{at}" + cond["value"] = v.strip() if isinstance(v, str) else v + return cond, None + + +def cond_fields(cond): + """Every field name a tree reads — the set a caller must have on the row image before the + answer means anything.""" + if not isinstance(cond, dict): + return set() + for key in COND_GROUP_KEYS: + if key in cond: + out = set() + for child in cond.get(key) or []: + out |= cond_fields(child) + return out + f = cond.get("field") + return {f} if f else set() +#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would +#: collide with the stage the label is a choice FOR, and the board could no longer tell a +#: routed card from a gated one. +# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the +# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a +# legacy stamped cell would read as the same lane while behaving as a different one. +def _lane_num(v): + try: + return float(str(v).replace(",", "")) + except (TypeError, ValueError): + return None + + +def lane_match(cond, row): + """Does `row` satisfy this condition TREE? None matches everything (the catch-all). + + Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 + caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping + this name rather than growing a second entry point that could disagree with it. + + ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse + as a number answers False — the record simply does not enter the lane — never "treat blank + as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank + cell is an unknown, and an unknown cannot be less than 10,000. + """ + if cond is None: + return True + if isinstance(cond, dict): + if "all" in cond: + return all(lane_match(c, row) for c in cond.get("all") or []) + if "any" in cond: + return any(lane_match(c, row) for c in cond.get("any") or []) + raw = (row or {}).get(cond.get("field")) + op = cond.get("op") + if op == "is_empty": + return raw in (None, "") + if op == "is_not_empty": + return raw not in (None, "") + want = cond.get("value") + if op in (">", ">=", "<", "<="): + a, b = _lane_num(raw), _lane_num(want) + if a is None or b is None: + return False + return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] + have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") + if op == "includes": + return want_s.lower() in have_s.lower() + if op == "not_includes": + return want_s.lower() not in have_s.lower() + a, b = _lane_num(raw), _lane_num(want) + same = (a is not None and b is not None and a == b) or have_s == want_s + return same if op == "=" else (not same) if op == "!=" else False + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. +# +# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. +# A `group` holds nested actions behind a condition — Airtable's "conditional action group", +# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's +# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this +# builder is a column, not a canvas). +# +# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a +# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / +# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty +# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready +# kind with a sentence, so the faded state is a wall and not a styling choice. +MAX_ACTIONS = 20 # per automation, counting nested ones +MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow +MAX_BRANCHES = 6 # per If / then +MAX_ACTION_VALUES = 20 # cells one update/create action may write +FIND_LIMIT_MAX = 100 +#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder` +#: and never by a literal list of group names: a client-side ordering is a second copy of this +#: table, and the way it fails is that a group added here renders last, or not at all, with +#: nothing red anywhere. +ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} +ACTION_CATALOG = [ + # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ──────── + # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row + # is a WALL, not a styling choice — and each `detail` says the true reason rather than a + # placeholder, because "coming soon" on five rows is how a menu stops being believed. + # + # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24 + # scheduled this build for wave 25 and wave 25's six owner items do not include it, so + # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its + # own promises has to be re-read by whoever misses the date, and a stale date is worse than + # none because it reads as a commitment somebody already broke. The replacement names the + # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule. + {"kind": "web_goto", "label": "Open a page", "group": "Web action", "ready": False, + "detail": "Needs the browser job — a real browser running inside an ephemeral worker"}, + {"kind": "web_fill", "label": "Fill a field", "group": "Web action", "ready": False, + "detail": "Needs the recorder that captures a selector instead of a coordinate"}, + {"kind": "web_click", "label": "Click something", "group": "Web action", "ready": False, + "detail": "Needs the recorder that captures a selector instead of a coordinate"}, + {"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": False, + "detail": "Needs the browser job — the scrape action reads static pages today"}, + {"kind": "web_repair", "label": "Repair a broken step", "group": "Web action", + "ready": False, + "detail": "Self-healing selectors arrive with the browser engine (Scrapling)"}, + # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── + {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, + "detail": "Write values onto the record walking the flow"}, + {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True, + "detail": "Add a row to another database"}, + {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True, + "detail": "Look rows up by condition; the run log opens them"}, + # ── 3. CONNECTED ───────────────────────────────────────────────────────────────────────── + # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND. + # `field_instagram` was an automation you created to fill one column; this is a step any flow + # can take, which is the shape it should always have had — enriching a profile is something + # you do TO a record, not a species of automation. + # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu, + # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`. + {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected", + "ready": True, "connector": "scraper", + "detail": "Fill this record's Instagram columns from its profile, and add a point to its " + "history"}, + {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, + "detail": "Needs a send scope on the Gmail connection"}, + {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, + "detail": "Needs the Slack connector"}, + # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── + # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism, + # and R8 made it a FORK with lettered branches, which is a thing people already have a name + # for. The KIND is untouched — renaming it would orphan every stored action for a caption. + {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True, + "detail": "Send the record down one of several branches, by condition"}, + {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", + "ready": False, "detail": "Run the same actions on every item in a list"}, + {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, + "detail": "Not built — a sandbox is its own decision"}, + {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", + "ready": False, "detail": "Needs an AI action implementation"}, +] + + +#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can +#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an +#: "Integrations" header, one row per connector, a chevron into that connector's own submenu). +#: +#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are +#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the +#: key and render the label, never join it against the directory. +#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key. +#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one +#: connector wearing two names on two screens somebody sees within a second of each other, and a +#: second literal is how that happens. This tuple says only WHICH connectors an action may name; +#: what they are CALLED has exactly one source. +ACTION_CONNECTORS = ("scraper",) + + +def _connector_meta(key): + """`{key, label}` for a connector key, from the one place either picker declares it.""" + key = str(key or "") + if key not in ACTION_CONNECTORS: + return None + return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) + + +def action_catalog(): + """The catalog as the wire carries it — a copy, because a caller that mutated the module + constant would change every later reader's answer. + + ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from + `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two + different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST + (not first) — a new group appearing above "Web action" because its order defaulted to 0 is + the failure that would look deliberate. + + ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client + nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two + rows in one nest cannot disagree about what that nest is called. A row naming an unknown + connector loses the stamp instead of inventing a nest with a raw slug for a title. + """ + last = max(ACTION_GROUP_ORDER.values()) + 1 + out = [] + for a in ACTION_CATALOG: + row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)} + conn = _connector_meta(a.get("connector")) + if conn: + row["connector"] = dict(conn) + else: + row.pop("connector", None) + out.append(row) + return out + + +def _action_label(act): + for row in ACTION_CATALOG: + if row["kind"] == act.get("kind"): + return row["label"] + return str(act.get("kind") or "Action") + + +def clean_actions(raw, depth=0, _seen=None, _count=None): + """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. + + Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder + survives a Save. A missing or colliding one is minted `act_`; minting on every clean would + move the selection under the person editing it. + """ + if raw in (None, ""): + return [], None + if not isinstance(raw, list): + return None, "actions must be a list" + seen = _seen if _seen is not None else set() + count = _count if _count is not None else [0] + out = [] + for entry in raw: + if not isinstance(entry, dict): + return None, "each action must be an object with a kind" + kind = _s(entry.get("kind"), 30).strip() + row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) + if row is None: + return None, (f"{kind or 'that action'!r} is not one of: " + + ", ".join(a["kind"] for a in ACTION_CATALOG)) + if not row["ready"]: + return None, (f"{row['label']!r} is on the menu but not built yet — " + f"{row['detail'][0].lower()}{row['detail'][1:]}") + count[0] += 1 + if count[0] > MAX_ACTIONS: + return None, f"an automation runs at most {MAX_ACTIONS} actions" + aid = _s(entry.get("id"), 40).strip() + if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: + n = 1 + while f"act_{n}" in seen: + n += 1 + aid = f"act_{n}" + seen.add(aid) + # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER. + # + # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't + # think you can even add Conditions at all. That's not how the Create Record works."* + # Airtable agrees, and so does the shape: every other action operates ON the record the + # flow is walking, so "run this only when …" is a question about something + # that exists. Create record MAKES one. There is nothing to test yet, which is why the + # picker on this panel was empty and why every condition saved against it named no field — + # the owner's *"the condition on action act_1 names no field"*. The condition belongs on + # the trigger, or on a conditional group wrapping the action. + # + # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored + # automation that already carries one — forever, with no way to edit it out, because the + # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a + # locked door. + # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26 + # removes the condition editor from this action kind entirely (C4), so there is no control + # whose value could appear to be ignored. If a `create_record` condition editor ever comes + # back, this needs a disclosure channel — `clean_actions` has none today. + if kind == "create_record": + when = None + else: + when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") + if cerr: + return None, cerr + cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} + cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count) + if cerr: + return None, cerr + out.append({"id": aid, "kind": kind, + "enabled": bool(entry.get("enabled", True)), + "when": when, "config": cfg}) + return out, None + + +def _clean_action_config(kind, cfg, depth, seen, count): + """One action's `config`, per kind. Returns `(config, error)`.""" + if kind == "group": + # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, + # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an + # if with no else, so expressing "otherwise" meant a second group carrying the negation + # by hand — two conditions a later edit could put out of step with each other. + if depth + 1 > MAX_GROUP_DEPTH: + return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — " + f"past that a flow is a flowchart and belongs on the board") + raw_branches = cfg.get("branches") + if not isinstance(raw_branches, list) or not raw_branches: + # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and + # the live automations carry it — refusing it would 400 every Save of a flow that + # was legal yesterday. Read-side, so nothing has to be rewritten in the store. + raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions")}] + if len(raw_branches) > MAX_BRANCHES: + return None, (f"an If / then has at most {MAX_BRANCHES} branches — past that the " + f"record is being routed, which is what the board's lanes are for") + out_branches, bseen = [], set() + for i, br in enumerate(raw_branches): + if not isinstance(br, dict): + return None, "each branch of an If / then is an object" + cond, cerr = clean_cond(br.get("cond"), where="the branch") + if cerr: + return None, cerr + if cond is None and i != len(raw_branches) - 1: + # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches + # and a null condition matches everything, so a catch-all above another branch + # would make every branch below it dead code — silently, and only for the + # records that reached it. Refused with the reason rather than reordered: this + # module does not quietly rewrite what somebody built. + return None, ("only the LAST branch of an If / then can be the Otherwise leg — " + "a catch-all above another branch makes the ones below it " + "unreachable") + kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count) + if kerr: + return None, kerr + if not kids: + return None, "a branch with no actions inside it does nothing" + bid = _s(br.get("id"), 40).strip() + if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen: + bid = f"b{i + 1}" + bseen.add(bid) + label = " ".join(_s(br.get("label"), 40).split()) or ( + "Otherwise" if cond is None else _branch_letter(i)) + out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids}) + return {"branches": out_branches}, None + if kind in ("update_record", "create_record"): + values = cfg.get("values") + if not isinstance(values, dict) or not values: + return None, f"the {kind.replace('_', ' ')} action writes no values" + if len(values) > MAX_ACTION_VALUES: + return None, f"one action writes at most {MAX_ACTION_VALUES} cells" + clean_vals = {} + for k, v in values.items(): + key = _s(k, 80).strip() + if not key: + return None, "a value is written to a field with no name" + if v is not None and not isinstance(v, (str, int, float, bool)): + return None, (f"the value for {key!r} must be text or a number — " + f"an automation writes cells, not objects") + clean_vals[key] = _s(v, 500) if isinstance(v, str) else v + out = {"values": clean_vals} + # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is + # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic + # "Create record". Follows the `review` arm below, which has carried `config.label` since + # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config` + # bag so no shared client type has to change for it. + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + if kind == "create_record": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the create record action names no database" + if not table.startswith(UT_PREFIX): + return None, (f"actions write to blank databases (ut_*) this wave — " + f"{table!r} is not one") + out["table"] = table + # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS + # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a + # Create record duplicates a row per run — silently, and worse every day. + # + # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a + # guess: a stored automation must not change meaning because this key arrived. The + # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`. + # + # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the + # action never sets means every incoming row carries a BLANK key — `upsert_rows` + # counts those as `skipped` and writes nothing at all. That is a control that reads + # as "no duplicates" and delivers "no rows", which is the worse failure by a distance. + # Same shape as `scrape_db`'s "the key field must be one of the mapped fields" + # (`clean_config`, above) — one precedent, not a second mechanism. + unique = _s(cfg.get("uniqueOn"), 80).strip() + if unique and unique not in clean_vals: + return None, (f"this action does not write {unique!r}, so it cannot keep records " + f"unique on it — it writes: " + ", ".join(sorted(clean_vals))) + out["uniqueOn"] = unique + return out, None + if kind == "enrich_instagram": + # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid + # rung, its fallback, the per-post engagement buy and the dry run — because they are the + # things that decide what a run COSTS and R4 moves the capability, not the controls. + # + # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than + # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a + # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a + # sentence at run (`profile_field_key`), which is the honest half — refusing the save + # would make the action unaddable on a database that has not been flagged yet. + # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent + # clamps on the same key is how the action and the legacy kind come to disagree about + # what "10 posts" means. + # + # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference + # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH, + # so a key being there is a person having typed it. This function gets the whole action + # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a + # `maxPosts` here is usually just the client echoing back what was already stored — and + # every action written before this wave stored the old default of 24. Refusing would + # therefore 400 those automations forever on a number nobody chose, which is D-65. + # The ceiling is still taught, in the place where a person can act on it: the panel's + # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more). + max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False) + if perr: + return None, perr + # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a + # working default. That is D-65's lesson applied ahead of time rather than after: an enrich + # action stored before this wave carries none of these, and it must keep saving and keep + # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT; + # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching. + # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door + # sees a person's typed number and can refuse politely, while the selection sees whatever + # is STORED — including configs written before the cap existed. Clamping only here would + # let an old 5000 through at run time. + try: + e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) + except (TypeError, ValueError): + return None, "how many records to enrich has to be a number" + if e_limit < 1: + return None, "an enrich step has to be allowed at least one record" + try: + e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) + except (TypeError, ValueError): + return None, "the number of days has to be a number" + groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts) + if gerr: + return None, gerr + return {"profileField": _s(cfg.get("profileField"), 80).strip(), + "fromView": _s(cfg.get("fromView"), 80).strip(), + "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT, + # Anything that is not "asc" is newest-first, so a typo cannot invert the order a + # person is spending against — it lands on the documented default instead. + "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc" + else "desc", + "limit": min(e_limit, MAX_ENRICH_PER_RUN), + "skipRecent": bool(cfg.get("skipRecent")), + "skipRecentDays": max(1, e_days), + # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in + # `clean_config` (R5 / C2): a stored action carrying them still saves and simply + # loses them, because refusing a retired key 400s every definition written before + # this wave (D-65). + # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for. + "postMetrics": bool(cfg.get("postMetrics")), + # The separate Comments dataset is more granular than a post read and stays off + # until the user intentionally enables it. + "commentMetrics": bool(cfg.get("commentMetrics")), + "dryRun": bool(cfg.get("dryRun")), + # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so + # every action stored before today keeps saving and keeps running unchanged. + "postGroups": groups, + "maxPosts": max_posts}, None + if kind == "find_records": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the find records action names no database" + cond, cerr = clean_cond(cfg.get("cond"), where="the find") + if cerr: + return None, cerr + try: + limit = int(cfg.get("limit") or 25) + except (TypeError, ValueError): + return None, "the find records limit must be a whole number" + if limit < 1 or limit > FIND_LIMIT_MAX: + return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" + return {"table": table, "cond": cond, "limit": limit}, None + return {}, None + + +def _branch_letter(i): + """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a + label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is + worse than an ugly name.""" + return chr(65 + i % 26) * (1 + i // 26) + + +def group_branches(act): + """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape. + + ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying + for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored + group might carry either shape, and the one that forgot would silently skip a live + automation's actions. A definition cleaned since wave 24 always carries `branches`; one read + straight out of the store (the runner reads definitions, not payloads) may not. + """ + cfg = (act or {}).get("config") or {} + brs = cfg.get("branches") + if isinstance(brs, list) and brs: + return [b for b in brs if isinstance(b, dict)] + return [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions") or []}] + + +def walk_actions(actions): + """Every action in the tree, depth-first, groups included. One walker, so "how many actions + does this flow have" and "which review stages exist" cannot answer differently. + + ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action + COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version + that still read `config.actions` would report a fork's contents as zero actions, hide every + review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing. + """ + for act in actions or []: + yield act + if act.get("kind") == "group": + for br in group_branches(act): + for kid in walk_actions(br.get("actions")): + yield kid + + +def interpolate(text, row): + """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an + action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would + put template syntax in front of a customer, and a blank is the visible failure.""" + if not isinstance(text, str) or "{{" not in text: + return text + def _sub(m): + v = (row or {}).get(m.group(1).strip()) + return "" if v is None else str(v) + return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. +# +# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits +# them once at the end (the 256-commits/hr budget every writer in this module respects). A +# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget +# on one busy flow. +# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the +# runner stamps its stage and stops walking THAT record — later actions belong to the branch a +# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. +# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the +# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write +# cannot fire an event trigger, its own or a sibling's. + +def _act_row_patch(patches, table, rid, values): + patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) + + +def profile_field_key(table, named=""): + """C3/C4: WHICH COLUMN on this database carries the Instagram handle. '' when none does. + + Resolution order, and the missing branch is the important one: + 1. the column the action NAMES — the person picked it, so it is not a guess; + 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`; + 3. otherwise **NOTHING**, and the run says so. + + ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url` + column" and is exactly right for `field_instagram`, whose whole configuration was a URL column + — but here it would silently bind the enrich step to whatever URL column happens to be first, + on a database where nobody has said which column is a profile. The failure would be a run that + reports success having enriched from the wrong column, which is worse than one that refuses: + a wrong number is harder to notice than a missing one, and it would be written into the + permanent history as if it had been measured. + """ + fields = (table or {}).get("fields") or [] + want = str(named or "").strip() + if want: + return want if any(str(f.get("key")) == want for f in fields) else "" + for f in fields: + p = f.get("profile") + if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG: + return str(f.get("key") or "") + return "" + + +def _flow_table(defn): + """The database a flow's actions and ending operate on: the automation's own target, or the + table its event trigger watches when the flow has no target of its own.""" + cfg = (defn or {}).get("config") or {} + if defn.get("kind") == "discover_instagram": + return cfg.get("targetTable") or DISCOVER_TABLE + return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") + + +#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN. +#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the +#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]` +#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own. +MAX_ENRICH_PER_RUN = 100 +DEFAULT_ENRICH_LIMIT = 25 +DEFAULT_ENRICH_COOLDOWN_DAYS = 30 +#: Which column the selection orders by when nobody has said. `first_found` newest-first is the +#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what +#: discovery just found rather than re-walking the oldest leads in the book. +DEFAULT_ENRICH_SORT = "first_found" + +#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL. +#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence +#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is +#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored. +#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with +#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by +#: accident, and `_commit_run`'s numeric filter is the second net under it. +RUN_NOTES_KEY = "_notes" +#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES +#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval. +#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence +#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so +#: the verdict should expire. But nothing about US changes on day 31 — the only new information +#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per +#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a +#: hundred stale handles pays a hundred times a month to be told the same thing. +#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors: +#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no +#: wiring at all, because the corrected handle simply is not the one we recorded; and +#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle +#: (a person saying "try it again") also re-arms. +#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered +#: them: a tombstone the owner cannot see is just a disappearance. +#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An +#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the +#: opposite mask ([[gate-answers-the-wrong-question]]). +PENDING_PROFILE_MAX_HOURS = 24 + + +def _order_key(value): + """Order one CELL. Numbers numerically, everything else as text. + + ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text — + which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting + paid for. Numbers are tried first and text is the fallback, never the other way round. + ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where + lexicographic order IS chronological order. If that format ever changes, this comment is the + thing that was relied on. + """ + s = str(value if value is not None else "").strip() + try: + return (0, float(s.replace(",", "")), "") + except ValueError: + return (1, 0.0, s.lower()) + + +def _days_since(day, today=None): + """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read. + + ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a + large number — the cooldown below reads an unreadable stamp as "never enriched", which spends + money rather than skipping. That is the right way round: a skipped record is invisible, and a + stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data. + """ + import datetime as _dt + s = str(day or "").strip()[:10] + if not s: + return None + try: + d = _dt.date.fromisoformat(s) + except ValueError: + return None + return ((today or _dt.date.today()) - d).days + + +def _hours_since(stamp): + """Whole hours since a full ISO stamp, or None when it cannot be read. + + ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date` + column and truncates to 10 characters. A pending snapshot's age is measured in hours and its + stamp carries a time, so reusing that function would read every entry as "queued at + midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid, + outstanding snapshot out of the queue. + """ + import datetime as _dt + s = str(stamp or "").strip() + if not s: + return None + try: + t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + except ValueError: + return None + now = _dt.datetime.now(_dt.timezone.utc) + if t.tzinfo is None: + t = t.replace(tzinfo=_dt.timezone.utc) + return max(0, int((now - t).total_seconds() // 3600)) + + +def _gone_key(row, handle): + """The identity a "this account does not exist" verdict is remembered under. + + `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on + Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what + every row on a preset profile table is and what `preset_cells` stamps. + """ + plat = str((row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower() + return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}" + + +def clear_gone(rt, table_key, handle, row=None): + """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle. + + Returns the number of automations whose memory was changed — 0 is the ordinary answer and is + not an error, because most cell edits are not on a dead handle. + + ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER. + The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to, + because it is a fact about what a run learned and paid for. But the thing that re-arms it is a + ROW event, and the row writer must not need to know the shape of an automation's state to + trigger it. So the seam is one call with the three things the writer already has, and every + walk of `all_definitions` stays on this side of the fence. + ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which + handle is this", so a verdict can never be recorded under a key this cannot find + ([[one-evaluator-per-question]]). + + ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first + door — correcting a typo — works with no wiring at all, because the key IS the handle and a + different handle is simply not the one we recorded. This second door only opens when the row + write path calls it, which lives in `routes_tables.patch_row` (another session's fence). + Until that line lands, re-typing the SAME dead handle stays skipped + ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later). + """ + key = _gone_key(row or {}, handle) + if not str(handle or "").strip(): + return 0 + changed = 0 + for auto_id, defn in (all_definitions(rt) or {}).items(): + if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key): + continue + known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {}) + if key not in known: + continue + known.pop(key, None) + # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what + # every other state writer here does and what keeps a definition from growing a graveyard + # of empty maps. + set_state(rt, str(auto_id), {"enrichNotFound": known or None}) + changed += 1 + return changed + + +def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch, + step=_no_step, log=print): + """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved. + + ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a + LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an + hour"); the only profile caller passed a single-element list from inside the per-record walk. + MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25 + vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand. + + ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from + `prefetch`, and the per-record rung then behaves exactly as it does today — including its + corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a + batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure + returns 0 instead of raising. + + ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now + serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they + share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the + handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch` + behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time. + """ + want, dests, handles = [], {}, [] + for rid in (chosen or []): + row = rows.get(str(rid)) or {} + h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower() + if not h or h in prefetch: + continue + want.append((h, str(rid))) + if h not in dests: + handles.append(h) + dests.setdefault(h, []).append(str(rid)) + if not handles: + return 0 + step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source") + deferrals = [] + try: + nodes, note = bd_profiles_batch(handles, deferred=deferrals) + except Exception as exc: # noqa: BLE001 — never fatal, see docstring + log(f"[aios-auto] profile batch failed, falling back to per-record reads: " + f"{type(exc).__name__}: {exc}") + return 0 + prefetch.update(nodes) + if note: + log(f"[aios-auto] profile batch note: {_s(note, 300)}") + for d in deferrals: + sid = str(d.get("snapshotId") or "") + if not sid: + continue + for u in (d.get("urls") or []): + h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower() + if not h or h in nodes: + continue + prefetch[h] = {DEFERRED_MARK: sid} + for rid in dests.get(h, []): + enrich["pendingProfiles"].append( + {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""), + "urls": [str(u)], "kind": "profile", "influencer": h, + "table": table, "rowId": rid, "requestedAt": _iso()}) + return len(nodes) + + +def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None): + """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order. + + Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did + and is surfaced in the run summary; `""` means there is nothing worth saying. + + ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's + ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is + enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every + skipped record until 30 have actually been enriched or the list runs out. The naive reading — + take the top 30, then filter — would have billed for 10 and reported success, and the number in + the box would have meant something different every run depending on how much of the top of the + list happened to be fresh. A quota is predictable; a filtered window is not. + + The order of operations, each step narrowing the one above: + 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view` + and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an + empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich + the entire database", at vendor prices. + 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions + (a blank is unknown, not smallest). + 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on. + 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`. + + ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is + nothing to enrich and it must not consume a slot somebody paid for. + """ + t = ut_get(rt, str(table_key or "")) + if t is None: + return [], f"{table_key!r} is not a database in this workspace" + rows = dict(t.get("rows") or {}) + notes = [] + + view_id = str(cfg.get("fromView") or "").strip() + if view_id: + tree, fields, problem = view_filter(rt, table_key, view_id) + if problem: + # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet + # fallback costs real money. + return [], f"the enrich step names a view it cannot read — {problem}" + # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS + # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE. + # + # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}], + # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/ + # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition — + # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on + # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`, + # got `""`, and answered **False for every row**: MEASURED on the owner's own automation, + # a view matching exactly one record selected NOTHING, reported NO error, and the run + # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual + # record... how come it says 51 records walked?"* + # + # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch + # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has + # always been correct. `_seed_event_state`'s own docstring states the law this line broke: + # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it + # is supposed to arm."* There were three. Now there are two callers of one function. + # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date + # as a date and a number as a number, which is the half `lane_match` could not have had. + import harness.filter_eval as filter_eval + rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)} + + sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT + newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc" + # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN". + # + # Owner: *"treat a missing first found as now, and manual entry should go first, instead of + # treated as last. I want to see my inayma manual entry works with automation enrichment."* + # And they are right about the semantics, not just the preference: on these tables the ONLY + # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it + # on every row it writes. So a blank is not missing data — it is a row that was first found + # today, by the person sitting in front of it, and the one they most want enriched. + # + # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position + # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one + # it never reached. + # + # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about + # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and + # treating it as the maximum would spend the budget on the rows we know least about. So a + # non-date sort keeps the old rule: unknown sorts last, in both directions. + # + # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real + # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at + # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One + # rule, both directions, no branch that can disagree with itself. + ftype = str(((next((f for f in (t.get("fields") or []) + if str(f.get("key")) == sort_field), None)) or {}).get("type") or "") + if ftype == "date": + ordered = sorted(rows.items(), + key=lambda x: (0, str(x[1].get(sort_field) or "").strip()) + if str(x[1].get(sort_field) or "").strip() else (1, ""), + reverse=newest_first) + else: + have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()] + blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()] + have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first) + blank.sort(key=lambda x: _rid_num(x[0])) + ordered = have + blank + + try: + quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT) + except (TypeError, ValueError): + quota = DEFAULT_ENRICH_LIMIT + quota = max(1, min(quota, MAX_ENRICH_PER_RUN)) + if cfg.get("limit") and quota != int(cfg.get("limit") or 0): + notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run") + + cooling = bool(cfg.get("skipRecent")) + try: + days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS) + except (TypeError, ValueError): + days = DEFAULT_ENRICH_COOLDOWN_DAYS + days = max(1, days) + + picked, cooled, blank_handle = [], 0, 0 + # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST. + # + # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT. + # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is + # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it + # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at + # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix + # removes that loop; only a memory of the verdict does. + # + # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the + # note goes into the run every single time, not only on the run that discovered it, because a + # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf". + skipped_gone = [] + for rid, r in ordered: + if len(picked) >= quota: + break + raw_handle = str(r.get(profile_key) or "").strip() + if not raw_handle: + blank_handle += 1 + continue + # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is + # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the + # owner reads to know when we last paid to be told this), it is simply not a clock. + if isinstance((gone or {}).get(_gone_key(r, raw_handle)), dict): + skipped_gone.append(raw_handle) + continue + if cooling: + since = _days_since(r.get("enriched_at"), today=today) + if since is not None and since < days: + cooled += 1 + continue + picked.append(str(rid)) + + # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the + # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and + # `run_plain` follow. Silence here would make a shrinking selection invisible. + if cooled: + notes.append(f"{cooled} skipped as enriched in the last {days} days") + if skipped_gone: + shown = ", ".join(f"@{h}" for h in skipped_gone[:5]) + # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can + # do, because nothing else will ever retry them — under R9 this note is the only path + # back from a tombstone, so a vaguer version would strand the row permanently. + notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account " + f"({shown}{', …' if len(skipped_gone) > 5 else ''}) — delete the row or " + f"correct the handle; they are not retried automatically") + if blank_handle: + notes.append(f"{blank_handle} skipped with no handle") + if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered): + notes.append(f"{len(picked)} of the {quota} asked for — the list ran out") + return picked, "; ".join(notes) + + +def _has_action(actions, kind): + """Does this flow contain `kind` ANYWHERE, forks included? + + ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under + `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an + enrich step somebody put inside an If — and the caller would then skip a schema top-up the + run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24 + recorded when four hand-rolled walks all forgot to descend. + """ + for a in (actions or []): + if not isinstance(a, dict): + continue + if a.get("kind") == kind: + return True + for br in ((a.get("config") or {}).get("branches") or []): + if _has_action((br or {}).get("actions") or [], kind): + return True + return False + + +def _actions_of_kind(actions, kind): + """Every action of `kind`, including actions nested inside If branches.""" + out = [] + for action in actions or []: + if not isinstance(action, dict): + continue + if action.get("kind") == kind: + out.append(action) + for branch in ((action.get("config") or {}).get("branches") or []): + out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind)) + return out + + +def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print, + step=_no_step): + """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. + + ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery + automation's review stage comes from `stages_for`, not from actions, + so an early return on an empty action list would have silently disabled the owner's loop + feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in + review; the tests all happened to pass a non-empty action list, which is why it was green. + """ + defn, _retired = _without_retired_board(defn) + flow = (defn or {}).get("flow") or {} + actions = flow.get("actions") or [] + # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE. + # + # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the + # "Save results" node) and THEN walked the flow over the rows it had just written — where + # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on + # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner + # ruling, same day) would have made the duplication permanent and unavoidable with it. + # + # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what + # it finds in a database" is exactly what the engine does. What it must not do is do it + # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the + # server still refuses to let it be removed. + if ig_action_pinned(defn, 0) and actions: + actions = actions[1:] + counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0} + # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected + # across the WHOLE walk and written once at the end, for the reason the module header states + # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on + # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an + # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts. + enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor) + # and handles a vendor has said do not exist. Both are collected across the whole + # walk and written once, for the same reason every other list here is. + "pendingProfiles": [], "gone": {}, + "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False} + # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them + # per record and re-reading the definition per row would be a store read per record. + known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) + # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD. + # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk + # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked + # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps + # in one flow are two independent budgets. + enrich_plan = {} + # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK. + # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a + # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was + # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever + # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are + # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth + # buying twice. + enrich_prefetch = {} + if not table_key: + return counts + table = str(table_key) + # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN. + # + # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled + # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a + # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door. + # So every preset the target table did not already happen to have was silently dropped, and + # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET + # FILLED"* — this is the half that was swallowing them. + # + # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`, + # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose + # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the + # feature worked on discovery tables purely because discovery creates them for its own reasons. + # + # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present: + # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady + # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no + # enrich step never touches the schema of the database it walks. + # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset + # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an + # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a + # step at it, and (b) add a SECOND profile column to a table that already flagged a different + # one, which `user_tables` forbids at both write doors. A table with no profile column is an + # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its + # own honest refusal, not something to paper over by inventing the binding. + _tbl_now = ut_get(rt, table) or {} + _enrich_actions = _actions_of_kind(actions, "enrich_instagram") + _bound = next((f for f in (_tbl_now.get("fields") or []) + if isinstance(f.get("profile"), dict)), None) + if _bound is None: + _named = next((str((a.get("config") or {}).get("profileField") or "").strip() + for a in _enrich_actions + if str((a.get("config") or {}).get("profileField") or "").strip()), "") + _bound = next((f for f in (_tbl_now.get("fields") or []) + if str(f.get("key") or "") == _named), None) + if _bound and _enrich_actions: + # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists — + # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a + # table whose profile column is `ig_handle` would otherwise gain a rival `handle`. + topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")} + if f.get("key") != _bound.get("key") else dict(f)) + for f in PRESET_PROFILE_FIELDS] + try: + if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions): + ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=table, profile_field=str(_bound.get("key") or "")) + else: + # Dry run keeps its original no-create contract; only the already-established + # Profile table is topped up, and no canonical related database is spawned. + ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table, + flow_tag=str(defn.get("id") or ""), lock_fields=True) + except Exception as exc: # noqa: BLE001 + # A schema top-up that cannot run must not stop the enrichment: the cells that DO + # have columns still land, which is strictly better than the pull being thrown away. + log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}") + rows = dict((ut_get(rt, table) or {}).get("rows") or {}) + if not rows: + return counts + # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The + # walk used to keep a `reached` map and stamp each record with the furthest action it made — + # a board column's worth of bookkeeping written into the tenant's own table on every run. + # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a + # cell the customer has to look at and a store commit we pay for on every scheduled run. + # The RUN LOG is the operational record of what happened now, and it is the only one. + patches, creates = {}, {} + + def _walk(acts, row, rid, depth=0): + """Walk the actions in order for ONE record. + + ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean + "this record was SUSPENDED by a review gate" — the one branch that could stop a walk + early. With review retired nothing suspends anything, so every record walks its whole + flow; the signature stays so a future gate-style action has an obvious place to say so. + """ + for act in acts: + if not act.get("enabled", True): + continue + if not lane_match(act.get("when"), row): + continue + kind = act.get("kind") + cfg = act.get("config") or {} + counts["actionsRun"] += 1 + if kind == "group": + # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs. + # Declaration order is priority order, the same rule `route_record` applies to + # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch + # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions` + # has already guaranteed a null condition can only be LAST. + for br in group_branches(act): + if lane_match(br.get("cond"), row): + if _walk(br.get("actions") or [], row, rid, depth + 1): + return True + break + elif kind == "update_record": + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + row.update(vals) # later actions see the write, as they must + _act_row_patch(patches, table, rid, vals) + counts["updated"] += 1 + elif kind == "create_record": + target = str(cfg.get("table") or "") + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions + # may legitimately write to ONE database on different keys — collapsing them onto + # the table would silently apply one action's uniqueness rule to the other's rows. + creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) + counts["created"] += 1 + elif kind == "enrich_instagram": + # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions + # `run_field_instagram` calls, not a second implementation of either. + pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField")) + if not pkey: + # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a + # property of the flow, so a 100-record run would otherwise put the same + # sentence in the log a hundred times and bury everything else. + if "unbound" not in enrich["notes"]: + enrich["notes"].append("unbound") + log("[aios-auto] enrich_instagram: no profile column on " + f"{table} — name one on the action, or mark a text column as an " + "Instagram profile") + continue + # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET? + # Resolved once (see `enrich_plan`) and then a membership test. A record outside + # the selection is NOT an error and NOT a skip worth logging per row — it is simply + # not this run's work, and the selection's own note already accounts for it. + aid = str(act.get("id") or "") + if aid not in enrich_plan: + chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone) + enrich_plan[aid] = set(chosen) + if sel_note: + enrich["notes"].append(sel_note) + # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else: + # this is the only place the full chosen set and the row bodies are both in + # hand. Anything it resolves the per-record rung below reads from memory. + _prime_enrich_batch(chosen, rows, pkey, table, enrich, + enrich_prefetch, step, log) + if str(rid) not in enrich_plan[aid]: + continue + handle_raw = str(row.get(pkey, "") or "").strip() + if not handle_raw: + continue # nothing to enrich on this record, not an error + # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED. + # A handle already in `enrich_prefetch` was bought by the batch above and is read + # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the + # batching win straight back (25 records = ~62 s of pure waiting). + if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \ + not in enrich_prefetch: + time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does + enrich["profiles"] += 1 + # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with + # the row it belongs to: the collector writes preset cells back onto THAT record, + # and a run-wide list would have no way to say which handle each snapshot was for. + pend_prof = [] + res = pull_profile(handle_raw, + max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, + log=log, + post_metrics=bool(cfg.get("postMetrics")), + comment_metrics=bool(cfg.get("commentMetrics")), + pending_metrics=(enrich["pending"] + if not cfg.get("dryRun") else None), + pending_profile=(pend_prof + if not cfg.get("dryRun") else None), + prefetch=enrich_prefetch) + # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels / + # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is + # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped. + # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this + # costs nothing and changes nothing until somebody configures it. + # ⚠ IT FILTERS THE WINDOW WE ALREADY BOUGHT, it does not widen it. The profile + # read is capped at 12 posts (MEASURED: asked 40, got 12), so "12 reels AND 12 + # images" needs `bd_profile_posts` — which is built and proven (30 rows in 90 s) + # but deliberately NOT wired here: the views top-up runs INSIDE `pull_profile` + # against the posts it captured, so swapping a wider list in afterwards would + # hand back posts with no view counts. Widening belongs ahead of that step, in + # the paid path, and that is its own change. See DEBT. + if cfg.get("postGroups"): + res = {**res, "posts": select_post_groups(res.get("posts") or [], + cfg.get("postGroups"))} + for _p in pend_prof: + _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso() + enrich["pendingProfiles"].extend(pend_prof) + pulled = _iso() + if res["state"] in ("ok", "partial"): + enrich["ok"] += 1 + snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) + enrich["snaps"].append(snap_row) + enrich["posts"].extend(idents) + enrich["psnaps"].extend(metric_rows) + enrich["comments"].extend(comment_rows) + # R3: LATEST onto the record itself. `row.update` too, so a LATER action in + # this same flow sees the enriched values — the rule `update_record` already + # follows, and without it "enrich, then route by follower count" would judge + # the record on the values it had before the pull. + cells = preset_cells(res, pulled) + if cfg.get("dryRun"): + enrich["dry"] = True + else: + row.update(cells) + _act_row_patch(patches, table, rid, cells) + else: + enrich["blocked"] += 1 + # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH. + # It used to be `_s(note, 90)` with no handle attached: three runs blocked + # the same record and the store could not say which record, let alone why. + # 90 characters also truncated the one measured sentence mid-clause. This is + # the most valuable thing the run produces and it now reaches the run entry. + enrich["notes"].append( + f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}") + # A vendor STATING that the account does not exist is remembered, so the next + # run stops paying to be told the same thing. + if res.get("gone"): + enrich["gone"][_gone_key(row, handle_raw)] = { + "at": _iso(), "handle": handle_raw, + "note": _s(res.get("note"), 300)} + elif kind == "find_records": + found = find_records(rt, cfg.get("table"), cfg.get("cond"), + int(cfg.get("limit") or 25)) + counts["found"] += len(found) + # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had + # already stopped being reachable one wave earlier — which is the part worth reading. + # `_without_retired_board` strips every `review` action out of a definition on the + # READ path, so `all_definitions` and this function have not seen one since the board + # was retired. What was left behind was a branch referencing `skey`, `stamp` and + # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not + # dead code that merely wasted space: it was a `NameError` held back by a migration + # rather than by a guard, and any change that let one stored `review` action through + # would have crashed the whole action walk for every record in that flow. + # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps + # review "as an AI decision without lanes", and the cheap-first provider ladder behind + # it is real, working, measured work. They are PARKED, deliberately and in writing, + # not orphaned; see their own notes. + return False + + if actions: + for rid in list(row_ids or [])[:FLOOD_LIMIT]: + row = dict(rows.get(str(rid)) or {}) + if not row: + continue + _walk(actions, row, str(rid)) + counts.update(_enrich_flush(rt, defn, username, enrich, log)) + # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was + # incremented once per create the flow decided to make; what landed is what the store says, + # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches + # every row it made last time — which is the whole point of the feature). + counts.update(_commit_action_writes(rt, table, patches, creates, username, log)) + return counts + + +def migrate_field_instagram(rt, defn): + """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the + `enrich_instagram` action. Returns `(definition, changed)`. + + R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live + `field_instagram` automation to a plain flow carrying it; delete the kind and its label." + + ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration + turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** — + so a live automation whose `urlField` is blank has been working off that fallback for months. + The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a + blank `urlField` verbatim would produce an automation that USED to work and now refuses. The + fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit + binding — which is also the honest outcome: the column stops being implicit. + + ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The + old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s + column; the action writes the C1 PRESET CELLS instead. The status column is left in place and + simply stops being written — deleting somebody's column as part of a migration would be data + loss, and a stale cell beside a fresh `enriched_at` is readable for what it is. + + ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can + run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN + against the real stored definition" is meant to be satisfied. + """ + if (defn or {}).get("kind") != "field_instagram": + return defn, False + cfg = dict(defn.get("config") or {}) + target = str(cfg.get("targetTable") or "") + bound = str(cfg.get("urlField") or "").strip() + if not bound and target: + bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "") + try: + max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL) + except (TypeError, ValueError): + max_posts = 24 + act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None, + "config": {"profileField": bound, + "postMetrics": bool(cfg.get("postMetrics")), + "commentMetrics": bool(cfg.get("commentMetrics")), + "dryRun": bool(cfg.get("dryRun")), + "maxPosts": max(1, min(max_posts, 200))}} + flow = dict(defn.get("flow") or {}) + out = dict(defn) + out["kind"] = DEFAULT_KIND + # The enrich step goes FIRST — it is what the automation was, and every action the owner + # added afterwards was written expecting the capture to have happened. + out["flow"] = {"actions": [act] + list(flow.get("actions") or [])} + out["config"] = {"targetTable": target, + "targetLabel": str(cfg.get("targetLabel") or "")} + return out, True + + +def _enrich_flush(rt, defn, username, acc, log): + """C4: the enrich action's canonical IG tables, written ONCE for the whole run. + + Returns the counts the run reports. Empty when no enrich action ran, so a flow without one + pays nothing and its run history is unchanged (`run_now` merges only non-zero keys). + + ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three + tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then + the write-through to the platform master. Forking any of those would give the enrich action a + history that the metric fields could not see — and it is the LAST step, the write-through to + the platform master, that they read (W29-T34: `compute_metric_cells` → `ig_master.series_for`, + never this tenant's `ut_ig_snapshots`), so dropping that one line is the version of this fork + that would look harmless. + """ + if "unbound" in (acc.get("notes") or []): + # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step + # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so + # this function used to return `{}`, `run_now` merged only non-zero keys, and the run + # committed **`ok` — "N records walked"** having written nothing at all. The sentence + # naming the fix went to `log()`, i.e. the server console, which no customer reads. That + # is "green over nothing" on the one path where somebody is waiting for data. + # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a + # special case — and so the run entry itself records that this happened. + return {"enrichUnbound": 1} + # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail. + # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate + # was skipped as a known-dead handle — so the sentence explaining why the automation appears + # to do nothing would have been dropped on precisely the runs that most need it, and the + # owner would be back at "0 records walked, wtf". The selection note is produced before any + # profile is read and must outlive that early return. + if not acc.get("profiles"): + notes = list(acc.get("notes") or []) + return {RUN_NOTES_KEY: notes} if notes else {} + out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]} + if acc.get("notes"): + out[RUN_NOTES_KEY] = list(acc["notes"]) + # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so + # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel. + if not acc.get("dry"): + # The vendor's not-found verdicts, merged into engine state (never a tenant column and + # never a status string — W25/R4 retired those). Merged rather than replaced: a run that + # walked one record must not forget what earlier runs learned about the others. + if acc.get("gone"): + merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}) + merged.update(acc["gone"]) + set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged}) + pending_profiles = queue_pending_profile_snapshots( + rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or []) + if pending_profiles: + out["enrichProfileBatchesPending"] = pending_profiles + queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), + acc.get("pending") or []) + if queued: + out["enrichMetricBatchesPending"] = queued + if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]): + # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES. + if acc.get("dry"): + out["enrichDryRun"] = acc["profiles"] + return out + tag = str(defn.get("id") or "") + profile_table = str(_flow_table(defn) or "") + graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table) + snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE], + graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE]) + missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key) + snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}), + acc["snaps"], "snapshot_key", cap=row_cap(snap_key)) + old_posts, collapsed_posts = dedupe_canonical_rows( + dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), + acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key)) + old_comments, collapsed_comments = dedupe_canonical_rows( + dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") + comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key", + cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps), + (comment_key, comments)): + if cur.get(k) is not None: + cur[k]["rows"] = rws + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables + # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which + # is the failure a chart cannot show you. + capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"] + if capped: + out["enrichCapped"] = capped + log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap") + if missing: + out["enrichMissingTables"] = len(missing) + log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}") + out["enrichPoints"] = len(acc["snaps"]) + out["enrichPosts"] = c_post["inserted"] + out["enrichComments"] = c_comments["inserted"] + # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same + # contract `run_field_instagram` follows, because a pooled history with silent holes is worse + # than none. + try: + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"], + acc["posts"], acc["psnaps"]) + if m_status == "error": + out["enrichMasterFailed"] = 1 + log(f"[aios-auto] enrich_instagram: the platform master copy FAILED — {m_note}; " + f"the tenant copy is complete and the next run re-appends") + elif m_status == "ok": + out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"]) + except Exception as e: # noqa: BLE001 + out["enrichMasterFailed"] = 1 + log(f"[aios-auto] enrich_instagram: master write-through raised " + f"{type(e).__name__}: {e}") + return out + + +_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS)) +_PENDING_METRIC_KINDS = frozenset(("posts", "comments")) + + +def _pending_metric_tasks(defn): + """Read validated, deduplicated paid metric snapshots from automation continuation state. + + A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally + stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or + the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor + endpoint, and the collector never starts a second scrape request. + """ + raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or [] + out, seen = [], set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + sid = str(item.get("snapshotId") or "").strip() + dataset = str(item.get("datasetId") or "").strip() + kind = str(item.get("kind") or "").strip() + handle = str(item.get("influencer") or "").strip().lstrip("@").lower() + if (not sid.startswith("sd_") or dataset not in _PENDING_METRIC_DATASETS + or kind not in _PENDING_METRIC_KINDS or not handle): + continue + key = (sid, dataset, kind, handle) + if key in seen: + continue + seen.add(key) + out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind, + "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""), + "lastChecked": str(item.get("lastChecked") or ""), + "lastNote": _s(item.get("lastNote"), 160)}) + return out + + +def queue_pending_metric_snapshots(rt, auto_id, pending): + """Durably retain new engagement snapshots without creating another paid provider request.""" + aid = str(auto_id or "").strip() + if not aid: + return 0 + incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}}) + if not incoming: + return 0 + added = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + definition = cur.get(aid) + if not isinstance(definition, dict): + return cur + state = definition.setdefault("state", {}) + existing = _pending_metric_tasks(definition) + known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"]) + for x in existing} + for task in incoming: + key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"]) + if key not in known: + existing.append(task) + known.add(key) + added[0] += 1 + state["pendingMetricSnapshots"] = existing + return cur + + rt.update(STORE_KEY, _up, flush="sync") + return added[0] + + +def _match_profile_row(rows, handle): + """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there. + + ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With + `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and + taking the first row would write ONE profile's followers onto all of them — a wrong number that + looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the + same field `_bd_profile` reads first. + + ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points + at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that + shape the task's own handle is still the best evidence. It applies only when the snapshot holds + ONE row, so it can never mis-assign inside a batch. + """ + want = str(handle or "").strip().lstrip("@").lower() + usable = [n for n in (rows or []) if isinstance(n, dict)] + for node in usable: + got = str(_first(node, "account", "username", default="") or "").strip() + if got.lstrip("@").lower() == want and want: + return node + if len(usable) == 1: + return usable[0] + return None + + +def _pending_profile_tasks(defn): + """Validated, deduplicated deferred PROFILE snapshots from automation state. + + ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather + than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and + `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to + zero by its own validator, and even if it survived, the collector would hand it to + `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile + row. Reusing the name would have shipped a green no-op of exactly the class this change + exists to remove. + + ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written + back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be + resolved from the handle alone: two databases may both hold `@x`. + """ + raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or [] + out, seen = [], set() + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + sid = str(item.get("snapshotId") or "").strip() + dataset = str(item.get("datasetId") or "").strip() + handle = str(item.get("influencer") or "").strip().lstrip("@").lower() + table = str(item.get("table") or "").strip() + row_id = str(item.get("rowId") or "").strip() + # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a + # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor + # endpoint at all. + if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle + or not table or not row_id): + continue + if sid in seen: + continue + seen.add(sid) + out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile", + "influencer": handle, "table": table, "rowId": row_id, + "requestedAt": str(item.get("requestedAt") or ""), + "lastChecked": str(item.get("lastChecked") or ""), + "lastNote": _s(item.get("lastNote"), 200)}) + return out + + +def queue_pending_profile_snapshots(rt, auto_id, pending): + """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many + were newly added.""" + aid = str(auto_id or "").strip() + if not aid: + return 0 + incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}}) + if not incoming: + return 0 + added = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + definition = cur.get(aid) + if not isinstance(definition, dict): + return cur + state = definition.setdefault("state", {}) + existing = _pending_profile_tasks(definition) + known = {x["snapshotId"] for x in existing} + for task in incoming: + if task["snapshotId"] not in known: + existing.append(task) + known.add(task["snapshotId"]) + added[0] += 1 + state["pendingProfileSnapshots"] = existing + return cur + + rt.update(STORE_KEY, _up, flush="sync") + return added[0] + + +def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step): + """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape. + + ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred` + list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the + vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown + away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a + 180 s budget, two abandoned `sd_…` snapshots in two runs. + + ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same + forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero + records ends the task with the vendor's reason attached — and, when the vendor blames the + target rather than itself, records the not-found verdict so the handle stops being re-bought. + """ + tasks = _pending_profile_tasks(defn) + if not tasks: + return ("ok", "No pending profile reads.", {}, [], {"source": "idle"}) + step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}") + remaining, patches, affected, notes = [], {}, [], [] + acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [], + "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0, + "notes": [], "dry": False} + ready = waiting = closed = 0 + # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME + # `snapshotId`, and asking the vendor about it once per task would spend the round trips the + # batching just saved. Memoised for this collection pass only — a snapshot's state must be + # re-read on the NEXT run, which is a fresh call and a fresh dict. + _progress_memo, _rows_memo = {}, {} + + def _progress_of(sid): + if sid not in _progress_memo: + _progress_memo[sid] = bd_snapshot_progress(sid) + return _progress_memo[sid] + + def _store_profile(task, profile, via): + """Write ONE collected profile — snapshot row + preset cells on the record that asked. + + ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the + two paths cannot drift into writing different things — the same rule `top_up_views` + follows for the inline/deferred split. + """ + res = {"state": "ok", "profile": profile, "posts": [], "comments": [], + "via": via, "note": ""} + pulled = _iso() + snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled) + acc["snaps"].append(snap_row) + acc["posts"].extend(idents) + acc["psnaps"].extend(metric_rows) + acc["comments"].extend(comment_rows) + _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled)) + affected.append(task["rowId"]) + + def _backup_profile(handle): + """The second rung, asked ONLY once the first has finished and delivered nothing. + + ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS + CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but + a profile that DEFERRED never came back through `pull_profile` — it came back here, and + this collector had no second rung at all. So the one path where the primary is most + likely to have failed was the one path with no fallback. + + ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary + answered is never re-bought from a second vendor — the mistake the capability split exists + to prevent. A refusal returns None and the caller reports blocked exactly as before. + """ + try: + import providers as _p + if not _p.PROVIDERS["apify"].can("ig_profile"): + return None + prof, note = apify_profile(str(handle)) + except Exception as exc: # noqa: BLE001 — a backup must not raise + log(f"[aios-auto] backup profile rung failed for @{handle}: " + f"{type(exc).__name__}: {exc}") + return None + if prof and prof.get("followers") is not None: + return prof + return None + + def _rows_of(sid): + if sid not in _rows_memo: + payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) + got = _bd_rows(payload) if not err else [] + if got and (_bd_deferral(payload) or + (len(got) == 1 and str(got[0].get("status") or "") in + ("running", "building", "collecting"))): + got = [] + _rows_memo[sid] = got + return _rows_memo[sid] + + for task in tasks: + stale_h = _hours_since(task.get("requestedAt")) + state, records, empty_note = _progress_of(task["snapshotId"]) + if state in ("done", "failed") and not records: + acc["profiles"] += 1 + # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing, + # so there is nothing left to wait for and no risk of buying the same record twice. + backup = _backup_profile(task["influencer"]) + if backup is not None: + ready += 1 + acc["ok"] += 1 + _store_profile(task, backup, "apify") + note = (f"@{task['influencer']}: the primary source returned nothing, so a backup " + f"source supplied the profile ({backup.get('followers')} followers)") + notes.append(note) + acc["notes"].append(note) + continue + closed += 1 + acc["blocked"] += 1 + note = f"@{task['influencer']}: {_s(empty_note, 240)}" + notes.append(note) + acc["notes"].append(note) + # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a + # profile request that means the account could not be reached at all, so the verdict + # is remembered and the selection stops re-buying it (R9: permanently, until a + # human edits the handle cell — see `clear_gone`). + # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about + # the query, and turning it into "this account does not exist" would silently retire + # live handles. + if state == "failed": + acc["gone"][_gone_key({}, task["influencer"])] = { + "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)} + continue + rows = _rows_of(task["snapshotId"]) if state != "running" else [] + if not rows: + # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of + # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never + # silently. An unbounded queue is the forever-loop this change removes, inverted. + if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS: + closed += 1 + note = (f"@{task['influencer']}: the source never finished the profile read " + f"queued {int(stale_h)}h ago ({task['snapshotId']}) — it was dropped; " + f"the next run will ask again") + notes.append(note) + acc["notes"].append(note) + continue + waiting += 1 + remaining.append({**task, "lastChecked": _iso(), + "lastNote": _s("still building", 200)}) + continue + # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot + # holds several profiles and `rows[0]` would write one creator's numbers onto every record + # in the chunk. + node = _match_profile_row(rows, task["influencer"]) + if node is None: + closed += 1 + acc["profiles"] += 1 + acc["blocked"] += 1 + note = (f"@{task['influencer']}: the source delivered {len(rows)} profile" + f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle — " + f"it was dropped from the batch and the next run will ask again") + notes.append(note) + acc["notes"].append(note) + continue + ready += 1 + acc["profiles"] += 1 + profile = _bd_profile(node, task["influencer"]) + if profile.get("followers") is None and profile.get("following") is None: + acc["blocked"] += 1 + note = (f"@{task['influencer']}: the source delivered the profile but no " + f"follower/following counts were readable in it") + notes.append(note) + acc["notes"].append(note) + continue + acc["ok"] += 1 + _store_profile(task, profile, "brightdata:deferred") + notes.append(f"@{task['influencer']}: collected the profile the source had already been " + f"paid for ({profile.get('followers')} followers)") + + set_state(rt, str(defn.get("id") or ""), + {"pendingProfileSnapshots": remaining or None}) + counts = _enrich_flush(rt, defn, username, acc, log) + counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below + counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {}, + username, log)) + counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting, + "profileBatchesEmpty": closed}) + if notes: + counts[RUN_NOTES_KEY] = notes + head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected" + if ready else "no deferred profile read was ready") + tail = "".join([f"; {closed} finished with nothing to collect" if closed else "", + f"; {waiting} still building" if waiting else ""]) + state = "ok" if ready and not closed else "partial" + return (state, head + tail, counts, affected, + {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"}) + + +def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log): + """Write a completed metric snapshot through the canonical Post/Comment graph once.""" + if not (idents or snapshots or comments): + return {"posts": 0, "snapshots": 0, "comments": 0} + profile_table = str(_flow_table(defn) or "") + if not profile_table: + raise Refused("the pending engagement snapshot has no Profile database to link to") + graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""), + profile_table=profile_table) + post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE], + graph[IG_COMMENTS_TABLE]) + old_posts, collapsed_posts = dedupe_canonical_rows( + dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at") + posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key)) + c_post["duplicates"] += collapsed_posts + psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots, + "post_snapshot_key", cap=row_cap(ps_key)) + old_comments, collapsed_comments = dedupe_canonical_rows( + dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key") + comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key", + cap=row_cap(comment_key)) + c_comments["duplicates"] += collapsed_comments + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)): + if cur.get(key) is not None: + cur[key]["rows"] = rows + _refresh_relations_inplace(cur, log=log) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + try: + import ig_master + status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots) + if status == "error": + log(f"[aios-auto] deferred post metrics master copy FAILED: {note}") + except Exception as exc: # noqa: BLE001 + log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}") + return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"], + "comments": c_comments["inserted"]} + + +def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step): + """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request.""" + tasks = _pending_metric_tasks(defn) + if not tasks: + return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"}) + step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}") + remaining, idents, snapshots, comments = [], [], [], [] + ready, waiting, closed, run_notes = 0, 0, 0, [] + for task in tasks: + # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be + # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued + # as "still building" on every tick — forever, because nothing about it would ever + # change. That is the same forever-loop the profile path was measured in, one dataset + # over, and it was latent here the whole time. + state, records, empty_note = bd_snapshot_progress(task["snapshotId"]) + if state in ("done", "failed") and not records: + # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect; + # keeping the entry would be a pending task that can never resolve. + closed += 1 + run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}") + continue + payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"}) + rows = _bd_rows(payload) if not note else [] + building = (state == "running" or not rows or _bd_deferral(payload) or + (len(rows) == 1 and str(rows[0].get("status") or "") in + ("running", "building", "collecting"))) + if building: + remaining.append({**task, "lastChecked": _iso(), + "lastNote": _s(note or "still building", 160)}) + waiting += 1 + continue + ready += 1 + pulled = _iso() + if task["kind"] == "posts": + mapped_posts = [] + for raw in rows: + post = _bd_post_metrics(raw) + if post: + # The snapshot was created from this Profile's canonical post URLs. Keep + # that explicit backlink even when a vendor response omits `user_posted`. + post["influencer_key"] = task["influencer"] + mapped_posts.append(post) + if mapped_posts: + # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the + # owner's `theresalearns` run filled every column except Views. + # + # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a + # Posts row physically cannot carry a view count — MEASURED on the stored + # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them. + # Views only ever comes from the Apify capability. That top-up lived INSIDE + # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait + # budget; when the batch deferred — which is routine, and what happened here — the + # rows came back through THIS function and Apify was never asked. + # ⇒ `top_up_views` is now one function with two callers rather than a copy, so + # the inline and deferred paths cannot answer this differently again. + # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which + # is what freezes the values into the post + snapshot rows. + v_note = top_up_views(mapped_posts, log=log) + if v_note: + run_notes.append(f"@{task['influencer']}: {v_note}") + _unused, post_rows, metric_rows, embedded = capture_rows( + {"state": "ok", "profile": {"username": task["influencer"]}, + "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled) + idents.extend(post_rows) + snapshots.extend(metric_rows) + comments.extend(embedded) + else: + for raw in rows: + comment = _bd_comment(raw, influencer_key=task["influencer"]) + if comment: + comments.append(comment) + + set_state(rt, str(defn.get("id") or ""), + {"pendingMetricSnapshots": remaining or None}) + written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log) + counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting, + "metricBatchesEmpty": closed, + "postEngagementSnapshots": written["snapshots"], + "commentsCollected": written["comments"]} + # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved + # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four + # other call sites disagree about the shape of a run — and `_commit_run` already drops + # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather + # than to a crash. + if run_notes: + counts[RUN_NOTES_KEY] = run_notes + # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is + # a real outcome the tenant paid for and it must read as an answer, not as a disappearance. + tail = (f"; {closed} finished with nothing to collect" if closed else "") + if waiting: + return ("partial", + f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; " + f"{waiting} still building and will be collected automatically{tail}", + counts, [], {"capture_posts": "partial", "write": "ok"}) + return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}", + counts, [], {"capture_posts": "ok", "write": "ok"}) + + +def ai_decide(rt, defn, act, row, row_id="", log=print): + """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" + to leave the card for a person. + + ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed + answer, or a label the review does not offer all return "" — and "" means the card sits at + the review gate exactly as it would with no AI at all. The feature can be broken, absent or + wrong and the worst outcome is a human doing the work. + + ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN + ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board + under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely + worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter → + anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is + MISSING is only the door — an action kind that asks a question and takes an answer, without a + stage column to write it into. **Do not delete this in a dead-code sweep without reading that + sentence first**; equally, do not treat it as shipped — nothing reaches it today. + """ + cfg = act.get("config") or {} + options = list(cfg.get("next") or []) + if not options: + return "" + try: + import ai_review + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") + return "" + fields = [f.get("key") for f in + (ut_get(rt, ((defn.get("config") or {}).get("targetTable") + or (defn.get("trigger") or {}).get("table") or "")) or {} + ).get("fields") or []] + choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, + row=row, fields=[f for f in fields if f], + label=cfg.get("label") or "Review") + if not choice: + if meta.get("problem"): + log(f"[aios-auto] ai review declined to answer: {meta['problem']}") + return "" + review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", + choice, meta.get("provider") or "ai", by="ai", + note=meta.get("reason") or "", model=meta.get("model") or "") + return choice + + +def find_records(rt, table_key, cond, limit=25): + """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts + the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" + rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} + out = [] + for rid, row in rows.items(): + if lane_match(cond, row or {}): + out.append(str(rid)) + if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): + break + return out + + +def _commit_action_writes(rt, table, patches, creates, username, log): + """The ONE write. Row patches merge into the target's rows; creates land in their own tables — + APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a). + + Returns the REALIZED create counts, and returning them is the point rather than a + convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only + this function knows how many of those became rows, how many matched one that was already + there, and how many the cap refused. A run that reported the attempt as "created" would be + the summary-disagrees-with-what-happened defect this module names in three other places. + + ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE + implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand + rolling a third inside this function is that debt returning with a new name — and it would + quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED, + NEVER DELETED, and `capped` is its own count rather than folded into `skipped`). + """ + out = {"created": 0, "createUpdated": 0, "createUnchanged": 0, + "createCapped": 0, "createSkipped": 0} + if not patches and not creates: + return out + # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one + # `ut_write_rows` per patched table, and that was free while the only patcher was a review + # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the + # step it reached, so the flow's own table is now patched on essentially every run — and a + # flow that also creates into that same table would have committed it TWICE per run, against + # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around. + # Staged into `staged` and handed to the creates pass, which writes each table exactly once. + staged = {} + for tkey, rowpatch in (patches or {}).items(): + cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) + for rid, vals in rowpatch.items(): + cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} + staged[str(tkey)] = cur + # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The + # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once + # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr + # repo budget this whole module is shaped around. + by_table = {} + for (tkey, unique), new_rows in (creates or {}).items(): + by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows)) + for tkey, groups in by_table.items(): + t = ut_get(rt, tkey) + if t is None: + n = sum(len(r) for _u, r in groups) + log(f"[aios-auto] create_record: {tkey} no longer exists — {n} skipped") + out["createSkipped"] += n + continue + # The PATCHED rows when this table was also stamped this run, so the creates land on top + # of the stamp rather than on a copy of the store that predates it. + cur = staged.pop(tkey, None) + cur = dict(t.get("rows") or {}) if cur is None else cur + cap = row_cap(tkey) + for unique, new_rows in groups: + if unique: + cur, c = upsert_rows(cur, new_rows, unique, cap=cap) + out["created"] += c["inserted"] + out["createUpdated"] += c["updated"] + out["createUnchanged"] += c["unchanged"] + out["createCapped"] += c["capped"] + # ⚠ `skipped` here means "this row had no value for the unique key", which for a + # create action is a mapped value that interpolated to nothing — worth surfacing, + # because the symptom is otherwise a run that says it created less than it walked. + out["createSkipped"] += c["skipped"] + if c["capped"]: + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — " + f"{c['capped']} row(s) not written") + continue + nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 + for i, vals in enumerate(new_rows): + if len(cur) >= cap: + # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is + # named, here and in the run's counts. This used to `break` with a log line + # that said the cap was hit and never said how much was lost. + out["createCapped"] += len(new_rows) - i + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — " + f"{len(new_rows) - i} row(s) not written") + break + cur[str(nxt)] = dict(vals) + nxt += 1 + out["created"] += 1 + ut_write_rows(rt, tkey, cur) + # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and + # once each, so "one store write per table" holds whether a table was patched, created into, + # or both. + for tkey, cur in staged.items(): + ut_write_rows(rt, tkey, cur) + return out + + +def _lane_sentence(cond, top=True): + """One condition tree → the sentence a step's `detail` carries on the canvas. + + C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a + label never claims a flat comparison the tree does not make. The SERVER composes it, for the + same reason it composes every other `detail` — a client paraphrase of a structure the engine + evaluates is a second implementation of the same sentence, free to drift from it. + + ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is + live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a + rename would be churn on a working function to fix a word. + """ + if cond is None: + return "Everything else" if top else "" + if isinstance(cond, dict): + for key, joiner in (("all", " and "), ("any", " or ")): + if key in cond: + parts = [_lane_sentence(c, False) for c in cond.get(key) or []] + parts = [p for p in parts if p] + if not parts: + return "" + inner = joiner.join(parts) + return inner if top or len(parts) == 1 else f"({inner})" + v = cond.get("value") + return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") + + +def _rid_num(rid): + return int(rid) if str(rid).isdigit() else 10 ** 9 + + +#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can +#: grow a definition without limit is a serialisation cost wearing a compliance hat. +MAX_REVIEWS = 100 + + +def review_audit(rt, auto_id, row_id, from_label, to_label, username, + by="user", note="", model=""): + """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to + the definition (newest first, bounded). + + ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that + made it and the one-line reason it gave. One audit log, not two — a reader asking "who + decided this card" must not have to know there are two places to look, and the moment an + AI decision is invisible beside a human one the log stops being an audit. + + ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone: + `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote + this shape off a stage field's `flowId` — a field the migration now drops. Kept because the + SHAPE is the contract a future decision action would write, and re-deriving an audit format + is how two of them end up existing. + """ + entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), + "from": _s(from_label, 60), "to": _s(to_label, 60), + "by": "ai" if by == "ai" else "user"} + if note: + entry["note"] = _s(note, 300) + if model: + entry["model"] = _s(model, 60) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] + return cur + + rt.update(STORE_KEY, _up, flush="sync") + + +# --------------------------------------------------------------------------------------------- +# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) +# --------------------------------------------------------------------------------------------- +# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — +# the value reflects every pull the platform has, not just this tenant's). Computed by the +# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files +# no session owns this wave), human-write-refused at the grid door, and every number drills to +# the exact snapshot rows behind it. +# +# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no +# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero +# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not +# readable") — the `_bd_posts_count` law, one layer up. + +METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") +METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") +#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each +#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; +#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). +PROFILE_MEASURES = ("followers", "avg_engagement") +METRIC_AGGS = ("avg", "sum", "latest") + + +def metric_value(series, measure, window, agg="", today=None): + """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. + + `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference + day; post-count windows count back from the newest post. `drill_rows` are the exact rows + the number came from, so the route can honour [[no-unverifiable-aggregates]] without + recomputing differently.""" + series = series or {} + today = today or _now() + if measure in PROFILE_MEASURES: + rows = [r for r in series.get("snapshots") or [] + if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] + if window in ("last_7d", "last_30d"): + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + rows = [r for r in rows + if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] + if not rows: + return None, [] + if window == "latest" or agg == "latest": + picked = [rows[-1]] + else: + picked = rows + vals = [_lane_num(r.get(measure)) for r in picked] + vals = [v for v in vals if v is not None] + if not vals: + return None, [] + out = vals[-1] if (window == "latest" or agg == "latest") else \ + (sum(vals) if agg == "sum" else sum(vals) / len(vals)) + if measure == "avg_engagement": + # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs + # transform-pct scar) — scaled exactly once, here. + return f"{out * 100:.2f}", picked + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked + # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. + posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] + posts.sort(key=lambda p: str(p.get("posted_at"))) + if window == "latest": + picked_posts = posts[-1:] + elif window == "last_3_posts": + picked_posts = posts[-3:] + else: + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + picked_posts = [p for p in posts + if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) + or _dt.datetime.min) >= floor] + vals, drill = [], [] + for p in picked_posts: + snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] + for snap in reversed(snaps): + raw = snap.get(measure) + v = _lane_num(raw) + if v is not None and str(raw).strip() != "": + vals.append(v) + drill.append(snap) + break + if not vals: + return None, [] + out = sum(vals) if (agg or "sum") == "sum" else \ + (vals[-1] if agg == "latest" else sum(vals) / len(vals)) + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill + + +def metric_fields_of_table(t): + return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] + + +def _table_handle(row, url_field): + return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) + or "").strip().lower() + + +def compute_metric_cells(rt, table_key, today=None): + """Recompute every metric cell on ONE table from the master series. One coalesced write, + only when something actually changed (the flush-ceiling law); zero reads when the table + has no metric fields or the master is off. Returns the number of rows touched.""" + t = ut_get(rt, table_key) + mfields = metric_fields_of_table(t) + if not mfields: + return 0 + import ig_master + if not ig_master.configured(): + return 0 + url_field = next((f.get("key") for f in (t.get("fields") or []) + if f.get("type") == "url"), "") + rows = t.get("rows") or {} + handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} + series = ig_master.series_for({h for h in handles.values() if h}) + changes = {} + for rid, row in rows.items(): + s = series.get(handles.get(rid) or "") + for f in mfields: + bag = f["metric"] + val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), + bag.get("agg") or "", today=today) + if s else (None, [])) + want = "" if val is None else str(val) + if str((row or {}).get(f["key"], "")) != want: + changes.setdefault(str(rid), {})[f["key"]] = want + if not changes: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ──────────────── +# +# Owner: *"adding relational database function with links and rollups… exactly like how Airtable +# does it… and this rollup needs to have formula that we can use to calculate things like average +# Views over last N posts."* +# +# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula +# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not +# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced +# write only when something changed — and inherits its flush-ceiling discipline for free. +# +# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching +# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is +# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as +# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED +# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The +# moment one writes a number nothing else can re-derive, it has become a third copy. + +def _ut(): + """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic. + + Pulling this module in at import time initialises the store layer earlier than + `automation_engine` used to, and the last time that happened it moved a store-commit COUNT + from three to four on an unrelated gate. The relational pass + needs three constants and two predicates from the field layer; it does not need to change + when this module is imported. + """ + import core.user_tables as _m + return _m + + +#: The fns by family — how a value is folded, and what a blank means in each. +_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"}) +_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"}) +#: The count/order family — folded by their own arms above the lanes. +_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"}) +#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how +#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED +#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm +#: claims fell past both lanes into the join below — so a "Std deviation" column rendered +#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now +#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question +#: nothing can answer; a comma-joined list is a different question's answer wearing this label). +_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) +#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED. +#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name +#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot +#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the +#: defect above, because the defect was that the two lists already disagreed. +ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS + | _ROLLUP_TEXT_FNS) +_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"}) +#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing. +_ROLLUP_JOIN = ", " +#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a +#: cell nobody can read is not an aggregate. +ROLLUP_MAX_CHARS = 4000 + + +def _sort_key(value, ftype): + """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell + is blank or does not parse as its declared type. + + ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a + string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides + WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently + averages the wrong twelve posts. + + ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION + SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")` + otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True` + flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent + real one out of the window. A sentinel inside the key cannot mean "last" in both directions, + because the direction is applied to the sentinel too. + ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as + `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at` + only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every + key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed + because its fixture posts all carried dates. + ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it + sorts, and "an unknown date is not a recent one" is finally what the code does. + """ + raw = "" if value is None else str(value).strip() + if raw == "": + return None + if ftype in ("int", "currency", "pct", "rating"): + n = _lane_num(raw) + return (n,) if n is not None else None + if ftype == "date": + d = _parse_iso(raw.replace(" ", "T")) + return (d.timestamp(),) if d is not None else None + return (raw.lower(),) + + +def _sample_stdev(nums): + """SAMPLE standard deviation (n-1) of `nums`, or None under two values. + + ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a + function: `_rollup_fold` RENDERS this number into a "Std deviation" column and + `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy + would let a column and the filter beside it disagree about the same word on the same set — + the exact drift `top_up_views` was extracted to prevent one module over + ([[one-evaluator-per-question]]). + ⛔ None means UNANSWERABLE and no caller may read it as 0. + """ + if len(nums) < 2: + return None + mean = sum(nums) / len(nums) + return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5 + + +def _rollup_ref_threshold(rows, field, sigmas): + """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1). + + ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`, + and NOT yet filtered by the conditions this threshold feeds. That order is the contract + (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders + produce a plausible number: computing it before `limit` answers "2 sigma of everything this + account ever posted" under a column that says "of the last 10", and computing it after the + filter makes the threshold depend on the rows it is choosing — a definition that chases + itself. + + Returns None when the window cannot answer — fewer than two numeric values in `field`. + ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question + with a permissive answer; letting an unanswerable leaf pass everything would silently turn + "the outliers" into "all of them", which is this module's worst failure mode wearing a filter. + """ + nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None] + sd = _sample_stdev(nums) + if sd is None: + return None + return (sum(nums) / len(nums)) + float(sigmas) * sd + + +def _rollup_fold(fn, values, ftype="text"): + """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING. + + Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's + oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a + question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement + that the creator gets no views. + ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records" + over an empty set is genuinely 0, not unknown. + + `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only + `min`/`max` read it today; it defaults to `text` so every existing caller and every test that + folds a bare list keeps its exact previous answer. + """ + if fn == "countall": + return str(len(values)) + if fn == "latest": + # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve + # a blank on the newest row rather than reaching backwards and presenting an older value + # as current. + return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS] + if fn == "counta": + return str(len([v for v in values if str(v or "").strip() != ""])) + if fn == "count": + # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them + # distinct is the whole reason both exist. + return str(len([v for v in values if _lane_num(v) is not None])) + # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN. + # + # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered + # BLANK forever while looking completely configured — the failure this module refuses + # everywhere else, arriving through the one fold that had no type awareness. "Earliest order" + # and "latest invoice due" are the two most ordinary date rollups there are, and + # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates + # and the fold did not. + # + # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a + # string-compare fallback would have worked on well-formed data and silently mis-ordered a + # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is + # already at this call site (`linked_types`), and `_sort_key` is already the one function that + # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather + # than growing a second idea of what a date is. + # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person + # wants in the column is the stored date string exactly as the source row spells it. + # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean + # of two timestamps is a number this product has no column type for, and inventing one here + # would be a value with no author. + if fn in ("min", "max") and ftype == "date": + keyed = [(k, str(v)) for k, v in + ((_sort_key(v, "date"), v) for v in values) if k is not None] + if not keyed: + return "" + return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS] + if fn in _ROLLUP_NUM_FNS: + nums = [n for n in (_lane_num(v) for v in values) if n is not None] + if not nums: + return "" + if fn == "stdev": + # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference + # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to + # be LINKED, which is a sample of an account's posting history and not its entirety. + # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION + # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently + # switched divisors. + # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the + # honest reason is upstream of the arithmetic: one measurement has no spread to + # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the + # single most confident thing this column can say, asserted from a single row. Same + # law as the blank `sum`, and it bites harder here. + out = _sample_stdev(nums) + if out is None: + return "" + else: + out = (sum(nums) if fn == "sum" else min(nums) if fn == "min" + else max(nums) if fn == "max" else sum(nums) / len(nums)) + return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}" + if fn in _ROLLUP_BOOL_FNS: + # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero". + flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values] + if not flags: + return "" + hit = (all(flags) if fn == "and" else any(flags) if fn == "or" + else sum(1 for f in flags if f) % 2 == 1) + return "1" if hit else "" + # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`. + # An fn no arm above recognises returns "" rather than a comma-joined dump of every value, + # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28. + if fn not in _ROLLUP_TEXT_FNS: + return "" + vals = [str(v).strip() for v in values if str(v or "").strip() != ""] + if fn == "arrayunique": + seen, uniq = set(), [] + for v in vals: + if v.lower() not in seen: + seen.add(v.lower()) + uniq.append(v) + vals = uniq + if not vals: + return "" + text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals)) + return text[:ROLLUP_MAX_CHARS] + + +def _link_from_key(fields, bag): + """Which column on THIS table supplies the join value. + + Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That + fallback chain is what makes an Instagram database link up with no configuration at all — + the "automatically" in the owner's instruction — and it is the same chain the grid already + uses to decide a table's identity column, rather than a second opinion about it. + """ + declared = str((bag or {}).get("from") or "").strip() + if declared: + return declared + prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None) + if prof: + return str(prof.get("key") or "") + pin = next((f for f in fields if f.get("pinned") is True), None) + return str(pin.get("key") or "") if pin else "" + + +def _linked_rows_by_join(linked, on_key): + """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE. + + ⚠ Lower-cased because the join values this exists for are Instagram handles, which the + profile flag already normalises to lower case on one side and which a hand-typed cell on the + other side may not. A join that misses on case is a relation that silently reports zero. + """ + idx = {} + for rid, row in ((linked or {}).get("rows") or {}).items(): + k = str((row or {}).get(on_key) or "").strip().lower() + if k: + idx.setdefault(k, []).append((str(rid), row or {})) + return idx + + +def _rollup_condition_matches(row, condition, field_types, ref_value=None): + """Evaluate one Airtable-style linked-record condition against a candidate row. + + `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the + caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed + here because this function sees one row and the statistic is a property of the whole window — + a version that reached for the set from inside would be recomputing the same mean once per + row, and would have to be handed the window anyway. + ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the + threshold helper for why the permissive reading is the dangerous one. + """ + field = str((condition or {}).get("field") or "") + op = str((condition or {}).get("op") or "") + raw = (row or {}).get(field) + text = str(raw or "").strip() + if op == "is_empty": + return text == "" + if op == "is_not_empty": + return text != "" + if (condition or {}).get("ref") is not None: + # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering + # ops, and a row whose cell is blank or unparseable has no position relative to a computed + # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key` + # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]). + left_num = _lane_num(text) + if ref_value is None or left_num is None: + return False + return ((op == "gt" and left_num > ref_value) + or (op == "gte" and left_num >= ref_value) + or (op == "lt" and left_num < ref_value) + or (op == "lte" and left_num <= ref_value)) + wanted = str((condition or {}).get("value") or "").strip() + if op == "contains": + return wanted.casefold() in text.casefold() + if op == "not_contains": + return wanted.casefold() not in text.casefold() + if op in ("eq", "neq"): + left_num, right_num = _lane_num(text), _lane_num(wanted) + equal = (left_num == right_num if left_num is not None and right_num is not None + else text.casefold() == wanted.casefold()) + return equal if op == "eq" else not equal + ftype = (field_types or {}).get(field, "text") + left = _sort_key(text, ftype) + right = _sort_key(wanted, ftype) + if left is None or right is None: + return False + return ((op == "gt" and left > right) or (op == "gte" and left >= right) + or (op == "lt" and left < right) or (op == "lte" and left <= right)) + + +def compute_relation_cells(rt, table_key, tables=None): + """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched. + + Zero store reads when the table declares neither kind — the same cheap-by-construction shape + `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table. + """ + store = tables if tables is not None else ut_all(rt) + t = (store or {}).get(table_key) + fields = list((t or {}).get("fields") or []) + links = [f for f in fields if _ut().is_derived_link(f)] + rollups = [f for f in fields if isinstance(f.get("rollup"), dict)] + if not links and not rollups: + return 0 + rows = (t or {}).get("rows") or {} + by_key = {str(f.get("key")): f for f in fields} + + # --- resolve every link field ONCE per table, not once per row. + # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]` + resolved, linked_types = {}, {} + for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]: + lk_key = str((f or {}).get("key") or "") + if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict): + continue + bag = f["link"] + linked = (store or {}).get(str(bag.get("table") or "")) or {} + linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text") + for lf in (linked.get("fields") or [])} + if bag.get("inverse"): + # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary + # link cell contains this row id. The source cell remains the one relationship truth. + source_rows = linked.get("rows") or {} + inverse_key = str(bag.get("inverse") or "") + inverse_index = {} + for source_id, source_row in source_rows.items(): + for target_id in [part.strip() for part in + str((source_row or {}).get(inverse_key) or "").split(",")]: + if target_id: + inverse_index.setdefault(target_id, []).append( + (str(source_id), source_row or {})) + resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows} + elif bag.get("on"): + idx = _linked_rows_by_join(linked, str(bag["on"])) + from_key = _link_from_key(fields, bag) + resolved[lk_key] = { + str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), []) + for rid, row in rows.items()} if from_key else {} + else: + # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds + # are dropped rather than carried — a link to a deleted row is not a link. + lrows = (linked.get("rows") or {}) + resolved[lk_key] = { + str(rid): [(i, lrows[i]) for i in + [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")] + if i and i in lrows] + for rid, row in rows.items()} + + changes = {} + for rid, row in rows.items(): + rid = str(rid) + row = row or {} + for f in links: + fk = str(f["key"]) + hits = (resolved.get(fk) or {}).get(rid) or [] + if f["link"].get("single"): + hits = hits[:1] + # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell + # is a projection of `resolved`, which is uncapped and is what every rollup below + # consumes — so a profile with 900 posts shows the first 500 ids and still averages + # over all 900. Same argument the `posts` window already makes: shedding is safe + # precisely because the authoritative store still holds everything. + want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS]) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + for f in rollups: + fk, bag = str(f["key"]), f["rollup"] + lk_key = str(bag.get("link") or "") + hits = list((resolved.get(lk_key) or {}).get(rid) or []) + # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING + # and therefore to a blank cell — never to a stale number. A column that keeps + # printing yesterday's answer after its input is gone is the worst of the options. + # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE + # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so + # "last 10 posts where views > X" meant *the 10 most recent of the posts over X* + # rather than *the ones over X among the last 10* — two different windows wearing one + # sentence. Harmless while a threshold was a literal; incoherent the moment a + # threshold is a statistic OF the window, because the set being described and the set + # doing the describing would be different sets. + # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s + # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is + # computed over that window, THEN the conditions filter it") and the validator half + # was written against it. + # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH + # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational` + # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall` + # with no limit), so the blast radius is user-built rollups only. + # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last + # 12 posts, we also want to make it so its last N record, where the record's Status is + # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose, + # so they select among the rows the window already kept. `where` selects WHICH rows + # the window is spent on. + # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per + # identity; run it first and a carousel could claim the slot its reel sibling needed, + # so the window would come up short for a reason nothing on screen explains. + # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this + # list, because at this point there is no fixed set for a statistic to be about. + where = list(bag.get("where") or []) + if where: + where_matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=None) + for condition in where] + if str(bag.get("whereConj") or "and") == "or": + hits = [pair for pair in hits if any(where_matches(pair))] + else: + hits = [pair for pair in hits if all(where_matches(pair))] + sort_by = str(bag.get("sortBy") or "") + if sort_by: + ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text") + # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not + # rankable, and it must land at the END whichever direction is asked for — which + # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel + # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its + # window on rows that HAVE the value before it falls back to ones that do not. + keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits] + rankable = [(p, k) for p, k in keyed if k is not None] + rankable.sort(key=lambda pk: pk[1], + reverse=str(bag.get("sortDir") or "desc") == "desc") + hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None] + distinct_by = str(bag.get("distinctBy") or "") + if distinct_by: + seen, unique = set(), [] + for pair in hits: + identity = str(pair[1].get(distinct_by) or "").strip().lower() + # A blank is not an identity. Keep it rather than collapsing every unknown + # record into one synthetic duplicate. + if identity and identity in seen: + continue + if identity: + seen.add(identity) + unique.append(pair) + hits = unique + limit = int(bag.get("limit") or 0) + if limit: + hits = hits[:limit] + # --- the window is now FIXED, so a set-statistic threshold has a set to be about. + conditions = list(bag.get("conditions") or []) + if conditions: + # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the + # window; computing it inside `matches` would recompute the same mean for every + # candidate and — worse — would invite computing it over a set that the filter is + # already shrinking underneath it. + # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond + # the mean") is falsy, and the `or` spelling would silently rewrite it to the same + # number by accident. It reads identically and is right for the wrong reason, + # which is how it survives a review. + refs = [_rollup_ref_threshold( + [r for _i, r in hits], str(c.get("field") or ""), + (c.get("ref") or {}).get("sigmas", 0.0)) + if isinstance(c, dict) and c.get("ref") is not None else None + for c in conditions] + matches = lambda pair: [ + _rollup_condition_matches(pair[1], condition, + linked_types.get(lk_key) or {}, ref_value=ref) + for condition, ref in zip(conditions, refs)] + if str(bag.get("conditionConj") or "and") == "or": + hits = [pair for pair in hits if any(matches(pair))] + else: + hits = [pair for pair in hits if all(matches(pair))] + src = str(bag.get("field") or "") + # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the + # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at + # all, so the default stands for it. + want = _rollup_fold(str(bag.get("fn") or ""), + [r.get(src) for _i, r in hits] if src else [1] * len(hits), + (linked_types.get(lk_key) or {}).get(src, "text")) + if str(row.get(fk, "")) != want: + changes.setdefault(rid, {})[fk] = want + if not changes: + return 0 + + # A run that just wrote linked rows passes its in-flight user_tables bucket here so the + # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps + # the public helper's old persist-on-change behaviour. + if tables is not None: + tt = tables.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return len(changes) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for r, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(r, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +def _refresh_relations_inplace(tables, log=print): + """Refresh every relation against one mutable user_tables bucket; perform no store write.""" + touched = 0 + for tk, table in list((tables or {}).items()): + fields = (table or {}).get("fields") or [] + if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields): + continue + try: + touched += compute_relation_cells(None, tk, tables=tables) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def refresh_relations(rt, log=print): + """The tick half of the relational pass — the twin of `refresh_metrics`. + + ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale + when table B gains a row, and A has no way to know that happened. Cheap by construction — a + table declaring neither kind costs one dict scan. + """ + snapshot = { + str(key): {**(table or {}), + "rows": {str(rid): dict(row or {}) + for rid, row in ((table or {}).get("rows") or {}).items()}} + for key, table in (ut_all(rt) or {}).items() + } + if not _refresh_relations_inplace(snapshot, log=log): + return 0 + actual = [0] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + actual[0] = _refresh_relations_inplace(cur, log=log) + return cur + + # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug). + # + # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and + # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget. + # A derived-cell recompute has no business forcing a commit on someone typing. + # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new + # row lives only in the store cache for 2-20s, and this call — on the SAME key + # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the + # result. `POST /rows` answered 201 and the row was gone. + # + # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it + # is what makes the other ~39 sync writers of this key safe. This line is not that fix and + # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both + # landed together on purpose: one is correctness, one is the hot path. + rt.update(UT_STORE_KEY, _up, flush="async") + return actual[0] + + +def refresh_metrics(rt, today=None, log=print): + """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window + metric can never go staler than one tick while a scheduler exists. Cheap by construction — + a table without metric fields costs a dict scan and nothing else.""" + touched = 0 + for tk, t in ut_all(rt).items(): + if metric_fields_of_table(t): + try: + touched += compute_metric_cells(rt, tk, today=today) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def purge_subject(rt, handle): + """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the + tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half + non-optional: a purge that missed the pooled copy would not be erasure). Returns + `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is + now ABSENT, which is the one aggregate whose drill is emptiness.""" + subject = str(handle or "").strip().lstrip("@").lower() + if not subject: + return {} + counts = {} + tables = ut_all(rt) + post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} + codes = {str(r.get("shortcode") or "") for r in post_rows.values() + if str((r or {}).get("influencer_key") or "").strip().lower() == subject} + + keeps = { + "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, + DISCOVER_TABLE: lambda r: str((r or {}).get("handle") + or "").strip().lower() != subject, + } + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for tk, keep in keeps.items(): + t = cur.get(tk) + if t is None: + continue + rows = t.get("rows") or {} + nxt = {rid: r for rid, r in rows.items() if keep(r)} + counts[tk] = len(rows) - len(nxt) + t["rows"] = nxt + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + import ig_master + for bucket, n in (ig_master.purge_handle(subject) or {}).items(): + counts[f"master:{bucket}"] = n + return counts + + +# --------------------------------------------------------------------------------------------- +# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) +# --------------------------------------------------------------------------------------------- +# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | +# webhook | email. The first two are what always existed (Run now; cron via the tick). The four +# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: +# +# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, +# not for being in it. Implemented as per-record ARMED state over successive evaluations +# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms +# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing +# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the +# disarmed set with everything currently matching, so already-matching records do not fire +# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). +# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only +# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers +# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never +# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by +# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as +# a statusNote) catches whatever construction did not foresee. +# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds +# instead of running — a partial run entry names the count and the deliberate way through +# (Run now). C4's discovery guard is this rule's special case. +# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record +# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); +# event_field fires every transition. + +# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── +# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that +# renames the same events is a second vocabulary to learn for no gain. +# +# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". +# Renaming the key would have orphaned every stored trigger in production for a caption; the key +# is the contract with the store, the label is the contract with the reader, and they are allowed +# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the +# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 +# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. +# +# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show +# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a +# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop +# class wearing a friendlier face; here the two lists are separate on purpose and the refusal +# names the state rather than pretending the key is unknown. +# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── +# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the +# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — +# it is the event this flow starts from. Picking it sets the definition's kind to +# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. +# +# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the +# schedule (or on Run now), so it stays out of both lists below and its node switch flips the +# CRON — see `TRIGGER_SCHEDULE_KEYS`. +# +# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and +# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in +# the owner's own words and has not changed; what changed is which tuple it sits in, because +# `clean_trigger` refuses the planned list with a sentence and accepts this one. +TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", + "enters_view", "webhook", "email", "form_submitted", "ig_profile_match", + "tiktok_profile_match") +#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS +#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses +#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper +#: section is not a section of one. +#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an +#: action that drives a browser are different builds; this row must not be read as progress on D-51. +TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed") +TRIGGER_LABELS = { + "manual": "Manual", + "schedule": "At a scheduled time", + "event_field": "When a record matches conditions", + "record_updated": "When a record is updated", + "record_created": "When a record is created", + "enters_view": "When a record enters a view", + "webhook": "When a webhook is received", + "email": "When an email arrives", + "form_submitted": "When a form is submitted", + "ig_profile_match": "When an Instagram profile fits a criteria", + "button_clicked": "When a button is clicked", + "comment_added": "When a comment is added", + "web_page_changed": "When a website page changes", + "tiktok_profile_match": "When a TikTok profile fits a criteria", +} + +# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ── +# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a +# distinction about where a trigger came FROM rather than about what a person is choosing. The +# question the picker actually asks is: does this fire on TIME, on your own DATA, or because +# something OUTSIDE said so. Three answers, and every trigger has exactly one. +# +# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map +# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the +# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS` +# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation +# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is +# the surface. So: +# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule +# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because +# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into +# Database. +# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, +# so the fallback is provably dead code in production and the classification is still +# mandatory. The fallback catches the accident; the gate stops it shipping. +TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, + "database": {"label": "Database", "order": 2}, + "connector": {"label": "Connector", "order": 3}, + "other": {"label": "Other", "order": 99}} +#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to +#: prevent — it is the soft landing, not a category anybody should be adding triggers to. +TRIGGER_GROUP_FALLBACK = "other" +TRIGGER_GROUP_OF = { + "manual": "time", "schedule": "time", + "event_field": "database", "record_updated": "database", "record_created": "database", + "enters_view": "database", "form_submitted": "database", + "button_clicked": "database", "comment_added": "database", + "email": "connector", "webhook": "connector", "ig_profile_match": "connector", + "web_page_changed": "connector", "tiktok_profile_match": "connector", +} +#: The SUB-group inside "Connector" — which connected thing this trigger comes through. +#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two +#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a +#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key +#: against `/connectors/directory` would match `scraper` and `tiktok` and miss `gmail` — so it +#: must group by it and render `label`, never look it up. Said here because the miss would be +#: silent and partial, which is the worst shape. +TRIGGER_CONNECTOR = { + "email": {"key": "gmail", "label": "Gmail"}, + "webhook": {"key": "webhooks", "label": "Webhooks"}, + # R9: the Scraper ships with ONE built trigger and one faded one. + "ig_profile_match": {"key": "scraper", "label": "Scraper"}, + "web_page_changed": {"key": "scraper", "label": "Scraper"}, + "tiktok_profile_match": {"key": "tiktok", "label": "TikTok"}, +} +#: Triggers that watch a database and therefore need one named before they can fire. +TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", + "form_submitted") +#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. +#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, +#: so flipping its node must flip the schedule. +#: +#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read +#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, +#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For +#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a +#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns +#: about eight lines into its own comment. Derived from one named set now, so a trigger added to +#: `TRIGGER_KEYS` cannot silently join the wrong side of it. +#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES, +#: and forgetting it is precisely the failure this constant's own note describes: it watches no +#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left +#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while +#: the schedule kept firing — a switch that lies. +TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match") +#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last. +#: +#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire +#: reason this constant exists rather than the tuple above being sent. That set answers "which +#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual +#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw +#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run +#: now": a control contradicting its own description. +#: +#: D-55's history is why it is DERIVED rather than listed: the client carried +#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact +#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face). +#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here. +TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"} +#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so +#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. +TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") +MAX_WATCH_FIELDS = 12 +#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, +#: and so would a deployment that prefers immediacy over coalescing. +EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) +FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses +FIRE_WINDOW_SECONDS = 300 +FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on +EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation +EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest +CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota + +EMAIL_FIELDS = [ + field_def("email_id", "Email id"), field_def("email_from", "From"), + field_def("email_subject", "Subject"), field_def("email_date", "Date"), + field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), +] + + +def clean_trigger(raw, previous=None): + """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and + means what it always meant: manual + whatever `schedule` says. + + ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry + with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT + rather than refused — the picker writes `{key}` first and the table/field after, the + wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the + wire so the surface says "finish setting this up" instead of snapping back to Manual. It + cannot fire while incomplete (the hooks match on the table it does not name), which is the + fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a + condition on a field the trigger does not watch) are still refused with the sentence — + incomplete is a state, wrong is not. + """ + if raw in (None, "", {}): + return (dict(previous) if isinstance(previous, dict) and previous else None), None + if not isinstance(raw, dict): + return None, "the trigger must be an object" + prev = previous if isinstance(previous, dict) else {} + key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), + 30).strip() + if key in TRIGGER_PLANNED: + # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence + # says WHY rather than "unknown trigger", because the picker legitimately showed it. + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — " + f"it renders so you can see it is coming, and it cannot be saved") + if key not in TRIGGER_KEYS: + return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) + if key == "schedule": + # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule` + # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored + # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it. + return None, None + if key == "manual": + # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.** + # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user + # can just press Run to make the full flow work."* + # + # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys + # together with one argument — *"storing a no-op trigger would be a second copy of that + # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has + # another home; **manual has none.** Nothing anywhere recorded "this automation is + # manual", so storing it is not a duplicate: it is the only record there has ever been. + # + # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so + # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the + # "nobody has decided yet" empty state, and Configuration — including the Database picker + # a plain automation cannot do without — never rendered. The owner reported it twice. The + # previous note reasoned that a manual option *"would bounce straight back to this state + # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion + # was that it should be STORED. + # + # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off + # (Run now always works, which is the whole of what it means) and a switch that governs + # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the + # cron stays reachable — picking Manual says how it fires today, never that it may not be + # scheduled tomorrow. + return {"key": "manual"}, None + out = {"key": key, + "enabled": bool(raw["enabled"]) if "enabled" in raw else + bool(prev.get("enabled", True)), + "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} + if key in TRIGGER_TABLE_KEYS: + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("event triggers watch blank databases (ut_*) this wave — " + f"{table!r} is not one") + out["table"] = table + if key == "event_field": + # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record + # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a + # second, quieter way to express the same narrowing, and the owner asked for one. + # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is + # dropped on this definition's next clean — silently, and exactly once, because nothing + # writes the key back. A refusal here would have 400'd the live automations that carry it. + cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), + where="the trigger") + if cerr: + return None, cerr + out["when"] = cond + if key == "record_updated": + # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty + # is the WIDER reading and it is the default there too, so it stays the default here. + watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") + if watch_raw in (None, ""): + watch = [] + elif not isinstance(watch_raw, list): + return None, "the watched-field list must be a list of field keys" + else: + watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] + if len(watch) > MAX_WATCH_FIELDS: + return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " + f"fields — leave the list empty to watch every field") + out["fields"] = watch + # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A + # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: + # "a record was updated" is an event, and asking it to also be a filter was the overlap + # with `event_field` the owner asked to end. Stated loudly in the contract AND here so + # nobody restores it as a bug fix. + # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` + # naturally returns "no gate" for it — the write itself becomes the event — rather than + # this needing a second removal anywhere. + if key == "enters_view": + out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), + 80).strip() + if key == "form_submitted": + # Blank = any form on that database. Naming one narrows to it, which is what a table + # carrying an intake form AND a correction form needs. + out["formToken"] = _s(raw.get("formToken") if "formToken" in raw + else prev.get("formToken"), 64).strip() + if key == "webhook": + # The token is MINTED here, once, and survives every later patch — rotating it on + # every Save would silently break the external caller the URL was given to. + out["token"] = _s(prev.get("token"), 64) or _secrets_token() + # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. + # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no + # map behaves exactly as it did — it fires the flow and reads nothing — so the live + # webhook automations are untouched. `webhook` deliberately stays OUT of + # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and + # every existing webhook trigger would go unconfigured and stop firing. + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("a webhook writes into a blank database (ut_*) — " + f"{table!r} is not one") + out["table"] = table + fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw + else prev.get("fieldMap")) + if ferr: + return None, ferr + out["fieldMap"] = fmap + if key == "email": + out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), + 200).strip() or "in:inbox is:unread" + out["configured"] = _trigger_configured(out) + return out, None + + +#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody +#: else's schema and is legitimately wider than an action's hand-written value list. +MAX_BODY_FIELDS = 40 +MAX_BODY_DEPTH = 5 + + +def clean_body_map(raw): + """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. + + Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated + rather than half-supported: `items.0.sku` would read as working for the first element and + silently write nothing the day a payload arrives with the list empty, which is the shape of + bug this module keeps paying for. A path that resolves to nothing writes nothing. + """ + if raw in (None, ""): + return {}, None + if not isinstance(raw, dict): + return None, "the webhook field map must be an object of {body path: field key}" + if len(raw) > MAX_BODY_FIELDS: + return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" + out = {} + for path, field in raw.items(): + p = _s(path, 200).strip() + if not p: + return None, "a webhook mapping has an empty body path" + if len(p.split(".")) > MAX_BODY_DEPTH: + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload — " + f"map a shallower value") + fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") + if not fk: + return None, f"the value at {p!r} is not mapped to a field" + out[p] = fk[:60] + return out, None + + +def body_value(body, path): + """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is + an object or a list answers None rather than a stringified `{...}` in a cell, because the + Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" + cur = body + for part in str(path or "").split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur if isinstance(cur, (str, int, float, bool)) else None + + +def webhook_row(rt, defn, body): + """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. + + ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same + `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, + from the only side the caller is on, from a payload whose paths did not resolve. That is the + D-11 class (a table that quietly stops growing), and the caller here is a machine that will + keep posting. The note rides the 200: the flow still fires, and the answer says why no row + was written. + + ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop + prevention law (A2(2)). A sibling automation watching this table does NOT fire on a + webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow + fires, because `hook_fire` fires it explicitly, which is the difference between "this + trigger fired" and "a write happened". + """ + trg = (defn or {}).get("trigger") or {} + table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) + if not table or not fmap or not isinstance(body, dict): + return "", 0, "" # no map configured — nothing to report + t = ut_get(rt, table) + if t is None: + return "", 0, f"{table} no longer exists, so nothing was written" + values = {} + for path, fkey in fmap.items(): + v = body_value(body, path) + if v is not None: + values[fkey] = _s(v, 500) if isinstance(v, str) else v + if not values: + return "", 0, ("none of the mapped paths resolved to a value in this payload — " + "check the paths against what you are sending") + rows = dict((t.get("rows") or {})) + if len(rows) >= row_cap(table): + return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " + f"written (the flow still ran)") + rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) + rows[rid] = values + ut_write_rows(rt, table, rows) + return rid, len(values), "" + + +def _trigger_configured(trg, config=None): + """Is this trigger complete enough to fire? One reader, because "configured" is asserted in + three places (the wire, the graph node, the hooks) and three copies of a boolean is how a + surface says "ready" about a trigger the engine skips. + + ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is + the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) + rather than on the trigger. `clean_trigger` calls this without it and so answers + conservatively (False); `clean_definition` calls it again with the validated config and + refines. Conservative-then-refined is the fail-closed order — the reverse would flash + "ready" on a trigger with nothing to search for. + """ + key = str((trg or {}).get("key") or "") + # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no + # filter is not a search, it is a request for the whole index. Named as a pair rather than + # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a + # boolean expression. + if key in ("ig_profile_match", "tiktok_profile_match"): + return bool((config or {}).get("predicates")) + if key in TRIGGER_TABLE_KEYS and not trg.get("table"): + return False + if key == "event_field": + # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, + # ever" — which is not a trigger, it is a description of the table. + # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone + # and carried no condition becomes `configured: false` on its next clean. It stops + # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it + # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. + return bool(trg.get("when")) + if key == "enters_view": + return bool(trg.get("viewId")) + return True + + +def _secrets_token(): + import secrets as _sec + return _sec.token_urlsafe(24) + + +# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the +# process would keep punishing an automation for a storm that ended with the restart. +_FIRES = {} +_FIRES_LOCK = threading.Lock() + + +def _breaker_trips(tenant, auto_id, now=None): + now = now if now is not None else time.time() + key = (tenant, str(auto_id)) + with _FIRES_LOCK: + log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] + log.append(now) + _FIRES[key] = log + return len(log) > FIRE_LIMIT + + +def _pause_trigger(rt, auto_id, note): + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = _s(note, 200) + return cur + rt.update(STORE_KEY, _up, flush="sync") + + +def trigger_fire(rt, tenant, auto_id, log=print, rows=None): + """One trigger firing — breaker first, then the ordinary async run. False when it did not + start (breaker, or already running — both are answers, not errors).""" + if _breaker_trips(tenant, auto_id): + note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " + f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's " + f"subject in a loop") + _pause_trigger(rt, auto_id, note) + log(f"[aios-auto] breaker: {auto_id} {note}") + return False + return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) + + +# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE +# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). +_SETTLE = {} +_SETTLE_LOCK = threading.Lock() + + +def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): + """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. + + ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave + dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the + reason is worth stating because it is invisible from this file: an ordinary ut cell typed at + the grid door lands in the editor's PER-USER OVERLAY stratum + (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` + definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for + the common case the value that just changed exists ONLY in this event, and a definition-row + read sees the pre-write value — the trigger would evaluate stale and never fire. + `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree + read the record's other columns) with the written cells on top. + """ + key = (tenant, str(auto_id)) + cell = {str(field): after} if field else {} + if EVENT_SETTLE_SECONDS <= 0: + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + _settle_eval(rt, tenant, auto_id, log=log) + return + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + timer = buf.get("timer") + if timer is not None: + timer.cancel() # the burst continues — push the window out + timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, + args=(rt, tenant, auto_id), kwargs={"log": log}) + timer.daemon = True + buf["timer"] = timer + timer.start() + + +def view_filter(rt, table_key, view_id): + """`(tree, fields, problem)` for one saved view on a user table — the substrate the + `enters_view` trigger tests membership against (C3-v2 / owner R2). + + Personal strata first (`find_view`), then the shared bucket, because a view somebody shared + is exactly the kind an automation gets pointed at. A view that has been deleted answers a + PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one + would turn "when a record enters Overdue" into "on every write", which is the widening this + module refuses everywhere else. + """ + key = str(table_key or "") + vid = str(view_id or "").strip() + if not key or not vid: + return None, [], "the trigger names no view" + try: + import core.table_store as table_store + tops = table_store.make(f"{key}_table_workspace", st=rt) + found = tops.find_view(vid) + view = (found[1] if found else None) or tops.shared_view(vid) + except Exception as e: # noqa: BLE001 + return None, [], f"the view could not be read ({type(e).__name__})" + if not isinstance(view, dict): + return None, [], f"view {vid!r} no longer exists on {key}" + cfg = view.get("config") or {} + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" + + +def _row_gate(rt, defn, trg): + """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. + + None means the trigger has NO match gate — the write itself is the event (wave 22's "the + field changed", and `record_updated` over any field). A problem means the gate cannot be + built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that + passes. + """ + key = trg.get("key") + if key == "enters_view": + tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) + if problem: + return None, problem + import harness.filter_eval as filter_eval + return (lambda row: filter_eval.matches(tree, row, fields)), "" + when = trg.get("when") + if when: + return (lambda row: lane_match(when, row)), "" + return None, "" + + +def _settle_eval(rt, tenant, auto_id, log=print): + """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" + with _SETTLE_LOCK: + buf = _SETTLE.pop((tenant, str(auto_id)), None) + written = dict((buf or {}).get("rows") or {}) + touched = list(written) + if not touched: + return + d = all_definitions(rt).get(str(auto_id)) + trg = (d or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True): + return + gate, problem = _row_gate(rt, d, trg) + if problem: + # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken + # automation, not a quiet no-op — the note is what the surface shows instead of "On". + if (d.get("statusNote") or "") != problem: + _pause_trigger(rt, auto_id, problem) + log(f"[aios-auto] trigger gate: {auto_id} {problem}") + return + fired = [] + if gate is not None: + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) + nxt = set(disarmed) + for rid in touched: + # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 + # tree can read the record's other columns, the just-written cells on top because + # for an ordinary ut column this event is the ONLY place that value exists yet. + if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), + **(written.get(rid) or {})}): + if rid not in disarmed: + fired.append(rid) # false→true THIS evaluation: the edge + nxt.add(rid) + else: + nxt.discard(rid) # left the state — re-armed (Airtable's rule) + if nxt != disarmed: + set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) + if not fired: + return + else: + fired = list(touched) # "the field changed" — the burst is the edge + if len(fired) > FLOOD_LIMIT: + _commit_run(rt, auto_id, "partial", + f"the trigger matched {len(fired)} records in one evaluation — more than " + f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " + f"process them deliberately", {"held": len(fired)}, True) + return + trigger_fire(rt, tenant, auto_id, log=log, rows=fired) + + +def _seed_event_state(rt, defn): + """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so + turning the trigger on fires nothing — the first fire needs a real false→true transition. + Evaluated over the definition rows (the machine-written truth this trigger class watches). + + Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation + reads. Two implementations of "does this row match" is how a seed disagrees with the edge it + is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. + """ + trg = (defn or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or not trg.get("configured"): + return + gate, problem = _row_gate(rt, defn, trg) + if gate is None or problem: + return # no match gate ⇒ nothing to arm; a broken gate seeds nothing + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) + set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) + + +def grid_hook(evt): + """THE listener the human write doors emit into (registered onto + `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may + import both sides). Never raises into a write path; a broken trigger listener must not + break typing into a cell.""" + try: + st = evt.get("st") + if st is None: + return + tenant = str(getattr(st, "key", "") or "royal-imports") + table = str(evt.get("table") or "") + kind = str(evt.get("type") or "") + defs = all_definitions(st) + field = str(evt.get("field") or "") + rid = str(evt.get("rowId") or "") + for aid, d in defs.items(): + trg = (d or {}).get("trigger") or {} + key = str(trg.get("key") or "") + if trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True) \ + or key not in TRIGGER_ROW_KEYS \ + or str(trg.get("table") or "") != table: + continue + if kind == "record_created" and key == "record_created": + hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 + ridn = int(rid) if rid.isdigit() else None + if ridn is None or ridn <= hw: + continue # once per record EVER (A2(4)) — undo-proof + set_state(st, aid, {"rcHighwater": ridn}) + trigger_fire(st, tenant, aid, rows=[rid]) + elif kind == "event_field" and key == "event_field": + # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every + # write on the watched table settles, and the CONDITION decides whether it fires + # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would + # now always take its first arm anyway; leaving a read of a key the validator no + # longer writes is the drift seat this migration exists to close. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif kind == "event_field" and key == "record_updated" and ( + not (trg.get("fields") or []) or field in (trg.get("fields") or [])): + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif key == "enters_view": + # BOTH kinds feed it: a row can enter a view by being edited into its filter or + # by being CREATED already inside it. Listening only to edits would silently miss + # every new record — the half of the definition a reader assumes is covered. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") + + +def form_fired(rt, table_key, row_id, values=None, form_token=""): + """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). + + One submitted form row → every `form_submitted` automation watching that database fires. + Returns the list of automation ids that started, so the door can log what it set off (and so + the gate can assert it, rather than asserting a side effect nobody can see). + + ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, + so it fires triggers exactly like typing into a cell does. The loop-prevention law is not + weakened by that — the engine's own writers still never reach here (only `routes_forms` calls + it), so an automation cannot create a form row and re-fire itself. + + `values` is accepted and unused today: the row is already written when this is called, and + the flow reads it from the table. It stays in the signature because the caller HAS it and a + later refire policy ("only when field X was submitted") needs it — a parameter added later + would mean changing D's call site in a wave that does not own it. + """ + started, table = [], str(table_key or "") + if not table: + return started + tenant = str(getattr(rt, "key", "") or "royal-imports") + for aid, d in all_definitions(rt).items(): + trg = (d or {}).get("trigger") or {} + if trg.get("key") != "form_submitted" or trg.get("paused") \ + or not trg.get("enabled", True) or not trg.get("configured", True): + continue + if str(trg.get("table") or "") != table: + continue + want = str(trg.get("formToken") or "") + if want and not hmac.compare_digest(want, str(form_token or "")): + continue # this automation watches a DIFFERENT form on that table + if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): + started.append(aid) + return started + + +def hook_fire(rt, tenant, auto_id, token, body=None): + """The webhook trigger's decision, separated from FastAPI so the gate can drive it. + Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, + 409 already running, 200 started. + + ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or + sent something that is not JSON. It is written onto a record BEFORE the flow fires — the + flow's actions walk the record the webhook just created, which is the whole point of mapping + it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. + """ + import hmac as _hmac + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return 404, {"error": "unknown_automation"} + trg = defn.get("trigger") or {} + want = str(trg.get("token") or "") + if trg.get("key") != "webhook" or not want: + return 403, {"error": "no_webhook", "message": + "this automation has no webhook trigger"} + if trg.get("paused") or not trg.get("enabled", True): + return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} + if not _hmac.compare_digest(want, str(token or "")): + return 403, {"error": "bad_token", "message": "that token is not valid"} + # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. + row_id, mapped, note = webhook_row(rt, defn, body) + started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) + out = {"started": bool(started), "at": _iso(), + # Answered even when zero, so a caller wiring a map up can see whether their paths + # resolved. Silence here would make "my JSON is not landing" undebuggable from the + # outside, which is the only side the caller is on. + "rowId": row_id or None, "mapped": mapped} + if note: + out["note"] = note # a 200 that wrote no row SAYS which reason it was + return 200, out + + +def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): + """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a + row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the + gate drives this without a network; production leaves them None. + + Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per + tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most + `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. + """ + trg = defn.get("trigger") or {} + if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): + return None + import oauth_connect + creator = str(defn.get("createdBy") or "").strip() or "admin" + token, err = oauth_connect.google_creds(rt, creator) + if err: + if (defn.get("statusNote") or "") != err: + def _note(cur): + cur = cur if isinstance(cur, dict) else {} + dd = cur.get(str(auto_id)) + if dd is not None: + dd["statusNote"] = _s(err, 200) + return cur + rt.update(STORE_KEY, _note, flush="sync") + return None + lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) + reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) + ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) + if lerr: + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}", + {}, True) + seen = set((defn.get("state") or {}).get("emailSeen") or []) + fresh = [m for m in ids if m not in seen] + if not fresh: + return None # nothing new is not a run — no history spam + if len(fresh) > FLOOD_LIMIT: + return _commit_run(rt, auto_id, "partial", + f"{len(fresh)} new emails matched in one poll — more than the " + f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " + f"the query, or press Run now after adjusting it", + {"held": len(fresh)}, True) + fresh = fresh[:EMAIL_MAX_PER_POLL] + rows_in, notes = [], [] + for mid in fresh: + row, rerr = reader(mid) + if row: + rows_in.append(row) + elif rerr: + notes.append(rerr) + table_key = (defn.get("config") or {}).get("targetTable") or "" + if not table_key: + return _commit_run(rt, auto_id, "error", + "the email trigger has nowhere to write — the automation names no " + "target database", {}, False) + ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", + EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) + ut_write_rows(rt, table_key, merged) + new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] + set_state(rt, auto_id, {"emailSeen": new_seen}) + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — " + f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") + if notes: + summary += f" — {notes[0][:100]}" + entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, + True, affected=list(merged)[:200]) + trigger_fire(rt, tenant, auto_id, log=log) + return entry + + +def compose_sentence(defn): + """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the + definition so it cannot lie about what runs.""" + cfg = defn.get("config") or {} + trg = defn.get("trigger") or {} + sched = defn.get("schedule") or {} + kind = defn.get("kind") + if trg.get("key") == "event_field": + # Law 4: no watched field any more, so the sentence stops naming one. It said + # "When None changes on ut_x" the moment the key stopped being stored. + head = f"When a record in {trg.get('table')} matches conditions" + elif trg.get("key") == "record_updated": + head = f"When a record in {trg.get('table')} is updated" + elif trg.get("key") == "ig_profile_match": + head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "record_created": + head = f"When a record is created in {trg.get('table')}" + elif trg.get("key") == "webhook": + head = "When the webhook is called" + elif trg.get("key") == "email": + head = f"When an email matches {trg.get('query')}" + elif sched.get("enabled"): + head = f"{_cron_label(sched.get('cron'))}" + else: + head = "When you press Run now" + if kind == "scrape_db": + host = urlparse(str(cfg.get("url") or "")).hostname or "the page" + body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" + elif kind == "field_instagram": + # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts + # first" / ", anonymous only" described a choice that no longer exists. + body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" + elif kind == "discover_instagram": + body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or DISCOVER_TABLE}") + else: + # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. + # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every + # plain automation would have introduced itself as "search Instagram for up to 0 profiles + # into ut_ig_candidates" — a sentence composed from a definition that says none of it, + # on the one surface whose whole promise is that it "cannot lie about what runs". + n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) + tbl = cfg.get("targetTable") or trg.get("table") or "" + body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) + if n else "do nothing yet — this automation has no actions") + lanes = cfg.get("lanes") or [] + tail = f", then route each record across {len(lanes)} lanes" if lanes else "" + return f"{head}, {body}{tail}." + + +def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it + directly. Returns the run entry, or None when it was already running (the 409).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None + if not _claim(tenant, auto_id): + return None + # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever + # happened to press Run — and above all not to the scheduler, which is not a person and + # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by + # whether a human or a cron got to the first run first. + owner = str(defn.get("createdBy") or "").strip() + if owner and username in MACHINE_OWNERS: + username = owner + try: + _step(tenant, auto_id, "running") + # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it + # first and do not start another profile scrape while it is outstanding: re-running the + # action would buy duplicate engagement reads and reintroduce the timeout this handoff + # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`. + # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one + # rung earlier: a profile snapshot the vendor is still building is paid work, and + # starting a fresh scrape for the same handle would buy the identical row a second time. + # Ahead of the metric collector because the profile IS the thing the run was asked for; + # the engagement batches hang off it. + if _pending_profile_tasks(defn): + state, summary, counts, affected, steps = collect_pending_profile_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + if _pending_metric_tasks(defn): + state, summary, counts, affected, steps = collect_pending_metric_snapshots( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + return _commit_run(rt, auto_id, state, summary, + {k: v for k, v in counts.items() if k != RUN_NOTES_KEY}, + state != "error", affected, steps, + notes=counts.get(RUN_NOTES_KEY)) + runner = RUNNERS.get(defn.get("kind")) + if runner is None: + return _commit_run(rt, auto_id, "error", + f"unknown automation kind {defn.get('kind')!r}", {}, False) + try: + # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. + # `status.step` was already on the wire and D's half renders it; measured against the + # code, it was set exactly ONCE ("running") and never again, so the word would have + # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a + # constant as a progress indicator is worse than rendering nothing: it looks like an + # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. + state, summary, counts, affected, steps = runner( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text), rows=rows) + except Refused as e: + return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") + return _commit_run(rt, auto_id, "error", + f"{type(e).__name__}: {str(e)[:200]}", {}, False) + # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is + # committed, over the records this run actually touched. ONE call site rather than three + # inside the runners: every kind gets actions and endings for free, and a fourth runner + # cannot forget to opt in. + # + # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, + # wired to the wire and covered by twelve gate checks that all called `apply_actions` + # DIRECTLY — so the whole feature was green and unreachable. A person would have built a + # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. + # + # A failing action must not fail the RUN: the machine steps already wrote their rows and + # reporting that as an error would misdescribe what happened. It degrades to `partial` + # with the reason in the summary — the cap_note discipline. + # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`, + # which now reads this name — an assignment only on the success path would turn any + # action failure into a NameError inside the handler that exists to prevent exactly that. + # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked + # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the + # run a person stares at wondering why the automation stopped doing anything. + run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or []) + try: + a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], + username=username, log=log, + step=lambda text: _step(tenant, auto_id, text)) + # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so + # the runner contract keeps its shape, and they must leave before the merge or they + # would be a "count" everywhere downstream. + run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or []) + counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} + if a_counts.get("enrichMetricBatchesPending"): + batches = int(a_counts["enrichMetricBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f" — {batches} post-engagement batch" + f"{'' if batches == 1 else 'es'} still building; Views and other " + "metrics will be collected automatically without another paid read") + if a_counts.get("enrichUnbound"): + # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is + # `partial` because it genuinely did part of its job — it walked the records — and + # the sentence names the ONE thing that has to change, in the two places a person + # can change it. The old behaviour was `ok` with an empty table. + state = "partial" if state != "error" else state + summary += (" — the Instagram step did not run: this database has no profile " + "column. Name one on the step, or mark a text column as the " + "Instagram profile") + if a_counts.get("enrichProfileBatchesPending"): + # ⭐ The paid profile the vendor is still building. Said out loud so a run that + # looks like a failure is read as the handoff it is — the tick finishes it. + n = int(a_counts["enrichProfileBatchesPending"]) + state = "partial" if state != "error" else state + summary += (f" — {n} profile read{'' if n == 1 else 's'} took longer than the " + "wait allows and will be collected automatically, at no extra cost") + if a_counts.get("enrichBlocked"): + state = "partial" if state != "error" else state + # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile + # read(s) were blocked" is the exact string the owner read three mornings running + # before asking "wtf is going on"; it names a quantity and withholds the one + # thing that would let anybody act. The first note is the vendor's own words. + blocked_note = next((n for n in run_notes if ": " in n), "") + summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + + (f". {_s(blocked_note, 220)}" if blocked_note else "")) + # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the + # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it + # logged the cap and rolled up `ok`, so a flow that had silently stopped writing + # looked exactly like one that had nothing to write. Same rule, same sentence shape. + if a_counts.get("createCapped"): + state = "partial" if state != "error" else state + summary = (f"{summary} — {a_counts['createCapped']} row(s) NOT created: a target " + f"database is at its row cap") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") + state = "partial" if state != "error" else state + summary = f"{summary} — the actions did not finish ({type(e).__name__})" + return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, + steps, notes=run_notes) + finally: + _release(tenant, auto_id) + + +def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Start a run on a background thread. False when one is already in flight (→ 409).""" + if running(tenant, auto_id): + return False + th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows), + daemon=True, name=f"automation-{tenant}-{auto_id}") + th.start() + return True + + +# --------------------------------------------------------------------------------------------- +# THE TICK + the in-process scheduler +# --------------------------------------------------------------------------------------------- + +def pending_collect_ids(rt): + """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building. + + ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking + forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and + stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is + true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false), + so `due_ids` never returns it, nothing ever collected it, and the results the account had + already been charged for sat ready at the vendor forever. The sentence was not wrong; it was + describing a run nobody had scheduled. + + A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it + regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something + new*, this says *collect what is already running*, and folding the second into the first would + make an unscheduled automation's paid result depend on someone remembering to press a button. + """ + out = [] + for aid, d in all_definitions(rt).items(): + if not isinstance(d, dict): + continue + if (d.get("trigger") or {}).get("paused"): + continue + pending_discovery = (d.get("kind") == "discover_instagram" + and str((d.get("state") or {}).get("pendingSnapshot") or "").strip()) + # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile + # deferral would be stored and never collected, which is the same defect it fixes wearing + # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was + # measured on is `trigger: manual`. A capability written down but never walked is what + # this whole change is about, so it must not be reintroduced one function later. + if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d): + out.append(aid) + return sorted(out) + + +def due_ids(rt, now=None): + """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" + return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now)) + + +def tick(rt, tenant, now=None, log=print): + """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the + ids started. The email polls are bounded and fail-quiet per automation — one broken + mailbox connection must not stop the tenant's schedules.""" + started = [] + # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the + # tenant has already been charged for; it is collected whether or not this automation is on a + # schedule. `_claim` makes the union safe — an id in both lists starts once. + for aid in dict.fromkeys(list(pending_collect_ids(rt)) + list(due_ids(rt, now))): + if run_async(rt, tenant, aid, username="scheduler", log=log): + started.append(aid) + for aid, d in all_definitions(rt).items(): + if (d.get("trigger") or {}).get("key") == "email": + try: + if email_poll(rt, tenant, aid, d, log=log) is not None: + started.append(aid) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") + # C7's amendment: date-window metrics advance with the tick, so `today` is never staler + # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan. + try: + refresh_metrics(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}") + # ⭐ 2026-08-07 — the relational pass, on the same tick and for the same reason. A rollup on + # table A goes stale when table B gains a row, and A cannot know that happened; the tick is + # the only place that sees both. ⚠ SEPARATE try/except from the metrics above deliberately — + # one pass failing must not silently cancel the other, which a shared block would do. + try: + refresh_relations(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] relation refresh {tenant} failed: {type(e).__name__}: {e}") + # ⭐ THE READ-THROUGH ROLLUPS, on the same tick and for a stronger version of the same reason. + # A linked rollup goes stale when the LINKED TABLE gains a row; a source-backed one goes stale + # when ODOO does — and its window moves on its own besides (a `ytd` column is wrong on 1 + # January without anybody writing anything). The tick is the only place that sees either. + # + # ⛔ ITS OWN try/except, like the two passes above, and the reason is the same one stated + # there: this pass REFUSES loudly on a truncated group set, and a shared block would let that + # honest refusal silently cancel the relational pass that had already succeeded. + # + # ⚠ IT WRITES NOTHING WHEN IT REFUSES — see `rollup_sql`'s header. A half-applied rollup mixes + # two vintages of one column and looks completely normal, which is why the refusal is total. + try: + import rollup_sql + for _tk in list((rt.get(UT_STORE_KEY) or {})): + if rollup_sql.source_fields((rt.get(UT_STORE_KEY) or {}).get(_tk) or {}): + # ⚠ `today` is DEFAULTED INSIDE `compute`, not passed: this scope has no + # such local, and `refresh_metrics` above resolves it the same way. + n = rollup_sql.compute(rt, _tk) + if n: + log(f"[aios-auto] source rollups {tenant}/{_tk}: {n}") + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] source rollup {tenant} failed: {type(e).__name__}: {e}") + if started: + log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") + return started + + +def tick_all(now=None, log=print): + """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop + the others' schedules.""" + from harness import runtime as _rt + out = {} + for slug in _rt.known_tenants(): + try: + out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}") + return out + + +#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer +#: would be a busy loop against a vocabulary that cannot express it. +TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) +_SCHEDULER = [None] + + +def scheduler_loop(log=print): + """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. + + Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and + `verify_seam` import this module through `main.py`, run against fake stores in seconds and + exit. A loop that ticked on entry would fire inside them. + """ + while True: + time.sleep(TICK_SECONDS) + try: + tick_all(log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") + + +def start_scheduler(log=print): + """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** + + ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave + brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 + seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires + two or three times inside a gate that imports `main.py` — and while a tick over a fake store + finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and + mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread + that can move a gate's subject is a flaky suite waiting to happen. + + So it takes the convention `main.py:353` already established for exactly this hazard — + `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup + event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the + external EventBridge tick (R5) does not depend on it either way, since that POSTs the + endpoint rather than riding this thread. + """ + if os.environ.get("AIOS_AUTOMATIONS") != "1": + return False + if _SCHEDULER[0] is not None: + return False + th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, + daemon=True, name="automation-scheduler") + _SCHEDULER[0] = th + th.start() + return True diff --git a/api/connectors_tt.py b/api/connectors_tt.py new file mode 100644 index 0000000000000000000000000000000000000000..d73c0725ed7683d2b42741a06892d332611ba751 --- /dev/null +++ b/api/connectors_tt.py @@ -0,0 +1,293 @@ +"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). + +Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an +automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file +exists at all rather than another thousand lines inside the engine. + +⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 +post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read +live from `GET /datasets/{id}/metadata` for **$0.00** and written down in +`.claude/wiki/research/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a +name below reads through a candidate list it is because the vendor has two names for one fact +(`biography`/`signature`, `region`/`country`), never because the name is uncertain. + +⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — +public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The +vendor key is a key to a SUPPLIER. + +⚠ **THE TRANSPORT IS BORROWED FROM `connectors_ig`, ON PURPOSE.** `bd_call`, `bd_scrape` and +`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not +Instagram's — so re-implementing them here would be a second copy of the deferral handling, the +truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. +The name is wrong and the code is right; extracting a `connectors_bd` transport module is booked +as a PENDING line in B's mailbox rather than done mid-wave, because moving those functions would +land in the middle of three other tickets' diffs. + +⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** + 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared + envelope keys (`timestamp`, `input`) that no metadata call mentions; + 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` + and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one + TikTok claim that matters most (`play_count`) is exactly a declaration. +""" +from __future__ import annotations + +import automation_engine as engine +from connectors_ig import ( # noqa: F401 (bd_call/bd_scrape are re-exported for the runners) + _bd_first_url, + _bd_flag, + _bd_list, + _bd_source_payload, + _first, + _ig_int, + bd_call, + bd_filter_start, + bd_key, + bd_ready, + bd_scrape, +) + +# --------------------------------------------------------------------------------------------- +# THE DATASETS +# --------------------------------------------------------------------------------------------- +# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET +# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a +# full field list, and the two DISCOVERY halves answer **404 for our key**: +# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). +# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's +# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. +TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records +TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields +TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields + +#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` +#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. +TT_POST_TYPE_VIDEO = "video" +TT_POST_TYPE_CONTENT = "content" + +#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept +#: beside the dataset ids because it is the same class of vendor fact. +#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical +#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own +#: batch calls, and the gate asserts the two agree rather than trusting that they do. +TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" + + +def tt_profile_url(handle): + """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" + h = str(handle or "").strip().lstrip("@") + return TT_PROFILE_URL.format(handle=h) if h else "" + + +# --------------------------------------------------------------------------------------------- +# THE FIELD MAPS +# --------------------------------------------------------------------------------------------- +# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared +# in `automation_engine`. The rules they all obey, stated once: +# +# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, +# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` +# returns `None` (never 0) when nothing matches, which is what makes that possible. +# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` +# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) +# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the +# vendor's side is preserved rather than silently discarded while our column model catches up. +# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` +# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they +# ride in `source_payload` where they make no claim. + + +def _tt_str(node, *names): + """The first non-empty string among `names`, or None when the vendor sent nothing. + + ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — + an empty string written into a cell claims "we looked and it is empty". + """ + v = _first(node, *names) + if v is None: + return None + s = str(v).strip() + return s or None + + +def _tt_pct(node, *names): + """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. + + ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to + the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — + measured on the Instagram side, and TikTok sends the same shape on all three of its rates. + """ + v = _first(node, *names) + if v is None: + return None + out = engine._pct100(v) + return out or None + + +def _tt_day(node, *names): + """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" + v = _first(node, *names) + if v is None: + return None + return engine._day(v) or None + + +def _drop_blanks(row): + """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" + return {k: v for k, v in row.items() if v is not None and v != ""} + + +def normalize_profile(node, handle=""): + """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. + + ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: + * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` + populated on 85% of rows and they carry the same text; + * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented + two-letter-ISO description, `country` has no description at all. + ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its + `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the + only count of its kind the dataset offers. + """ + node = node if isinstance(node, dict) else {} + account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "handle": account, + "full_name": _tt_str(node, "nickname"), + "tt_id": _tt_str(node, "id"), + "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), + "bio": _tt_str(node, "biography", "signature"), + "external_url": _bd_first_url(_first(node, "bio_link")), + "verified": _bd_flag(node, "is_verified"), + "is_private": _bd_flag(node, "is_private"), + # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* + # on their own description. It is the closest thing TikTok has to Instagram's + # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so + # the approximation only ever fills a cell the vendor actually answered. + "is_business": _bd_flag(node, "is_commerce_user"), + "followers": _ig_int(_first(node, "followers")), + "following": _ig_int(_first(node, "following")), + "posts_count": _ig_int(_first(node, "videos_count")), + # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no + # nulls). It is not a post-level number and it is not our `likes` column, which is why it + # is stored under a different name. + "likes_received": _ig_int(_first(node, "likes")), + "avg_engagement": _tt_pct(node, "awg_engagement_rate"), + "like_engagement": _tt_pct(node, "like_engagement_rate"), + "comment_engagement": _tt_pct(node, "comment_engagement_rate"), + "country_code": _tt_str(node, "region", "country"), + "region": _tt_str(node, "region"), + "predicted_lang": _tt_str(node, "predicted_lang"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT + # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, + # which is why the append law dates a snapshot by when WE read it. + "account_created_at": _tt_day(node, "create_time"), + "source_payload": _bd_source_payload(node), + }) + + +def tt_post_type(node): + """A TikTok post row → one of OUR three type options, or None. + + ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / + `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the + vendor's token would fail `_clean_field`'s option check on the way in and would put an + untranslated API word in front of a user on the way out. + + So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the + row carries more than one `carousel_images` entry, which is what a carousel IS on either + network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the + token cannot express it, so inferring `carousel` from the word would be inventing a fact. + ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and + that is exactly what would make the wrong default invisible). + """ + raw = str((node or {}).get("post_type") or "").strip().lower() + if raw == TT_POST_TYPE_VIDEO: + return "video" + if raw != TT_POST_TYPE_CONTENT: + return None + images = (node or {}).get("carousel_images") + return "carousel" if isinstance(images, list) and len(images) > 1 else "image" + + +def normalize_post(node): + """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. + + ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and + Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same + fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and + `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would + manufacture a second measurement that a rollup could average or double-count, which is a worse + outcome than the missing column it would paper over. + + ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and + the probe measured them as the same shape. That equality is what lets the comments→posts link + join with NO normaliser, which the Instagram side never had. + ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). + ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and + countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares + no equivalent, so `paid_partnership`/`partner` have no column on this family at all. + """ + node = node if isinstance(node, dict) else {} + shortcode = _tt_str(node, "shortcode", "post_id") + if not shortcode: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "shortcode": shortcode, + "influencer_key": _tt_str(node, "profile_username"), + "posted_at": _tt_day(node, "create_time"), + "type": tt_post_type(node), + "caption": _tt_str(node, "description"), + "url": _tt_str(node, "url"), + "hashtags": _bd_list(node, "hashtags"), + "tagged_location": _tt_str(node, "commerce_info"), + "views": _ig_int(_first(node, "play_count")), + "likes": _ig_int(_first(node, "digg_count")), + "comments": _ig_int(_first(node, "comment_count")), + "shares": _ig_int(_first(node, "num_share_count")), + "saves": _ig_int(_first(node, "collect_count")), + "video_duration": _ig_int(_first(node, "video_duration")), + "source_payload": _bd_source_payload(node), + }) + + +def normalize_comment(node): + """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. + + ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR + `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in + `source_payload`. Reading the array's length instead would be a second, disagreeing answer to + a question the vendor already answers — and it would disagree, because a page of replies is not + all of them. + ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is + text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor + that changes its mind cannot change ours. + ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged + PII) stay in `source_payload` and are promoted to no column, which is the same posture the + Instagram comment schema takes for the same D-24 reason. + """ + node = node if isinstance(node, dict) else {} + comment_key = _tt_str(node, "comment_id") + if not comment_key: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "comment_key": comment_key, + "shortcode": _tt_str(node, "post_id"), + "commented_at": _tt_day(node, "date_created"), + "likes": _ig_int(_first(node, "num_likes")), + "replies": _ig_int(_first(node, "num_replies")), + "source_payload": _bd_source_payload(node), + }) + + +#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them +#: rather than naming three functions, and so adding a fourth dataset is one entry. +TT_NORMALIZERS = { + "tt_profile": normalize_profile, + "tt_post_metrics": normalize_post, + "tt_comments": normalize_comment, +} diff --git a/api/main.py b/api/main.py index dd666f170eca42f46522f164d41ec27f60bbbf04..efe1afba1822a5fdcfff73f45242e532ee1594fb 100644 --- a/api/main.py +++ b/api/main.py @@ -1,532 +1,545 @@ -"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. - -It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, -reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the -JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. - -WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: - - * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared - `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same - anonymous principal and no route could scope anything. Now a signed stateless cookie carries - a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the - user RECORD. `/api/health` stays unauthenticated (liveness only). - * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home - for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). - * **The SSL shim is env-gated.** It used to run at import, unconditionally. - -⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the -session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This -is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB -commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. -""" -import os -import sys -from pathlib import Path - -# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- -# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an -# unverified default context; the HF Space and the container are unaffected. This used to run -# unconditionally at import — i.e. the shipped container disabled TLS verification for every -# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a -# man-in-the-middle away from being someone else's data. It is now opt-in, must be set -# deliberately, and is never on by default. -# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. -if os.environ.get("AIOS_INSECURE_SSL") == "1": - import ssl - ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 - -# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container -# (where the layout differs from the local sibling-dir default). --- -_HERE = Path(__file__).resolve() -_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) -for _p in (str(_RI), str(_HERE.parent)): - if _p not in sys.path: - sys.path.insert(0, _p) - -from dotenv import load_dotenv # noqa: E402 -load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed - -from fastapi import Body, Depends, FastAPI, Request # noqa: E402 -from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 -from fastapi.responses import JSONResponse # noqa: E402 -from fastapi.staticfiles import StaticFiles # noqa: E402 -from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 - -import aios_session # noqa: E402 -import routes_admin # noqa: E402 -import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) -import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) -import routes_auth # noqa: E402 -import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) -import routes_customers # noqa: E402 -import routes_grid # noqa: E402 -import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) -import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) -import routes_nav # noqa: E402 -import routes_pages # noqa: E402 -import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) -import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) -import routes_records # noqa: E402 -import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) -import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) -import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) -import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) -import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) -import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) -import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) -from core import grid_events # noqa: E402 -from deps import Session, module_gate # noqa: E402 - -_WEB_DIST = _HERE.parents[1] / "web" / "dist" - -# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- -# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never -# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the -# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so -# embed == standalone by construction. -# -# TWO REASONS THIS IS HERE and not folded into the routes: -# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR -# layout is broken, and discovering that from a 500 on the first customer request instead of -# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a -# rewrite dropped it and this restores it.) -# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and -# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this -# API have not drifted. The rewrite removed them and the referee went red; retargeting the -# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do -# not delete — but only when the subject genuinely moved. Here it should not have moved). -# -# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from -# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and -# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone -# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the -# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. -import json as _json_contract # noqa: E402 -_FIELDS_PATH = _RI / "aios_grid_fields.json" -if not _FIELDS_PATH.is_file(): - raise FileNotFoundError( - f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " - "platform root (it also carries the data layer this API imports)." - ) -_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) -FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract -# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. -# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be -# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. -_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS - if f["source"] == "odoo" and f["type"] in ("text", "status", "select", - "date")} - -app = FastAPI(title="AIOS web API") - -# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with -# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the -# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. -app.add_middleware(GZipMiddleware, minimum_size=1024) - - -@app.exception_handler(StarletteHTTPException) -def _error_shape(request: Request, exc: StarletteHTTPException): - """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. - - `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a - 422 from a malformed path param) is wrapped here so a client never has to branch on two - different error bodies. - """ - detail = exc.detail - if isinstance(detail, dict) and "error" in detail: - body = detail - else: - body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} - return JSONResponse(body, status_code=exc.status_code, - headers=getattr(exc, "headers", None)) - - -@app.exception_handler(grid_events.StoreUnavailable) -def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): - """THE SAFETY NET for "the store is down" → 503, from anywhere. - - ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a - `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own - try/except — surfaced as a 500. A store outage is a normal operational state and every route - here touches the store at least twice (the workspace read, then the write), so "remember to - wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler - means a store outage can only ever be a 503, whichever call raised it. - - The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this - adapter, which has no durable session dict to degrade into. A 200 over a write that - evaporated is the failure the whole rule exists to prevent. - """ - return JSONResponse( - {"error": {"code": "store_unavailable", - "message": "the tenant store is unavailable — no change was saved"}}, - status_code=503) - - -@app.get("/api/health") -def health(): - """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not - reveal anything about the deployment beyond "the process is up". No version, no tenant list, - no config: a health endpoint is the one URL every scanner finds first. - - ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned - release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated - caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind - a session — and the authoritative copy is the `VERSION` file in the Space repo, which - `deploy_web.space_version()` reads without needing the app to be up at all.""" - return {"ok": True} - - -app.include_router(routes_auth.router) -app.include_router(routes_nav.router) -app.include_router(routes_customers.router) -app.include_router(routes_products.router) -app.include_router(routes_assets.router) -app.include_router(routes_tables.router) -app.include_router(routes_grid.router) -app.include_router(routes_records.router) -# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user -# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so -# a new page is a builder module plus one registry line, and nothing here changes. -app.include_router(routes_pages.router) -app.include_router(routes_admin.router) -app.include_router(routes_automation.router) -app.include_router(routes_keychain.router) -app.include_router(routes_statements.router) -# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. -# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path -# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a -# platform-operator route ever shadowing each other. Every path it declares is gated by -# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. -app.include_router(routes_platform_admin.router) -# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are -# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is -# something every user does with their own views/folders/databases. -app.include_router(routes_alerts.router) -app.include_router(routes_shares.router) -app.include_router(routes_uploads.router) -# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this -# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: -# a router included AFTER it answers 404 forever while importing fine, type-checking fine and -# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. -# -# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished -# features that would have shipped dead. The workers each posted a mount ask and said they would -# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The -# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the -# control is that the enumeration must NAME every router in this file.** -app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated -app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory -# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its -# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so -# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token -# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot -# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. -app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door -# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, -# because the wave-23 scar is exactly this: three finished routers shipped with no include_router -# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks -# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. -app.include_router(routes_odoo_tables.router) - - -# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) -# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic -# behavior. An alias that kept the old auth would be a bypass of everything above it. -_GATE = module_gate(routes_customers.MODULE) - - -@app.get("/api/customers", deprecated=True) -def _customers_alias(session: Session = Depends(_GATE)): - return routes_customers._payload(session) - - -@app.patch("/api/customers/{pid}", deprecated=True) -def _patch_alias(pid: int, body: dict = Body(default=None), - session: Session = Depends(_GATE)): - return routes_customers.patch_customer(pid, body, session) - - -def _assert_contract_floor(): - """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. - - This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the - canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and - the embedded host are serving two different schemas and the grid's own contract has forked — - the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup - on whatever machine is actually running. - """ - import aios_grid - base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] - if [f["key"] for f in base] != [f["key"] for f in FIELDS]: - raise RuntimeError( - "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " - "disagree on the base field set — embed and standalone would serve different schemas. " - "Run aios-web/verify_fields_contract.py.") - - -def _startup_notes(): - """Say the two things an operator must know, once, at import — never in a response body.""" - if aios_session.EPHEMERAL_SECRET: - print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process " - "key. Sessions will not survive a restart and will not work across workers. " - "Set it in production.") - if os.environ.get("AIOS_INSECURE_SSL") == "1": - print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound " - "requests. Local development only; never in a deployed environment.") - - -_assert_contract_floor() -_startup_notes() - - -# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. -# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login -# page cannot live behind a password prompt, so the shell's own assets must be public — which they -# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. -# -# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also -# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are -# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to -# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` -# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), -# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same -# answer as any other path that is not part of the app. -# -# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); -# this guard is what makes the API safe regardless of what the bundle happens to contain. -_DEV_FIXTURES = {"sample_customers.json"} - - -class _AppStatic(StaticFiles): - async def get_response(self, path, scope): - if Path(path).name in _DEV_FIXTURES: - raise StarletteHTTPException(status_code=404, detail="Not Found") - resp = await super().get_response(path, scope) - # Vite content-hashes everything under assets/ (a change is a NEW url), so those are - # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. - # index.html must stay revalidated or a deploy would strand returning browsers on the old - # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. - # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, - # and `startswith("assets/")` silently skipped every asset (measured on the local probe). - if path.replace("\\", "/").lstrip("/").startswith("assets/"): - resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" - else: - resp.headers["Cache-Control"] = "no-cache" - return resp - - -# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets -if _WEB_DIST.is_dir(): - app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") - - -# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- -# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in -# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's -# default envelope; scoped users still pay their own scope's first build, once. -# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends -# run against fakes) can never fire a live Odoo pull. -def _seed_and_sync_store(): - """Bring the analytical store (harness.datastore) LIVE for this container — the app.py - bootstrap, mirrored (2026-07-31, owner item 1). - - `harness.datastore` powers every measure column/condition, and ONLY app.py used to call - `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store - that never existed and every measure cell served blank. Seeding alone was NOT enough either, - and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless - `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands - EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced - forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the - write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded - passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still - serve. - """ - import time as _t - t0 = _t.time() - try: - from harness import datastore as _ds - if _ds.ensure_seed(): - print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") - # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's - # search_reads must never interleave on the shared client's xmlrpc transport. - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - res = {} - for i in range(12): - res = _ds.sync_all(log=lambda *a, **k: None) - print(f"[aios-api] datastore sync pass {i + 1}: " - + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) - if all(v.get("phase") == "live" for v in res.values()): - break - # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see - # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — - # one reconcile pass at boot removes both classes of phantom row before the first - # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of - # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. - try: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception: - pass - print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " - f"(ready={_ds.ready()})") - # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. - # - # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside - # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a - # freshly booted container could spawn the four locked databases was T+30 MINUTES. The - # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s - # across a ~20-minute window over two boots returned the pre-wave schema on all 20 - # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like - # this — a thing that never ran, mistaken for a thing that ran and failed. - # - # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is - # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, - # then the delete reconcile. Calling it from the other thread would race the seed and hit - # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which - # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild - # refuses — correctly, but for a reason that reads like a data loss scare. - # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. - _rebuild_odoo_relational("boot") - except Exception as e: # noqa: BLE001 - print(f"[aios-api] datastore seed/sync skipped: {e}") - - -def _rebuild_odoo_relational(why): - """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or - 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone - could have asked" are different diagnoses. - - ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased - for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran - and was fine" from "the rebuild was never reached". Three failure markers and no success - marker means silence is ambiguous; now it is not. - - ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The - caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and - royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks - straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME - process-global, and rebinding it in a daemon thread under live readers can serve one tenant's - rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. - """ - try: - import odoo_relational as _rel - from harness import runtime as _runtime - if not _rel.is_royal("royal-imports"): - return - _rt = _runtime.get_runtime("royal-imports") - counts = _rel.refresh(_rt, "royal-imports") - # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. - # - # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field - # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to - # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a - # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every - # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the - # tables simply look finished and answer nothing. - # - # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count - # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. - # Calling the inner one here would return a plausible number and change no cell. - # - # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source - # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest - # refusal silently cancel a relational pass that had already succeeded. - try: - import automation_engine as _engine - _engine.refresh_relations(_rt, log=lambda *_a: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") - try: - import rollup_sql as _rollup - for _b, _key, _l, _f in _rel.TABLES: - _rollup.compute(_rt, _key) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") - print(f"[aios-api] odoo relational rebuild done ({why}): " - + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) - except Exception as e: # noqa: BLE001 - # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only - # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a - # `BinderException` about a missing column is a different action from a timeout or an auth - # failure, and the text alone often does not say which it was. - print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") - - -def _store_resync_loop(): - """Keep the analytical mirror current — the API-process stand-in for the re-sync the - Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a - refreshed pool re-reads the freshly synced store.""" - import time as _t - passes = 0 - while True: - _t.sleep(1800) - passes += 1 - try: - from harness import datastore as _ds - import core.odoo as _odoo - try: - _odoo._tlocal.client = _odoo.OdooClient() - except Exception: - pass - _ds.sync_all(log=lambda *a, **k: None) - # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot - # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; - # without it deleted Odoo lines inflate every sum on the mirror FOREVER. - if passes % 4 == 0: - _ds.reconcile_deletes(log=lambda *a, **k: None) - except Exception as e: # noqa: BLE001 - print(f"[aios-api] store resync failed: {e}") - # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt - # AFTER the mirror they are derived from, in the same pass and in that order — deriving - # from a mirror this loop is about to advance would publish a worklist one cycle stale - # every single time. - # - # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must - # not swallow the sync's error message, and — worse the other way — a sync failure must - # not skip a rebuild that had nothing wrong with it. Two independent failures, two - # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) - # - # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` - # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so - # "the wiring exists" and "the data is current" are different claims and only the stamp - # can tell a user which one they are looking at. - # - # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which - # is what made the boot path silent for 30 minutes after every restart; it is now the - # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here - # would be a second thing to keep in step, and the two would answer differently on the - # next ruling — the exact shape the Views top-up was just fixed for on the other side of - # this wave. - _rebuild_odoo_relational("resync") - - -def _prewarm(): - import time as _t - t0 = _t.time() - # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can - # take many minutes of XML-RPC to close, and the first live probe of this arrangement - # showed the pool warm (42s) silently queued behind it — the whole app cold for every - # visitor while a background column channel caught up. Measure cells retry via the - # transient rule until the sync lands; nothing else waits on it. - import threading as _th - _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() - try: - from harness import runtime as _runtime - rt = _runtime.get_runtime("royal-imports") - routes_customers.warm_default(rt) - import pages as _pages - _pages.warm_default(rt) - print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") - except Exception as e: # noqa: BLE001 — boot must not die on a warm-up - print(f"[aios-api] prewarm skipped: {e}") - - -if os.environ.get("AIOS_PREWARM") == "1": - import threading as _threading - _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start() - _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start() +"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at. + +It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical, +reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the +JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`. + +WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing: + + * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared + `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same + anonymous principal and no route could scope anything. Now a signed stateless cookie carries + a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the + user RECORD. `/api/health` stays unauthenticated (liveness only). + * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home + for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c). + * **The SSL shim is env-gated.** It used to run at import, unconditionally. + +⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the +session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This +is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB +commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared. +""" +import os +import sys +from pathlib import Path + +# --- the local Odoo SSL quirk, now BEHIND A GATE ------------------------------------------------- +# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an +# unverified default context; the HF Space and the container are unaffected. This used to run +# unconditionally at import — i.e. the shipped container disabled TLS verification for every +# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a +# man-in-the-middle away from being someone else's data. It is now opt-in, must be set +# deliberately, and is never on by default. +# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop. +if os.environ.get("AIOS_INSECURE_SSL") == "1": + import ssl + ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323 + +# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container +# (where the layout differs from the local sibling-dir default). --- +_HERE = Path(__file__).resolve() +_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform")) +for _p in (str(_RI), str(_HERE.parent)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from dotenv import load_dotenv # noqa: E402 +load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed + +from fastapi import Body, Depends, FastAPI, Request # noqa: E402 +from fastapi.middleware.gzip import GZipMiddleware # noqa: E402 +from fastapi.responses import JSONResponse # noqa: E402 +from fastapi.staticfiles import StaticFiles # noqa: E402 +from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402 + +import aios_session # noqa: E402 +import routes_admin # noqa: E402 +import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox) +import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery) +import routes_auth # noqa: E402 +import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A) +import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A) +import routes_customers # noqa: E402 +import routes_grid # noqa: E402 +import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces) +import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit) +import routes_nav # noqa: E402 +import routes_pages # noqa: E402 +import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane) +import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav) +import routes_records # noqa: E402 +import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases) +import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file) +import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire) +import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router) +import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router) +import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router) +import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router) +from core import grid_events # noqa: E402 +from deps import Session, module_gate # noqa: E402 + +_WEB_DIST = _HERE.parents[1] / "web" / "dist" + +# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ---------- +# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never +# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the +# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so +# embed == standalone by construction. +# +# TWO REASONS THIS IS HERE and not folded into the routes: +# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR +# layout is broken, and discovering that from a 500 on the first customer request instead of +# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a +# rewrite dropped it and this restores it.) +# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and +# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this +# API have not drifted. The rewrite removed them and the referee went red; retargeting the +# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do +# not delete — but only when the subject genuinely moved. Here it should not have moved). +# +# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from +# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and +# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone +# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the +# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from. +import json as _json_contract # noqa: E402 +_FIELDS_PATH = _RI / "aios_grid_fields.json" +if not _FIELDS_PATH.is_file(): + raise FileNotFoundError( + f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the " + "platform root (it also carries the data layer this API imports)." + ) +_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8")) +FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract +# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded. +# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be +# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage. +_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS + if f["source"] == "odoo" and f["type"] in ("text", "status", "select", + "date")} + +app = FastAPI(title="AIOS web API") + +# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with +# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the +# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead. +app.add_middleware(GZipMiddleware, minimum_size=1024) + + +@app.exception_handler(StarletteHTTPException) +def _error_shape(request: Request, exc: StarletteHTTPException): + """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`. + + `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a + 422 from a malformed path param) is wrapped here so a client never has to branch on two + different error bodies. + """ + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + body = detail + else: + body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}} + return JSONResponse(body, status_code=exc.status_code, + headers=getattr(exc, "headers", None)) + + +@app.exception_handler(grid_events.StoreUnavailable) +def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): + """THE SAFETY NET for "the store is down" → 503, from anywhere. + + ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a + `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own + try/except — surfaced as a 500. A store outage is a normal operational state and every route + here touches the store at least twice (the workspace read, then the write), so "remember to + wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler + means a store outage can only ever be a 503, whichever call raised it. + + The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this + adapter, which has no durable session dict to degrade into. A 200 over a write that + evaporated is the failure the whole rule exists to prevent. + """ + return JSONResponse( + {"error": {"code": "store_unavailable", + "message": "the tenant store is unavailable — no change was saved"}}, + status_code=503) + + +@app.get("/api/health") +def health(): + """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not + reveal anything about the deployment beyond "the process is up". No version, no tenant list, + no config: a health endpoint is the one URL every scanner finds first. + + ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned + release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated + caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind + a session — and the authoritative copy is the `VERSION` file in the Space repo, which + `deploy_web.space_version()` reads without needing the app to be up at all.""" + return {"ok": True} + + +app.include_router(routes_auth.router) +app.include_router(routes_nav.router) +app.include_router(routes_customers.router) +app.include_router(routes_products.router) +app.include_router(routes_assets.router) +app.include_router(routes_tables.router) +app.include_router(routes_grid.router) +app.include_router(routes_records.router) +# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user +# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so +# a new page is a builder module plus one registry line, and nothing here changes. +app.include_router(routes_pages.router) +app.include_router(routes_admin.router) +app.include_router(routes_automation.router) +app.include_router(routes_keychain.router) +app.include_router(routes_statements.router) +# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view. +# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path +# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a +# platform-operator route ever shadowing each other. Every path it declares is gated by +# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it. +app.include_router(routes_platform_admin.router) +# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are +# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is +# something every user does with their own views/folders/databases. +app.include_router(routes_alerts.router) +app.include_router(routes_shares.router) +app.include_router(routes_uploads.router) +# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this +# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by: +# a router included AFTER it answers 404 forever while importing fine, type-checking fine and +# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause. +# +# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished +# features that would have shipped dead. The workers each posted a mount ask and said they would +# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The +# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the +# control is that the enumeration must NAME every router in this file.** +app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated +app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory +# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its +# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so +# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token +# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot +# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug. +app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door +# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed, +# because the wave-23 scar is exactly this: three finished routers shipped with no include_router +# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks +# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED. +app.include_router(routes_odoo_tables.router) +# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=` +# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep +# copies), which is what lets a filtered view pick up a row created in another tab, by an automation +# or by a connector sync without re-downloading the world. +# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE: +# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes` +# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48 +# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no +# mounted route — which is precisely the control the wave-23 scar above was written to install. +# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this +# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500). +app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line + + +# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) +# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic +# behavior. An alias that kept the old auth would be a bypass of everything above it. +_GATE = module_gate(routes_customers.MODULE) + + +@app.get("/api/customers", deprecated=True) +def _customers_alias(session: Session = Depends(_GATE)): + return routes_customers._payload(session) + + +@app.patch("/api/customers/{pid}", deprecated=True) +def _patch_alias(pid: int, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + return routes_customers.patch_customer(pid, body, session) + + +def _assert_contract_floor(): + """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from. + + This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the + canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and + the embedded host are serving two different schemas and the grid's own contract has forked — + the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup + on whatever machine is actually running. + """ + import aios_grid + base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")] + if [f["key"] for f in base] != [f["key"] for f in FIELDS]: + raise RuntimeError( + "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " + "disagree on the base field set — embed and standalone would serve different schemas. " + "Run aios-web/verify_fields_contract.py.") + + +def _startup_notes(): + """Say the two things an operator must know, once, at import — never in a response body.""" + if aios_session.EPHEMERAL_SECRET: + print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process " + "key. Sessions will not survive a restart and will not work across workers. " + "Set it in production.") + if os.environ.get("AIOS_INSECURE_SSL") == "1": + print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound " + "requests. Local development only; never in a deployed environment.") + + +_assert_contract_floor() +_startup_notes() + + +# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose. +# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login +# page cannot live behind a password prompt, so the shell's own assets must be public — which they +# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing. +# +# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also +# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are +# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to +# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData` +# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"), +# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same +# answer as any other path that is not part of the app. +# +# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox); +# this guard is what makes the API safe regardless of what the bundle happens to contain. +_DEV_FIXTURES = {"sample_customers.json"} + + +class _AppStatic(StaticFiles): + async def get_response(self, path, scope): + if Path(path).name in _DEV_FIXTURES: + raise StarletteHTTPException(status_code=404, detail="Not Found") + resp = await super().get_response(path, scope) + # Vite content-hashes everything under assets/ (a change is a NEW url), so those are + # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle. + # index.html must stay revalidated or a deploy would strand returning browsers on the old + # bundle; ETag/304 makes that revalidation a header exchange, not a transfer. + # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes, + # and `startswith("assets/")` silently skipped every asset (measured on the local probe). + if path.replace("\\", "/").lstrip("/").startswith("assets/"): + resp.headers["Cache-Control"] = "public, max-age=31536000, immutable" + else: + resp.headers["Cache-Control"] = "no-cache" + return resp + + +# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets +if _WEB_DIST.is_dir(): + app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web") + + +# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) ----------------------- +# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in +# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's +# default envelope; scoped users still pay their own scope's first build, once. +# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends +# run against fakes) can never fire a live Odoo pull. +def _seed_and_sync_store(): + """Bring the analytical store (harness.datastore) LIVE for this container — the app.py + bootstrap, mirrored (2026-07-31, owner item 1). + + `harness.datastore` powers every measure column/condition, and ONLY app.py used to call + `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store + that never existed and every measure cell served blank. Seeding alone was NOT enough either, + and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless + `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands + EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced + forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the + write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded + passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still + serve. + """ + import time as _t + t0 = _t.time() + try: + from harness import datastore as _ds + if _ds.ensure_seed(): + print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s") + # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's + # search_reads must never interleave on the shared client's xmlrpc transport. + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + res = {} + for i in range(12): + res = _ds.sync_all(log=lambda *a, **k: None) + print(f"[aios-api] datastore sync pass {i + 1}: " + + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items()))) + if all(v.get("phase") == "live" for v in res.values()): + break + # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see + # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut — + # one reconcile pass at boot removes both classes of phantom row before the first + # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of + # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran. + try: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception: + pass + print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s " + f"(ready={_ds.ready()})") + # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE. + # + # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside + # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a + # freshly booted container could spawn the four locked databases was T+30 MINUTES. The + # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s + # across a ~20-minute window over two boots returned the pre-wave schema on all 20 + # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like + # this — a thing that never ran, mistaken for a thing that ran and failed. + # + # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is + # the exact line where the mirror has finished advancing — seed, up to twelve sync passes, + # then the delete reconcile. Calling it from the other thread would race the seed and hit + # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which + # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild + # refuses — correctly, but for a reason that reads like a data loss scare. + # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. + _rebuild_odoo_relational("boot") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] datastore seed/sync skipped: {e}") + + +def _rebuild_odoo_relational(why): + """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or + 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone + could have asked" are different diagnoses. + + ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased + for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran + and was fine" from "the rebuild was never reached". Three failure markers and no success + marker means silence is ambiguous; now it is not. + + ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The + caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and + royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks + straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME + process-global, and rebinding it in a daemon thread under live readers can serve one tenant's + rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify. + """ + try: + import odoo_relational as _rel + from harness import runtime as _runtime + if not _rel.is_royal("royal-imports"): + return + _rt = _runtime.get_runtime("royal-imports") + counts = _rel.refresh(_rt, "royal-imports") + # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. + # + # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field + # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to + # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a + # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every + # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the + # tables simply look finished and answer nothing. + # + # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count + # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes. + # Calling the inner one here would return a plausible number and change no cell. + # + # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source + # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest + # refusal silently cancel a relational pass that had already succeeded. + try: + import automation_engine as _engine + _engine.refresh_relations(_rt, log=lambda *_a: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") + try: + import rollup_sql as _rollup + for _b, _key, _l, _f in _rel.TABLES: + _rollup.compute(_rt, _key) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") + print(f"[aios-api] odoo relational rebuild done ({why}): " + + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) + except Exception as e: # noqa: BLE001 + # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only + # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a + # `BinderException` about a missing column is a different action from a timeout or an auth + # failure, and the text alone often does not say which it was. + print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") + + +def _store_resync_loop(): + """Keep the analytical mirror current — the API-process stand-in for the re-sync the + Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a + refreshed pool re-reads the freshly synced store.""" + import time as _t + passes = 0 + while True: + _t.sleep(1800) + passes += 1 + try: + from harness import datastore as _ds + import core.odoo as _odoo + try: + _odoo._tlocal.client = _odoo.OdooClient() + except Exception: + pass + _ds.sync_all(log=lambda *a, **k: None) + # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot + # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; + # without it deleted Odoo lines inflate every sum on the mirror FOREVER. + if passes % 4 == 0: + _ds.reconcile_deletes(log=lambda *a, **k: None) + except Exception as e: # noqa: BLE001 + print(f"[aios-api] store resync failed: {e}") + # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt + # AFTER the mirror they are derived from, in the same pass and in that order — deriving + # from a mirror this loop is about to advance would publish a worklist one cycle stale + # every single time. + # + # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must + # not swallow the sync's error message, and — worse the other way — a sync failure must + # not skip a rebuild that had nothing wrong with it. Two independent failures, two + # independent logs. (`_rebuild_odoo_relational` carries its own handlers.) + # + # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed` + # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so + # "the wiring exists" and "the data is current" are different claims and only the stamp + # can tell a user which one they are looking at. + # + # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which + # is what made the boot path silent for 30 minutes after every restart; it is now the + # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here + # would be a second thing to keep in step, and the two would answer differently on the + # next ruling — the exact shape the Views top-up was just fixed for on the other side of + # this wave. + _rebuild_odoo_relational("resync") + + +def _prewarm(): + import time as _t + t0 = _t.time() + # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can + # take many minutes of XML-RPC to close, and the first live probe of this arrangement + # showed the pool warm (42s) silently queued behind it — the whole app cold for every + # visitor while a background column channel caught up. Measure cells retry via the + # transient rule until the sync lands; nothing else waits on it. + import threading as _th + _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start() + try: + from harness import runtime as _runtime + rt = _runtime.get_runtime("royal-imports") + routes_customers.warm_default(rt) + import pages as _pages + _pages.warm_default(rt) + print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s") + except Exception as e: # noqa: BLE001 — boot must not die on a warm-up + print(f"[aios-api] prewarm skipped: {e}") + + +if os.environ.get("AIOS_PREWARM") == "1": + import threading as _threading + _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start() + _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start() diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 64c8895fc05650b53978df0b9067f5d1c9a6ba1b..ca2f5090764d4ddd6c19471c0b8df08078a59795 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -713,13 +713,40 @@ def read_products(cur): def read_customers(cur, excluded=None): - """[(row dict)] — every partner Odoo has TRANSACTED with, keyed on the `res.partner` id. - - ⭐ THE POPULATION IS THE UNION OF THE TWO DOCUMENT UNIVERSES, and both legs are load-bearing: - Amazon books as direct invoices with no sale order (the `odoo-api` gotcha), so a sale-order - leg alone would silently drop a real customer. Partners with no document at all are left out - on purpose — a row that can never appear in any topic has nothing to roll up and would render - as an all-blank line. + """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id. + + ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing. + + The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale + order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer. + + ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 — + the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old + docstring said partners with no document are *"left out on purpose — a row that can never + appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same + join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records** + (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer + a salesperson has not sold to yet is exactly the row a prospecting view needs. + + ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the + document legs for the rank leg would drop **16 partners that hold posted documents** (7 + archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry + `partner_id` LINKS straight back here — so those links would dangle with nothing reporting + it. When other tables point AT a population, a widening must be a SUPERSET. + + ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same + `columns()`/`_col` discipline every other optional column here uses — but note the difference + honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the + POPULATION back to the document union. It degrades to today's behaviour rather than to an + empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the + column is missing so the narrowing can never pass for done. + + ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above + states the rule for this module and it applies with more force to a POPULATION than to a name + list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its + mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this + time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the + weather. One source, always present at spawn time: the mirror. ⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR partners; sourcing a customer registry from its own receivables is what kept most Odoo ids @@ -732,6 +759,11 @@ def read_customers(cur, excluded=None): agent = ("ag.name" if "agent_id" in have else "NULL") agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL") join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "") + # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an + # archived prospect out, and a rank test without it would re-admit the 47 archived partners + # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. + rank_leg = (" OR (p.customer_rank > 0 AND p.active) " + if {"customer_rank", "active"} <= have else "") sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " "FROM res_partner p " @@ -740,7 +772,8 @@ def read_customers(cur, excluded=None): f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL " " UNION " " SELECT partner_id FROM account_move " - f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)") + f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)" + f"{rank_leg}") out = [] for r in cur.execute(sql).fetchall(): (pid, name, city, state, country, agent, agent_id) = r diff --git a/api/providers.py b/api/providers.py index 0eecba5f274bee8868a4086274519abaa07fa186..5e1a78fdcf84345be650175a85ea17986f9d9d9b 100644 --- a/api/providers.py +++ b/api/providers.py @@ -100,6 +100,26 @@ PROVIDERS: dict[str, Provider] = { "creator (identical for two shortcodes in one call, ticking upward between " "calls) and never populates video_play_count. Their own documented example " "account no longer reproduces it."), + # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our + # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own + # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. + "tt_profile": Capability(True, _COST_BRIGHTDATA, + "40 fields incl. bio/engagement rates/region"), + # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. + # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says + # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post + # record and a separate view rung would be a second bill for a number we already have. + # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: + # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an + # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it + # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a + # quiet fallback bolted onto this one. + "tt_post_metrics": Capability( + True, _COST_BRIGHTDATA, + "likes + comments + shares + saves, and play_count inline (DECLARED " + "no-empties-or-zeros, UNPROVEN until one live pull)"), + "tt_comments": Capability(True, _COST_BRIGHTDATA, + "separate paid dataset, opt-in — 17 fields"), }), "apify": Provider( key="apify", label="Apify", key_env="AIOS_APIFY_KEY", @@ -121,6 +141,17 @@ DEFAULT_CHAINS: dict[str, tuple] = { # would be a lie that costs a wasted call on every single post. "ig_post_views": ("apify",), "ig_comments": ("brightdata",), + # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. + # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a + # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data + # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap + # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here + # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, + # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; + # it is not declared until a runner walks it. + "tt_profile": ("brightdata",), + "tt_post_metrics": ("brightdata",), + "tt_comments": ("brightdata",), } diff --git a/api/routes_automation.py b/api/routes_automation.py index 2695ff6760f55dffce3f60192c4a31105f6f6982..e002faa05be91844a0df3baa69ad31de2e42c15b 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -127,6 +127,12 @@ def _triggers_vocab(session): # still ACCEPTS it either way — readiness is a deployment fact, not a validity one, which # is the same split `email` already makes. "ig_profile_match": (engine.bd_ready(), "" if engine.bd_ready() else "configure_brightdata"), + # ⭐ WAVE 29 (D-9 / R1) — TikTok, live. ⛔ IT NEEDS ITS OWN ROW EVEN THOUGH THE ANSWER IS + # IDENTICAL, and the reason is the `.get(k, (True, ""))` default below: a trigger this dict + # forgets is reported READY, so a deployment with no vendor key would offer TikTok search + # as configured and the search would find nothing. Same key, same readiness bit, stated. + "tiktok_profile_match": (engine.bd_ready(), + "" if engine.bd_ready() else "configure_brightdata"), } #: ⭐ WAVE 24 — the server's own one-line description per trigger. C-TYPES: the picker renders #: THIS under the option, because a CLIENT paraphrase of a server vocabulary is a second copy @@ -212,6 +218,14 @@ def _tick_state(): "source": "in-process" if inproc else ("external" if ext else "")} +#: W29-T01 — the Board retirement ran, per tenant, this process. Same shape and same reasoning as +#: `_C8_MIGRATED` below: a one-way cleanup of data no living code path creates any more, whose +#: cost is a full walk of every row-cell in the tenant and whose result on pass 2..N is always +#: "nothing to do". ⚠ A module-global keyed by tenant is deliberately NOT reset by a table create +#: or delete — a new database cannot contain the legacy stage cells this retires. +_BOARD_RETIRED = set() + + @router.get("/automations") def list_automations(session: Session = Depends(_GATE)): """`{automations: [...], kinds: [...], cronPresets: [...]}` — the rail's whole payload. @@ -219,10 +233,29 @@ def list_automations(session: Session = Depends(_GATE)): The vocabularies ride WITH the list rather than sitting in a client constant: the cron presets and the kind list are the server's, and a client copy of either is a thing that goes stale silently (the editor would offer a preset the parser rejects).""" - # Idempotent Board retirement removes only engine-marked stage fields and legacy Board state. - # User-created Status/Stage columns remain intact. - if session.runtime.available(): - engine.retire_automation_board_state(session.runtime) + # ⭐⭐ WAVE 29 (W29-T01, owner item 5: "Automation takes a while to appear"). THE COMPLAINT WAS + # THIS ROUTE, and the cost was never the automations: it read the whole `user_tables` bucket + # THREE TIMES per call — once for the stage-field retirement scan, twice more inside + # `discover_default_table`. That bucket's documented ceiling is 35.8 MB / ~1.4 s to serialize + # (`automation_engine.py` header) and `core/store.py` re-serializes on EVERY `.get()`, hit or + # miss, UNDER THE STORE LOCK — so the reads also serialize behind each other. ~4 s of deep + # copying to render a rail that shows a name and a toggle. + # + # Now: ONE read, lent to both callers. And the retirement SCAN — O(all row-cells in the + # tenant), which hits its own `if not stage_keys: return` only AFTER walking every row of + # every table — runs once per tenant per process, mirroring the `_C8_MIGRATED` guard below. + # + # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent": + # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s + # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can + # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either — + # wave 26's R6 deleted the board that made them. + tables = engine.ut_all(session.runtime) if session.runtime.available() else None + if tables is not None and session.tenant not in _BOARD_RETIRED: + _BOARD_RETIRED.add(session.tenant) + # Idempotent Board retirement removes only engine-marked stage fields and legacy Board + # state. User-created Status/Stage columns remain intact. + engine.retire_automation_board_state(session.runtime, tables=tables) defs = engine.all_definitions(session.runtime) items = [_wire(d, session.tenant) for _, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] @@ -272,6 +305,13 @@ def list_automations(session: Session = Depends(_GATE)): # validator refuses. "seedSources": list(engine.SEED_SOURCES), "seedMaxRows": engine.SEED_MAX_ROWS, + # W29-T01: `tables` is the snapshot read once at the top of this handler. + # ⚠ IT WAS TAKEN BEFORE THE RETIREMENT WROTE, and that is deliberate and + # harmless: retirement only removes `stage_auto_*` FIELDS and pops those + # same keys off rows, and the election reads the preset-profile vocabulary + # (`handle` + N preset keys) and whether a table has ANY rows. A stage key + # is in neither set, and no row is ever deleted — so the pre-write + # snapshot and the post-write bucket cannot elect different tables. # 2026-08-10 — the OFFER must name the table the SAVE will actually use. # This was the bare `DISCOVER_TABLE` constant while `create`/`patch` now # resolve a targetless discovery flow to the profile database the tenant @@ -279,7 +319,7 @@ def list_automations(session: Session = Depends(_GATE)): # save would have written somewhere else — a default that disagrees with # itself across two panels, which is the shape wave 25's R2 fixed for # `targetTable` vs the action's `table`. - "table": (engine.discover_default_table(session.runtime) + "table": (engine.discover_default_table(session.runtime, tables=tables) or engine.DISCOVER_TABLE)}, "storeAvailable": bool(session.runtime.available()), # Wave 22 C3 — the trigger vocabulary, session-scoped because email readiness is a @@ -299,7 +339,19 @@ def list_automations(session: Session = Depends(_GATE)): "maxCondDepth": engine.MAX_COND_DEPTH, "maxCondChildren": engine.MAX_COND_CHILDREN, "maxGroupDepth": engine.MAX_GROUP_DEPTH, - "maxActions": engine.MAX_ACTIONS}, + "maxActions": engine.MAX_ACTIONS, + # ⭐ W29-T09 (owner item 11) — THE POST-GROUP VOCABULARY, on the wire for the + # same reason `cronPresets` is: `clean_post_groups` REFUSES a type it does + # not know, so a client that invented one would build a config the save door + # rejects with a sentence about a word the person never typed. Both halves + # ride — the stored key and the label a person reads — because a client-side + # translation of `video` into "Reels" is a second copy of this list. + "postTypes": [{"key": t, "label": engine.POST_TYPE_LABELS.get(t, t)} + for t in engine.POST_TYPES], + # The ceiling a group's limit is judged against. `clean_post_groups` bounds a + # group by the action's OWN `maxPosts`, not by a constant, so the control can + # only warn honestly if it reads the same number the validator uses. + "maxPostsPerPull": engine.MAX_POSTS_PER_PULL}, # Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see # `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie. "tick": _tick_state()} diff --git a/api/routes_changes.py b/api/routes_changes.py new file mode 100644 index 0000000000000000000000000000000000000000..3ec0a9d785429f9a4d52c06b6547e0a3d57c1ed4 --- /dev/null +++ b/api/routes_changes.py @@ -0,0 +1,159 @@ +"""routes_changes.py — THE CHANGE TOKEN (wave 29, item 20 / ruling R11, contract C6). + + GET /api/v1/changes?scope= + -> {"scope": "ut_leads", + "tokens": {"rows": "b3f1c9:14", "overlay": "b3f1c9:3"}, + "backend": "hf"} + +The owner's report: *a record created in another tab, by an automation, or by a connector sync +does not appear in a filtered view until you reload.* The client had no way to ask whether +anything had changed except to re-download everything, so it never asked. + +⛔ WHY THIS ROUTE EXISTS INSTEAD OF THE OBVIOUS "JUST POLL THE ROWS ENDPOINT", and the arithmetic +is the whole argument. `GET /tables/{key}/rows` costs THREE full-tenant deep copies under ONE +lock, on ONE uvicorn process (the container runs no `--workers`), against a documented ceiling of +35.8 MB / ~1.4 s per bucket ⇒ roughly 4.2 s of lock-held CPU per poll, so TWO TABS SATURATE THE +SERVER and every unrelated write queues behind them. This handler performs ZERO `store.get()` +calls: `revision_of` below reads an in-memory counter that `Store.put`/`Store.update` bump at the +moment a write is accepted. `verify_live_workspace.py`'s server half counts +`get`/`_read_strict`/`_download` on this path and goes RED at one — because a token endpoint that +deep-copies pays the identical 1.4 s and wins nothing. + +⭐ IT REACHES ALL THREE WRITERS THE OWNER NAMED, and that is a property of the QUESTION, not of +the plumbing: it asks *"did this bucket change"*, never *"did somebody emit an event"*. The event +seam (`user_tables.ROW_HOOKS`) fires only from HUMAN write doors by construction — the automation +engine's own writer deliberately stays silent, and `verify_automation.py` has a negative control +enforcing that silence. A push channel could therefore only ever carry 1 of the 3 writers. A +counter on the store sees the automation and the connector sync for free, and the loop-prevention +law is untouched. + +WHAT WALLS THIS, and why it is not `may_open`: + + * The TENANT is the wall, and it is structural. Every bucket name is namespaced through the + session's OWN runtime (`TenantRuntime.store_key`), so a scope naming another tenant's table + resolves to a bucket this tenant does not have and reads 0. There is no cross-tenant answer + to give. + * A per-table `may_open` check would cost the very deep copy this route exists to avoid — + `user_tables.get()` reads the whole `user_tables` bucket to answer it. + * ⭐ And it would protect nothing, because `user_tables.STORE_KEY` is ONE bucket holding EVERY + `ut_*` table in the tenant: the rows token for `ut_a` and `ut_b` is literally the same + number. There is no per-table fact here to withhold. (Which also means a write to any one of + them invites every open tab to re-read its own table — over-signalling, i.e. the safe + direction: the cost is one extra fetch, never a missed change.) + +So: a valid session, a scope sanitised to a bucket-safe shape, and nothing else. +""" +from fastapi import APIRouter, Depends + +import core.shared_overlay as shared_overlay +import core.store as store +import core.user_tables as user_tables +import modules.customer_data as customer_data +import modules.product_data as product_data + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: A scope reaches a STORE KEY, so it is sanitised as strictly as one: lowercase, digits and +#: underscores only. Not paranoia about SQL — `Store` addresses files and `PgStore` binds +#: parameters — but a scope is user-supplied text that becomes part of a path in the HF backend, +#: and "it cannot escape today" is a property of the current backend rather than of this route. +_SCOPE_MAX = 64 + + +def revision_of(handle, name): + """`{'rev','updated_at','token'}` for bucket `name` as THIS caller addresses it — or None. + + ⭐ ONE resolver, and the gate calls the HANDLER rather than reproducing this dance, so the + route and its control cannot disagree about which physical key was read. A control that + re-derives the address is how a gate ends up certifying a key nobody serves. + + `handle` is whatever the caller holds — a tenant runtime, a `Store`, or `core.store` itself. + The runtime is recognised DUCK-TYPED (`store_key` + `store_handle`) rather than imported: + `store_key()` is the ONE place a tenant's namespace prefix is applied, and applying it here a + second time is how one tenant reads another's counter. + + ⛔ None — never a fabricated token — when the active backend publishes no revision. `PgStore` + maintains `rev` in `store_kv` and does not yet expose it, so under `STORE_BACKEND=pg` this + answers None, the client stops polling, and the app behaves exactly as it does today. A token + frozen at 0 would instead promise liveness the backend is not delivering. + """ + target, key = handle, str(name) + namespaced = getattr(handle, "store_key", None) + if callable(namespaced): # a tenant runtime owns the namespace prefix + key = namespaced(name) + bound = getattr(handle, "store_handle", None) + target = bound if bound is not None else store._d() + fn = getattr(target, "revision", None) + if not callable(fn): + return None + try: + return fn(key) + except Exception: + # A backend that raises here must not take the poll down with it: the caller degrades to + # "no token for this bucket", which is the same honest null the pg path returns. + return None + + +def _clean_scope(raw): + s = str(raw or "").strip().lower() + if not s or len(s) > _SCOPE_MAX or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789_" + for c in s): + return "" + return s + + +def buckets_for(scope): + """`{label: store bucket}` for one topic scope — the map the CLIENT never gets to write. + + ⛔ Every value is imported from the module that WRITES it, never re-spelled here. + `user_tables.STORE_KEY`, `customer_data.TABLE_KEY` and `product_data.TABLE_KEY` are the + literal keys those modules read and update, so a rename moves this map with it. A hand-copied + `'customer_table_workspace'` would keep answering 200 with a token for a bucket nobody writes + — a poll that costs nothing, changes never, and looks exactly like "there were no changes". + + An absent label means "no store residency": `customer` and `product` rows come from Odoo + through a 15-minute server cache, which R11 deliberately leaves alone. The client reads a + missing/null token as "do not poll this", never as "unchanged". + """ + s = _clean_scope(scope) + if not s: + return None + #: The shared stratum's bucket is asked FOR rather than spelled, so C5's `BUCKET_SUFFIX` has + #: exactly one definition and a rename cannot leave the poller watching a dead key. + if s.startswith(user_tables.KEY_PREFIX): + workspace = f"{s}_table_workspace" + elif s == "customer": + workspace = customer_data.TABLE_KEY + elif s == "product": + workspace = product_data.TABLE_KEY + else: + return {} + names = {"overlay": workspace, "shared": shared_overlay.bucket(workspace)} + if s.startswith(user_tables.KEY_PREFIX): + names["rows"] = user_tables.STORE_KEY + return names + + +@router.get("/changes") +def changes(scope: str = "", session: Session = Depends(require_session)): + """One tiny read per tab per interval. The response is small, uncacheable and boring.""" + names = buckets_for(scope) + if names is None: + raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer") + if not names: + # ⛔ A well-formed scope this map does not know is a 404, NOT an empty `tokens` object. + # An empty map is indistinguishable from "nothing here ever changes", so a topic added + # later would poll forever against silence and read as a working live view that never + # updates. Loud beats plausible. + raise err(404, "unknown_scope", f"nothing here watches {_clean_scope(scope)!r}") + tokens = {} + for label, bucket in names.items(): + rev = revision_of(session.runtime, bucket) + # ⛔ A NULL IS AN ANSWER, not an omission. The pg backend maintains `rev` in `store_kv` + # and does not yet expose it, so after D-4 flips this reads None until `PgStore` grows a + # `revision()`. Sending null and letting the client stop polling is the honest failure; + # a token frozen at 0 would promise liveness the backend is not delivering. + tokens[label] = rev.get("token") if isinstance(rev, dict) else None + return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()} diff --git a/api/routes_forms.py b/api/routes_forms.py index 3fc769ea39091243657794f4cefb37775bfb33e1..545819fa7d97216f1136881ba4633413abc7aa44 100644 --- a/api/routes_forms.py +++ b/api/routes_forms.py @@ -32,15 +32,32 @@ bypassed. Booked as DEBT rather than described as a guarantee — a rate limit t being one is worse than none. """ import hmac +import os import secrets import time -from fastapi import APIRouter, Request +from fastapi import APIRouter, Depends, Request -from deps import err +from deps import Session, err, require_session router = APIRouter(prefix="/api/v1") +#: ⭐⭐ WAVE-29 R7 — THE TOKEN INDEX, one bucket per tenant: `{token: {table, view}}`. +#: +#: ⛔ WHY THE TOKEN IS NOT IN THE FORM SPEC, which is where wave 23 assumed it would live. The spec +#: rides `view.config.display.form`, which a BROWSER writes — so a token stored beside it is a +#: token a browser can choose. `_resolve` walks tenants and answers with the FIRST match, so a +#: tenant that set its token to a value it had seen elsewhere would quietly receive another +#: tenant's submissions. Nothing about that is exotic: it needs one `PATCH` and a token somebody +#: pasted into a chat. Minting server-side into a bucket the client cannot write closes it by +#: construction rather than by validation, and `aios_grid._clean_form` drops a wire `token`. +#: +#: ⭐ It is also FASTER than what it replaces. The old resolution walked every table's workspace, +#: every user's stratum and every view of every tenant looking for a matching token — booked in +#: this file's own header as "a FULL SCAN, deliberately, and its cost is BOOKED". Now it is one +#: small dict read per tenant, and the view is loaded ONCE, after the token has already matched. +TOKENS_KEY = "form_tokens" + #: The mint. 24 bytes url-safe = 32 characters, 192 bits — the same shape the automation hook's #: per-automation token uses, so there is ONE token strength in the product rather than two. TOKEN_BYTES = 24 @@ -126,35 +143,40 @@ def _same(a: str, b: str) -> bool: # index becomes worth its risk when either number grows. -def _iter_form_views(rt): - """Yield `(table_key, view_id, form_spec)` for every form-enabled view in one tenant. +def _find_view(rt, table_key: str, view_id: str): + """`(view, owner_username)` for one view in one table's workspace, or `(None, None)`. - A form spec lives at `view.config.display.form` and carries the token — see W23-C9. Reads - only; nothing here writes, so a malformed stratum is skipped rather than repaired. + The bucket is `{username: {views: {id: view}}}` — one stratum per user plus the shared one — + and a view id is unique across them, so the first match is the view. """ - import core.user_tables as user_tables - try: - tables = user_tables.all_tables(st=rt) or {} + bucket = rt.get(f"{table_key}_table_workspace") or {} except Exception: # noqa: BLE001 - return - for key in list(tables): - try: - bucket = rt.get(f"{key}_table_workspace") or {} - except Exception: # noqa: BLE001 - continue - if not isinstance(bucket, dict): + return None, None + if not isinstance(bucket, dict): + return None, None + for owner, ws in bucket.items(): + if not isinstance(ws, dict): continue - for ws in bucket.values(): - if not isinstance(ws, dict): - continue - for view_id, view in (ws.get("views") or {}).items(): - if not isinstance(view, dict): - continue - display = ((view.get("config") or {}).get("display") or {}) - spec = display.get("form") - if isinstance(spec, dict) and spec.get("token"): - yield str(key), str(view_id), spec + view = (ws.get("views") or {}).get(view_id) + if isinstance(view, dict): + return view, str(owner) + return None, None + + +def _spec_of(view) -> dict: + """The form spec on a view, or `{}`. One place knows the key path.""" + spec = (((view or {}).get("config") or {}).get("display") or {}).get("form") + return spec if isinstance(spec, dict) else {} + + +def _tokens(rt) -> dict: + """This tenant's `{token: {table, view}}` index, always a dict.""" + try: + found = rt.get(TOKENS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} def _resolve(token: str): @@ -164,6 +186,12 @@ def _resolve(token: str): tenant whose store answered would make "the third tenant's form" depend on the first two being healthy, and a form that stops working because somebody else's store blipped is the kind of failure nobody can reproduce. + + ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. A token that resolves to a view which has since + been DELETED, or whose form spec has been removed, answers the same 403 as a token that never + existed — the index is written by the mint route and nothing deletes the tenant's views on its + behalf, so "the row is in the index" cannot be allowed to mean "the form is live". A stale + pointer outliving its view is exactly how a door that was closed stays open. """ from harness import runtime as _rt @@ -174,9 +202,17 @@ def _resolve(token: str): rt = _rt.get_runtime(slug) except Exception: # noqa: BLE001 continue - for table_key, _view_id, spec in _iter_form_views(rt): - if _same(str(spec.get("token") or ""), token): - return rt, slug, table_key, spec + for stored, where in _tokens(rt).items(): + if not _same(str(stored), token) or not isinstance(where, dict): + continue + table_key = str(where.get("table") or "") + view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) + spec = _spec_of(view) + # No view, no questions, no form. Checked HERE rather than by the caller so every + # public route inherits it from one place. + if not table_key or not spec.get("fields"): + return None + return rt, slug, table_key, spec return None @@ -192,6 +228,7 @@ def _public_form(rt, table_key: str, spec: dict) -> dict: somebody arrange these questions, and re-deriving the order from the schema would silently rearrange a form every time a column was added. """ + import aios_grid import core.user_tables as user_tables defn = user_tables.get(table_key, st=rt) or {} @@ -204,6 +241,24 @@ def _public_form(rt, table_key: str, spec: dict) -> dict: # and never an error: losing one column must not take a live form offline. if not f or f.get("source") not in (None, "overlay"): continue + # ⭐ WAVE-29 R7 — TWO WALLS, AND NEITHER IS REDUNDANT. + # + # (1) The TYPE allow-list: a form may only ask questions this door can answer honestly. + # `_clean_values` below coerces twelve types and passes everything else through as + # text, so an `image`, a `json` document or a `link` reached through a form would be + # a string typed into a column that means something else. + # (2) `is_computed_cell`: the SERVER owns that number. And it catches what the type list + # structurally cannot — a METRIC bag rides an ordinary `int`, so a machine-computed + # column passes the allow-list by type and must still be refused. + # + # ⚠ Enforced HERE rather than only in the builder, because the builder is a courtesy and + # this is the wall: the spec is a stored bag, and a field can become computed (a formula, + # a rollup, a metric binding) long after it was added to a live form + # ([[schema-role-is-not-a-value-wall]]). + if str(f.get("type") or "text") not in aios_grid.FORM_FIELD_TYPES: + continue + if user_tables.is_computed_cell(f): + continue row = {"key": str(f.get("key")), "label": str(f.get("label") or f.get("key")), "type": str(f.get("type") or "text"), "required": str(key) in required} options = f.get("options") @@ -218,9 +273,32 @@ def _public_form(rt, table_key: str, spec: dict) -> dict: "submitLabel": str(spec.get("submitLabel") or "")[:60] or "Submit", "fields": out, "honeypot": HONEYPOT_FIELD, + # ⛔ THE FLAG, NEVER THE LIST. `requireEmail` tells the page to ask for an address; the + # ADDRESSES stay server-side. Returning them would hand an unauthenticated caller the + # guest list — every colleague's work email, to anyone who has the link — which is a + # worse leak than the door this restriction exists to close. It is also the only reason + # this key can be computed here rather than shipped as `spec['emails']`. + "requireEmail": spec.get("access") == "emails", } +def _invited(spec: dict, email: str) -> bool: + """May this address submit? Case- and space-insensitive against the stored list. + + ⚠ **IDENTIFICATION, NOT AUTHENTICATION, and the honest boundary is stated rather than + implied** (this file's header does the same for the token). Nothing here proves the submitter + owns the address they typed: anyone holding the link who knows an invited address can pass. + What it buys is real but bounded — a link forwarded outside the invited group stops working + for whoever cannot name a member of it, and every submission carries the address it claimed. + A per-recipient link, which WOULD prove possession, is booked rather than half-built. + """ + if spec.get("access") != "emails": + return True + wanted = str(email or "").strip().lower() + return bool(wanted) and wanted in {str(e).strip().lower() + for e in (spec.get("emails") or [])} + + # --- validation ------------------------------------------------------------------------------- @@ -290,6 +368,135 @@ def _clean_values(fields: list, raw: dict): # --- the routes ------------------------------------------------------------------------------- +# --- the AUTHENTICATED half: minting, reading and revoking a form's share link ---------------- +# +# ⚠ PATH: `/form-link`, NOT `/forms/link`. The public routes are `/forms/{token}`, and a literal +# segment beside a path parameter is decided by declaration ORDER — a reader (or a later +# refactor that moves a decorator) would have to know that to know which handler answers. A +# different noun cannot be shadowed by a token somebody chooses. + + +def _public_base() -> str: + """The origin a share link is built on. Empty = the client builds it from its own location. + + `AIOS_PUBLIC_BASE` is the same variable the OAuth callback resolves against, so a domain move + takes the form links with it. Absent, we return a PATH rather than guessing a hostname: + a link with the wrong origin looks right, gets pasted into an email and 404s for the reader. + """ + return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") + + +def _link_of(token: str) -> str: + return f"{_public_base()}/#/form/{token}" + + +def _may_administer_view(session: Session, table_key: str, view_id: str): + """`(view, spec)` when this caller may publish THIS view, else raises. + + ⭐ `_may_administer`, deliberately, not `_may_edit`. Minting a public link is a PERMISSIONS + act — it opens a write door onto the tenant's table for anybody holding the URL — and + `table_store` already draws that line for views: a collaborator may edit a shared view, only + its creator or an admin may change who reaches it. Publishing is the strongest form of that + question, so it takes the narrower answer. + ⚠ A view sitting in the caller's OWN stratum is theirs even with no `createdBy` stamp (a + personal view predating the stamp has none), which `_may_administer` alone would refuse. + """ + import core.table_store as table_store + + if not table_key.startswith("ut_"): + # `add_row` is the only write a form can perform and it is a `ut_*` door; an Odoo-backed + # grid has no such door, so a link over one could never be anything but a 500 later. + raise err(400, "not_a_database", "forms collect into your own databases only") + view, owner = _find_view(session.runtime, table_key, view_id) + if not isinstance(view, dict): + raise err(404, "no_such_view", "that view no longer exists") + # ⛔ `uname` / `admin`, NOT `username` / `is_admin`. `Session` has no such attributes, and + # reading one raises OUTSIDE any handler's own try: — which is exactly D-107, where four + # finished Odoo routes were mounted, uncallable, and answered a bare plain-text 500 for a day + # ([[mounted-is-not-callable]]). Written wrong here first; caught by this file's own gate. + if not (owner == session.uname + or table_store._may_administer(view, session.uname, session.admin)): + raise err(403, "not_yours", "only this view's creator or an admin can publish it") + return view, _spec_of(view) + + +@router.get("/form-link") +def read_form_link(topic: str, view: str, session: Session = Depends(require_session)): + """This form's current share link, or `{"token": null}` — never a mint. + + ⛔ A GET MUST NOT MINT. The builder reads this every time the panel opens, and a route that + created a token on read would publish a form the moment somebody looked at the tab. + """ + _v, spec = _may_administer_view(session, topic, view) + token = "" + for stored, where in _tokens(session.runtime).items(): + if isinstance(where, dict) and where.get("table") == topic and where.get("view") == view: + token = str(stored) + break + return { + "token": token or None, + "url": _link_of(token) if token else None, + # ⭐ THE SERVER'S OWN VIEW OF THE SPEC, and it exists to make ONE failure visible rather + # than silent: the builder's `spec` prop comes from the client's `cleanDisplay`, so if the + # client ever stops carrying `display.form`, the panel would render an empty form over a + # stored one and the next save would erase real questions. Reporting what the SERVER holds + # lets the panel say so instead ([[read-path-cannot-witness-write-path]]). + "stored": ({"fields": len(spec.get("fields") or []), + "access": spec.get("access") or "public", + "title": str(spec.get("title") or "")[:120]} if spec else None), + } + + +@router.post("/form-link") +def mint_form_link(body: dict, session: Session = Depends(require_session)): + """Mint (or regenerate) this view's share link. `{topic, view, regenerate?}`. + + Idempotent without `regenerate`: opening the panel twice must not invalidate the link + somebody already emailed. With it, the previous token is DROPPED in the same write — a + "regenerate" that leaves the old link alive has revoked nothing. + """ + topic, view = str((body or {}).get("topic") or ""), str((body or {}).get("view") or "") + _v, spec = _may_administer_view(session, topic, view) + if not spec.get("fields"): + raise err(400, "empty_form", "add at least one question before sharing this form") + fresh = secrets.token_urlsafe(TOKEN_BYTES) + keep = not (body or {}).get("regenerate") + + minted = {"token": fresh} + + def _set(cur): + cur = dict(cur or {}) + for stored, where in list(cur.items()): + if isinstance(where, dict) and where.get("table") == topic and where.get("view") == view: + if keep: + minted["token"] = str(stored) + return cur + cur.pop(stored, None) # regenerate: the old link dies in this same write + cur[fresh] = {"table": topic, "view": view} + return cur + + # `sync`: a share link the user is about to paste into an email must be durable before the + # response says it exists. This is not a hot path. + session.runtime.update(TOKENS_KEY, _set, flush="sync") + return {"token": minted["token"], "url": _link_of(minted["token"])} + + +@router.delete("/form-link") +def revoke_form_link(topic: str, view: str, session: Session = Depends(require_session)): + """Turn the link off. The form spec stays; only the door closes.""" + _may_administer_view(session, topic, view) + + def _drop(cur): + cur = dict(cur or {}) + for stored, where in list(cur.items()): + if isinstance(where, dict) and where.get("table") == topic and where.get("view") == view: + cur.pop(stored, None) + return cur + + session.runtime.update(TOKENS_KEY, _drop, flush="sync") + return {"ok": True} + + @router.get("/forms/{token}") def get_form(token: str, request: Request): """Render-time payload for a public form. Rate-limited like the POST: an unauthenticated GET @@ -363,6 +570,19 @@ async def submit_form(token: str, request: Request): form = _public_form(rt, table_key, spec) if not form["fields"]: raise _refuse() + + # ⭐ WAVE-29 R7 — the authorized-by-email door. Checked BEFORE `_clean_values` so an + # uninvited caller learns nothing about the form's questions from the shape of its refusal, + # and before the daily cap so strangers cannot spend a live form's allowance. + # ⚠ NOT `_refuse()`: this one is deliberately a DIFFERENT answer. `_refuse` exists so a wrong + # token cannot be told apart from a missing form; here the caller already holds a valid link + # and the only useful thing we can say is "not with that address" — telling them nothing + # would leave a colleague staring at a form that silently does nothing. + submitter = str((body or {}).get("email") or "").strip().lower()[:254] + if not _invited(spec, submitter): + raise err(403, "not_invited", + "this form only accepts submissions from invited email addresses") + clean, problem = _clean_values(form["fields"], values) if problem: raise err(400, "invalid_submission", problem) @@ -384,7 +604,10 @@ async def submit_form(token: str, request: Request): # `record_created` row event, so every trigger and audit downstream sees `form:` # rather than a blank actor. Booked as a dated amendment in the split doc; a persistent # per-row stamp needs `add_row` to grow one, which is SESSION A's file. - actor = f"form:{token[:8]}" + # ⭐ WAVE-29 — an invited submitter is stamped BY ADDRESS rather than by token prefix: on an + # email-restricted form the address is the most identifying thing we honestly have, and it is + # what makes `record_created` legible in a run log ("form:ana@…" beats "form:Kx2p9Lm4"). + actor = f"form:{submitter or token[:8]}"[:120] row_id = user_tables.add_row(table_key, clean, username=actor, st=rt) if not row_id: raise err(409, "not_accepted", "that submission could not be saved — try again later") diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 658f16e60fdaab292a168070fd4fb21fa8cabfcc..78b419c01cb4ee102bb4ef4b44d0f915c2252969 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -32,7 +32,18 @@ def _redirect_uri(request: Request, provider: str) -> str: """The redirect URI this deployment registers at the provider — env-pinned when the container sits behind a proxy that rewrites the scheme (the HF Space), else derived from the request. MUST match a console-registered URI verbatim, so it is computed in exactly - one place.""" + one place. + + ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to + the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and + the request-derived fallback below is effectively dev-only. + ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host + this returns is where the provider sends the user BACK, and the session cookie is host-only + (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host + the user actually browsed plants the session on the wrong hostname and they return logged + out. Moving the app to a new hostname means moving this value AND re-registering the + resulting URI in the provider console; one without the other fails closed. + Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") if not base: base = f"{request.url.scheme}://{request.url.netloc}" diff --git a/api/routes_products.py b/api/routes_products.py index cb83bff460bd787fdae4ee6afb4a408c8d7f5e80..d198f28dbb2d4da95f2c80af35004ccb22a85705 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -63,6 +63,17 @@ def _pool_for(rt, team_id): # naming the pause when there is none. Minting a product snapshot here would be a new # persistence format at a wave tail; refusing with a sentence is the answer the user can act # on, and it is the same 503 the customer path already gives for an unsnapshotted scope. + # ⭐ WAVE 29 / R12 — THE POOL IS THE CATALOGUE NOW (5,875 active products, not the 2,717 that + # happened to sell in a revenue window). Two consequences that land HERE rather than in + # `product_data`: + # · the 503 below now also covers a TRUNCATED catalogue read. `modules.product_data. + # _catalogue_by_code` RAISES instead of degrading, precisely so a short pull arrives as a + # refusal a user can act on rather than as a plausible smaller grid nobody questions; + # `pool_unavailable` names the cause in its message. + # · the first build for a scope got slower (MEASURED 2026-08-11: ~44 s consolidated, of + # which the catalogue read is ~3 s; a BU-scoped build is ~9 s). Only a scope's FIRST-ever + # build blocks — the stale-while-refresh discipline above is what keeps that off the + # request path afterwards. import routes_keychain if routes_keychain.odoo_paused(rt): hit = rt.pool_cache.get(key) diff --git a/api/routes_tables.py b/api/routes_tables.py index b4b6cb6fc581d734417212728d794c3c894b0407..e35c9accd9cb2f44e306e21c1abfaf5bf2d58d0f 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -684,6 +684,56 @@ def add_row(table_key: str, body: dict = Body(default=None), return {"rid": rid, "pid": int(rid)} +@router.post("/tables/{table_key}/rows/import", status_code=201) +def import_rows(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write. + + Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a + spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record, + nothing is matched or overwritten, and the dialog says so before the button is pressed. + + ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate + the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in + an import is not a stray to be tidied away — it means the client offered a target it should + not have, and silently dropping it would leave the user looking for a column of values that + never arrived. The refusal names the column. + + ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every + profile cell validates, because a half-imported file is the worst outcome available: the user + cannot tell which rows landed without reconciling the spreadsheet by hand. + """ + _records_or_refuse(session, table_key) + ut = _ut() + rows_in = (body or {}).get("rows") + if not isinstance(rows_in, list) or not rows_in: + raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects") + if any(not isinstance(r, dict) for r in rows_in): + raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}") + defn = _defn_or_refuse(session, table_key) + by_key = {f["key"]: f for f in (defn.get("fields") or [])} + asked = {k for r in rows_in for k in r} + unknown = sorted(k for k in asked if k not in by_key) + if unknown: + raise err(400, "unknown_field", + f"this database has no column {unknown[0]!r}") + computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k])) + if computed: + label = by_key[computed[0]].get("label") or computed[0] + raise err(400, "computed_field", + f"{label!r} is worked out from other columns, so it cannot be imported into") + try: + made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) + except Exception: + raise err(503, "store_unavailable", "nothing was imported — the store refused") + if made is None: + raise err(400, "refused", + f"nothing was imported — {len(rows_in)} rows would take this database past " + f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") + _refresh_relations(session) + return {"imported": len(made), "pids": [int(r) for r in made]} + + # --------------------------------------------------------------------------------------------- # THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2) # --------------------------------------------------------------------------------------------- @@ -897,11 +947,18 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), except grid_events.StoreUnavailable: raise err(503, "store_unavailable", "the tenant store is unavailable — your change was not saved") - # ⚠ THE READ-BACK SPANS BOTH STRATA, and it has to (wave 25, C3-A1). An ordinary cell lands - # in this user's OVERLAY; a PROFILE cell writes THROUGH to the shared DEFINITION row. Reading - # only the overlay — which is what this did — reports a perfectly accepted profile write as - # `refused`, because the key is simply not in the bucket it looked in. Same merge order the - # rows envelope uses (`table_rows`): definition under, overlay over. + # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. + # + # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH + # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket. + # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's + # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began + # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells` + # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring + # claiming otherwise is what makes the next reader look for a merge bug that is not here. + # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY + # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took` + # asks whether THIS write landed, and this write goes to the definition. stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) .get(str(pid)) or {}) accepted = {k: stored.get(k) for k in updates if k in stored} diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 8dfd737177be5d9eeed5fb271be61bc03dda72fa..01c569b2ea282c41e974cf12dd0217757ea3be06 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -673,7 +673,10 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ "source": "overlay", "default": bool(field.get("default", True)), "custom": True, - "agg": field.get("agg") if field.get("agg") in {"sum"} else None, + # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a + # picker offering Average against a projection that only passes Sum through is the + # silent half of this feature. + "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None, "note": str(field.get("note") or "")[:2000], **({"options": options} if ftype in ("select", "multiselect") else {}), **(_choice_appearance(field, options) @@ -880,8 +883,109 @@ def _clean_rhs(raw, valid_keys): #: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an #: unknown mode, so a client that offers a mode this set does not carry lets a user build a view #: that silently reverts to a grid on the next read. +#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has +#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never +#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write — +#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door +#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at +#: anything. The mirror is one name, and it is the half nobody could see was missing because +#: BOTH sides were individually consistent. +#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out +#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law +#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by +#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered +#: one is red too, so the hold cannot outlive its reason the way wave 27's did). DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', - 'timeseries', 'catalog', 'swipe'} + 'timeseries', 'catalog', 'swipe', 'form'} + +#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at +#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read +#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a +#: legal mode and this function had no branch for the key, so every spec a client sent was dropped +#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one. +#: +#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a +#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match — +#: so one tenant setting its token to another tenant's value would silently receive that tenant's +#: submissions. The token is minted server-side and lives in a bucket no client write can reach; +#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. +FORM_ACCESS = ('public', 'emails') +MAX_FORM_FIELDS = 60 +MAX_FORM_EMAILS = 200 +MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60 +MAX_FORM_EMAIL = 254 + +#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the +#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing +#: from here is a question the builder cannot ask yet; a type wrongly present is a public door +#: writing values the column cannot mean (an `image` with no upload channel, a `json` document +#: typed into a text box, a `link` naming a record id a stranger guessed). +#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before: +#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an +#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks +#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather +#: than re-derived ([[one-evaluator-per-question]]). +#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares +#: the two files name-for-name. +FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date', + 'checkbox', 'phone', 'email', 'url', 'rating') + +#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in +#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the +#: expensive failure, not an odd address that gets in. +_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$') + + +def _clean_form(raw, valid_keys): + """One form spec, fail-closed. Returns None when nothing is configured. + + ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the + builder let somebody arrange these questions, and sorting them by column order would silently + rearrange a live form every time a column was added (`_public_form` states the same rule from + the serving end). + + ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A + swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck + that can never write. A form is a LIST of questions — losing the column behind question three + costs the asker question three, and taking the whole form away because one field was deleted + would be a far larger loss than the one that happened. + """ + if not isinstance(raw, dict): + return None + fields, seen = [], set() + for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]: + if k in valid_keys and k not in seen: + seen.add(k) + fields.append(k) + out = {} + if fields: + out['fields'] = fields + # A required flag on a question the form no longer asks is not a rule, it is a trap: the + # submitter can never satisfy it and the sentence names a field they cannot see. + req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen] + if req: + out['required'] = req + for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC), + ('submitLabel', MAX_FORM_SUBMIT)): + text = str(raw.get(key) or '').strip()[:cap] + if text: + out[key] = text + # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only + # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which + # is why `emails` is what gets written rather than a `public: false`. + if raw.get('access') == 'emails': + out['access'] = 'emails' + emails = [] + for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]: + e = str(e or '').strip().lower()[:MAX_FORM_EMAIL] + if _FORM_EMAIL.match(e) and e not in emails: + emails.append(e) + # Kept even while `access` is public: a person toggling the door open to test it and back + # again must not lose the list of people they typed. It is never served publicly. + if emails: + out['emails'] = emails + return out or None + #: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a #: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same @@ -937,6 +1041,35 @@ _LEGACY_MODES = {'dashboard': 'chart'} #: GroupTableView over salesParity.tableFromSpec. CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'} CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'} + +#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is +#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is +#: the picker's order; membership tests read it as a tuple perfectly well. +#: +#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS` +#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`): +#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole +#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum +#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are +#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both +#: would have to be the intersection of what each can express. +#: +#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS` +#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`, +#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would +#: give the product two words for one operation on two menus a click apart. +#: +#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column +#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours. +#: +#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells. +#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not, +#: and must not grow a second spelling of it. +#: +#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by +#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to +#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them. +FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop MAX_CHART_TITLE = 60 @@ -1379,6 +1512,10 @@ def _clean_display(raw, valid_keys): # `kanbanClamp` law) wearing a control that promises a choice. if left and right and left.casefold() != right.casefold(): out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} + # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. + form = _clean_form(raw.get('form'), valid_keys) + if form: + out['form'] = form return out diff --git a/platform/core/shared_overlay.py b/platform/core/shared_overlay.py new file mode 100644 index 0000000000000000000000000000000000000000..3f99d3b6d97cbeac8c610daa309747304aa111cc --- /dev/null +++ b/platform/core/shared_overlay.py @@ -0,0 +1,264 @@ +"""The TENANT-WIDE overlay stratum — one value per (row, column) for the whole workspace. + +Wave 29, item 20 / owner ruling R11, contract C5. The sibling of `core/table_store.py`, which +holds the PER-USER strata (`store.get(key)[username]`) and always has. + +⛔ THE DEFECT THIS EXISTS FOR IS NOT "SHARING WOULD BE NICE" — IT IS THAT A SHARED VIEW SILENTLY +WIDENS. `modules/product_data.py` already states it, in the comment above the supplier master: + + "A user-created field and its values live in the PER-USER strata; only VIEWS are shared. So a + shared 'Buy list' view that filtered on a user-created column would, for every OTHER account, + name a column that does not exist — and an unknown column is an INACTIVE condition in the + tri-state engine, which IGNORES it and therefore WIDENS. The buy list would silently show the + whole catalogue to everyone but its author." + +That is why the four supplier columns were frozen as read-only CONTRACT columns rather than made +editable, and why the owner's ask ("turn this Excel sheet into a User created Field that we can +edit") has been parked for two waves. A column whose value is the same for every reader makes the +shared view mean ONE thing, which is the precondition for editing it at all. + +──────────────────────────────────────────────────────────────────────────────────────────────── +⛔⛔ THIS MODULE IS NOT A PERMISSION WALL, AND MUST NEVER BECOME ONE BY ACCIDENT. + +It refuses no reader and no writer. The caller has already answered "may this session open this +surface" (`user_tables.may_open`, `Session.require`, the BU pool) and this module answers only +"what is stored". Two different questions; one of them belongs upstream, where the session is. + +What it DOES enforce is the one thing a caller can get wrong silently: + + ⭐ `cells(table_key, pids)` — `pids` IS REQUIRED, POSITIONALLY, AND THERE IS NO "EVERYTHING" + CALL. A default of "all rows" is the widening hazard above wearing a friendly face: the + caller passes the row set it has ALREADY scoped, so a cell for a row this reader may not see + cannot come back to be merged. There is deliberately no `all_cells()` to reach for; a caller + that genuinely needs the lot passes the lot, in writing, where a reviewer can see it. + +The row wall then holds twice over, because `aios_grid.rows_from_pool` iterates the POOL and looks +each pid UP in the overlay — never the reverse. A cell for a row outside the pool has nothing to +attach to. This module is built to keep that true rather than to re-implement it. +──────────────────────────────────────────────────────────────────────────────────────────────── + +RESIDENCY. Its own bucket, `__shared`, beside the per-user one — never a `__shared__` +member inside it. Two reasons, and the first is `product_data.py`'s own argument for separate +topic keys ("separate store keys make that structurally impossible, which is the whole reason the +table-page factory exists"): a per-user reader iterating usernames cannot encounter shared data +when there is no shared data in that bucket to encounter. The second is cost — the per-user bucket +carries every user's views, and a cell edit should not read-modify-write all of it. + +SHAPE, chosen to be drop-in: + + {"fields": {field_key: Field}, # the tenant-wide column DEFINITIONS + "cells": {"": {field_key: value}}} # exactly `rows_from_pool(overlays=...)`'s shape + +`cells()` returns STRING pid keys for that reason — `rows_from_pool` does `overlays.get(str(pid))`, +so a caller merges the shared stratum over the per-user one with one `dict.update` and cannot get +the key type wrong. (`table_store` stores `{str(pid): {...}}` too; one shape, three readers.) +""" +import core.store as store + +#: The suffix that turns a topic's per-user workspace key into its shared one. Callers pass the +#: key they ALREADY hold (`product_data.TABLE_KEY`, `f'{ut_key}_table_workspace'`) — one +#: identifier for both strata, so there is no second naming convention to get wrong. +BUCKET_SUFFIX = '__shared' + +#: One cell value's ceiling, matching the `json` field kind's. A shared cell is read by every user +#: in the tenant, so an unbounded one is an unbounded cost for all of them. +MAX_VALUE_CHARS = 32_768 + + +def bucket(table_key): + """The store key this topic's shared stratum lives under.""" + key = str(table_key or '').strip() + if not key: + raise ValueError('shared_overlay: a table_key is required — it names the bucket') + return f'{key}{BUCKET_SUFFIX}' + + +def _st(st): + return st if st is not None else store + + +def _read(table_key, st=None): + """The whole stratum, always in its full two-key shape. Lenient like every other display + read: an unreachable store degrades to "nothing shared yet", never to an exception on a page + that would otherwise render.""" + try: + data = _st(st).get(bucket(table_key)) or {} + except Exception: + data = {} + return {'fields': dict(data.get('fields') or {}), + 'cells': dict(data.get('cells') or {})} + + +def _write(table_key, change, st=None, flush='async'): + """Read-modify-write one shared stratum. + + `flush='async'` by default for the same reason `table_store._update` uses it: a typed cell + lands here inside the request round-trip, and the historical synchronous hub commit cost + seconds per keystroke. Structural writes (a field definition) pass `flush='sync'` — they are + rare, and a lost column definition is a worse failure than a lost keystroke. + """ + def _up(data): + data = data if isinstance(data, dict) else {} + data.setdefault('fields', {}) + data.setdefault('cells', {}) + change(data) + return data + return _st(st).update(bucket(table_key), _up, flush=flush) + + +def _pid(pid): + """Pids are ints everywhere in the grid and strings in JSON. ONE coercion, here, so a caller + passing either cannot write a row that a reader keyed the other way never finds.""" + return str(int(pid)) + + +def _value(value): + """A cell holds a SCALAR. The Row contract is scalar on every surface — the grid, the filter + engine, formulas, export — and the `json` kind is a validated STRING rather than an object. + + ⛔ A dict or a list RAISES rather than being dropped. This is called by our own code, so a + non-scalar is a programming error, and silently storing nothing would surface later as "the + shared column is blank for everyone" with no failure anywhere near the cause. + """ + if isinstance(value, (dict, list, tuple, set)): + raise ValueError(f'shared_overlay: a cell holds a scalar, not {type(value).__name__}') + if value is None or isinstance(value, bool) or isinstance(value, (int, float)): + return value + text = str(value) + return text[:MAX_VALUE_CHARS] + + +# ----------------------------------------------------------------- the column DEFINITIONS +def fields(table_key, st=None): + """`{field_key: Field}` — the columns this topic shares tenant-wide. + + Unscoped on purpose, and it is the one thing here that is: a shared column's EXISTENCE is + tenant-wide by definition — that is the whole feature, and it is what stops a shared view + naming a column half the workspace lacks. Its VALUES are scoped by `cells()`. + """ + return _read(table_key, st)['fields'] + + +def is_shared(table_key, field_key, st=None): + """Is this column's value tenant-wide? The predicate a caller uses to decide WHICH stratum to + read and write — one evaluator, so the read path and the write path cannot disagree about + where a given column lives ([[one-evaluator-per-question]]).""" + return str(field_key or '') in _read(table_key, st)['fields'] + + +def put_field(table_key, field_key, defn, st=None): + """Declare (or redefine) a shared column. Returns what was stored.""" + key = str(field_key or '').strip() + if not key: + raise ValueError('shared_overlay: a field needs a key') + entry = dict(defn or {}) + entry['key'] = key + + def _add(data): + data['fields'][key] = entry + + _write(table_key, _add, st, flush='sync') + return entry + + +def drop_field(table_key, field_key, st=None): + """Remove a shared column AND every stored value for it. + + ⛔ The cells go with the definition, exactly as `table_store.delete_field` scrubs its own: + orphaned values would silently resurface if the key were ever reused — and a resurrected + value in a TENANT-WIDE stratum reappears for everybody at once. + """ + key = str(field_key or '').strip() + if not key: + return False + hit = [False] + + def _drop(data): + hit[0] = data['fields'].pop(key, None) is not None + for row in data['cells'].values(): + if isinstance(row, dict): + row.pop(key, None) + # ⚠ AND DROP THE ROWS THAT ARE NOW EMPTY. `cells()` already skips a blank row, so this + # changes nothing a reader sees — but this bucket is read by every user in the tenant, + # and a `{}` per pid that once held a since-deleted column is dead weight that only ever + # grows. Caught by `prune()` reporting rows nobody could see as dropped. + for pid in [p for p, row in data['cells'].items() if not row]: + data['cells'].pop(pid, None) + + _write(table_key, _drop, st, flush='sync') + return hit[0] + + +# ----------------------------------------------------------------------------- the VALUES +def cells(table_key, pids, st=None): + """`{"": {field_key: value}}` for the rows named by `pids` — and ONLY those. + + ⛔ `pids` IS REQUIRED. See the module header: this is the row wall expressed as a signature, + so a caller cannot get "every shared cell in the tenant" by forgetting an argument. Pass the + pool you have already scoped for this session. + + An empty `pids` legitimately returns `{}` — a reader with no rows sees no cells, which is the + correct answer rather than a special case. + """ + if pids is None: + raise TypeError('shared_overlay.cells: pids is required — pass the row set you have ' + 'already scoped for this reader (there is deliberately no "all" call)') + wanted = {_pid(p) for p in pids} + if not wanted: + return {} + stored = _read(table_key, st)['cells'] + return {pid: dict(row) for pid, row in stored.items() + if pid in wanted and isinstance(row, dict) and row} + + +def put_cell(table_key, pid, field_key, value, st=None): + """Write ONE shared cell. Returns the value as stored (which may be truncated).""" + return put_cells(table_key, pid, {field_key: value}, st=st).get(str(field_key or '')) + + +def put_cells(table_key, pid, values, st=None): + """Write several shared cells on one row, in one store update. Returns what was stored.""" + row_id = _pid(pid) + clean = {str(k): _value(v) for k, v in dict(values or {}).items() if str(k)} + if not clean: + return {} + + def _patch(data): + data['cells'].setdefault(row_id, {}).update(clean) + + _write(table_key, _patch, st) + return clean + + +def clear_row(table_key, pid, st=None): + """Forget every shared cell on one row. True when there was something to forget.""" + row_id = _pid(pid) + hit = [False] + + def _clear(data): + hit[0] = data['cells'].pop(row_id, None) is not None + + _write(table_key, _clear, st) + return hit[0] + + +def prune(table_key, live_pids, st=None): + """Drop shared cells for rows that no longer exist. Returns how many rows were dropped. + + ⛔ NOT a scoped read wearing another name, and the difference is the whole reason `cells()` + refuses to serve everything: `live_pids` here means *every row this TABLE has*, which is a + fact about the data, whereas `pids` in `cells()` means *every row this READER may see*, which + is a fact about the session. Calling this with one session's pool would delete the shared + values of every row that session cannot see. The name says which is which; so does this note. + """ + keep = {_pid(p) for p in live_pids} + dropped = [0] + + def _prune(data): + gone = [pid for pid in data['cells'] if pid not in keep] + for pid in gone: + data['cells'].pop(pid, None) + dropped[0] = len(gone) + + _write(table_key, _prune, st, flush='sync') + return dropped[0] diff --git a/platform/core/store.py b/platform/core/store.py index 332610d7227fca6ddb624ec8ee029cc22803fc4e..a062cc3169a225d3d853ef5122d25b29a5f6c510 100644 --- a/platform/core/store.py +++ b/platform/core/store.py @@ -33,6 +33,16 @@ REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data') _FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit _FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget) +#: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP. +#: +#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a +#: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude +#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a +#: change, which is the truthful answer: the client's copy predates this process entirely. +#: +#: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality. +REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}' + def _token(): return os.environ.get('HF_TOKEN') or None @@ -63,6 +73,12 @@ class Store: self._flushers = {} # name -> live worker thread self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance self._last_flush = {} # name -> monotonic time of the last completed upload + # --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------ + # name -> how many accepted writes this process has applied to that bucket, and when. + # Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for + # why that constraint is the whole feature rather than an optimisation. + self._revs = {} + self._rev_at = {} def available(self): """True when a token is configured — callers degrade gracefully without one.""" @@ -131,6 +147,49 @@ class Store: except Exception: return True + # ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6) + def _bump(self, name): + """Advance one bucket's write counter. THE CALLER HOLDS THE LOCK. + + Called from the two places an accepted write becomes visible to readers of this process + — `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT + from `_flush_worker`: that uploads data a bump already accounted for, and counting it + again would report a change to every browser each time the coalescing worker woke. + """ + n = str(name) + self._revs[n] = self._revs.get(n, 0) + 1 + self._rev_at[n] = _time.time() + + def revision(self, name): + """`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else. + + ⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why + this method exists at all. A browser asking "has anything changed" by re-fetching rows + costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way + in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s + per bucket — so two tabs polling saturate the server and every unrelated write queues + behind them. This performs no download, no `get()`, and no copy of the value. + ⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace` + counts `get`/`_read_strict`/`_download` on the serving path and goes red on one). + + `rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend + ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops + being inert the moment D-4 flips, with the wire shape unchanged. + + ⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write + by a SECOND container against the same HF dataset is invisible here. That is not a limit + this introduces — `get()` caches a bucket for the life of the process and never re-reads + it, so the other container's write is invisible to every reader on this backend already. + The token is exactly as fresh as the store it reports on. Postgres has no such gap. + ⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted: + an attachment is fetched on explicit user action, never polled. + """ + n = str(name) + with self._lock: + rev = int(self._revs.get(n, 0)) + at = self._rev_at.get(n) + return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'} + def _upload(self, name, data): """The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra API round-trip on every put(), pure latency after the first success.""" @@ -193,6 +252,7 @@ class Store: self._cache[name] = json.loads(json.dumps(data)) self._owned[name] = True self._last_flush[name] = _time.monotonic() + self._bump(name) # C6 — AFTER the upload: a raise here is not a write return self._cache[name] def _flush_worker(self, name): @@ -322,6 +382,7 @@ class Store: raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') self._cache[name] = json.loads(json.dumps(data)) self._owned[name] = True + self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time self._schedule_flush(name) return data self.put(name, data) @@ -475,6 +536,20 @@ def update(name, fn, flush='sync'): return _d().update(name, fn, flush=flush) +def revision(name): + """The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing. + + ⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS + HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely + so a capability one backend lacks cannot be reached through the shared door and die at the + cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change + and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half + had no owner: a method is a capability the HF store has, a module function is a promise both + backends keep. + """ + return _d().revision(name) + + def _flush_all_at_exit(): with _INSTANCES_LOCK: instances = list(_INSTANCES.values()) diff --git a/platform/core/store_backend.py b/platform/core/store_backend.py index d8bda2c3dae751315225835737d92fcf37178968..24185f19b9875fd7312ac2c0c62a738644741ddd 100644 --- a/platform/core/store_backend.py +++ b/platform/core/store_backend.py @@ -55,9 +55,13 @@ def active(): return _hf -# --- the nine delegates. Written out rather than generated with setattr so that reading this file +# --- the TEN delegates. Written out rather than generated with setattr so that reading this file # tells you the whole interface, and so a new store operation cannot be added to core/store.py # without something here failing to match (verify_store_pg.py compares the signatures). +# +# ⭐ That guard did its job in wave 29: `revision()` landed in `core/store.py` and `core/store_pg.py` +# and the gate went red HERE, on the one file of the four that the change had missed. A generated +# delegate list would have covered it silently and left this file unable to describe itself. def available(): return active().available() @@ -92,3 +96,13 @@ def delete_path(path_in_repo): def flush(name_=None, timeout=30.0): return active().flush(name_, timeout=timeout) + + +def revision(name_): + """The bucket's change token (wave 29, C6) — `{'rev', 'updated_at', 'token'}`. + + Both backends answer, and they answer with the same keys and the same units (`updated_at` is + epoch seconds on each). That parity is the reason the change-token endpoint needs no branch on + the backend, and it is asserted rather than assumed by `verify_store_pg`'s interface section. + """ + return active().revision(name_) diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index ddc303c909d30661c3be16415bcad2c4db3c5361..50edd2df6e0f8b0fa270a9ad9a8896466c29e73d 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -233,6 +233,37 @@ def delete_path(path_in_repo, tenant_slug=None): return True +def revision(name, tenant_slug=None): + """`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6. + + ⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and + `update` above already ends `updated_at = now(), rev = {bare}.rev + 1`; all this adds is the + read door. Which is why C6 insisted the HF side use these two column NAMES for its in-memory + counter: the wire shape does not change at the cutover, so the client keeps polling the same + endpoint and the token simply stops being per-process. + + ⭐ AND HERE THE TOKEN IS DURABLE, which is the real upgrade. `core.store`'s counter lives in + one process, so it cannot see a write by a second container and it restarts at zero — hence + the process stamp in its token. `rev` is a column: it survives a restart, and it is shared by + every process pointed at this schema. The prefix below is therefore a constant rather than a + per-process value, and a backend flip changes the token exactly once (one refetch, correct). + + An absent key is `rev 0`, not an error: a bucket nobody has written yet has changed zero times, + which is a fact a client can hold a baseline against. + """ + with _pool().connection() as con: + row = con.execute( + _q('SELECT rev, updated_at FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug), + (str(name),)).fetchone() + rev = int(row[0]) if row and row[0] is not None else 0 + stamp = row[1] if row else None + # ⚠ EPOCH SECONDS ON BOTH BACKENDS. `core.store` records `time.time()`; this column is a + # `timestamptz`. Same key, two types, is how a consumer that works on one backend breaks on + # the other — the exact class this module's interface parity exists to prevent. + at = stamp.timestamp() if hasattr(stamp, 'timestamp') else None + return {'rev': rev, 'updated_at': at, 'token': f'pg:{rev}'} + + def flush(name=None, timeout=30.0): """A no-op that returns True: every write above is already committed when it returns. Kept in the interface because QA gates and shutdown hooks call it and must not branch on the backend.""" @@ -306,6 +337,9 @@ class PgStore: def update(self, name, fn, flush='sync'): return update(name, fn, flush=flush, tenant_slug=self.tenant_slug) + def revision(self, name): + return revision(name, tenant_slug=self.tenant_slug) + def upload_bytes(self, path_in_repo, data, message=None): return upload_bytes(path_in_repo, data, message=message, tenant_slug=self.tenant_slug) diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index c54f123c8fbff466b9feab7ce7a27071fed90320..34a3821dad7bc41b85809b0e884f7e781b083f02 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -165,7 +165,12 @@ MAX_JSON_CELL = 32 * 1024 * 1024 #: Where a profile handle points. A CLOSED vocabulary, for the reason every other closed enum #: here is closed: a source nobody wrote a reader for renders as a column that promises a link. -PROFILE_SOURCES = ('instagram',) +#: +#: ⭐ WAVE-29 CONTRACT C2 (owner R1/R2, 2026-08-11): `tiktok` joins it, so a `ut_tt_*` row can +#: carry a profile flag the enrich action resolves. **A source is not a NAME, it is three rules** +#: (below) — the vocabulary and the rules move together, because adding the word alone would +#: accept a TikTok handle and then hand it an `instagram.com` link from `profile_url`. +PROFILE_SOURCES = ('instagram', 'tiktok') #: Hosts whose `//` path IS an Instagram profile. `www.` is stripped before the lookup. _IG_HOSTS = ('instagram.com', 'instagr.am') @@ -179,6 +184,55 @@ _IG_RESERVED = frozenset({'p', 'reel', 'reels', 'tv', 'stories', 'explore', 'acc #: Instagram's own handle rule: 1–30 of letters, digits, period, underscore. _IG_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{1,30}$') +#: TikTok's profile hosts. ⛔ `vm.tiktok.com` is deliberately ABSENT: those are short links that +#: usually resolve to a VIDEO, and accepting one would store a "handle" (the opaque code) that can +#: never enrich — the same failure `_IG_RESERVED` exists to prevent, one layer earlier. +_TT_HOSTS = ('tiktok.com', 'm.tiktok.com') + +#: TikTok's non-handle path segments. `tiktok.com/@user/video/123` is already refused by the +#: one-segment rule; these are the ONE-segment paths that are still not people. +_TT_RESERVED = frozenset({'video', 'tag', 'music', 'discover', 'search', 'foryou', 'explore', + 'live', 'upload', 'about', 'legal', 'privacy', 'terms', 'effect', + 'business', 'ads', 'node', 'login', 'signup'}) + +#: TikTok's own handle rule: 2–24 of letters, digits, period, underscore. ⚠ NOT Instagram's — +#: the charset matches but the bounds do not, and a shared regex would accept a 30-character +#: TikTok "handle" that the vendor answers nothing for. +_TT_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{2,24}$') + +#: ⭐ C2 — the per-source rules, ONE table, so `PROFILE_SOURCES` cannot grow a member that no +#: normaliser knows. Every function below reads THIS rather than branching on the word: a source +#: with no entry refuses at the door instead of silently taking Instagram's rules and Instagram's +#: URL, which is what "add the word and ship" would have produced. +#: +#: `at_handle` = the profile path wears a leading `@` (TikTok's `/@name`; Instagram's `/name`). +#: It is stripped before validation and re-added by `profile_url`, so the STORED truth is the bare +#: handle in both families and a filter on the column cannot miss half of it. +_PROFILE_RULES = { + 'instagram': { + 'hosts': _IG_HOSTS, + 'reserved': _IG_RESERVED, + 'handle': _IG_HANDLE_RE, + 'at_handle': False, + 'url': 'https://www.instagram.com/{handle}/', + }, + 'tiktok': { + 'hosts': _TT_HOSTS, + 'reserved': _TT_RESERVED, + 'handle': _TT_HANDLE_RE, + 'at_handle': True, + 'url': 'https://www.tiktok.com/@{handle}', + }, +} + + +def _profile_rules(source): + """The rule bag for a source, or None — the ONE resolution of "is this source real".""" + key = str(source or '').strip().lower() + if key not in PROFILE_SOURCES: + return None + return _PROFILE_RULES.get(key) + #: The C1 preset keys a profile field OWNS — the cells R6 clears when its handle is blanked. #: #: MIRRORED from the engine's C1 set as local literals, the same discipline (and for the same @@ -227,17 +281,24 @@ def _clean_profile(raw): def normalize_profile(value, source='instagram'): """A typed profile cell → `(handle, ok)`. THE cell validator the flag promises (C3). - Accepts a bare handle (`name`), an at-handle (`@name`) or an Instagram profile URL in any of - the forms a person actually pastes (`https://www.instagram.com/name/?hl=en`, - `instagram.com/name`). Returns the BARE HANDLE — the URL form is derived for display, so the - stored truth has exactly one spelling and a filter on it cannot miss half the column. + Accepts a bare handle (`name`), an at-handle (`@name`) or a profile URL of THIS SOURCE in any + of the forms a person actually pastes (`https://www.instagram.com/name/?hl=en`, + `instagram.com/name`, `https://www.tiktok.com/@name`). Returns the BARE HANDLE — the URL form + is derived for display, so the stored truth has exactly one spelling and a filter on it cannot + miss half the column. + + ⛔ C2: the host, the reserved segments and the handle rule all come from `_PROFILE_RULES[source]`. + A TikTok URL pasted into an Instagram-flagged column is REFUSED, and vice versa — a cell that + names the wrong network is not a handle this row's automation can enrich, and quietly accepting + it would defer the failure to a run that then answers nothing. `('', True)` for a blank: clearing a profile cell is legal and is the whole of R6. `(None, False)` for anything else, and the caller refuses the write with a sentence — a value that is neither is not a handle we can enrich, and storing it would put the failure off until the automation runs and answers nothing. """ - if str(source or '').strip().lower() not in PROFILE_SOURCES: + rules = _profile_rules(source) + if rules is None: return (None, False) raw = str(value if value is not None else '').strip() if not raw: @@ -259,16 +320,19 @@ def normalize_profile(value, source='instagram'): host = (parts.netloc or '').split('@')[-1].split(':')[0].strip().lower() if host.startswith('www.'): host = host[4:] - if host in _IG_HOSTS: + if host in rules['hosts']: segs = [s for s in (parts.path or '').split('/') if s] - if len(segs) != 1 or segs[0].lower() in _IG_RESERVED: - # Two segments is a post/reel/highlight, zero is the site itself. Refused rather + # C2: TikTok's profile path is `/@name`; Instagram's is `/name`. The `@` is stripped + # here and re-added by `profile_url`, so both families STORE a bare handle. + first = segs[0].lstrip('@') if segs else '' + if len(segs) != 1 or not first or first.lower() in rules['reserved']: + # Two segments is a post/reel/video, zero is the site itself. Refused rather # than taking the first segment, which is what turns `/p/Cxyz` into handle `p`. return (None, False) - cand = segs[0] + cand = first elif host: - return (None, False) # a URL, but not an Instagram one - if not _IG_HANDLE_RE.match(cand) or cand.lower() in _IG_HOSTS: + return (None, False) # a URL, but not one of THIS source's + if not rules['handle'].match(cand) or cand.lower() in rules['hosts']: # The host check catches a bare `instagram.com` — it matches the handle charset, so # without this it would normalise to a "handle" that can only ever resolve to nothing. return (None, False) @@ -277,11 +341,18 @@ def normalize_profile(value, source='instagram'): def profile_url(handle, source='instagram'): """The DERIVED display form of a stored handle (C3: *"the URL form is derived for display"*). - Derived and never stored, so the two spellings cannot drift apart.""" + Derived and never stored, so the two spellings cannot drift apart. + + ⛔ C2 — the template comes from the SOURCE's own rules. This function used to hardcode + `instagram.com` while its guard merely checked membership of `PROFILE_SOURCES`, so the moment + that tuple grew a second member it would have handed every TikTok handle an Instagram link — + a guard that passes and a body that lies ([[hardcoded-fallback-not-hardcoded-key]]). + """ h = str(handle or '').strip().lstrip('@') - if not h or str(source or '').strip().lower() not in PROFILE_SOURCES: + rules = _profile_rules(source) + if not h or rules is None: return '' - return f'https://www.instagram.com/{h}/' + return rules['url'].format(handle=h) def _profile_of(fields, exclude=None): @@ -355,6 +426,54 @@ FORMAT_TZ = ('local', 'utc') FORMAT_MAX_DECIMALS = 4 +#: ⭐ WAVE-29 CONTRACT C7 — the COLUMN-SUMMARY vocabulary a field's `agg` may take. +#: +#: MIRRORED from `aios_grid.FIELD_AGGS` (session C publishes it) and from the client's +#: `aggregations.FIELD_AGGS`, as a local literal for the reason `METRIC_MEASURES` and +#: `PROFILE_PRESET_KEYS` are local literals here: `core/` must stay importable without the layers +#: above it. Three copies, one law, held in step by C's parity gate rather than by an import. +#: +#: ⛔ NOT `CHART_AGGS` (which spells it `avg` and gatekeeps STORED chart values with live data) +#: and NOT `ROLLUP_FNS` (16 fold names, overlapping but not the same list). Merging any two of the +#: three silently turns live stored charts into sums. +FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') + +#: A choice colour, as the client's editor spells one. `#RRGGBB`, upper-cased on the way in so a +#: stored value has exactly ONE spelling and two fields cannot disagree about `#ebf6ef`. +_OPTION_COLOR_RE = re.compile(r'#[0-9A-F]{6}') + + +def _clean_option_colors(raw, options): + """`{option label → #RRGGBB}`, capped to the field's own option vocabulary (wave-29 C1). + + ⛔ MIRRORED from `aios_grid._clean_option_colors` as a LOCAL function rather than imported, for + the reason `METRIC_MEASURES` and `PROFILE_PRESET_KEYS` are local literals here: `core/` must + stay importable without the layers above it, and `aios_grid` sits above this module. The pair + is held in step by a gate leg, never by an import. + + ⚠ THE CAP IS THE POINT, not tidiness: colours are keyed by LABEL, so a renamed or deleted + option would otherwise leave a colour behind that nothing renders and no editor can reach — + and on the next rename that orphan could re-attach to a different option with the same name. + Case-insensitive on the way in, canonical on the way out, so `Done` and `done` cannot both + claim the same option. + """ + if not isinstance(raw, dict): + return {} + supplied = {} + for label, color in raw.items(): + if not isinstance(label, str) or not isinstance(color, str): + continue + clean = color.strip().upper() + if _OPTION_COLOR_RE.fullmatch(clean): + supplied[label.strip().lower()] = clean + out = {} + for option in options or []: + color = supplied.get(str(option).strip().lower()) + if color: + out[str(option)] = color + return out + + def _clean_format(raw): """One `format` bag → the stored shape, or None. @@ -489,10 +608,12 @@ def clean_fields(raw): if entry is None: seen.discard(key) continue - # `agg` decides whether the grid's totals row SUMS the column. A rollup that declares it - # through one door and not the other totals differently depending on who made it. - if f.get('agg') in {'sum'}: - entry['agg'] = 'sum' + # `agg` decides HOW the grid's totals row summarises the column. A rollup that declares it + # through one door and not the other summarises differently depending on who made it. + # ⭐ WAVE-29 C7: the vocabulary is `FIELD_AGGS`, not the single word `sum` — see its note. + agg_in = str(f.get('agg') or '').strip() + if agg_in in FIELD_AGGS: + entry['agg'] = agg_in out.append(entry) return out or None @@ -702,6 +823,72 @@ def add_row(table_key, values=None, username=None, st=None, rid=None): return rid +def add_rows(table_key, rows_in, username=None, st=None): + """⭐⭐ WAVE-29 T25 (owner item 6) — APPEND MANY ROWS IN ONE STORE WRITE. Returns the list of + new row ids, or None if the table refuses records at all. + + ⛔ WHY THIS EXISTS RATHER THAN A LOOP OVER `add_row`. Every `add_row` is a read-modify-write of + the WHOLE table document plus a row event; importing a 2,000-row spreadsheet that way is 2,000 + full-document copies under one lock, which is the same shape as the 1.4 s-per-bucket problem + item 20 exists to avoid — on the one uvicorn process this product runs. One `update()` writes + them all. + + ⛔ AND WHY NOT `automation_engine.upsert_rows`, which this module's own note (below) says to + reuse: that function is UPSERT-BY-KEY shaped, and v1 of the import door is deliberately + APPEND-ONLY (no key to merge on, no silent overwrite of a row somebody edited). It also lives + in the API layer, and `core/` may not import upward. When the import door grows an "update + matching rows" mode, the engine's is the one to lift — the note stands, it just is not this. + + ⚠ THE CAP IS CHECKED AGAINST THE WHOLE BATCH, not per row: a partial import that stops at + `MAX_ROWS` leaves the user reconciling a spreadsheet against a table, which is exactly what + the refusal-with-a-sentence exists to prevent. Nothing is written if the batch does not fit. + + ⚠ Ids continue the table's own sequence, and a PROFILE column validates every cell exactly as + `add_row` does — a row born with an unusable handle is a row the enrich action can never + answer, and the failure would otherwise surface as an automation quietly returning nothing. + """ + if not is_user_table(table_key, st) or not records_mutable(table_key, st): + return None + batch = [dict(r) for r in (rows_in or []) if isinstance(r, dict)] + if not batch: + return [] + defn = get(table_key, st) or {} + rows = defn.get('rows') or {} + if len(rows) + len(batch) > MAX_ROWS: + return None + valid = {f['key'] for f in (defn.get('fields') or [])} + pf = _profile_of(defn.get('fields') or []) + next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + made = [] + for values in batch: + clean = {k: str(v) for k, v in values.items() if k in valid} + if pf and pf['key'] in clean: + handle, ok = normalize_profile(clean[pf['key']], pf['profile'].get('source')) + if not ok: + return None # atomic: one bad handle refuses the FILE + clean[pf['key']] = handle + next_id += 1 + made.append((str(next_id), clean)) + + def _add(cur): + t = cur.get(str(table_key)) + if t is not None: + bucket = t.setdefault('rows', {}) + for rid, clean in made: + bucket[rid] = clean + return cur + + _st(st).update(STORE_KEY, _add, flush='sync') + # ⚠ ONE EVENT PER ROW, deliberately: `record_created` is a trigger substrate, and an import of + # 500 rows genuinely IS 500 created records. The engine's own flood hold is what decides + # whether that runs an automation 500 times — a decision that belongs there, not to a writer + # that quietly emits less than it did. + for rid, _clean in made: + emit_row_event({'type': 'record_created', 'table': str(table_key), 'rowId': rid, + 'st': _st(st), 'user': str(username or '')}) + return [rid for rid, _ in made] + + # ⛔ `upsert_rows` USED TO LIVE HERE, AND DELETING IT IS THE FIX (DEBT D-6, wave 20). # # There were TWO bulk-upsert implementations — this one and `automation_engine.upsert_rows` — @@ -1150,6 +1337,57 @@ def _clean_field(raw, previous=None): opts = [' '.join(str(o).split())[:60] for o in (opts_raw or []) if str(o).strip()][:50] if opts: out['options'] = opts + # ⭐⭐ WAVE-29 CONTRACT C1 (owner item 13) — **THE OPTION COLOURS, WHICH THIS DOOR DROPPED.** + # + # The client has carried `optionColors`/`colorCodeOptions` since wave 14 (`types.ts`, the + # `BRAND_SWATCHES` editor, `cells.optionTint`, the kanban card) and `aios_grid._clean_field` + # keeps both — but THIS validator, the one every `ut_*` create and patch goes through, had + # zero occurrences of either key. So the owner picked a colour on a user database, the pane + # saved, and the colour was gone on the next read: the same crossed-validator shape as item 2a, + # in a different pair of doors. `grid_events._FIELD_ECHO_PROPS` already carried both, so the + # echo wire was plumbed for a value nothing stored. + # + # ⛔ SIBLING KEYS, and `options` IS NOT RESHAPED. Every consumer treats options as `string[]` + # (this function flattens each to a bare `str` twelve lines up); folding colour into the option + # entries would be a wire change across the grid, the kanban, the editor and the filter panel. + # + # ⛔ OMIT-WHEN-ABSENT, exactly like `format`/`metric`/`link`, because `verify_api.py` and + # `verify_automation.py` assert `_clean_field(f) == f` over every engine-seeded field list — + # a branch that emitted `optionColors: {}` would redden both gates for every preset column in + # the product. ([[default-must-pass-its-own-guard]]) + # + # ⚠ INHERITED ACROSS A PATCH, for `agg`'s reason: a PATCH that sends only a label must not + # silently strip the colours somebody chose, which is precisely how a money column stopped + # totalling. + # + # ⚠ A HEX IS VALIDATED; BRAND MEMBERSHIP IS NOT, and that is deliberate. `_clean_option_colors` + # (`aios_grid.py:200`) — the sibling this mirrors — accepts any `#RRGGBB`, and the client + # renders a stored colour through `choiceColors.brandedTint`, which maps it to its NEAREST + # brand swatch at DISPLAY time while leaving what is stored byte-stable. Refusing a non-brand + # hex here would refuse values the other validator accepts on the same UI, and would break a + # column that moved between the two. + if ftype in ('select', 'multiselect'): + colors_raw = (raw.get('optionColors') if 'optionColors' in raw + else prev.get('optionColors')) + colors = _clean_option_colors(colors_raw, out.get('options') or []) + if colors: + out['optionColors'] = colors + color_code = (raw.get('colorCodeOptions') if 'colorCodeOptions' in raw + else prev.get('colorCodeOptions')) + if isinstance(color_code, bool): + out['colorCodeOptions'] = color_code + # ⭐ WAVE-29 T22's own repair: a `rating` column's star count. Grid-created columns on a `ut_*` + # database now come through this door rather than the per-user overlay, and this validator had + # no `max` — so a 3-star or 10-star column silently became 5-star on the way in. Bounded rather + # than refused, mirroring `aios_grid._clean_rating_max`: the field still holds its 1..max + # integers whatever the number was, so an unusable value is a default and never a lost column. + if ftype == 'rating': + max_raw = raw.get('max') if 'max' in raw else prev.get('max') + if max_raw is not None: + try: + out['max'] = max(2, min(int(max_raw), 10)) + except (TypeError, ValueError): + pass role = str(raw.get('editRole') or prev.get('editRole') or 'admins').strip().lower() out['editRole'] = role if role in FIELD_EDIT_ROLES else 'admins' if prev.get('default') is True or raw.get('default') is True: @@ -1182,8 +1420,19 @@ def _clean_field(raw, previous=None): # which found `agg` instead. # ⚠ Sticky across a patch for the same reason as `default` and `pinned`: a client PATCH that # omits the key must not un-total a column as a side effect of renaming its header. - if str(raw.get('agg') or prev.get('agg') or '').strip() == 'sum': - out['agg'] = 'sum' + # ⭐ WAVE-29 C7 — the vocabulary widened from the single word `sum` to `FIELD_AGGS`. The + # STICKINESS is unchanged and now applies to every name: a PATCH that omits `agg` keeps + # whatever the column had, because dropping it is how a money column silently stopped + # totalling. ⛔ An unknown name is DROPPED rather than stored — a summary nothing can compute + # would render blank under a picker that says otherwise. + # ⚠ `'agg' in raw`, not `raw.get('agg') or …` — the same idiom `format` and `description` use + # two screens up, and it is what makes the difference between OMITTING the key (keep what is + # stored: the wave-28 stickiness that stops a rename un-totalling a money column) and sending + # it EMPTY (clear the summary, which the picker's "None" row has to be able to do). Written + # the loose way, "None" would have been a no-op with the menu cheerfully showing the change. + agg_name = str((raw.get('agg') if 'agg' in raw else prev.get('agg')) or '').strip() + if agg_name in FIELD_AGGS: + out['agg'] = agg_name # An automation column carries the binding the engine reads; it is preserved across a patch # rather than re-declared, because the automation editor owns it and this route does not. auto = raw.get('automation') if 'automation' in raw else prev.get('automation') diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index 21a1d00a1e088d387cc4afc263ebff309aaa26b8..390bd20f9c366621c901b9109ceda4633918d414 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -170,9 +170,20 @@ ENTITIES = { # table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent") # marks agents who came from staff; the owner ruling on whether those count as agents is # OUTSTANDING, so both flags are synced and neither reading is baked in. + # ⭐ WAVE 29 (W29-T54, amendment A4). `customer_rank` is the ONLY predicate that + # distinguishes "a partner Odoo has TRANSACTED with" (today's population, 2,465) from + # "a customer" (3,614 with `rank>0 active`) — a 1,149-partner drop, the same join-drop + # class as item 22's SKU pool. It cannot be derived from the document tables, because + # the missing partners are precisely the ones with no document. + # ⛔ It is synced HERE rather than read from live Odoo because `read_customers` also runs + # at BOOT off a hydrated snapshot, where Odoo may be unreachable — sourcing it live would + # make the population "whatever source answered this time", changing size with the network. "fields": ["name", "team_id", "city", "state_id", "country_id", "active", - "agent", "salesman_as_agent", "write_date"], - "backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28 + "agent", "salesman_as_agent", "customer_rank", "write_date"], + # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next + # sync_all() instead of needing a reseed. + "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28 + "customer_rank"], # added 2026-08-11 (W29-T54) # m2m fields land as _id (the FIRST linked id — the Customers-module convention: # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be @@ -256,6 +267,16 @@ BATCH = 2000 # from "unknown". BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free") +# ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence +# shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec. +# `_cols` types anything it does not recognise as VARCHAR, so `customer_rank` landed as text, the +# mirror populated correctly, every row looked right — and `odoo_relational`'s own predicate +# `customer_rank > 0` died with `Binder Error: Cannot compare values of type VARCHAR and type +# INTEGER_LITERAL`. ⛔ THE COLUMN WAS PRESENT AND THE DATA WAS CORRECT; only its TYPE was wrong, +# which is why "is the field in the mirror?" answered yes and the feature still could not run. +# ⇒ A new numeric field belongs HERE, exactly as a new boolean belongs in BOOL_FIELDS above. +INT_FIELDS = ("customer_rank", "supplier_rank") + import threading as _thr @@ -444,6 +465,8 @@ def _cols(spec): cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"] elif f in BOOL_FIELDS: cols.append(f"{f} BOOLEAN") + elif f in INT_FIELDS: + cols.append(f"{f} BIGINT") elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed", "price_subtotal", "margin", "purchase_price", "product_uom_qty", "standard_price", "debit", "credit", "balance", "amount"): diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index ada17a006d90572237c1205aa310c1eb4d3e8d21..f84b4ccadef38b0b4ad40b5c25c30ae77cb67892 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -30,11 +30,32 @@ template and this deliberately mirrors its signature and its row shape. Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it does not pretend to cover columns that are absent. + +4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12, + 2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the + union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid + showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise: + + · ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date + window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics + that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because + ~2,550 active SKUs have never sold in wholesale scope at all. + · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the + revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and + not this one, and that zero is a measurement. A row that appears in NO revenue read + carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the + client renders as an empty cell. Blank is an admission; zero is a measurement. + · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where + that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW + UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product` + carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two + sources are JOINED rather than merged. """ import json import zlib from pathlib import Path +import core.odoo as O import core.periods as P import core.table_store as table_store import modules.products as products @@ -155,31 +176,66 @@ def _inventory_by_code(t): return {} +def _catalogue_by_code(): + """`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from. + + ⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point. + Inventory degrades to `{}` because a product table with blank stock columns beats one that + will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that + failed quietly would drop the grid straight back to the sold-only 2,717 with every gate + green and nothing on screen saying so, which is the defect this seam exists to end + ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise + into a 503 that names the cause, so the loud failure has somewhere honest to land. + + It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the + same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs + to build a pool without Odoo. + """ + return products.catalogue() + + def pool(team_id=None, t=None): - """One row per SKU across the scoped book — the PRODUCT analogue of `customer_data.pool`. + """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`. + + CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set + is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs + that sold in its window and contributes NO rows of its own. `team_id` shapes the revenue columns exactly as it does for customers (`products.directory` passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep - driving it rather than a post-filter deciding the BU. + driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a + catalogue has no team. """ t = t or P.today() consolidated = team_id is None inv = _inventory_by_code(t) if consolidated else {} sup = supplier_master() + cat = _catalogue_by_code() + # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue + # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those + # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue + # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY). + rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} rows = [] - for r in products.directory(t=t, team_id=team_id): - code = r["code"] + for code, meta in cat.items(): + r = rev.get(code) row = { "pid": sku_pid(code), "code": code, - "product": r.get("product") or code, - "category": r.get("category") or "(uncategorized)", - "rev_ytd": r.get("rev_ytd", 0.0), - "rev_ly": r.get("rev_ly", 0.0), - "yoy_pct": r.get("yoy_pct"), - "qty_ytd": r.get("qty_ytd", 0.0), - "orders_ytd": r.get("orders_ytd", 0), + # ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's + # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active + # record's `display_name` is the current truth, and it is the same string for every + # SKU that is not re-SKUed. `directory()` derives the category identically. + "product": meta.get("product") or code, + "category": meta.get("category") or "(uncategorized)", + # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev` + # keeps its measured numbers including a real 0.0 (it sold last year, not this one). + "rev_ytd": r.get("rev_ytd", 0.0) if r else None, + "rev_ly": r.get("rev_ly", 0.0) if r else None, + "yoy_pct": r.get("yoy_pct") if r else None, + "qty_ytd": r.get("qty_ytd", 0.0) if r else None, + "orders_ytd": r.get("orders_ytd", 0) if r else None, } # Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock). s = sup.get(code) or {} @@ -227,6 +283,10 @@ def pool(team_id=None, t=None): row["buy_now"] = None rows.append(row) + # `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU + # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored + # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured + # nothing sold". rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0)) _assert_no_pid_collision(rows) return rows @@ -277,20 +337,68 @@ def validate(team_id=None, t=None): """Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number which does not tie to Odoo does not ship. - The check is deliberately narrow and says so: Σ per-SKU YTD revenue must equal the SKU - module's own YTD total over the same scope. It does NOT cover the inventory columns (they are - a different module's reconciliation, and they are absent on a scoped pull anyway). + ⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING + (wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the + same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could + not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave. + No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either. + That is the leg below, and it is the control that would have caught it + ([[no-unverifiable-aggregates]]). + + ⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join + lose money?") and is still the right shape for that one. What changed is that it now has to + account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived + products out of the grid; the total is decomposed rather than compared loosely, so a growing + "outside the grid" figure shows up as a number rather than as slack in a tolerance. + + Still deliberately NOT covered, and said rather than implied: the inventory columns are a + different module's reconciliation, and they are absent on a scoped pull anyway. """ t = t or P.today() rows = pool(team_id=team_id, t=t) - ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2) - yf, yt = P.ytd(t) - theirs = round(sum(v.get("rev", 0.0) - for v in (products._sku_rev(yf, yt, team_id) or {}).values()), 2) + + # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active + # products, asked fresh — not a `len()` over anything this module or `products.catalogue()` + # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it. + n_active = products.catalogue_count() checks = [{ - "check": "Σ per-SKU YTD revenue == modules.products._sku_rev over the same window", - "ours": ours, "theirs": theirs, "ok": abs(ours - theirs) < 0.01, + "check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT " + "Odoo search_count('product.product', active=True)", + "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active, }] + + yf, yt = P.ytd(t) + sku_rev = products._sku_rev(yf, yt, team_id) or {} + in_pool = {r["code"] for r in rows} + outside = {c: v for c, v in sku_rev.items() if c not in in_pool} + ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2) + theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2) + dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2) + checks.append({ + "check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product " + "carries == modules.products._sku_rev over the same window", + "ours": round(ours + dropped, 2), "theirs": theirs, + "ok": abs((ours + dropped) - theirs) < 0.01, + "detail": {"in_the_grid": ours, "outside_the_grid": dropped, + "codes_outside": len(outside)}, + }) + # ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is + # trivially true when checked against the dict that did the dropping — that is the + # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE + # product carries means `products.catalogue()` missed a row that has revenue, which is the + # 2,717 defect returning in a smaller costume. + # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the + # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all. + coded_outside = sorted(c for c in outside if not str(c).startswith("pid:")) + still_active = (O.get_odoo().search_count( + 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)]) + if coded_outside else 0) + checks.append({ + "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived " + "stay out) — asked of Odoo, never of the catalogue that did the dropping", + "ours": still_active, "theirs": 0, "ok": still_active == 0, + "detail": {"codes": coded_outside[:10], "n_codes": len(outside)}, + }) # WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every # member of it must be re-derivable from the two columns beside it. A signal somebody buys # stock on cannot be "mostly right": the failure that matters is a row that says OK because diff --git a/platform/modules/products.py b/platform/modules/products.py index b5b6f1435abd29d24aeea8498422793ee840a087..e996a9895c5eecddff30c3a8febf5a07cf8e36f8 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -166,6 +166,62 @@ def directory(t=None, team_id=None): return rows +def catalogue(): + """`{sku_code: {'product', 'category'}}` — EVERY ACTIVE product, sold or not. + + ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s + row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU + that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`, + `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe — + and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of + 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in + the consumer, never a window removed from `directory()`: removing the window alone lands at + 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all. + + ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one + catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than + merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold. + + Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active + records that carry none — so the join against `directory()` is code-for-code with no + normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields + off a sale line, so a never-sold row wears the same format as a sold one. + + ⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull + would put the grid back at a plausible wrong number with every gate green — the exact failure + this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard + `modules/backorders.py:161` already uses). + """ + dom = [('active', '=', True)] + prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'], + limit=50000) + n = O.get_odoo().search_count('product.product', dom) + if len(prods) != n: + raise ValueError( + f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against " + f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with " + f"nothing reporting it; raise the limit before shipping this.") + code_cat = _code_category() + out = {} + for p in prods: + code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") + # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero + # active products share a code, so this branch is a guard, not a merge policy. + out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code, + 'category': code_cat.get(code, '(uncategorized)')}) + return out + + +def catalogue_count(): + """The INDEPENDENT population oracle: Odoo's own count of active products. + + Deliberately a bare `search_count` and not a `len()` over anything this module built — the + 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides + of its reconciliation from `_sku_rev`, so the oracle could never see a missing row. + """ + return O.get_odoo().search_count('product.product', [('active', '=', True)]) + + def categories(t=None, team_id=None): """Sorted distinct category names present in the SKU directory (for the drill-down filter).""" return sorted({r['category'] for r in directory(t, team_id)}) diff --git a/web/src/automation/AutomationBuilder.tsx b/web/src/automation/AutomationBuilder.tsx index 9ace4a3e0bf722b16460656f5b63ba667899be12..aab76973023167a6ad9d10beb4fc5df936636117 100644 --- a/web/src/automation/AutomationBuilder.tsx +++ b/web/src/automation/AutomationBuilder.tsx @@ -2544,6 +2544,95 @@ export function ActionProps({ At most 12 — the provider returns a profile’s top 12 posts and no more.

+ {/* + ⭐⭐ W29-T09 (owner item 11: *"the last 12 reels"*) — THE ONLY DOOR TO `config.postGroups`. + The server side has been whole since wave 28 — `clean_post_groups` validates it, the + enrich runner applies it — and `postGroups` had **zero occurrences** anywhere in + `web/src`, so NO USER COULD ASK FOR REELS-ONLY AT ALL. That is why this is a blocker for + the reels routing work and not a nicety: those tickets' negative controls need a config + a person can actually produce. + + ⛔ THE KEYS ARE THE SERVER'S AND SO ARE THE WORDS. `vocab.postTypes` carries both; a + client that translated `video` into "Reels" locally would be a second copy of a + vocabulary `clean_post_groups` refuses deviations from. Absent vocabulary renders + NOTHING rather than three guessed names. + + ⛔ ABSENT IS OFF, AND OFF IS THE DEFAULT FOREVER. An enrich action stored before this + control carries no `postGroups`, and the runner returns the post list unchanged when the + key is missing — so unchecking every box must DELETE the key, never store `[]`-with- + meaning or a group of zero. Deleting it is what keeps an old automation capturing + exactly what it always captured. + + ⚠ ONE WRITER FOR ONE KEY [W-12]. This is the only control in `web/src` that writes + `postGroups`; the `maxPosts` box above writes `maxPosts` and nothing else. The two are + related only in that a group cannot keep more posts than the pull captures — which is + the server's rule, printed here rather than re-implemented. + */} + {(vocab?.postTypes || []).length ? ( +
+

Keep only certain posts

+ {(vocab?.postTypes || []).map((pt) => { + const groups = + ((cfg as { postGroups?: { type: string; limit: number }[] }).postGroups) || []; + const mine = groups.find((g) => g && g.type === pt.key); + const cap = Math.max( + 1, + Number((cfg as { maxPosts?: number }).maxPosts || 10) || 10 + ); + /* ⛔ WRITE THROUGH ONE FUNCTION, so the "no groups left ⇒ remove the key" rule + exists once. Two call sites each deciding it is how `[]` starts meaning + "capture nothing". */ + const write = (next: { type: string; limit: number }[]) => + setCfg({ postGroups: next.length ? next : undefined }); + return ( + + ); + })} +

+ Leave these unticked to keep every post. Ticked, only the kinds you name are + kept — and a group cannot keep more than the {" "} + {Number((cfg as { maxPosts?: number }).maxPosts || 10) || 10} posts this + enrichment captures. +

+
+ ) : null} {/* ⭐ WAVE 28 · R6/R7 — THE TWO CHECKBOXES THAT STOOD HERE MOVED UP INTO THE INCLUDE GROUP, KEYS AND ALL (`postMetrics`, `commentMetrics`). They are not deleted features: diff --git a/web/src/automation/AutomationSurface.tsx b/web/src/automation/AutomationSurface.tsx index 5254c2469a730c59b326ec061aa9b06715c44c67..a4f925f5b42428bc4312a0999a33545f0c69a173 100644 --- a/web/src/automation/AutomationSurface.tsx +++ b/web/src/automation/AutomationSurface.tsx @@ -216,7 +216,7 @@ export default function AutomationSurface() { * asked for" and "what is on screen" are written in one statement and cannot drift apart. * A resolution that wants to steer the rail compares itself against `wantedId` instead. */ - const select = useCallback((id: string, _stage = "") => { + const select = useCallback((id: string, _stage = "") => { wantedId.current = id; setActiveId(id); }, []); @@ -527,7 +527,21 @@ export default function AutomationSurface() { time a third kind shipped, and printed inside a `--lp-rail-w` column. What the kinds are belongs to the create form, which is one click away and lists all of them from the server. */} - {!items.length && !error ? ( + {/* ⭐ WAVE 29 (W29-T01) — A LOADING RAIL IS NOT AN EMPTY ONE. `data` is null until the + first `listAutomations()` resolves, and this branch read only `items.length`, so + every visit to Automation asserted "No automations yet." to a person who has + several — then replaced it with their list. Half of owner item 5's "takes a while + to appear" is that sentence: the wait is real, but being told you own nothing is + what makes it read as broken rather than slow. `data === null` is the one state + that means NOBODY HAS LOOKED, and it gets skeleton rows, not a claim. */} + {data === null && !error ? ( + + ) : null} + {data !== null && !items.length && !error ? (

No automations yet.

) : null} diff --git a/web/src/automation/TriggerPicker.tsx b/web/src/automation/TriggerPicker.tsx index 7a05510a501a71bf3606f554f672ff25744aa08f..ba792d6aa2b5e53261d1b5a359d38e1daa17109c 100644 --- a/web/src/automation/TriggerPicker.tsx +++ b/web/src/automation/TriggerPicker.tsx @@ -154,8 +154,13 @@ export function TriggerMark({ kind = "" }: { kind?: string }) { ); // A MUSICAL NOTE inside the same search frame as `ig_profile_match`: same act (search a corpus - // of profiles), different corpus. Drawing it as a second person-in-a-magnifier would say the - // two are interchangeable, which is exactly the claim D-9 is still open about. + // of profiles), different corpus. Drawing it as a second person-in-a-magnifier would say the two + // are interchangeable, and they are not — the two networks have different schemas and, per wave + // 29's R2, different `ut_*` families behind them. + // ⚠ WAVE 29: this row is LIVE now (D-9 closed the trigger half). Nothing in this file changed to + // make it so — `planned` rides the wire, which is the property that makes a server vocabulary + // worth having. The sentence that used to end this note ("the claim D-9 is still open about") + // was true when it was written and would have quietly aged into a lie. if (kind === "tiktok_profile_match") return ( diff --git a/web/src/automation/automationApi.ts b/web/src/automation/automationApi.ts index ccfafafcffe25857a81b315246dd0327edc06f71..52fe1dba17b0c2713d32ac6194423c53823e3f69 100644 --- a/web/src/automation/automationApi.ts +++ b/web/src/automation/automationApi.ts @@ -472,6 +472,18 @@ export interface FlowVocab { maxCondChildren: number; maxGroupDepth: number; maxActions: number; + /** + * ⭐ W29-T09 (owner item 11: *"the last 12 reels"*) — the post kinds a capture group may name, + * with the words a person reads. OPTIONAL, like every other vocabulary here: absent means a + * server older than this client, and the control renders nothing rather than inventing three + * names the save door would refuse. + * ⛔ `key` is what `config.postGroups` STORES; `label` is only ever painted. Writing the label + * would build a filter `clean_post_groups` rejects with a sentence naming a word nobody typed. + */ + postTypes?: { key: string; label: string }[]; + /** The ceiling `clean_post_groups` judges a group's limit against (the action's own maxPosts + * is the real bound; this is the absolute one, so a hint can be honest about both). */ + maxPostsPerPull?: number; /** Legacy read compatibility only; the server no longer sends Board review vocabulary. */ reviewDeciders?: string[]; /** Legacy read compatibility only; the server no longer sends Board review readiness. */ diff --git a/web/src/customer-grid/ColumnMenu.tsx b/web/src/customer-grid/ColumnMenu.tsx index 819309eb539f29f8b525cf39ce850306530d853f..704dbfbd499edc9348d486309d3796c9c8dd2afe 100644 --- a/web/src/customer-grid/ColumnMenu.tsx +++ b/web/src/customer-grid/ColumnMenu.tsx @@ -7,13 +7,14 @@ import { CODE_LANGUAGE_LABELS, CODE_LANGUAGES, CREATABLE_TYPES, choiceOptions, c codeLanguageOf, directionLabel, isDerivedLink, isMachineOwned, isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS, ROLLUP_REF_OPS } from "./types"; -import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn, +import type { AggName, Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn, RollupRefOp, RollupSource, Viewer } from "./types"; import type { LinkTarget, RollupSourceOffer } from "./apiBridge"; import type { WindowSpec } from "./windows"; import { normalizeWindow, windowLabel } from "./windows"; import { WindowPicker } from "../filter-kit"; import { TYPE_LABELS } from "./iconShapes"; +import { AGG_LABELS, aggOptions } from "./aggregations"; import { validateFormula } from "./formulaEngine"; import { BRAND_SWATCHES, defaultOptionColor, nearestSwatch, normalizeOptionColor, OPTION_PALETTE } from "./choiceColors"; @@ -238,6 +239,10 @@ interface ColumnMenuProps { /** Wave-5 item 10 — save a display format. Supplied for number/currency/formula and * date/created_time fields. */ onFormat?: (format: FieldFormat) => void; + /** ⭐ WAVE-29 T33 (owner item 17) — set (or clear, with `undefined`) this column's SUMMARY: the + * value its group footers and the table's totals row show. Supplied wherever a field write is + * possible; the pane offers only what `aggregations.aggOptions` allows for the type. */ + onAggregate?: (agg: AggName | undefined) => void; /** Wave-5 item 1 — Sort by this field (replaces the view's sort), and the conditional * clears (item 3). All of these mutate the EXISTING view config — no new host surface. */ onSort: (dir: "asc" | "desc") => void; @@ -273,7 +278,7 @@ type CreatePosition = "left" | "right" | "end"; * choices/stars/formula config), and the Change-field control that swaps the column to show * another field. */ -type MenuPane = "menu" | "note" | "edit" | "permissions" | "format"; +type MenuPane = "menu" | "note" | "edit" | "permissions" | "format" | "summary"; /** * How each creatable type reads in the menu. DERIVED from CREATABLE_TYPES rather than listed @@ -1738,6 +1743,7 @@ export default function ColumnMenu({ onDuplicate, onPermissions, onFormat, + onAggregate, onSort, onClearSort, onFilterBy, @@ -3146,6 +3152,63 @@ export default function ColumnMenu({ } // ------------------------------------------------------------------ FORMAT pane (item 10) + /** + * ⭐⭐ WAVE-29 T33 (owner item 17) — THE COLUMN SUMMARY PANE. + * + * A flat list of the six aggregates plus "None", because that is the whole decision — there is + * no draft to reconcile and no Save button to press, so the click IS the answer and the pane + * closes on it (the same shape as Sort, one group up, and unlike Format, which composes four + * settings into one bag). + * + * ⛔ THE OPTIONS COME FROM `aggOptions`, never from a list written here. `sum · average · + * median · min · max` are numeric-only and `count` counts ROWS, so a text column offers exactly + * one row — and it offers it, rather than hiding the pane, because "how many records is this + * group" is the summary a text column CAN answer. + */ + if (pane === "summary" && onAggregate) { + const options = aggOptions(field); + return ( + + +
+ + {options.map((name) => ( + + ))} +
+ + ); + } + if (pane === "format" && formatKind) { return ( )} + {/* ⭐ WAVE-29 T33 (owner item 17) — the COLUMN SUMMARY. Offered on every column, because + `count` applies to any type; the numeric folds are filtered per type inside the pane + by `aggOptions`. ⚠ NOT gated on `schemaLocked`: a summary is a reading of the column, + not a change to its schema, so it belongs on a pre-set column too — the same reason + Sort and Filter sit below without that gate. */} + {onAggregate && ( + + )}
diff --git a/web/src/customer-grid/CustomerGrid.tsx b/web/src/customer-grid/CustomerGrid.tsx index 22d697a89667b76c352c97eef5ab73c4ed1726ee..dff2701b26b4911b1c84f332edf495322847148a 100644 --- a/web/src/customer-grid/CustomerGrid.tsx +++ b/web/src/customer-grid/CustomerGrid.tsx @@ -1,6270 +1,6570 @@ -import { - memo, - useCallback, - useEffect, - useLayoutEffect, - useMemo, - useRef, - useState, -} from "react"; -import { DataEditor, GridCellKind } from "@glideapps/glide-data-grid"; -import type { - CellClickedEventArgs, - DataEditorRef, - EditableGridCell, - GridKeyEventArgs, - GridMouseEventArgs, - GridSelection, - DrawHeaderCallback, - HeaderClickedEventArgs, - Item, - Rectangle, - Theme, -} from "@glideapps/glide-data-grid"; -import "@glideapps/glide-data-grid/dist/index.css"; - -import { useCustomerData } from "./useCustomerData"; -import type { SurfaceScope } from "./apiBridge"; -import { defaultViewConfig, useGridColumns } from "./useGridColumns"; -import { activeMeasureRuleIds, pendingMeasures, runPipeline, unresolvedConditions, useVisibleRows } - from "./useVisibleRows"; -import type { CohortSets } from "./useVisibleRows"; -import type { MeasureSets } from "./useVisibleRows"; -import { useGetCellContent } from "./useGetCellContent"; -import { useGridSelection } from "./useGridSelection"; -import Toolbar from "./Toolbar"; -import JsonViewer from "./JsonViewer"; -import RecordDetail from "./RecordDetail"; -import ViewSidebar from "./ViewSidebar"; -import ColumnMenu from "./ColumnMenu"; -import type { ColumnMenuState } from "./ColumnMenu"; -import { HEADER_ICONS } from "./iconShapes"; -import { emitHostEvent, eventId } from "./hostBridge"; -// ⭐ WAVE 27 item 21 (R14) — which of this table's views carry an alert. The SERVER's -// list is the only answer; the rail's alert door is a one-way create. -import { fetchAlerts } from "../alerts/alertsApi"; -import { NAV_MINIMIZE_EVENT, ROWS_STALE_EVENT, TOAST_EVENT, VIEW_OPEN_EVENT, - WORKSPACE_STALE_EVENT, signal } - from "../apiContract"; -import type { ViewOpenDetail } from "../apiContract"; -import { addTableField, addTableRow, deleteTableField, deleteTableRow, fetchLinkTargets, - fetchRollupSources, patchTableField } from "./apiBridge"; -import type { LinkTarget, RollupSourceOffer } from "./apiBridge"; -import SelectFromFile from "./SelectFromFile"; -// Owner item 16 / R4 / C-UNDO — the stack and every inverse. Pure, so a node gate can run it. -import { describe, directed, popRedo, popUndo, pushUndo, stackFor } from "./undoStack"; -import type { CellChange, UndoBook, UndoEntry, UndoRow, UndoValue } from "./undoStack"; -import { exportFilename, runExport, triggerDownload } from "./export"; -import type { ExportFormat } from "./export"; -// owner item 3 (2026-08-03) — a time-series view exports its SHEET, not the rows under it. -import { buildTsCsv, tsSheetToTable } from "./timeSeriesData"; -import type { TsSheet } from "./timeSeriesData"; -import { echoReemit, reconcileEchoView } from "./viewEcho"; -import { pruneStamps, reconcileFields } from "./optimism"; -import { applyViewOrder, newFolderId, pruneFolderStamps, reconcileFolders, - resolveFolderId } from "./folders"; -import type { FolderStamps } from "./folders"; -// ⭐ WAVE 27 item 22 — the cells the active view's filter forces on a record added under it. -import { filterSeedValues } from "./filterSeed"; -import { adoptNewFields, adoptNewViews, pruneTombstones, seedLocalViews, stampTombstone } - from "./liveWorkspace"; -import type { Tombstones } from "./liveWorkspace"; -import type { GridFolder, ViewPermissions } from "./types"; -import type { FieldStamps } from "./optimism"; -import { evalFormula, orderFormulas, parseFormula } from "./formulaEngine"; -import type { FormulaAst } from "./formulaEngine"; -import { CalendarView, KanbanView, ListView, ModeSwitch } from "./viewModes"; -import { SwipeView } from "./SwipeView"; -import type { SwipeSpec } from "./SwipeView"; -// The chart engine lives in `viz/` since EXIT wave 2 (W2-5/Y3) — the grid is now -// one of its two callers, the Y1 page envelope being the other. -import { DashboardView } from "../viz/DashboardView"; -import { cleanCharts } from "../viz/chartData"; -import type { ChartSpec } from "../viz/chartData"; -import { MapView } from "./MapView"; -/* ═══ W18-C CATALOG ═══ (owner item 4, contract C6) */ -import { CatalogView } from "./CatalogView"; -import { CATALOG_CODE_FIELD } from "./catalogData"; -import type { CatalogSpec } from "./types"; -/* ═══ end W18-C CATALOG ═══ */ -import { lightTheme, STATUS_ROW_THEME, HOVER_ROW_THEME, HOVER_NEUTRAL, - ACTIVE_ROW_NEUTRAL, CUSTOM_FIELD_MARK } from "./theme"; -import { avatarInitials, formatDisplay, imageCellRenderer, ratingCellRenderer, setAvatarRepaint, - userCellRenderer } from "./cells"; -import { optionTint, pickTint } from "./choiceColors"; -import { - EMPTY_GRID_COPY_PROVENANCE, - markGridCopy, - observeCopyEvent, - planFieldPaste, - pasteRowCount, -} from "./clipboard"; -import { cellTipText, expandButtonRect, GROUP_HEADER_FONT, GROUP_LABEL_PAD, headerMarkLayout, - headerMarkSizes, tipLeft } from "./overlayPlacement"; -import { AnchoredOverlay, BodyPortal, useOverlayLayer } from "./OverlaySurface"; -import type { AnchorRect } from "./OverlaySurface"; -import { StarIcon } from "./Stars"; -import { ALL_VIEW_ID, MAX_CALENDAR_METRICS, allViewName, - MAX_FROZEN, choiceOptions, choiceVocabulary, clampFrozenCount, cleanDisplay, formulaOf, - topicForScope, - isDateFamilyType, isFilterGroup, isGroupableField, isMachineOwned, isMachineWritten, - isDerivedLink, - isNumericFieldType, - isPickType, mayEditField, - isModeFrozen, isUndeletableView, mayEditView, mayToggleViewLock, - measureColumnIndex, ratingMax, ruleColumnKeys, - tableMode, uniqueDisplayName } from "./types"; -import type { - DisplayMode, - DisplaySpec, - Field, - HostEvent, - FieldScope, - FieldType, - FilterNode, - FilterRule, - Row, - RowHeightMode, - SavedView, - ViewConfig, -} from "./types"; -// Item 7 (C-TS). Default-exported because it is a leaf VIEW like MapView, and because RECORD -// imports the same default for the record-detail Insights tab (C-EMBED). -import TimeSeriesPanel from "./TimeSeriesPanel"; -import type { WindowSpec } from "./windows"; -import { windowLabel } from "./windows"; - -const ROW_PX: Record = { short: 28, medium: 34, tall: 48 }; - -/** - * Wave-14 R7 — **a button label is ONE LINE.** Every button row in this file is a flex row with - * no width reservation, so on a narrow grid box the items shrink and their labels wrap: - * "Remove from cohort" becomes two lines, the row grows, and the bar stops being a bar. It only - * happens below a width no fixed-size screenshot is taken at, which is why it survived. - * - * ⚠ Applied INLINE rather than on `.cg-btn`, where it belongs: `index.css` is PANEL's fence this - * wave. A mailbox line asks them to hoist it onto `.cg-btn` globally — after which this is - * redundant, and harmless, because it says exactly the same thing. - */ -const ONE_LINE = { whiteSpace: "nowrap" } as const; - -/** - * How many VISIBLE columns glide freezes. Module-level and used twice on purpose (item 15): the - * `freezeColumns` prop and the group bar's label fit must read ONE answer, because the label is - * clipped to exactly this strip and a second copy of the clamp would let the two disagree by a - * column — which shows up as a label cut early for no visible reason. - */ -function frozenCountOf(config: ViewConfig): number { - return Math.min(MAX_FROZEN, Math.max(1, config.frozenCount ?? 1)); -} -/* WAVE 21 item 3 (R6): the id and the NAME both moved to `types.ts` — the id because a second - copy of a pinned literal is the drift class this repo gates against, the name because it is - now topic-derived and the host mints the same string. */ - -/** - * owner item 2 (2026-08-03) — the measure-cell skeleton's pulse, and its ceiling. - * - * 140ms against the 4-step colour ramp is a ~0.6s cycle: a wait, not a strobe. The ceiling is - * ~60s, after which the cells fall back to ordinary blanks — a measure that has not resolved in - * a minute is not "still loading", and a shimmer that never ends promises a number that is not - * coming. Shared constant so the two are read in one place rather than tuned apart. - */ -const PULSE_MS = 140; -const PULSE_MAX_TICKS = Math.round(60_000 / PULSE_MS); -/** Referentially stable, so the fallback does not change `getCellContent`'s identity per render - * and repaint the canvas forever. */ -const NO_PENDING_KEYS: ReadonlySet = new Set(); - -/** - * I2 — measure a cell's text in GLIDE'S OWN font, so "is this cut off?" is a fact rather than - * a character-count guess (a guess is wrong in both directions: "IIIII" is narrow, "WWWWW" is - * wide, and being wrong means either a missing tip or a tip over text you can already read). - * One lazily-built offscreen context for the whole module — `measureText` is cheap, but - * creating a canvas per mouse-move would not be. - */ -let _tipCtx: CanvasRenderingContext2D | null | undefined; -function measureCellText(text: string): number { - if (_tipCtx === undefined) { - _tipCtx = document.createElement("canvas").getContext("2d"); - // glide's default `baseFontStyle` is 13px; the family comes from our own theme. - if (_tipCtx) _tipCtx.font = `13px ${lightTheme.fontFamily ?? "Inter, sans-serif"}`; - } - return _tipCtx ? _tipCtx.measureText(text).width : 0; -} -/** - * Item 15 — the GROUP BAR's measurer, and it is a SECOND context on purpose. - * - * `measureCellText` above is set to plain "13px", which is right for ordinary cells and WRONG for - * a group bar: that row carries `baseFontStyle: GROUP_HEADER_FONT` (semibold), and glide paints a - * cell with the merged theme's font. Measuring the fit with the lighter weight under-truncates - * and the label overflows into the frozen strip's hard clip — the exact failure `fitGroupLabel` - * exists to prevent. Font string built from the SAME constant the cell's themeOverride uses, so - * the two cannot drift. - */ -let _groupCtx: CanvasRenderingContext2D | null | undefined; -function measureGroupText(text: string): number { - if (_groupCtx === undefined) { - _groupCtx = document.createElement("canvas").getContext("2d"); - if (_groupCtx) - _groupCtx.font = `${GROUP_HEADER_FONT} ${lightTheme.fontFamily ?? "Inter, sans-serif"}`; - } - return _groupCtx ? _groupCtx.measureText(text).width : 0; -} -/** - * Owner item 8 (2026-07-27): the first render shows this many rows; "See more" reveals the rest. - * A DISPLAY cap only — the pipeline still runs over the whole book, the toolbar count is still - * the full matched count, and the cap is stated beside the control that lifts it, which is what - * keeps [[no-unverifiable-aggregates]] satisfied: nothing is silently truncated. - */ -const DISPLAY_PAGE = 50; -const DISPLAY_STEP = 250; - -/* ═══════════════════════════════════════════════════════════════════════════════════════════ - ═══ W18-B VOID ═══ (wave 18, owner item 1b) — the two constants the void geometry needs - that belong to the BROWSER and to GLIDE rather than to us. Both are mirrors, and both say - here what they are mirroring, because a mirror that does not name its original is how the - two copies stop agreeing. - ═══════════════════════════════════════════════════════════════════════════════════════════ */ - -/** The header band. `headerHeight={36}` at the DataEditor mount is the original. - * Not `+ groupHeaderHeight`: glide adds that only when `enableGroups` is on, which it turns on - * when a COLUMN carries a `group` — `useGridColumns` never sets one (scrolling-data-grid.js:13). */ -const HEADER_PX = 36; - -/** glide's own row-marker width ladder, `data-editor.js:103`, verbatim. - * - * ⚠ It is passed BACK to glide as `rowMarkerWidth` rather than merely predicted here. The - * marker column is part of the content width the void's left edge is measured from, so a - * four-pixel disagreement between what glide draws and what we compute is a four-pixel seam of - * white against the tint — visible, and invisible to every gate. Pinning makes the two the same - * number by construction; mirroring the ladder (rather than pinning one constant) is what keeps - * the pin from changing the marker column's width on tables of more than 100 rows. */ -function rowMarkerPx(rows: number): number { - return rows > 10_000 ? 48 : rows > 1000 ? 44 : rows > 100 ? 36 : 32; -} - -/** The scrollbar gutter this browser steals from a scrollable box, measured once. - * - * ⛔ WHY THE VOID HAS TO KNOW. The void rectangles sit ON TOP of glide's scroller, so a - * rectangle drawn to the box's own edge paints over the scrollbar and hides it. The two cases - * are CROSSED, which is the part that is easy to get backwards: the BOTTOM void has to stop - * short when the columns overflow (a horizontal scrollbar), and the RIGHT void has to stop - * short when the rows overflow (a vertical one). Both happen in ordinary use — twelve columns - * filtered to five rows is exactly the owner's case. - * - * 0 on overlay-scrollbar platforms (macOS, touch), which is correct: nothing is stolen there. */ -let scrollbarGutterPx: number | null = null; -function scrollbarGutter(): number { - if (scrollbarGutterPx !== null) return scrollbarGutterPx; - if (typeof document === "undefined") return 0; - const probe = document.createElement("div"); - probe.style.cssText = - "position:absolute;top:-9999px;width:100px;height:100px;overflow:scroll"; - document.body.append(probe); - scrollbarGutterPx = probe.offsetWidth - probe.clientWidth; - probe.remove(); - return scrollbarGutterPx; -} -/* ═══ end W18-B VOID (module scope) ═══ */ - -interface LocalWorkspace { - fields: Field[]; - views: SavedView[]; - activeViewId: string; - /** When this copy was written (browser clock — same-machine freshness only, - * never date semantics). Absent on copies from before the echo reconcile. */ - savedAt?: number; - /** Wave-6 item 3c — per-key freshness for this browser's own field-def - * writes (edits + tombstones), consumed by reconcileFields at init. */ - fieldStamps?: FieldStamps; - /** BUG-1 (wave 11) — viewId → the filter tree already pushed back to the host, so the - * echo re-emit happens once per user edit rather than once per remount. NOT a timestamp: - * `savedAt` beside it is rewritten on every init, so any time-based bound is refreshed by - * the remount it is meant to bound. See viewEcho.echoReemit. */ - reemitted?: Record; - /** 2026-08-04 — viewId → when THIS browser deleted it. The views half of the tombstone - * rule fields and folders already had, needed the moment the rail adopts host views - * after mount (liveWorkspace.ts). Persisted, not a ref, because a remount inside the - * echo window would otherwise re-adopt a view the user deleted a second ago. */ - viewTombstones?: Tombstones; - /** D-19 (wave 20) — viewId → when THIS browser last WROTE it. The counterpart of the - * tombstones above: they stop a deleted view coming back, this stops a just-created one - * being dropped by the host-list rule at init, before its upsert has round-tripped. - * Same window, same prune, same browser-clock caveat. */ - viewWrites?: Tombstones; -} - -function sameConfig(a: ViewConfig, b: ViewConfig): boolean { - return JSON.stringify(a) === JSON.stringify(b); -} - -function normalizeConfig(config: Partial | undefined, fields: Field[]): ViewConfig { - const base = defaultViewConfig(fields); - return { - ...base, - ...config, - filters: Array.isArray(config?.filters) ? config.filters : [], - // Legacy views persisted a flat rule array with an implicit AND — anything - // other than an explicit "or" normalizes to "and", so old views are unchanged. - filterConj: config?.filterConj === "or" ? "or" : "and", - sorts: Array.isArray(config?.sorts) ? config.sorts : [], - order: Array.isArray(config?.order) && config.order.length ? config.order : base.order, - visible: - Array.isArray(config?.visible) && config.visible.length - ? config.visible - : base.visible, - widths: config?.widths && typeof config.widths === "object" ? config.widths : {}, - memberPids: Array.isArray(config?.memberPids) ? config.memberPids : [], - // Wave-6 items 10/11. Both normalize junk to ABSENT (grid / 1 frozen), so - // a legacy view is byte-identical to itself after a round trip. W13 (C4 as - // amended): cleanDisplay keeps grid-carried field picks. - display: cleanDisplay(config?.display), - frozenCount: clampFrozenCount(config?.frozenCount), - }; -} - -/** - * The pinned system view, as this browser mints it when no host copy has arrived. - * - * WAVE 21 item 3 (R6) — the NAME is the scope's now (`allViewName`), not the literal "All - * customers" this minted on every topic including `ut_*` databases of anything at all. The - * ID is unchanged and deliberately so (see `ALL_VIEW_ID`). - */ -function allRecordsView(fields: Field[], scope: string): SavedView { - return { - id: ALL_VIEW_ID, - name: allViewName(scope), - kind: "system", - locked: true, - config: defaultViewConfig(fields), - }; -} - -function readLocal(storageKey: string): LocalWorkspace | null { - try { - const raw = localStorage.getItem(`aios-grid:${storageKey}`); - return raw ? (JSON.parse(raw) as LocalWorkspace) : null; - } catch { - return null; - } -} - -function writeLocal(storageKey: string, value: LocalWorkspace): void { - try { - localStorage.setItem(`aios-grid:${storageKey}`, JSON.stringify(value)); - } catch { - // Storage can be blocked in hardened/opaque iframes. The host bridge remains durable. - } -} - -function nextViewId(): string { - return `view_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`; -} - -/** A multiselect cell's SET, out of its comma-joined string (the `multi` contract). */ -function splitMulti(v: string): string[] { - return v - .split(",") - .map((s) => s.trim()) - .filter((s) => s !== ""); -} - -/** Wave-5 item 3 — does any leaf of the filter tree name this column? - * - * ⚠ Wave-20 item 2 RETARGETED the note that stood here ("measure leaves carry measure keys, - * which are never field keys, so they cannot false-positive"). True about false POSITIVES, - * and precisely why it was a false NEGATIVE: a measure column's own condition names the - * MEASURE, so this answered "not filtered" for a column the user had visibly filtered, and - * the menu never offered "Don't filter by this field" on it. Both doors now go through - * `ruleColumnKeys` — the same resolution the tint uses, so the menu and the colour can never - * disagree about which columns a filter is about. */ -function treeNamesField( - nodes: FilterNode[], - key: string, - measureCols: Map -): boolean { - for (const n of nodes) { - if (isFilterGroup(n)) { - if (treeNamesField(n.children, key, measureCols)) return true; - } else if (ruleColumnKeys(n as FilterRule, measureCols).includes(key)) return true; - } - return false; -} - -/** Wave-5 item 3 — "Don't filter by this field": drop every leaf naming the column, prune - * groups that end up empty. Returns new arrays throughout (the config is state). */ -function dropFieldFromTree( - nodes: FilterNode[], - key: string, - measureCols: Map -): FilterNode[] { - const out: FilterNode[] = []; - for (const n of nodes) { - if (isFilterGroup(n)) { - const children = dropFieldFromTree(n.children, key, measureCols); - if (children.length) out.push({ ...n, children }); - } else if (!ruleColumnKeys(n as FilterRule, measureCols).includes(key)) { - out.push(n); - } - } - return out; -} - -// The header sprite map moved to iconShapes.ts (wave-8 I20/I21) — it is data + -// SVG-source builders with no React in it, which is what lets verify_icons.py -// import it under node and assert the (i) actually contrasts. - -function slugify(label: string): string { - return ( - label - .toLowerCase() - .replace(/[^a-z0-9]+/g, "_") - .replace(/^_|_$/g, "") - .slice(0, 28) || "field" - ); -} - -interface FieldBuildExtra { - formula?: string; - max?: number; - scope?: FieldScope; - label?: string; - colorCodeOptions?: boolean; - optionColors?: Record; - /** - * ⭐⭐ 2026-08-09 — THE RELATIONAL BAGS, WHICH THIS INTERFACE USED TO DROP ON THE FLOOR. - * - * `ColumnMenu.FieldConfigExtra` has carried them since 2026-08-07 and `extraFor` builds a - * complete bag; this type — the receiving end of the same `extra` argument — never declared - * either key, so `buildOverlayField` could not copy what it could not see. The measured result - * on nurilab: a column stored as - * `{key:'custom_video_views_90j26', label:'Video Views', type:'rollup', source:'overlay'}` - * with NO bag, in one user's workspace stratum, which `compute_relation_cells` never reads. - * Named, configured, rendering, and permanently blank — the owner's *"the Rollup doesn't - * work"*. ⛔ Both halves were needed: this type, and the `createField` route below. - */ - link?: Record; - rollup?: Record; - /** Item 15 (C-RENAME) — the option renames this save carries, by row identity. Consumed by - * `retypeField` (which emits `choice_rename`) and ignored by every create path: a field - * being CREATED has no values to migrate. */ - renames?: { from: string; to: string }[]; -} - -/** - * One constructor for a user-created field, shared by "insert field" and Change-field's - * "New field" half (wave-2 item 5) so the two doors cannot drift. A `multiselect` carries its - * declared options AND `multi: true` — the cell is a comma-joined SET, and the flag is what the - * grouping contract keys on (belt to the type-derived brace in groupRows). - * - * Wave-5: `formula` and `created_time` are built in the HOST's OWN emission shape (source - * 'odoo' + derived — aios_grid.READONLY_CUSTOM_TYPES), so the optimistic local def and the - * next payload's echo are byte-identical and nothing restyles on the round trip. `rating` - * carries top-level `max`, `formula` carries top-level `formula` (the ~20:20 contract - * amendment: options stays the select-family list). Wave-6 item 6: both types are now - * `filterable: true` — the host flipped its emission, and conditions on them evaluate in the - * client engine over computedRows (this table's counts are client-mode; a windowed table's - * columns come from the semantic model, so the SQL engine never sees these types). - */ -/** - * ⭐⭐ 2026-08-09 (owner ruling) — DOES THE PRE-SET LOCK CLOSE THIS COLUMN'S SCHEMA? - * - * Owner: *"No rollup field should be uneditable, everything is custom and changeable always."* - * A ROLLUP holds no data of its own — it is a question asked of other rows, re-askable at any - * time, and re-asking it costs nothing because the answer is recomputed from the authoritative - * store on the next pass. `preset` still closes every column that HOLDS something: retyping a - * pre-set `followers` column would strand real measurements in a column that can no longer read - * them. - * - * ⛔ THE CLIENT MIRROR OF `core.user_tables.preset_editable`, and `verify_rollup_editor.py` - * holds the two in step. This check has THREE enforcement points (here, `may_edit_field`, and - * `routes_tables._field_or_refuse`'s sentence); a client that kept hiding the editor while the - * route allowed the PATCH would read as "you fixed nothing", which is the failure mode a fix in - * one of three places always wears. - */ -function isSchemaLocked(field: Field): boolean { - return field.automation?.preset === true && field.type !== "rollup"; -} - -function buildOverlayField( - label: string, - type: FieldType, - options?: string[], - extra?: FieldBuildExtra -): Field { - const key = `custom_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`; - if (type === "formula" || type === "created_time") { - return { - key, - label, - type, - source: "odoo", - derived: true, - filterable: true, - default: true, - custom: true, - ...(type === "formula" - ? { agg: "sum", ...(extra?.formula ? { formula: extra.formula } : {}) } - : {}), - }; - } - return { - key, - label, - type, - source: "overlay", - default: true, - custom: true, - agg: ["currency", "int"].includes(type) ? "sum" : undefined, - // The relational bags ride the local definition too, so the optimistic field and the - // server's echo describe the same column. Without them the grid held a bagless twin of a - // field the store had configured, and `mayEditField`/`isDerivedLink` read the twin. - ...(extra?.link ? { link: extra.link as Field["link"] } : {}), - ...(extra?.rollup ? { rollup: extra.rollup as Field["rollup"] } : {}), - ...((type === "select" || type === "multiselect") && options?.length ? { options } : {}), - ...((type === "select" || type === "multiselect") - ? { - colorCodeOptions: extra?.colorCodeOptions !== false, - ...(extra?.optionColors && Object.keys(extra.optionColors).length - ? { optionColors: extra.optionColors } - : {}), - } - : {}), - ...(type === "multiselect" ? { multi: true } : {}), - ...(type === "rating" ? { max: extra?.max ?? 5 } : {}), - }; -} - -/** The surface this grid draws. `cohort` is the same table over hand-curated SETS — the server - * confirms it by stamping `workspace.cohortMode`, which is the only thing the body below reads. - * - * MEMOIZED (owner item 1, 2026-07-31): the shell re-renders on every chrome state flip — nav - * collapse, toast, settings — and an unmemoized grid re-rendered its whole 3,400-line tree - * each time. Measured cost: 200ms-5s of main-thread block, which ate the 240ms rail-fold - * transition whole (the fold froze, then SNAPPED — the exact "static" the owner named). The - * props surface is one stable string, so memo makes chrome state changes free; the grid still - * re-renders for its own state (edits, resize) and remounts on route change via `key`. */ -interface CustomerGridProps { - scope?: SurfaceScope; - /** Read-only Grid view embedded in a linked-record modal. It keeps the standard - * filter/sort/search toolbar while withdrawing schema and row mutations. */ - embedded?: boolean; - /** Exact linked pids to project from the target database. */ - embeddedRecordIds?: readonly number[]; - /** Selection mode used by an editable ordinary-Link modal. */ - embeddedSelectable?: boolean; - embeddedSelectedIds?: readonly number[]; - onEmbeddedSelectionChange?: (recordIds: number[]) => void; -} - -interface LinkGridModalProps { - label: string; - table: SurfaceScope; - recordIds: readonly number[]; - editable?: boolean; - single?: boolean; - onSave?: (recordIds: number[]) => void; - onClose: () => void; -} - -function LinkGridModal({ label, table, recordIds, editable = false, single = false, - onSave, onClose }: LinkGridModalProps) { - const panelRef = useRef(null); - const [selectedIds, setSelectedIds] = useState(() => [...recordIds]); - const changeSelectedIds = useCallback((ids: number[]) => { - const next = single ? ids.slice(-1) : ids; - setSelectedIds((current) => - current.join(",") === next.join(",") ? current : next - ); - }, [single]); - useOverlayLayer({ - panelRef, - onDismiss: onClose, - dismissOnOutside: true, - initialFocus: "[data-overlay-autofocus]", - trapFocus: true, - }); - return ( - -
-
-
-
-
{label}
-
- {(editable ? selectedIds.length : recordIds.length).toLocaleString()} linked{ - single ? " (one allowed)" : "" - } {(editable ? selectedIds.length : recordIds.length) === 1 ? "record" : "records"} -
-
- -
-
- -
- {editable ? ( -
- - -
- ) : null} -
-
-
- ); -} - -function CustomerGrid({ - scope = "customer", - embedded = false, - embeddedRecordIds, - embeddedSelectable = false, - embeddedSelectedIds = [], - onEmbeddedSelectionChange, -}: CustomerGridProps = {}) { - // Wave 16 C-TOPIC: which TABLE this tree is drawing, derived from the one scope prop. - const topic = topicForScope(scope); - const { - fields: payloadFields, - rawRows: fetchedRows, - payload, - loading, - overlayEdits, - setOverlayEdits, - patchOverlay, - } = useCustomerData(scope, { - bindSurface: !embedded, - includeWorkspace: !embedded, - writable: !embedded, - }); - const embeddedIdsKey = embeddedRecordIds?.join(",") ?? ""; - /** - * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — **ROWS THIS BROWSER JUST CREATED**, held locally - * until the server's own copy comes back. - * - * ⛔ THE OWNER'S WORDS: *"adding a new record visually takes too long, I need to be able to - * spam it."* The old path awaited the POST, then fired `ROWS_STALE_EVENT` and re-read the - * WHOLE table — so every "+" cost a round trip plus a full refetch before anything moved, and - * pressing it five times queued five full refetches of a table that grew by five rows. - * - * ⚠ ADD-ONLY AND SELF-PRUNING, which is `liveWorkspace.ts`'s discipline applied to rows: a - * pending row is merged in only while the fetched payload does NOT carry its pid, and the - * effect below drops it the moment the server's copy arrives. There is no third state and no - * merge of values — the server's row wins outright, because by then it IS this row. - */ - const [pendingRows, setPendingRows] = useState([]); - const rawRows = useMemo(() => { - const base = embeddedRecordIds - ? fetchedRows.filter((row) => new Set(embeddedRecordIds).has(row.pid)) - : fetchedRows; - if (!pendingRows.length) return base; - const known = new Set(base.map((r) => r.pid)); - const extra = pendingRows.filter((r) => !known.has(r.pid)); - return extra.length ? [...base, ...extra] : base; - // The scalar key keeps this stable when a caller reconstructs the same id list. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [fetchedRows, embeddedIdsKey, pendingRows]); - useEffect(() => { - // ⚠ PRUNED AGAINST THE FETCHED PAYLOAD, not against `rawRows` — `rawRows` contains the - // pending rows, so it can never report that one has been absorbed. Pruning against the - // memo's own output is the loop that keeps a placeholder alive forever. - if (!pendingRows.length) return; - const known = new Set(fetchedRows.map((r) => r.pid)); - if (!pendingRows.some((r) => known.has(r.pid))) return; - setPendingRows((cur) => cur.filter((r) => !known.has(r.pid))); - }, [fetchedRows, pendingRows]); - /** - * The rows as of the last render, for callbacks that must not be rebuilt per payload. - * `appendRow` reads it to mint the next row id the way the server does (`max + 1`); putting - * `rawRows` in that callback's dependency list would give the trailing "+" a new identity on - * every refetch, which is a re-render of the grid for a value only used at click time. - */ - const rowsRef = useRef(rawRows); - rowsRef.current = rawRows; - /** - * ⭐ WAVE 27 · ITEM 2 — THE NEXT ROW ID, ADVANCED SYNCHRONOUSLY AT MINT TIME. - * - * ⛔ A RENDER-TIME SNAPSHOT IS NOT ENOUGH FOR THE GESTURE THIS ITEM IS ABOUT. `rowsRef.current` - * is assigned during RENDER, so two "+" presses that land before React commits the first - * `setPendingRows` both read the same list and both mint the same id. The owner's words were - * *"I need to be able to spam it"* — that IS the failing case, and the damage is not a - * duplicate row: `minted` is the identity key the failure and re-anchor branches filter and - * map on, so ONE refused write would withdraw BOTH rows and one taken id would rewrite both. - * - * ⚠ SEEDED FROM THE ROWS ON EVERY MINT, then advanced: `Math.max(seen, storeMax) + 1`. Reading - * the store's max each time is what keeps the client in step after a refetch brings rows in - * from elsewhere; the ref is what stops a burst from colliding with itself in between. - */ - const nextRidRef = useRef(0); - const mintRid = useCallback(() => { - const storeMax = rowsRef.current.reduce((m, r) => Math.max(m, r.pid), 0); - nextRidRef.current = Math.max(nextRidRef.current, storeMax) + 1; - return nextRidRef.current; - }, []); - /** - * Withdraw rows this browser is still holding locally, because they have been DELETED. - * - * ⛔ THE PRUNE EFFECT CANNOT DO THIS AND THAT IS THE WHOLE BUG IT FIXES. Pruning is "the - * server's payload now CONTAINS this row, so drop my copy" — a deleted row will never appear - * in a payload again, so a pending row that is deleted before its first refetch stays merged - * into `rawRows` for the life of the mount. Add a record, press Ctrl+Z, and the row sits there - * looking undeleted while the store has already lost it. - * ⚠ CALLED AT EVERY `deleteTableRow` SITE (the selection delete and undo's `rowDelete` - * inverse), because those are the only two doors that remove a row — and the undo one is - * exactly the path that produces a still-pending victim. - */ - const withdrawPending = useCallback((rids: (string | number)[]) => { - if (!rids.length) return; - const gone = new Set(rids.map((r) => Number(r))); - setPendingRows((cur) => (cur.some((r) => gone.has(r.pid)) - ? cur.filter((r) => !gone.has(r.pid)) - : cur)); - }, []); - /** - * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — WHICH VIEWS ON THIS TABLE CARRY AN ALERT. - * - * ⛔ THE SERVER'S LIST IS THE ONLY ANSWER. The rail's "Alert me about new records…" is a - * one-way CREATE (it emits an event the shell turns into a POST) and nothing in the grid - * hears the outcome — a `no_filter` refusal is a real answer, so an optimistic local set - * would paint a badge on a view that has no alert. `GET /alerts` is the state. - * - * ⚠ RE-READ ON FOCUS, the same idiom the shell's own inbox uses. An alert created or deleted - * in another surface (the Alerts pane owns the delete door) reaches this rail on the next - * focus rather than instantly; the alternative was for the grid to guess at the outcome of a - * request another component made. - * ⚠ EMBEDDED GRIDS DO NOT ASK. A linked-record grid inside a modal is not a table anyone - * alerts on, and a second fetch per relation cell would be a workspace read per click. - */ - const [alerted, setAlerted] = useState([]); - useEffect(() => { - if (embedded) return; - let live = true; - const pull = () => { - void fetchAlerts().then((r) => { - if (!live || !r.ok) return; // a failed read leaves the badges as they were - setAlerted(r.value.filter((a) => a.topic === scope).map((a) => a.viewId)); - }); - }; - pull(); - window.addEventListener("focus", pull); - return () => { - live = false; - window.removeEventListener("focus", pull); - }; - }, [embedded, scope]); - - const [fields, setFields] = useState([]); - const [views, setViews] = useState([]); - const [activeViewId, setActiveViewId] = useState(ALL_VIEW_ID); - const [config, setConfig] = useState(() => defaultViewConfig([])); - const [workspaceReady, setWorkspaceReady] = useState(false); - const [saveState, setSaveState] = useState<"saved" | "saving">("saved"); - const [search, setSearch] = useState(""); - const [collapsed, setCollapsed] = useState>(new Set()); - const [hoverRow, setHoverRow] = useState(undefined); - /** - * Owner item 19 — the hover-only Expand button: which record it opens and where it sits, in - * VIEWPORT coordinates (glide's `getBounds` space, the same one `.cg-header-tip` uses). - * - * A real DOM button rather than a mark painted into the canvas, and that is the whole design: - * a drawn affordance needs a second hit-test that drifts out of step with the drawing, and - * neither `tsc` nor a screenshot can see the drift — the button is painted, and clicking it - * does nothing ([[ui-invisible-to-assertions]]). An element IS its own hit test. - */ - const [expandAt, setExpandAt] = useState< - { pid: number; x: number; y: number; size: number } | null - >(null); - const [detailPid, setDetailPid] = useState(null); - const [columnMenu, setColumnMenu] = useState(null); - /** Open choice list for a `select` / `user` cell — see onCellClicked. */ - const [picker, setPicker] = useState<{ - pid: number; - fieldKey: string; - anchor: AnchorRect; - } | null>(null); - /** Owner item 5 (2026-07-31) — closing a picker RETURNS FOCUS TO THE GRID, so Enter keeps - * navigating (pick → Enter → next record) instead of stranding focus on a dead overlay. */ - const closePicker = useCallback(() => { - setPicker(null); - requestAnimationFrame(() => gridRef.current?.focus()); - }, []); - /** ⭐ Wave-23 C7 — which `json` cell the big viewer is open on. NOT anchored like the picker - * above: a document is not a choice list, so it opens as a centred modal (the record drawer's - * surface) rather than a popover the size of the cell it came from. */ - const [jsonAt, setJsonAt] = useState<{ pid: number; fieldKey: string } | null>(null); - const closeJson = useCallback(() => { - setJsonAt(null); - requestAnimationFrame(() => gridRef.current?.focus()); - }, []); - /** A relation opens as the target database's real Grid view, projected to the linked ids. - * This is intentionally distinct from JsonViewer: filters, sorts, search, column visibility, - * and the standard cell renderers all remain available inside the large modal. */ - const [linkAt, setLinkAt] = useState<{ pid: number; fieldKey: string } | null>(null); - const closeLink = useCallback(() => { - setLinkAt(null); - requestAnimationFrame(() => gridRef.current?.focus()); - }, []); - /** WAVE 21 item 11 (R10) — is "Select records from a list" open? Opened from the view - * rail's "…" and closed by the dialog; the SELECTION it produces outlives it. */ - const [selectFromFile, setSelectFromFile] = useState(false); - /** The selection bar's "Add to cohort" popover (owner item 2's purpose for the checkboxes). */ - const [selAddOpen, setSelAddOpen] = useState(false); - const [selListName, setSelListName] = useState(""); - const selAddRef = useRef(null); - /** Owner item 9 (wave 20) — its counterpart: "Remove from cohort…", in ORDINARY views. */ - const [selRemoveOpen, setSelRemoveOpen] = useState(false); - const selRemoveRef = useRef(null); - /** Owner item 4 / C-ADDROW — the pid the trailing "+" just created, so the cursor can land on - * it once the re-read actually brings it back (the row does not exist locally before that). */ - const [newRowPid, setNewRowPid] = useState(null); - /** - * ⭐ WARN-THEN-ALLOW on delete (owner, 2026-08-06) — the SIGNATURE of the selection currently - * armed, or `""`. - * - * A signature and not a boolean, deliberately: it is the pid list itself, so changing the - * selection DISARMS automatically. A boolean would stay true while you re-selected, and the - * second Delete would then destroy a set nobody had been warned about — which is worse than no - * warning at all, because the warning is what made it feel checked. - */ - const [delArmed, setDelArmed] = useState(""); - /** Wave-2 item 2c — cohort mode: the active cohort and the "+ Add customers" picker. Picks - * ACCUMULATE across searches (search, tick, search again, confirm once). */ - const [activeCohortId, setActiveCohortId] = useState(null); - const [addCustOpen, setAddCustOpen] = useState(false); - const [addCustQuery, setAddCustQuery] = useState(""); - const [addCustPicked, setAddCustPicked] = useState>(new Set()); - const addCustRef = useRef(null); - /** Wave-5 item 1 — "Filter by this field": a monotonic signal the Toolbar consumes (it - * appends the seeded condition and opens the builder). Wave-6 item 6c: a MEASURE-carrying - * column seeds the equivalent measure condition instead (same measure key + window). */ - const [filterSeed, setFilterSeed] = useState< - | { key?: string; measure?: { key: string; window: WindowSpec }; n: number } - | null - >(null); - /** Wave-6 item 4 — the header-row "+" (insert at end): anchors the create-only menu. */ - const [plusMenu, setPlusMenu] = useState(null); - /** - * Wave-6 item 3c — per-key freshness of this browser's own field-def writes, persisted with - * the workspace copy and consumed by reconcileFields at init. A ref, not state: stamps ride - * along with the setFields that caused them; nothing renders from a stamp. - */ - const fieldStampsRef = useRef<{ edited: Record; deleted: Record }>( - { edited: {}, deleted: {} } - ); - /** Wave-5 item 6 — the header description tip. Floated, pointer-events:none — a tooltip - * that can receive the pointer swallows the NEXT click ([[ui-invisible-to-assertions]]). */ - const [headerTip, setHeaderTip] = useState<{ text: string; x: number; y: number } | null>( - null - ); - /** Wave-9 I2 — the CELL tip: hovering a cell whose text is cut off reveals it in full. - * Same contract as the header tip above (floated, pointer-events:none, aria-hidden) for - * the same reason — a tooltip that can receive the pointer swallows the NEXT click, which - * this surface has already paid for once ([[ui-invisible-to-assertions]]). */ - const [cellTip, setCellTip] = useState<{ text: string; x: number; y: number } | null>(null); - - /** BUG-1 — viewId → the filter tree we have already pushed back to the host. Persisted with - * the local workspace because the remount it guards against destroys any ref. */ - const reemittedRef = useRef>({}); - /** 2026-08-04 — views THIS browser deleted, by id. Read by the live adopt below AND by - * init, so the mount path and the after-mount path answer "was this deleted here?" the - * same way. Persisted with the local workspace for the same reason `reemitted` is. */ - const viewTombstonesRef = useRef({}); - /** D-19 — viewId → when this browser last wrote it (see LocalWorkspace.viewWrites). */ - const viewWritesRef = useRef({}); - const initializedKey = useRef(null); - const saveTimer = useRef(null); - const gridRef = useRef(null); - const gridBoxRef = useRef(null); - /** Internal clipboard provenance. The browser exposes pasted text, not its source column; - * recording the grid's own copy keeps cross-field paste fail-closed without blocking - * ordinary external text pasted into an editable field. */ - const copyProvenanceRef = useRef(EMPTY_GRID_COPY_PROVENANCE); - useEffect(() => { - const onCopy = () => { - // The native copy event follows our grid key event. Preserve that provenance once; - // every later copy (address bar, another app surface, another input) invalidates it. - copyProvenanceRef.current = observeCopyEvent(copyProvenanceRef.current, Date.now()); - }; - const onWindowBlur = () => { - copyProvenanceRef.current = EMPTY_GRID_COPY_PROVENANCE; - }; - document.addEventListener("copy", onCopy, true); - window.addEventListener("blur", onWindowBlur); - return () => { - document.removeEventListener("copy", onCopy, true); - window.removeEventListener("blur", onWindowBlur); - }; - }, []); - const [gridSize, setGridSize] = useState({ width: 800, height: 600 }); - - /** - * ⛔ WAVE 21 item 3 (3c) — THE FALLBACK KEY IS THE SCOPE'S, and the literal it replaces is - * the cross-table bleed the owner reported as "RI fields on a new database". - * - * MEASURED, not theorised. `fetchWorkspace` collapses 403 / 5xx / network / unreadable into - * one silent `null` — deliberately, because the workspace is an ENHANCEMENT of the table and - * an older host simply does not serve one. But `null` also meant "no `storageKey`", and the - * fallback was the LITERAL `"customer-grid-standalone"` — ONE localStorage bucket for every - * scope this component can be mounted at. So on a tenant whose `/workspace` 403s for `ut_*` - * (routes_grid gated the route on `module_gate("customer_data")` — C4 fixes that half), the - * FIRST user table to be opened writes its views and custom fields into that bucket and the - * SECOND one reads them back as its own. Two databases with nothing to do with each other, - * sharing a schema, with no error anywhere. - * - * Scoping the key makes that structurally impossible: a bucket belongs to one surface, so - * the worst a failed workspace read can now do is show THIS table its own last-known local - * copy. The prefix marks it as the local-only stratum — a key the server never issues, so it - * can never collide with a real `storageKey` (those are `_table_workspace`-shaped). - * - * ⚠ Existing browsers lose the contents of the old shared bucket. That is the point: every - * byte in it is a workspace some other surface persisted, and there is no way to tell whose. - */ - const storageKey = payload?.workspace?.storageKey ?? `local:${scope}`; - - // Initialize once per permission/data scope. Host state wins; local state - // fills only missing objects and keeps the standalone path useful. - useEffect(() => { - if (!payload || payloadFields.length === 0 || initializedKey.current === storageKey) return; - const local = embedded ? null : readLocal(storageKey); - // Item 3c: the def half of the no-blip layer. A RECENT local stamp beats a - // lagged host echo (rename survives, retype holds, a delete stays deleted); - // a caught-up echo returns host objects byte-identical (see optimism.ts). - const stamps = pruneStamps(local?.fieldStamps, Date.now()); - fieldStampsRef.current = { - edited: { ...(stamps.edited ?? {}) }, - deleted: { ...(stamps.deleted ?? {}) }, - }; - const initialFields = reconcileFields( - payloadFields, - local?.fields ?? [], - stamps, - Date.now(), - payload.workspace != null - ); - const hostViews = embedded ? [] : payload.workspace?.views ?? []; - const byId = new Map(); - byId.set(ALL_VIEW_ID, allRecordsView(initialFields, scope)); - // D-19 — the HOST'S LIST DECIDES WHICH VIEWS EXIST. A local copy the host no longer - // names is a ghost (deleted elsewhere, share revoked, store moved) and is dropped here, - // guarded by this browser's own recent writes so an optimistic create survives. - viewWritesRef.current = pruneTombstones(local?.viewWrites, Date.now()); - for (const view of seedLocalViews(local?.views ?? [], hostViews, viewWritesRef.current, - Date.now(), payload.workspace != null)) - byId.set(view.id, view); - // Host state wins — EXCEPT when it is a lagged echo of this browser's own - // in-flight edit, where taking it would turn a just-completed measure rule - // valueless: inactive, no pending marker, whole-book count. A rerun replaces - // the iframe, so this init runs after every host round trip; the reconcile - // is what keeps a rule with a typed rhs PENDING rather than inactive across - // that remount (see viewEcho.ts). - // - // BUG-1 (wave 11) — which views carry a FILTER TREE the host copy does not have, and have - // not already been told about. `echoReemit` owns both guards (see viewEcho.ts): identity - // alone would also fire on a display-only reconcile, and a time-based bound cannot work - // because the local-workspace effect below restamps `savedAt` on every init. - const reemit = new Map(); - // 2026-08-04 — the views tombstone map, seeded from the persisted copy so a remount - // INSIDE the echo window cannot walk back a view this browser just deleted. Init had - // no such guard before (fields and folders both did), which was survivable only while - // every host round trip was itself a remount. - const tombstones = pruneTombstones(local?.viewTombstones, Date.now()); - viewTombstonesRef.current = tombstones; - for (const view of hostViews) { - // Both doors, the `reconcileFields` rule: a tombstone drops the host copy AND any - // local copy, so a delete that raced localStorage cannot re-enter through either. - if (tombstones[view.id] !== undefined) { - byId.delete(view.id); - continue; - } - const merged = reconcileEchoView( - view, byId.get(view.id), local?.savedAt, Date.now() - ); - const key = echoReemit(merged, view, local?.reemitted?.[view.id]); - if (key !== null) reemit.set(view.id, key); - byId.set(view.id, merged); - } - const initialViews = [...byId.values()].map((view) => ({ - ...view, - config: normalizeConfig(view.config, initialFields), - })); - const requested = - local?.activeViewId ?? payload.workspace?.activeViewId ?? ALL_VIEW_ID; - const active = initialViews.find((view) => view.id === requested) ?? initialViews[0]; - setFields(initialFields); - setViews(initialViews); - setActiveViewId(active.id); - setConfig(active.config); - setWorkspaceReady(true); - initializedKey.current = storageKey; - - /** - * BUG-1 (wave 11) — TELL THE HOST. The reconcile above was right about the state and silent - * about the consequence: it restored a rule this browser completed and the host copy does - * not have, and then nothing sent it. The autosave effect cannot: init seeds `views` AND - * `config` from the same reconciled object, so `sameConfig` reports "saved" and no upsert - * is ever emitted. The rule stays ACTIVE client-side, PENDING forever, and unknown to - * `_cl_resolve_measures` — which is exactly W11-HOST's live trace, a pending view carrying - * no measure rule at all while the saved-view path resolves correctly. - * - * ⚠ THIS TERMINATES, and NOT because the host will store what we send. It might not: a - * measure key outside this user's BU offer is stripped by `clean_filter_tree`, so the host - * copy never gains the rule and `lagged` stays true forever. Termination comes from the - * IDEMPOTENCE KEY — the filter tree itself, persisted per view — so we tell the host about - * a given tree exactly once and only speak again when the USER edits the conditions. - * ⚠ A ref cannot hold that: the remount this exists to survive destroys it. It rides in - * localStorage beside the views, and `writeLocal` below carries it forward. - * - * Emitted from the NORMALIZED copy (`initialViews`), never from the raw host view: that is - * the config the client is actually filtering with, and sending anything else would ask the - * host to resolve a question nobody on screen is asking. - */ - if (!embedded && reemit.size > 0) { - const stamped = { ...(local?.reemitted ?? {}) }; - for (const view of initialViews) { - const key = reemit.get(view.id); - if (key === undefined) continue; - viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id, - Date.now()); - viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id, Date.now()); - emitHostEvent({ id: eventId("view"), type: "view_upsert", view }); - stamped[view.id] = key; - } - reemittedRef.current = stamped; - } else { - reemittedRef.current = { ...(local?.reemitted ?? {}) }; - } - }, [payload, payloadFields, storageKey, embedded, scope]); - - useEffect(() => { - if (!workspaceReady || embedded) return; - writeLocal(storageKey, { - fields, - views, - activeViewId, - savedAt: Date.now(), - fieldStamps: pruneStamps(fieldStampsRef.current, Date.now()), - // BUG-1 — carried forward, never recomputed here. Dropping it would restore the loop: - // the next init would find no record of what it already told the host and say it again. - reemitted: reemittedRef.current, - // Pruned on the way out, like every other stamp map here: the persisted blob is a - // recent window, never an archive of everything this tab ever deleted. - viewTombstones: pruneTombstones(viewTombstonesRef.current, Date.now()), - // D-19's other half — without persisting these, a reload inside the echo window would - // drop a view this browser created seconds ago. - viewWrites: pruneTombstones(viewWritesRef.current, Date.now()), - }); - }, [workspaceReady, storageKey, fields, views, activeViewId, embedded]); - - /** - * ⭐ THE LIVE WORKSPACE (owner report, 2026-08-04) — what has APPEARED since we mounted. - * - * The init effect above runs ONCE per mount (`initializedKey`), which under Streamlit was - * indistinguishable from "once per host round trip" because a rerun replaces the iframe. - * The standalone shell has no rerun: this component stays mounted until `key={active.key}` - * changes in Shell.tsx — switching modules. So a cohort created here reached the store, - * came back in the very next `/workspace` re-read, sat in `payload.workspace.views`, and - * never reached the rail. That is the owner's report, exactly: it appears after you visit - * another module and come back, because that is the only thing that remounts this tree. - * - * ⚠ FIELDS FIRST, AND THE ORDER IS LOAD-BEARING. `fields_from_workspace(ws, - * cohorts=bool(cohort_lists))` emits the derived "Locked views" column only once a cohort - * EXISTS — so a user's FIRST cohort changes the field contract in the same payload that - * carries the new view. The adopted views are normalized against `nextFields`, not against - * the `fields` state (which React has not committed yet), or a projected `config.order` - * naming the new column would be normalized against a list that does not have it. - * - * ⚠ Add-only, tombstoned, and identity-stable — see liveWorkspace.ts for why each of those - * is a correctness property rather than a nicety. - */ - const hostWorkspaceViews = payload?.workspace?.views; - useEffect(() => { - if (!workspaceReady || embedded) return; - const now = Date.now(); - const nextFields = adoptNewFields( - fields, payloadFields, fieldStampsRef.current.deleted, now - ); - if (nextFields !== fields) setFields(nextFields); - setViews((current) => - adoptNewViews(current, hostWorkspaceViews, viewTombstonesRef.current, now, - (config) => normalizeConfig(config, nextFields)) - ); - }, [workspaceReady, hostWorkspaceViews, payloadFields, fields, embedded]); - - /** Item 3c — stamp a def write / a delete. Pruned at every touch so the persisted blob - * stays a recent window, never an archive. */ - const stampFieldEdit = useCallback((key: string) => { - const s = pruneStamps(fieldStampsRef.current, Date.now()); - fieldStampsRef.current = { - edited: { ...(s.edited ?? {}), [key]: Date.now() }, - deleted: { ...(s.deleted ?? {}) }, - }; - }, []); - const stampFieldDelete = useCallback((key: string) => { - const s = pruneStamps(fieldStampsRef.current, Date.now()); - const edited = { ...(s.edited ?? {}) }; - delete edited[key]; - fieldStampsRef.current = { - edited, - deleted: { ...(s.deleted ?? {}), [key]: Date.now() }, - }; - }, []); - - const updateConfig = useCallback( - (next: ViewConfig) => { - if (next.groupBy !== config.groupBy) setCollapsed(new Set()); - setConfig(next); - }, - [config.groupBy] - ); - - const { - visibleCols, - fieldByKey, - order, - visible, - lockedKey, - onColumnResize, - onColumnMoved, - onColumnProposeMove, - setColumnVisible, - insertColumn, - } = useGridColumns(fields, config, updateConfig); - - /** - * ⭐ Wave-20 owner item 4 (ruling R8, contract C-ADDROW) — **THE GHOST ROW.** - * - * A trailing "+" row at the bottom of the grid, the way Airtable grows a table, replacing the - * "Add record" button that used to sit in the shell's header bar (S4 deleted it this wave). - * - * ⛔ USER DATABASES ONLY. A connector's rows are read-synced from Odoo; a "+" there could only - * refuse, and R8 names that a fake affordance. The test is the SCOPE (`ut_*`), which is also - * the only scope with a rows endpoint to POST to — so the affordance and the capability come - * from the same fact rather than from two lists that can drift. - * - * ⚠ THE ROW IS NOT ADDED LOCALLY. `rawRows` is the server's answer, and a client-invented row - * would have no rid, no defaults and no place in anyone else's copy. The POST clears the rows - * cache and fires `ROWS_STALE_EVENT`, `useCustomerData` re-reads, and the row arrives with the - * id the store gave it. The cursor then follows it (`newRowPid` + the effect below) — which is - * why the pid is remembered instead of glide's `"bottom"` being returned here: at the moment - * this resolves, the appended row does not exist yet, so "bottom" would land on the last OLD - * row. - */ - const isUserTable = !embedded && scope.startsWith("ut_"); - const recordsMutable = payload?.recordsMutable !== false; - const canMutateRecords = isUserTable && recordsMutable; - /** - * ⭐ 2026-08-07 — the databases a `link` column may point at, fetched when the column menu - * OPENS rather than on every render of the grid. - * - * ⚠ ON DEMAND IS THE WHOLE DESIGN. `GET /tables` walks every table in the tenant and returns - * their full field lists; hanging that off the grid's mount would put a workspace-wide read - * behind every page view for a picker almost nobody opens. The column menu is the only surface - * that needs it, so it is the surface that asks. - * ⚠ The list is left standing once fetched — a database created in another tab mid-session is - * a staleness a menu re-open corrects, and re-fetching per open would spend the same read - * repeatedly for a list that changes about once a week. - */ - const [linkTargets, setLinkTargets] = useState([]); - useEffect(() => { - // ⛔⛔ 2026-08-09 — `plusMenu` TOO, AND THAT OMISSION WAS THE WHOLE BUG. Owner: *"I am not - // able to see Choose Column for Post rows link"*. There are TWO doors into this editor — - // a column's own menu (`columnMenu`) and the "+" Add-field button (`plusMenu`) — and this - // effect knew about one. Via "+", `linkTargets` stayed `[]`, so `target` never resolved and - // the rollup Column picker held nothing but its placeholder, permanently. Editing an - // EXISTING column worked, which is what made it look like a rollup bug rather than a fetch - // that never fired. MEASURED on the live app: 5 selects rendered, "Column to roll up" with - // exactly 1 option while `GET /tables` returns `ut_ig_posts` with 22 fields. - // ⚠ A picker that is EMPTY and a picker whose data never loaded look identical, which is - // why this survived: both render "Choose a column…" over nothing. - if ((!columnMenu && !plusMenu) || linkTargets.length) return; - let live = true; - void fetchLinkTargets().then((t) => { - if (live) setLinkTargets(t); - }); - return () => { - live = false; - }; - }, [columnMenu, plusMenu, linkTargets.length]); - /** - * ⭐⭐ 2026-08-09 — the READ-THROUGH rollup's offer, fetched on the same terms as the link - * targets above: only when the column menu opens, and once per session. - * - * ⚠ THE GUARD IS A SEPARATE `asked` FLAG, NOT `topics.length`. An empty offer is the CORRECT - * and common answer (a tenant with nothing connected), so guarding on the length would re-ask - * on every single menu open for exactly the workspaces where the answer can never change — - * the one case the "fetch once" rule exists for. - */ - const [rollupSourceOffer, setRollupSourceOffer] = - useState({ topics: [], windows: [] }); - const rollupOfferAsked = useRef(false); - useEffect(() => { - // ⛔ SAME TWO DOORS as the link targets above. This one is less visible because an empty - // offer is the correct answer for a tenant with nothing connected — so a source offer that - // never loaded is indistinguishable from one that is legitimately empty, and the mode switch - // simply never appears. Fixing only the link half would have left that asymmetry in place. - if ((!columnMenu && !plusMenu) || rollupOfferAsked.current) return; - rollupOfferAsked.current = true; - let live = true; - void fetchRollupSources().then((o) => { - if (live) setRollupSourceOffer(o); - }); - return () => { - live = false; - }; - }, [columnMenu, plusMenu]); - /** - * ⭐ WAVE 27 · OWNER ITEM 22 — the values the ACTIVE VIEW forces on a new record. - * - * ⛔ THE DEFECT: "+" posted `{values: {}}` on every view, so a record added while a filter was - * on was created, was real, and was invisible — the button read as broken and the row the user - * then typed into did not exist as far as they could see. `filterSeed.ts` owns the derivation - * and its whole design is how much it REFUSES to derive (negations, ranges, `or` branches, - * contradictions, machine columns — each with its reason at the clause). - * - * ⚠ ONE FUNCTION, TWO CALLERS, deliberately: item 2's optimistic add must insert the row - * carrying these same cells, or the row it paints locally fails the filter it was added under - * and disappears on the next read — the exact blip C2 exists to prevent. - */ - const seedValues = useCallback( - () => filterSeedValues(config.filters, config.filterConj, fields, payload?.viewer), - [config.filters, config.filterConj, fields, payload?.viewer] - ); - /** - * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — the "+" paints its row NOW. - * - * ⛔ THE ID IS MINTED THE WAY THE SERVER MINTS IT — `max(numeric row id) + 1`, which is - * `core.user_tables.add_row`'s own rule — and then POSTed as `{rid, values}` through the door - * the undo path already opened (C-ADDROW). That is what makes the optimism honest rather than - * hopeful: on the overwhelmingly common single-writer path the server stores exactly the id - * this browser drew, so the row on screen and the row in the store are the same record from - * the first frame. - * - * ⚠ AND WHEN THEY ARE NOT, IT RE-ANCHORS. Two people adding at once means the second POST - * finds the id taken; `add_row` falls back to `max+1` and ANSWERS with what it actually wrote - * (its own note says so), so the local row is re-keyed to the returned pid. Assuming the - * requested id came back is the one way this could leave a row on screen that no longer - * matches anything in the store. - * - * ⛔ NO `ROWS_STALE_EVENT` ON THE HAPPY PATH. That event re-reads the whole table, which is - * the cost item 2 exists to remove — and re-reading would also un-paint the row for the - * duration of the fetch, which is the NO-BLIP law's exact subject. The pending row is pruned - * by the effect above when a payload that already contains it arrives, from whatever cause. - * - * ⚠ THE NARROW RACE, STATED: a cell PATCHed within the POST's round trip could reach the - * server before the row exists. The window is now ~20ms (A moved the relation refresh off - * this path), it needs a keystroke inside it, and the PATCH's failure is a rolled-back - * overlay edit rather than lost data — the alternative, blocking the paint until the POST - * resolves, is the defect being fixed. - */ - const appendRow = useCallback(async (): Promise => { - if (!canMutateRecords) return undefined; - const seeded = seedValues(); - const minted = mintRid(); - const optimistic: Row = { ...seeded, pid: minted }; - setPendingRows((cur) => [...cur, optimistic]); - setNewRowPid(minted); - const made = await addTableRow(scope, seeded, minted); - if (!made) { - // `addTableRow` already said WHY, in the server's own words (a row cap, a refused profile - // cell, an unreachable host). The row is withdrawn rather than left standing: a row that - // survives its own failed write is the lie this whole path has to avoid. - setPendingRows((cur) => cur.filter((r) => r.pid !== minted)); - setNewRowPid(null); - return undefined; - } - if (made.pid !== minted) { - setPendingRows((cur) => - cur.map((r) => (r.pid === minted ? { ...r, pid: made.pid } : r)) - ); - setNewRowPid(made.pid); - } - // R4 — the append is undoable: Ctrl+Z deletes the row it just created, and a redo restores - // it under the SAME rid (the server's `{rid}` passthrough, C-ADDROW). - // ⛔ ITEM 22 MOVED THIS LINE. It read `values: {}` with a comment saying "an empty new record - // carries no values" — true until this wave, and now false: a row added on a filtered view is - // born holding the cells that filter forces. `redo` replays `r.values` verbatim - // (`:1569`), so leaving the literal would have made Ctrl+Z → Ctrl+Y restore the row under the - // right id with its seeded cells silently dropped — [[undo-capture-before-the-write]] in its - // quietest form, since the id and the row count would both be right. - undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { - kind: "rowAdd", table: scope, rows: [{ rid: made.rid, values: seeded }], - }); - // ⛔ NO `signal(ROWS_STALE_EVENT)` HERE ANY MORE (item 2 / C2). It re-read the whole table - // on every "+", which is the whole cost the owner reported — and every OTHER caller of it - // still fires, so a row created by an automation, another tab or an undo still arrives. - // The one thing that used to depend on this refetch was the cursor landing on the new row, - // and the row is now on screen before the POST resolves, so it lands immediately. - // `undefined` is returned rather than glide's "bottom": "bottom" is resolved against the - // rows glide knew about when the gesture started, which is the row BEFORE this one. - return undefined; - }, [canMutateRecords, scope, seedValues, mintRid]); - - /* wave20 item 2 — measure key + window -> the columns that display it. Built ONCE per field - list and handed to every consumer of "which column is this rule about", so the tint - (`columnTones`, inside the hook above) and the column menu's filter doors resolve a - measure condition the same way. See `types.ruleColumnKeys`. */ - const measureCols = useMemo(() => measureColumnIndex(fields), [fields]); - - // Airtable behavior: configuration changes to the active view autosave. - useEffect(() => { - if (!workspaceReady || embedded) return; - const active = views.find((view) => view.id === activeViewId); - if (!active || sameConfig(active.config, config)) { - setSaveState("saved"); - return; - } - setSaveState("saving"); - if (saveTimer.current !== null) window.clearTimeout(saveTimer.current); - saveTimer.current = window.setTimeout(() => { - const updated = { ...active, config }; - setViews((current) => - current.map((view) => (view.id === updated.id ? updated : view)) - ); - viewWritesRef.current = stampTombstone(viewWritesRef.current, updated.id, - Date.now()); - emitHostEvent({ id: eventId("view"), type: "view_upsert", view: updated }); - setSaveState("saved"); - saveTimer.current = null; - }, 420); - return () => { - if (saveTimer.current !== null) window.clearTimeout(saveTimer.current); - }; - }, [workspaceReady, activeViewId, config, views, embedded]); - - // Numeric dimensions force a glide relayout when either the component frame - // or Streamlit's main column changes width (notably sidebar collapse). - useLayoutEffect(() => { - const element = gridBoxRef.current; - if (!element) return; - const measure = () => { - const rect = element.getBoundingClientRect(); - if (rect.width > 0 && rect.height > 0) { - setGridSize({ - width: Math.floor(rect.width), - height: Math.floor(rect.height), - }); - } - }; - measure(); - const observer = new ResizeObserver(measure); - observer.observe(element); - window.addEventListener("resize", measure); - return () => { - observer.disconnect(); - window.removeEventListener("resize", measure); - }; - }, [workspaceReady]); - - // CG-3: one mode per table, decided by whether the payload is a window. See types.ts - // `TableMode` for why both engines running at once is the failure this prevents. - const mode = tableMode(payload?.counts); - const serverWindowed = mode === "server-windowed"; - // Owner items 4+6 — the Cohort page's grid renders without the Views sidebar. - const hideViews = embedded || payload?.workspace?.hideViews === true; - // Wave-6 item 10 — how this view displays. A WINDOWED table is always the grid: list/ - // calendar/kanban compute over the whole matched set, and one page is not it (CG-3's rule, - // the same reason grouping is off there). - const displaySpec = cleanDisplay(config.display); - const displayMode: DisplayMode = serverWindowed ? "grid" : displaySpec?.mode ?? "grid"; - - // CG-8 — measure conditions, answered by the host as `{ruleId: pid[]}`. Converted to Sets - // once here rather than per row: the pipeline runs this over every row of the book. - // - // ⚠ REPLACED, not merged. Merging looks like the safer choice — "hold the previous result" - // is the design — but it holds it FOREVER: the host omits a rule's id when it could not - // resolve it (a warming store, a scope change mid-flight), and a merged map keeps yesterday's - // answer under today's question with `pendingMeasures` reporting zero, so the marker never - // shows. That is the widening sin wearing a confident count. - // - // Holding the previous result across a LOCAL edit does not need the merge and never did: the - // memo is keyed on `payload.measureSets` identity, which does not change while the user is - // typing, so the ref already carries the last answers through to the next payload. Merge and - // replace therefore differ only in the dangerous case. - const measures = useMemo(() => payload?.measures ?? [], [payload?.measures]); - const measureSetsRef = useRef({}); - const measureSets = useMemo(() => { - const incoming = payload?.measureSets; - if (incoming) { - const next: MeasureSets = {}; - for (const [ruleId, pids] of Object.entries(incoming)) next[ruleId] = new Set(pids); - measureSetsRef.current = next; - } - return measureSetsRef.current; - }, [payload?.measureSets]); - const pendingMeasureCount = useMemo( - () => pendingMeasures(config.filters, measureSets), - [config.filters, measureSets] - ); - // BUG-1 (wave 11) — the two key sets, published for the handshake. See the `.cg-shell` - // attributes below for why they are DOM attributes and not console logs. - const measureRuleKeys = useMemo( - () => activeMeasureRuleIds(config.filters).join(","), - [config.filters] - ); - const measureSetKeys = useMemo(() => Object.keys(measureSets).join(","), [measureSets]); - - // Owner item 5. Cohort membership rides `workspace.lists`, the same array the "Add to list" - // picker already reads, so the names offered and the sets tested come from one place. A - // cohort with no `pids` is a target you can add to but cannot filter on; the engine treats it - // as unanswerable (matches nothing) rather than absent (matches everything). - const lists = useMemo(() => payload?.workspace?.lists ?? [], [payload?.workspace?.lists]); - /* wave17 R1 / C-LOCKV — `railCohortId` is gone. It held the TRANSIENT lock the retired - Cohorts section applied on click: a lock that lived in component state and was never - persisted. A locked view now carries its own `config.cohortLock`, so SELECTING the view - is the lock and the saved config is the only source. One mechanism, and it survives a - reload — which the transient one never did. */ - const cohortSets = useMemo(() => { - const out: CohortSets = {}; - for (const l of lists) if (l.pids) out[l.id] = new Set(l.pids); - return out; - }, [lists]); - // The TENANT'S today. Never `new Date()`: a browser a day ahead of the server would resolve - // "the past month" to a different month than the host summed, and the count would deny the - // rows with nothing erroring. - const today = payload?.today; - // Wave-5 item 1 — who is looking (host-computed). ONE editability verdict for every edit - // door (cells, pickers, drawer): types.mayEditField — stratum + read-only-by-nature + - // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown. - const viewer = payload?.viewer; - const canEditField = useCallback( - (f: Field): boolean => !embedded && recordsMutable && mayEditField(f, viewer), - [embedded, recordsMutable, viewer] - ); - const unresolvedCount = useMemo( - () => unresolvedConditions(config.filters, { cohortSets, today }), - [config.filters, cohortSets, today] - ); - - // Wave-5 items 9/11 — CLIENT-COMPUTED cells, injected AT their field keys so the whole - // pipeline (filter/search/sort/group), the cells and the drawer read them like any other - // value. Formulas parse ONCE per definition change (never per row); evaluation reads the - // row WITH this session's overlay edits layered, so editing a referenced field recomputes - // live. A formula that does not parse, or errors on a row, yields BLANK — never a wrong - // number (formulaEngine.ts). `created_time` copies the row's `_created`. - // 2026-07-31 (owner item 2): formulas may reference OTHER formulas now, so parse order is - // TOPOLOGICAL (orderFormulas) — a formula runs after the formulas it reads, cycle members - // never run (blank, never a stale number), and each row's results feed the next formula - // through a per-row scope. - const formulaAsts = useMemo(() => { - const sources = new Map(); - for (const f of fields) { - if (f.type !== "formula") continue; - const src = formulaOf(f); - if (src) sources.set(f.key, src); - } - const { order, cyclic } = orderFormulas(sources); - const out: { key: string; ast: FormulaAst }[] = []; - for (const key of order) { - if (cyclic.has(key)) continue; - const p = parseFormula(sources.get(key)!); - if (p.ok) out.push({ key, ast: p.ast }); - } - return out; - }, [fields]); - const createdTimeKeys = useMemo( - () => fields.filter((f) => f.type === "created_time").map((f) => f.key), - [fields] - ); - const computedRows = useMemo(() => { - if (formulaAsts.length === 0 && createdTimeKeys.length === 0) return rawRows; - const env = { today: payload?.today }; - return rawRows.map((r) => { - const edits = overlayEdits[r.pid]; - const scope: Row = edits ? { ...r, ...edits } : { ...r }; - const out: Row = { ...r }; - for (const k of createdTimeKeys) { - out[k] = (r._created as string | undefined) ?? null; - scope[k] = out[k]; - } - for (const { key, ast } of formulaAsts) { - const v = evalFormula(ast, (k) => scope[k], env); - out[key] = v; - scope[key] = v; // later formulas read this one's result — the topo order above - } - return out; - }); - }, [rawRows, overlayEdits, formulaAsts, createdTimeKeys, payload?.today]); - - // Wave-2 item 2c — COHORT MODE (the Cohort page). The host serves the WHOLE pool (rows + - // derived values over the pool); the ACTIVE cohort scopes the table to its pids CLIENT-side. - const cohortMode = payload?.workspace?.cohortMode === true; - useEffect(() => { - if (!cohortMode) return; - if (activeCohortId && lists.some((l) => l.id === activeCohortId)) return; - setActiveCohortId(lists[0]?.id ?? null); - }, [cohortMode, lists, activeCohortId]); - const activeCohort = cohortMode - ? lists.find((l) => l.id === activeCohortId) ?? null - : null; - const cohortMemberSet = useMemo( - () => new Set(activeCohort?.pids ?? []), - [activeCohort] - ); - // The FIXED subset is the scope. A cohort with no membership shows nothing — falling back to - // the whole pool would put 1,500 rows under a rail entry that says 12, which is the widening - // sin wearing a sidebar. Everything downstream (pipeline, counts, selection, detail) runs - // over this scope, so the toolbar count is the cohort's matched count by construction. - const scopedRows = useMemo( - () => - cohortMode - ? computedRows.filter((r) => cohortMemberSet.has(r.pid)) - : computedRows, - [cohortMode, computedRows, cohortMemberSet] - ); - - const { visibleRows, pidToIndex } = useVisibleRows( - scopedRows, - fields, - config.filters, - search, - config.sorts, - // grouping a WINDOW would headline a page's count/subtotals as the group's - serverWindowed ? null : config.groupBy, - collapsed, - config.memberPids, - config.filterConj, - serverWindowed, - measureSets, - cohortSets, - today, - // Item 12 (C-LOCK) — the 14th positional, RECORD's engine input. The lock intersects - // FIRST, so everything downstream (conditions, memberPids, the ranking domain) operates - // inside the cohort. Absent = today's behaviour exactly. - // wave17 R1 / C-LOCKV: the view's SAVED lock is now the only source. A locked view is a - // saved view whose `config.cohortLock` names its own id, so opening it IS applying the - // lock — the wave-15 transient rail pick that used to outrank this is gone with its rail. - config.cohortLock - ); - - // Owner item 8 — the DISPLAY cap. The pipeline above still ran over the whole book (counts, - // "Add to list" and selection-by-pid all depend on that); only what glide PAINTS is sliced. - // pidToIndex is rebuilt over the slice because every index consumer (selection, detail - // navigation, hover) is positional against what is actually on screen. - const [displayCap, setDisplayCap] = useState(DISPLAY_PAGE); - const capped = visibleRows.length > displayCap; - const displayRows = useMemo( - () => (capped ? visibleRows.slice(0, displayCap) : visibleRows), - [capped, visibleRows, displayCap] - ); - const displayPidToIndex = useMemo(() => { - if (!capped) return pidToIndex; - const m = new Map(); - displayRows.forEach((vr, i) => { - if (vr.kind === "data" && !m.has(vr.record.pid)) m.set(vr.record.pid, i); - }); - return m; - }, [capped, displayRows, pidToIndex]); - - // C-AVATAR (wave-14 item 11) — profile photos, absent until HOST serves them. The tick is the - // async half: `cells.setAvatarRepaint` fires it when an Image finishes decoding, and it is a - // dep of `getCellContent`, which is the only thing glide watches. See useGetCellContent. - const [avatarTick, setAvatarTick] = useState(0); - useEffect(() => { - setAvatarRepaint(() => setAvatarTick((t) => t + 1)); - return () => setAvatarRepaint(undefined); - }, []); - const userAvatars = payload?.workspace?.userAvatars; - /** - * Item 15 — the frozen strip's usable width, which is what a GROUP BAR's label is clipped to - * now that the first column is pinned in every mode. `null` when nothing is grouped (no bar to - * clip) — see `fitGroupLabel` for why the clip exists and why the count outranks the name. - */ - const groupLabelSpace = useMemo(() => { - if (!config.groupBy) return null; - const n = Math.min(frozenCountOf(config), visibleCols.length); - let px = 0; - // `GridColumn` is a union — only its SIZED member declares `width`, and `useGridColumns` - // always builds that one. Narrowed rather than asserted so an auto-sized column (which we - // do not create) would read 0 and simply not be counted, instead of throwing in a memo. - for (let i = 0; i < n; i += 1) - px += (visibleCols[i] as { width?: number } | undefined)?.width ?? 0; - return Math.max(0, px - GROUP_LABEL_PAD); - }, [config, visibleCols]); - /** - * ⭐ owner item 2 (2026-08-03) — WHICH MEASURE COLUMNS ARE STILL BEING CALCULATED. - * - * A measure column's numbers are resolved by the server (one aggregate over the whole book) - * and reach the browser on the workspace re-read that follows the create — seconds later. - * In between, the column is on screen with nothing in it, which the owner correctly read as - * an error rather than as a wait. - * - * ⛔ `editRequestId` IS THE TEST, and it is the honest one. It is set optimistically at the - * moment the create is emitted and cleared by `reconcileFields` when the host's own copy of - * that field comes back — and the host's copy travels in the SAME `/workspace` response as - * `derived`, which is where the values are. So the flag is true across exactly the window - * where the column exists and its numbers do not, and false the instant they land. - * - * The alternative — "no value anywhere in the column" — cannot tell a pending column from - * one that permanently failed to resolve (a BU-scoped caller on a company-level measure, - * say), and would spin forever on the second. This one resolves either way: when the echo - * arrives with no values, the flag clears and the cells go honestly blank. - */ - const pendingMeasureKeys = useMemo(() => { - const out = new Set(); - for (const f of fields) - if (f.editRequestId && f.key.startsWith("measure_")) out.add(f.key); - return out; - }, [fields]); - /** - * The skeleton's pulse. A canvas cell cannot hold a CSS animation, so the motion is repaints: - * this counter is a dependency of `getCellContent`, which is the only thing glide watches. - * - * ⚠ IT RUNS ONLY WHILE SOMETHING IS PENDING, and the effect's own guard is what stops it — - * an interval left running would repaint the whole canvas ~7×/s forever, on every grid, to - * animate nothing. 140ms × the 4-step ramp is a ~0.6s cycle: a wait, not a strobe. - */ - const [pulse, setPulse] = useState(0); - /** - * ⛔ AND IT GIVES UP. A skeleton that never resolves is worse than the blank it replaced: a - * blank cell is at least honest about having no number, while a permanent shimmer promises - * one that is never coming. - * - * The window it guards is narrow but real. `editRequestId` clears when `reconcileFields` takes - * the host's copy — and that function has a branch (`hostAuthoritative: false`, i.e. a payload - * with no workspace) whose `{...local, ...host}` spread would PRESERVE the flag forever. Today - * that branch cannot be reached with a measure column on screen (measures are offered through - * the workspace, so a payload without one cannot have produced this field), which is an - * argument about the current call graph and not a property of the code. This bound holds - * whether or not the argument stays true, and it costs one boolean. - */ - const [pendingGaveUp, setPendingGaveUp] = useState(false); - useEffect(() => { - setPendingGaveUp(false); - if (pendingMeasureKeys.size === 0) return; - let n = 0; - const id = window.setInterval(() => { - n += 1; - if (n > PULSE_MAX_TICKS) { - window.clearInterval(id); - setPendingGaveUp(true); // fall back to ordinary blank cells - return; - } - setPulse((p) => p + 1); - }, PULSE_MS); - return () => window.clearInterval(id); - }, [pendingMeasureKeys]); - const activePendingKeys = pendingGaveUp ? NO_PENDING_KEYS : pendingMeasureKeys; - const getCellContent = useGetCellContent( - displayRows, - visibleCols, - fieldByKey, - overlayEdits, - canEditField, - userAvatars, - avatarTick, - groupLabelSpace, - measureGroupText, - activePendingKeys, - pulse - ); - const { gridSelection, selectedPids, onGridSelectionChange, selectPids, togglePid, - setActiveCell, clearSelection } = - useGridSelection( - displayRows, - displayPidToIndex, - visibleCols.length, - embeddedSelectable ? embeddedSelectedIds : [] - ); - const embeddedSelectedKey = embeddedSelectedIds.join(","); - const selectedPidKey = [...selectedPids].join(","); - useEffect(() => { - if (!embeddedSelectable) return; - if (selectedPidKey !== embeddedSelectedKey) - selectPids(embeddedSelectedIds as number[], "replace"); - }, [embeddedSelectable, embeddedSelectedKey, embeddedSelectedIds, selectedPidKey, selectPids]); - useEffect(() => { - if (!embeddedSelectable) return; - onEmbeddedSelectionChange?.([...selectedPids]); - }, [embeddedSelectable, onEmbeddedSelectionChange, selectedPids]); - /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════ - THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one - thing it cannot: reading the value a cell held BEFORE the write, which only exists at the - call site. Everything below funnels through `patchAndRecord` / `patchManyAndRecord`, so a - write path that forgets to record is a write path that does not reach the store either. - - ⚠ A REF, NOT STATE. Nothing on screen depends on the stack in v1 (no undo button), so - keeping it in state would repaint the grid on every keystroke of a paste for no pixels. - Per SCOPE (a Ctrl+Z on the Customer grid must never rewrite a user table's cell) and per - tab (nothing persists it — a stack restored into a session that did not make those edits - would undo somebody else's work). */ - const undoBook = useRef({}); - const applyingUndo = useRef(false); - - /** The value a cell holds RIGHT NOW: the overlay stratum wins over the payload row, exactly - * as `useGetCellContent` renders it — so what undo restores is what was on screen. */ - const rowByPid = useMemo(() => { - const map = new Map(); - for (const r of rawRows) map.set(r.pid, r); - return map; - }, [rawRows]); - const currentValue = useCallback( - (pid: number, key: string): UndoValue => { - const edited = overlayEdits[pid]?.[key]; - const raw = edited !== undefined ? edited : rowByPid.get(pid)?.[key]; - return raw == null ? null : (raw as UndoValue); - }, - [overlayEdits, rowByPid] - ); - - const recordCells = useCallback( - (changes: CellChange[], label: string) => { - if (applyingUndo.current || changes.length === 0) return; - undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { - kind: "cells", label, changes, - }); - }, - [scope] - ); - - /** ONE cell write, recorded. Every editor, picker and drag goes through this. */ - const patchAndRecord = useCallback( - (pid: number, updates: Partial, label = "an edit") => { - const changes: CellChange[] = Object.entries(updates).map(([key, value]) => ({ - pid, key, - before: currentValue(pid, key), - after: (value ?? null) as UndoValue, - })); - patchOverlay(pid, updates); - recordCells(changes, label); - }, - [currentValue, patchOverlay, recordCells] - ); - - /** MANY cells, ONE stack entry — a paste and a bulk clear are each one user action (R4). */ - const patchManyAndRecord = useCallback( - (writes: { pid: number; updates: Partial }[], label: string) => { - const changes: CellChange[] = []; - for (const w of writes) - for (const [key, value] of Object.entries(w.updates)) - changes.push({ - pid: w.pid, key, - before: currentValue(w.pid, key), - after: (value ?? null) as UndoValue, - }); - for (const w of writes) patchOverlay(w.pid, w.updates); - recordCells(changes, label); - }, - [currentValue, patchOverlay, recordCells] - ); - - /** - * Apply one stack entry in one direction. The INVERSE lives in `undoStack.directed` — this - * only knows how to WRITE each op, and it writes through the same doors the user does - * (`patchOverlay`, the rows endpoint), so an undone edit is persisted exactly like the edit - * was. Nothing here is optimistic-only: a Ctrl+Z that reverted the screen and not the store - * would come back on the next reload. - */ - const applyEntry = useCallback( - async (entry: UndoEntry, dir: "back" | "forward") => { - const op = directed(entry, dir); - if (op.kind === "cells") { - const byPid = new Map>(); - for (const c of op.changes) { - const at = byPid.get(c.pid) ?? {}; - at[c.key] = (c.after ?? "") as Row[string]; - byPid.set(c.pid, at); - } - applyingUndo.current = true; - try { - for (const [pid, updates] of byPid) patchOverlay(pid, updates); - } finally { - applyingUndo.current = false; - } - } else if (op.kind === "rowAdd") { - // Restore under the OLD id where the store still has it free; the server answers with - // what it actually wrote and the re-read is what puts the rows back on screen. - // - // ⚠ SEQUENTIAL, not `Promise.all`. `add_row` picks `max(id)+1` when the requested id is - // taken, and it reads the store to do it — firing ten restores concurrently is ten - // readers racing one counter, which is how two rows end up sharing an id. - let any = false; - let moved = 0; - for (const r of op.rows) { - const made = await addTableRow(op.table, r.values as Record, r.rid); - any = any || !!made; // a refusal already said why, in the server's words - // ⚠ THE ID IS PART OF WHAT IS BEING UNDONE, and `add_row` falls back to `max(id)+1` - // when the one it was asked for is no longer free — which happens if anything created - // a row in the gap. The restore is still the right thing to do, but it is no longer - // the SAME record to anything that named the old id (a cohort, a comment, a filter), - // and a redo would then look for an id that is not there. Said out loud rather than - // discovered later. - if (made && String(made.rid) !== String(r.rid)) moved += 1; - } - if (!any) return; - if (moved) - signal( - TOAST_EVENT, - `${moved} restored record${moved === 1 ? "" : "s"} came back under a new id — ` + - `something had taken the original while it was gone.` - ); - signal(ROWS_STALE_EVENT); - } else if (op.kind === "rowDelete") { - let any = false; - for (const r of op.rows) any = (await deleteTableRow(op.table, r.rid)) || any; - if (!any) return; - // ⭐ ITEM 2 — the row may still be one of THIS browser's pending copies (add, then - // Ctrl+Z). A deleted row never reappears in a payload, so the prune effect can never - // absorb it and it would sit on screen looking undeleted. See `withdrawPending`. - withdrawPending(op.rows.map((r) => r.rid)); - signal(ROWS_STALE_EVENT); - } else if (op.kind === "choiceRename") { - // Item 15 — the inverse mapping. The host rewrites the values and the saved views that - // name them, exactly as it did on the way out; `directed` already turned the pairs - // around, so this emits what it is given. - emitHostEvent({ - id: eventId("choicerename"), - type: "choice_rename", - key: op.fieldKey, - renames: op.renames, - }); - } - signal(TOAST_EVENT, describe(entry, dir)); - }, - [patchOverlay, withdrawPending] - ); - - /** - * Ctrl+Z / Ctrl+Shift+Z (and Ctrl+Y, which is the same request on Windows). - * - * ⚠ ON `window`, IN CAPTURE, and it steps aside for real text fields. The grid is a canvas — - * glide's key handling only fires while the canvas has focus, so a Ctrl+Z after clicking the - * toolbar would do nothing, which is exactly the "sometimes it works" the owner would report - * next. But an with a cursor in it has its OWN undo that belongs to the browser, and - * stealing that would be worse than not having ours: the search box, the rename field and - * glide's own cell editor are all inputs, so `activeElement` decides. - * - * Nothing to undo says so out loud rather than silently ignoring the key — an undo that - * appears to do nothing is indistinguishable from one that is broken. - */ - useEffect(() => { - const onKey = (event: KeyboardEvent) => { - if (!(event.ctrlKey || event.metaKey) || event.altKey) return; - const key = event.key.toLowerCase(); - if (key !== "z" && key !== "y") return; - const el = document.activeElement as HTMLElement | null; - if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) - return; - const forward = key === "y" || event.shiftKey; - event.preventDefault(); - const book = undoBook.current; - const state = stackFor(book, scope); - const { state: next, entry } = forward ? popRedo(state) : popUndo(state); - if (!entry) { - signal(TOAST_EVENT, forward ? "Nothing to redo." : "Nothing to undo."); - return; - } - book[scope] = next; - void applyEntry(entry, forward ? "forward" : "back"); - }; - window.addEventListener("keydown", onKey, true); - return () => window.removeEventListener("keydown", onKey, true); - }, [scope, applyEntry]); - - - /** - * R4 — **BULK BACKSPACE/DELETE IS ONE ACTION.** glide's own delete walks the selection and - * calls `onCellEdited` per cell, which would put forty entries on the stack for one keypress; - * returning `false` takes the whole operation over so it lands as one. - * - * ⚠ PRESETS ARE NEVER ATTEMPTED (the contract says so, and it is also the only honest - * behaviour): `canEditField` is the same verdict the editor and the paste path use, so a - * selection spanning read-only columns clears the editable ones and leaves the rest exactly - * as they were — rather than firing writes the server will refuse one by one. - */ - /** - * ⭐ 2026-08-06 (owner) — DELETE RECORDS, with Ctrl+Z. - * - * Owner, verbatim: *"I should always be able to delete records that I created myself manually - * (not from automation), because right now it only shows 'Add to cohort'… make sure I can use - * 'Delete/Backspace' to delete the record and use Ctrl Z if i want to undo."* - * - * Everything under this was already built and had no door: `DELETE /tables/{key}/rows/{rid}`, - * `deleteTableRow`, and `undoStack`'s `rowDelete` — whose inverse (`rowAdd` under the SAME rid) - * is exactly "put it back". The only missing piece was something that calls them. - * - * ⛔ THE VALUES ARE CAPTURED BEFORE THE DELETE, and this is the line the whole feature rests on. - * `deleteTableRow` answers a boolean; it does not hand the row back. Push the undo entry after - * the round trip and there is nothing left to read, so Ctrl+Z would faithfully restore an EMPTY - * row under the right id — a silent data loss wearing the costume of a working undo. - * - * ⚠ ONE ENTRY FOR THE WHOLE GESTURE (R4's "bulk Backspace = ONE grouped stack entry"), which is - * why `UndoEntry` carries a row LIST now. Ten rows deleted must be one Ctrl+Z, not ten. - * - * ⚠ NO CONFIRM DIALOG, deliberately. The owner named Ctrl+Z as the safety net in the same - * sentence as the delete; a modal on top of a working undo is friction that teaches people to - * dismiss modals. - */ - const deleteRecords = useCallback( - async (pids: number[]): Promise => { - if (!canMutateRecords || !pids.length) return false; - // The row as it stands NOW, straight off the rendered records — the same values the - // reader can see, so a restore puts back what they watched disappear. - const byPid = new Map(); - for (const vr of displayRows) - if (vr.kind === "data") byPid.set(vr.record.pid, vr.record); - const captured: UndoRow[] = []; - /* - * ⛔ A SKIPPED ROW IS COUNTED AND NAMED, never dropped quietly ([[no-unverifiable-aggregates]]). - * `displayRows` is FILTERED and CAPPED, and the selection is not: Select-from-file matches - * on a value and can tick pids the current view does not render — `fileSelect` has a whole - * bucket for exactly that ("not in this view"). Capturing only what is on screen and - * `continue`ing past the rest would delete two rows and report three, which is the silent-cap - * defect in the one place it is least forgivable. - * - * REFUSED WHOLE rather than partially applied: a delete that half-happened leaves the reader - * reconciling a count against a table, and the fix (scroll or clear the filter, then select - * again) is one sentence away. - */ - const offscreen: number[] = []; - for (const pid of pids) { - const rec = byPid.get(pid); - if (!rec) { - offscreen.push(pid); - continue; - } - const values: Record = {}; - for (const f of fields) { - const v = rec[f.key]; - if (v !== undefined && v !== null && v !== "") - values[f.key] = v as UndoValue; - } - captured.push({ rid: pid, values }); - } - if (offscreen.length) { - signal( - TOAST_EVENT, - `Nothing was deleted — ${offscreen.length} of the ${pids.length} selected ` + - `record${pids.length === 1 ? " is" : "s are"} not shown in this view. Clear the ` + - `filter (or scroll them into view) and select again.` - ); - return false; - } - if (!captured.length) return false; - - /* - * ⭐ WARN THEN ALLOW (owner ruling, 2026-08-06) — the arm fires ONLY where something is - * actually at stake, so deleting your own scratch rows stays one keypress. - * - * ⛔ THE PREDICATE IS "WHAT WOULD BE LOST", NOT "WHO CREATED THIS". There is no stored - * creator on a row, and deriving one from a stage column would flip meaning as the - * automation runs. But the question the warning answers is a different and answerable one: - * *has an automation written to this row* — because that is exactly what a re-find cannot - * give back. MEASURED in `run_discover_instagram`: the seen-before test reads the rows - * CURRENTLY IN THE TABLE, so a deleted candidate returns as NEW — `found_count` back to 1, - * `first_found` re-stamped, and its stage reset to Review. Deleting a candidate somebody - * had already judged throws that judgement away. - * - * ⚠ This paragraph used to name a "Declined" stage. WAVE 26 / R6 DELETED THE BOARD'S - * BUILT-IN TERMINALS — the lanes are whatever the user defined now — so the warning names - * the mechanism (the stage resets to Review) and no longer a stage that does not ship. - */ - /* ⭐ WAVE 27 · ITEM 12 / R3 — `f.automation?.stageField` SURVIVES THE STAGE DELETION, and - that is SESSION B's recommendation taken rather than an oversight (their ASK ->C, in the - wave doc): the engine stops WRITING the flag, but the guarded migration that removes an - existing stage column rides `ut_ensure`, so a tenant's table is only cleaned the next - time something writes to it. Until then a legacy stage column still exists on real - data, and this reader is exactly what keeps it read-only. Deleting it would hand a - machine-written column back to the user for the length of that window — which is the - one thing R3's deletion must not do on its way out. */ - const machineKeys = fields - .filter((f) => isMachineWritten(f) || f.automation?.stageField) - .map((f) => f.key); - // `captured` holds only NON-EMPTY values, so a key being present IS "a machine wrote here". - const risky = captured.filter((r) => machineKeys.some((k) => r.values[k] !== undefined)); - const sig = captured.map((r) => r.rid).join(","); - if (risky.length && delArmed !== sig) { - setDelArmed(sig); - signal( - TOAST_EVENT, - `${risky.length} of these ${risky.length === 1 ? "records was" : "records were"} ` + - `written by an automation. Deleting ${risky.length === 1 ? "it" : "them"} resets ` + - `first-found and times-found, and puts the stage back to Review — so a card ` + - `somebody had already moved on comes back undecided. Press Delete again ` + - `(or click again) to confirm.` - ); - return false; - } - setDelArmed(""); - // SEQUENTIAL, matching the restore path: the store is one document and ten concurrent - // read-modify-writes against it is how a delete silently misses a row. - const gone: UndoRow[] = []; - for (const r of captured) - if (await deleteTableRow(scope, r.rid)) gone.push(r); - if (!gone.length) { - signal(TOAST_EVENT, "Nothing was deleted — the server refused."); - return false; - } - undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { - kind: "rowDelete", table: scope, rows: gone, - }); - // ⭐ ITEM 2 — same reason as the undo path: a row added and then deleted in one session - // is still a PENDING copy here, and no future payload will ever prune it. - withdrawPending(gone.map((r) => r.rid)); - clearSelection(); - signal(ROWS_STALE_EVENT); - // ⚠ THE CONSEQUENCE IS NAMED IN FULL, and the first version of this sentence undersold it. - // It said only that a matching record "will return", which is true and reassuring and - // leaves out the part that actually costs something. - // - // MEASURED in `run_discover_instagram`: the seen-before test is built from the rows - // CURRENTLY IN THE TABLE (`seen` comes from `existing2`). A deleted profile is therefore - // not "seen again" on the next run — it is NEW. `found_count` resets to 1, `first_found` - // is re-stamped, and the stage goes back to Review, because a new candidate's card starts - // at the human gate. So deleting a candidate somebody had already moved out of Review puts - // it back there undecided — the human judgement is the thing the re-find cannot give back. - // (W26/R6 deleted the board's built-in terminals, so there is no named stage to cite here.) - // - // (The follower SNAPSHOT history is safe either way — it lives in the platform master keyed - // by handle, a different store, which only `purge_handle` touches.) - // Same survival, same reason as `machineKeys` above (item 12 / R3, B's ASK ->C): the flag - // is no longer WRITTEN, and a table that has not been migrated yet still carries it. - const fedByAutomation = fields.some((f) => f.automation?.stageField); - signal( - TOAST_EVENT, - `Deleted ${gone.length} record${gone.length === 1 ? "" : "s"}. Ctrl+Z to undo.` + - (fedByAutomation - ? " If this automation finds one again it comes back as a NEW candidate — first-found" + - " and times-found reset, and its stage back to Review. Undo keeps all of that." - : "") - ); - return true; - }, - [canMutateRecords, displayRows, fields, scope, clearSelection, delArmed, - withdrawPending] - ); - - const onGridDelete = useCallback( - (sel: GridSelection): boolean => { - /* - * ⭐ 2026-08-06 (owner) — A ROW SELECTION + Delete/Backspace DELETES THE RECORDS. - * - * Checking rows and pressing Delete used to CLEAR every editable cell in them, which on the - * owner's Instagram table now clears nothing at all: every column but one is machine-written - * and refuses the write. So the key appeared to do nothing, which is what they reported. - * - * ⚠ ROWS ONLY, AND ONLY WHEN NO **MULTI-CELL** RANGE IS DRAWN. Checking rows is an explicit - * gesture about RECORDS; DRAGGING a range is a gesture about CELLS, and a Delete that - * destroyed records because some rows happened to be checked would feel unrecoverable even - * with an undo behind it. When both are present, the narrower reading (clear the cells) wins. - * - * ⛔ A SINGLE ACTIVE CELL IS NOT A RANGE, and the first version of this got it wrong. - * `sel.current.range` is ALWAYS set when a cell is merely focused — a 1×1 rect — so - * `!sel.current?.range` meant the feature worked only if you had never clicked a cell. - * Caught by driving it live rather than by the gate: click a cell, then check a row, then - * press Delete, and the key silently went back to clearing cells. The gesture the rule is - * actually about is a DRAG, which is width or height greater than one. - */ - const dragged = !!sel.current - && (sel.current.range.width > 1 || sel.current.range.height > 1); - if (canMutateRecords && sel.rows.length > 0 && !dragged) { - const pids: number[] = []; - for (const rowIndex of sel.rows) { - const vr = displayRows[rowIndex]; - if (vr && vr.kind === "data") pids.push(vr.record.pid); - } - // ⛔ `true`, NOT `false`, WHEN THIS BRANCH DOES NOT DO THE WORK — the scar the comment at - // the bottom of this handler already records, pointed at a second case. `false` cancels - // glide's own delete, so bailing with it would leave Delete doing nothing at all on a - // selection this branch declined. `deleteRecords` is async and the keypress cannot wait - // for it, so the branch commits here and reports the outcome through its own toast. - if (!pids.length) return true; - void deleteRecords(pids); - return false; - } - const byPid = new Map>(); - const clear = (rowIndex: number, colIndex: number) => { - const vr = displayRows[rowIndex]; - if (!vr || vr.kind !== "data") return; - const column = visibleCols[colIndex]; - const field = column ? fieldByKey.get(column.id!) : undefined; - if (!field || !canEditField(field)) return; - const at = byPid.get(vr.record.pid) ?? {}; - at[field.key] = ""; - byPid.set(vr.record.pid, at); - }; - for (const rowIndex of sel.rows) - for (let c = 0; c < visibleCols.length; c++) clear(rowIndex, c); - const range = sel.current?.range; - if (range) - for (let y = range.y; y < range.y + range.height; y++) - for (let x = range.x; x < range.x + range.width; x++) clear(y, x); - const writes = [...byPid.entries()].map(([pid, updates]) => ({ pid, updates })); - if (!writes.length) return true; - patchManyAndRecord(writes, "clearing cells"); - // ⚠ `false` ONLY WHEN THIS ACTUALLY DID THE WORK. Returning it unconditionally cancels - // glide's own delete for cases this handler does not cover — a COLUMN selection, which - // glide deletes from `toDelete.columns` and the loops above never look at — so Delete - // would silently clear nothing. Handing the keypress back when there is nothing to - // group is strictly safer than swallowing it: the per-cell path still refuses read-only - // fields (`onCellEdited`'s own `canEditField`), it just does not arrive as one entry. - return false; - }, - [canMutateRecords, deleteRecords, displayRows, visibleCols, fieldByKey, canEditField, - patchManyAndRecord] - ); - - /** - * Owner item 4 / C-ADDROW — the cursor FOLLOWS the appended row, once it exists. - * - * The append is a server round trip, so at click time there is nothing to focus; this waits - * for the re-read to bring the pid back and then lands the active cell on its first column, - * scrolled into view. `newRowPid` is cleared either way — a row the re-read never produced - * (a refused write, a filter that excludes it) must not leave a cursor waiting forever. - */ - useEffect(() => { - if (newRowPid === null) return; - const index = displayPidToIndex.get(newRowPid); - if (index === undefined) return; - setActiveCell(0, index); - gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" }); - setNewRowPid(null); - }, [newRowPid, displayPidToIndex, setActiveCell]); - - /** Owner item 23 — the fields in the view's column order, for the Hide-fields panel. Built - * from `order` (already reconciled by useGridColumns) rather than from `config.order` so the - * panel and the grid can never disagree about which fields exist or where they sit. */ - const orderedFields = useMemo( - () => order.map((key) => fieldByKey.get(key)).filter((f): f is Field => !!f), - [order, fieldByKey] - ); - /** Owner item 17 — the row of the ACTIVE cell, i.e. what the last click highlighted. Read off - * `gridSelection` rather than tracked separately so keyboard navigation moves the wash too; - * `useGridSelection` already guards this index against a grid that shrank this render. */ - const activeRow = gridSelection.current?.cell[1]; - - const statusValues = useMemo(() => { - const map: Record = {}; - for (const field of fields) { - // A `user`'s people come from the HOST, never from the rows — an assignee nobody has been - // given yet is still assignable. - if (field.type === "user") { - map[field.key] = payload?.userOptions ?? []; - continue; - } - // ⭐ Owner item 24 — every other choice column goes through ONE rule (`choiceVocabulary`): - // the DECLARED list when the field has one, the values seen in the data when it does not. - // This used to branch on the type, which sent `stock_bucket` — a `select` whose vocabulary - // is computed server-side and declares no `options` — down the declared path to an empty - // list, and an empty supplied list WINS in the filter panel. See the function's own note; - // it is pure so the gate can run it, which nothing inside this file can be. - if (field.type === "select" || field.type === "multiselect" || field.type === "status") - map[field.key] = choiceVocabulary(field, rawRows); - } - return map; - }, [fields, rawRows, payload?.userOptions]); - - const rowPx = ROW_PX[config.rowHeightMode]; - // DISTINCT customers, not painted data rows. Grouping a MULTI field (Cohorts) puts the same - // customer under every group it belongs to, so counting rows would report more records than - // there are customers — a number nobody could reconcile against the book. `pidToIndex` is - // first-wins per pid, so its size IS the distinct count, and it equals the row count for every - // non-multi grouping and for no grouping at all. - // - // ⚠ The FULL pipeline's map, never the display slice's: the toolbar count must state what the - // view MATCHES. What is painted is the "Showing first N" bar's job (owner item 8). - const recordCount = pidToIndex.size; - const shownRecords = displayPidToIndex.size; - /* wave17 GRID — item 2 / owner R5. The whole-table row band is GONE, and with it the - `bandTone` / `bandMask` pair that used to sit here: the winning control's tone, and the - alternating-row mask built from the DATA-ROW ORDINAL so group headers could not flip the - stripe at a boundary. R5 keeps the involved-COLUMN wash and the header tints — those are - `useGridColumns.columnTones` + `COLUMN_TONE_THEME` and are untouched. What follows is now - three row states, not four. */ - const getRowThemeOverride = useCallback( - (row: number): Partial | undefined => { - const visibleRow = displayRows[row]; - if (!visibleRow || visibleRow.kind !== "data") return undefined; - // Owner item 17 — the row holding the ACTIVE cell stays washed after the pointer moves - // on. Ordered AFTER hover on purpose: the pointer is the more immediate signal, and a - // hovered-and-active row reading as merely active would make hover look broken. - if (row === hoverRow) { - if (!config.colorBy) return HOVER_NEUTRAL; - return ( - HOVER_ROW_THEME[String(visibleRow.record[config.colorBy] ?? "").toLowerCase()] ?? - HOVER_NEUTRAL - ); - } - // A colour-by wash already marks this row with meaning the user chose; overlaying the - // active tint on top would blend two hues into a third that means neither. There, glide's - // accent ring is the active marker and this stays out of the way. - if (!config.colorBy) { - if (row === activeRow) return ACTIVE_ROW_NEUTRAL; - return undefined; - } - return STATUS_ROW_THEME[ - String(visibleRow.record[config.colorBy] ?? "").toLowerCase() - ]; - }, - [displayRows, config.colorBy, hoverRow, activeRow] - ); - const onItemHovered = useCallback( - (args: GridMouseEventArgs) => { - const row = args.kind === "cell" ? args.location[1] : undefined; - setHoverRow((previous) => (previous === row ? previous : row)); - // Owner item 19 — the hover-only Expand. Placed from glide's OWN bounds for the primary - // cell of this row, so freeze, horizontal scroll and row-height mode are handled by the - // component that owns them rather than re-derived here. - // - // ⚠ Only ever CLEARED from here for a different row, never for "the pointer left the - // canvas". Moving the pointer ONTO the button leaves the canvas, so clearing on - // out-of-bounds would unmount the control between the mouse arriving and the click - // landing — an affordance that vanishes exactly when you reach for it, and one that - // every assertion still sees because it exists in every state except the one that - // matters. `.cg-grid-box`'s own onMouseLeave is what dismisses it. - const vrow = row !== undefined ? displayRows[row] : undefined; - if (args.kind === "cell" && vrow?.kind === "data" && row !== undefined) { - const pid = vrow.record.pid; - const b = gridRef.current?.getBounds(0, row); - // Clamped to the grid's own box. ⚠ RETARGETED wave-14 item 15: this used to say "with a - // grouping active `freezeColumns` is 0, so the primary column scrolls away". It no - // longer does — the first column is now frozen in EVERY mode. The clamp stays because - // its other half is still live: glide's bounds are VIEWPORT coordinates, so a row - // scrolled under the header or past the bottom still reports a rect outside the grid, - // and a `position: fixed` button with no clamp would paint over the chrome on a row - // nobody can see. The horizontal legs are now defence in depth (a column drag, a box - // narrower than the frozen strip) rather than the everyday case. - const at = b - ? expandButtonRect(b, gridBoxRef.current?.getBoundingClientRect()) - : null; - // Recomputed EVERY move, not memoised on the pid: a scroll can leave the pointer over - // the same record at a new y, and a button that keeps the pid but not the position - // floats over the wrong row. Referential stability is preserved by comparing values, - // which is the cheap half — `getBounds` is arithmetic over glide's own layout. - setExpandAt((prev) => - !at - ? null - : prev && prev.pid === pid && prev.x === at.x && prev.y === at.y - ? prev - : { pid, x: at.x, y: at.y, size: at.size } - ); - } - // Wave-5 item 6 — hovering a header whose field carries a description floats the text - // under the header. The tip element is pointer-events:none and aria-hidden: it can - // NEVER become the click target, which is the tooltip trap this page has already paid - // for once ([[ui-invisible-to-assertions]]). - if (args.kind === "header") { - const colDef = visibleCols[args.location[0]]; - const field = colDef ? fieldByKey.get(colDef.id!) : undefined; - const note = field ? field.note || field.description : undefined; - // Owner item 16 — a title the fit SHORTENED must still be readable somewhere, and the - // tip is where. `colDef.title !== field.label` IS the truncation test: `fitHeaderTitle` - // returns the label by identity when it fits, so this asks the renderer what it did - // rather than re-running the measurement and hoping the two agree. - const cut = !!field && !!colDef && colDef.title !== field.label; - const text = cut && field ? (note ? `${field.label} — ${note}` : field.label) : note; - const bounds = args.bounds; - if (text && bounds) { - setHeaderTip((prev) => - prev && prev.text === text && prev.x === bounds.x - ? prev - : { text, x: bounds.x, y: bounds.y + bounds.height + 4 } - ); - return; - } - } - setHeaderTip((prev) => (prev === null ? prev : null)); - - // I2 — the cell half. Only text that is genuinely CUT OFF gets a tip: a tooltip on - // every cell just repeats what is already legible and covers the row under it. - // Truncation is MEASURED with glide's own font rather than guessed from a character - // count, which is what makes "e.g. a Note" work and a short currency cell stay quiet. - if (args.kind === "cell" && row !== undefined) { - const vr = displayRows[row]; - const cellCol = visibleCols[args.location[0]]; - // Group headers and footers carry no cell text of their own — a tip over the group - // bar would repeat the label glide has already drawn across it. - if (vr?.kind === "data" && cellCol) { - const field = fieldByKey.get(cellCol.id!); - const text = field ? formatDisplay(field, vr.record[field.key]) : ""; - const bounds = args.bounds; - const tip = bounds ? cellTipText(text, measureCellText(text), bounds.width) : null; - if (tip && bounds) { - const x = tipLeft(bounds.x, window.innerWidth, measureCellText(tip) + 20); - const y = bounds.y + bounds.height + 4; - setCellTip((prev) => - prev && prev.text === tip && prev.x === x && prev.y === y - ? prev - : { text: tip, x, y } - ); - return; - } - } - } - setCellTip((prev) => (prev === null ? prev : null)); - }, - [visibleCols, fieldByKey, displayRows] - ); - const rowHeight = useMemo( - () => - config.groupBy - ? (row: number) => (displayRows[row]?.kind === "group-header" ? 32 : rowPx) - : rowPx, - [config.groupBy, rowPx, displayRows] - ); - - /* ═══ W18-B VOID ═══ (wave 18, owner item 1b) — WHERE THE TABLE ENDS, in canvas pixels. - Consumed by the two `.cg-grid-void` rectangles at the DataEditor mount; see the CSS region - of the same name for why the void is painted in the DOM rather than in the glide theme. - - ⭐ THE PROPERTY THAT MAKES THIS ARITHMETIC AND NOT SCROLL-TRACKING, and the reason there is - no `onVisibleRegionChanged` handler anywhere near it: **a void can only exist in an axis the - content does not overflow.** glide's scroll extent is EXACTLY the content — `scrollWidth = - nonGrowWidth (+ overscrollX)`, `scrollHeight = header + rows (+ overscrollY)`, and we pass - neither overscroll prop (scrolling-data-grid.js:12-24) — so at maximum scroll the last - row/column lands flush on the client edge. Wherever there is something to paint, the scroll - offset in that axis is 0 and cannot become anything else. The rectangles therefore depend on - the columns, the rows and the box, and on nothing that moves while the user drags. - - Every column carries an explicit `width` (`useGridColumns` sets `config.widths[key] ?? - DEFAULT_WIDTH`) and none carries `grow`, so glide's column sizer passes them through - untouched and this sum is the width it actually lays out. */ - const gridVoid = useMemo(() => { - const gutter = scrollbarGutter(); - let colsPx = rowMarkerPx(displayRows.length); - for (const c of visibleCols) colsPx += (c as { width?: number }).width ?? 0; - let rowsPx = HEADER_PX; - if (typeof rowHeight === "number") rowsPx += displayRows.length * rowHeight; - else for (let i = 0; i < displayRows.length; i++) rowsPx += rowHeight(i); - // ⚠ Owner item 4 — THE GHOST ROW IS PART OF THE TABLE, and this line is what makes it - // visible. glide draws its trailing row AFTER the last data row, but `displayRows` (our - // rows) does not contain it, so the void started exactly where the ghost row does and - // painted flat #F6F8FC straight over it. The "+" was still clickable the whole time - // (`.cg-grid-void` is `pointer-events: none`), which is the worst version of this bug: - // the gate went green on an affordance nobody could see. Found by READING THE SCREENSHOT - // ([[ui-invisible-to-assertions]], [[finalize-visual-review-sop]]). - if (canMutateRecords) - rowsPx += typeof rowHeight === "number" ? rowHeight : rowHeight(displayRows.length); - - /* Fit is tested against the client box the OTHER axis's scrollbar leaves behind — the same - `clientWidth`/`clientHeight` glide's own scroll handler reads (infinite-scroller.js: - 110-116), so this branch and glide's cannot disagree about whether a bar is there. The - two `if`s resolve the circularity in the only direction it can run: a bar in one axis can - CREATE one in the other, but two bars can never un-create each other. */ - let vBar = rowsPx > gridSize.height; - let hBar = colsPx > gridSize.width; - if (vBar && !hBar) hBar = colsPx > gridSize.width - gutter; - if (hBar && !vBar) vBar = rowsPx > gridSize.height - gutter; - const clientW = gridSize.width - (vBar ? gutter : 0); - const clientH = gridSize.height - (hBar ? gutter : 0); - - /* `null` = the content reaches that edge, so there is no void and no rectangle. The - comparison is strict: a table ending exactly on the edge has nothing past it. */ - return { - clientW, - clientH, - below: rowsPx < clientH ? rowsPx : null, - right: colsPx < clientW ? colsPx : null, - }; - }, [visibleCols, displayRows, rowHeight, gridSize, canMutateRecords]); - /* ═══ end W18-B VOID (geometry) ═══ */ - - // The record drawer resolves positions against what the MODE paints: the display slice for - // grid/list (the cap is real there), the FULL pipeline for calendar/kanban/map (a month, a - // stack or a pin reaches past the cap by design — a drawer that refused those pids would - // close itself on a card the user can plainly see). - // ⭐ WAVE-27 item 8 (C3) — `swipe` belongs here for calendar/kanban/map's own reason: the deck - // is derived from `modeDataRows` (the FULL pipeline), so a card past the display cap is one - // the user can plainly see, and a drawer that refused its pid would close itself on open. - const fullSetMode = - displayMode === "calendar" || displayMode === "kanban" || displayMode === "map" || - displayMode === "swipe"; - const detailRows = fullSetMode ? visibleRows : displayRows; - const detailIdxMap = fullSetMode ? pidToIndex : displayPidToIndex; - const detailIndex = detailPid !== null ? detailIdxMap.get(detailPid) : undefined; - useEffect(() => { - if (detailPid !== null && detailIndex === undefined) setDetailPid(null); - }, [detailPid, detailIndex]); - const detailRecord = useMemo(() => { - if (detailIndex === undefined) return null; - const row = detailRows[detailIndex]; - if (!row || row.kind !== "data") return null; - return overlayEdits[row.record.pid] - ? { ...row.record, ...overlayEdits[row.record.pid] } - : row.record; - }, [detailIndex, detailRows, overlayEdits]); - const dataPosition = useMemo(() => { - if (detailIndex === undefined) return 0; - return detailRows - .slice(0, detailIndex + 1) - .filter((row) => row.kind === "data").length; - }, [detailIndex, detailRows]); - const neighborExists = useCallback( - (delta: -1 | 1) => { - if (detailIndex === undefined) return false; - for ( - let index = detailIndex + delta; - index >= 0 && index < detailRows.length; - index += delta - ) - if (detailRows[index]?.kind === "data") return true; - return false; - }, - [detailIndex, detailRows] - ); - const go = useCallback( - (delta: -1 | 1) => { - if (detailIndex === undefined) return; - for ( - let index = detailIndex + delta; - index >= 0 && index < detailRows.length; - index += delta - ) { - const row = detailRows[index]; - if (row?.kind !== "data") continue; - setDetailPid(row.record.pid); - if (displayMode === "grid") - gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" }); - return; - } - }, - [detailIndex, detailRows, displayMode] - ); - - /** - * Wave-9 I3 — the description "(i)", drawn RIGHT-ALIGNED and vertically centred with the - * field name. - * - * ⚠ This is the header-draw alternative, taken because glide's `overlayIcon` PROVABLY - * cannot do it: `drawHeaderInner` paints an overlay at a hard-coded `drawX + 9` / - * `(height - 18) / 2 + 6` — a badge on the bottom-right corner of the TYPE mark at the far - * LEFT of the header — and no prop moves it. `drawHeader` is glide's supported escape - * hatch: it hands over the ctx, the rect, the menu bounds and the sprite manager, plus a - * `drawContent()` that runs its own rendering first. So glide still draws the header it - * always drew (type mark, title, menu); only the (i) is ours, and `overlayIcon` is no - * longer set on any column. - * - * The geometry is `infoMarkRect()` in overlayPlacement.ts, so "right-aligned and centred" - * is asserted numerically by a gate rather than judged from one screenshot at one width. - */ - const drawGridHeader = useCallback( - (args, drawContent) => { - const field = args.column.id ? fieldByKey.get(args.column.id) : undefined; - // ⚠⚠ WAVE-14 ITEM 2 — THE ONE LINE THAT MAKES AN INVOLVED HEADER BOLD, and it is not a - // font setting anywhere near where you would look for one. - // - // glide's `drawGridHeaders` does, verbatim: - // if (theme !== outerTheme) ctx.font = theme.baseFontFull; // :38-40 - // i.e. the moment a column carries ANY `themeOverride` its header title is painted in the - // CELL font ("13px") instead of the header font ("600 13px"). That is why the owner's - // filtered/sorted/grouped headers had already lost their weight — nothing in our code - // asked for it, and setting `headerFontStyle` on the involved theme alone would have - // changed precisely nothing, because glide never reads it on that path. Re-asserting the - // MERGED theme's header font on the ctx here is what puts it back, for every column, - // uniformly: on a column with no override this is a no-op (glide set the same value at - // :21), and glide's own save/restore around each column stops it leaking to the next. - // - // Built from `headerFontStyle` + `fontFamily` rather than reading `headerFontFull`, - // which is real at runtime but is declared on `FullTheme`, not the public `Theme` the - // callback is typed with — same string, no cast, and it is exactly how - // `mergeAndRealizeTheme` composes it. - // - // Bonus fix, worth knowing: `drawHeaderInner` passes `theme.headerFontFull` to - // `getMiddleCenterBias`, which measures with the CURRENT ctx font but CACHES under the - // string it was handed. Before this line, an overridden column measured its baseline bias - // in the 13px font and filed it under the "600 13px" key, so whichever column drew first - // decided the vertical centring for all of them. - args.ctx.font = `${args.theme.headerFontStyle} ${args.theme.fontFamily}`; - drawContent(); - if (!field) return; - const hasInfo = !!(field.note || field.description); - // ⭐ OWNER ITEM 4 (2026-08-06): *"why is the Field still have the Dot at the header to - // mark its a custom editable field?"* Because the predicate was `source === "overlay"` - // alone — true of columns a person added BESIDE the Odoo ones, and true of literally - // every column in a `ut_*` database, including the ones an automation spawns and fills. - // So on the owner's Instagram table the dot marked all 23 columns as "yours", which is - // both meaningless (it never varies) and wrong (they are not yours to edit). - const isCustom = field.source === "overlay" && !isMachineOwned(field); - // ONE layout for both marks, from the same `headerMarkSizes` order `useGridColumns` used - // to reserve the label's room — so the number of marks drawn and the number reserved for - // cannot disagree. null = the column is too narrow to carry them without covering its own - // label, and then NOTHING is drawn (all-or-nothing; see headerMarkLayout). - const marks = headerMarkLayout( - args.rect, - args.menuBounds.width, - headerMarkSizes(hasInfo, isCustom) - ); - if (!marks) return; - let slot = 0; - if (hasInfo) { - const at = marks[slot++]; - args.spriteManager.drawSprite( - "aiosInfo", - "normal", - args.ctx, - at.x, - at.y, - at.size, - args.theme - ); - } - if (isCustom) { - // Wave-14 item 1 / R11 — the user-created-field marker that REPLACES the yellow header - // wash. A muted dot: quiet enough to ignore while reading, present enough to answer - // "which of these columns are mine?" at a glance. Never a background — the owner killed - // the wash, and a paler wash would have been the same answer in a lower voice. - const at = marks[slot++]; - const ctx = args.ctx; - ctx.save(); - ctx.beginPath(); - ctx.arc(at.x + at.size / 2, at.y + at.size / 2, at.size / 2, 0, Math.PI * 2); - ctx.fillStyle = CUSTOM_FIELD_MARK; - ctx.fill(); - ctx.restore(); - } - }, - [fieldByKey] - ); - - const onCellClicked = useCallback( - (cell: Item, event: CellClickedEventArgs) => { - const row = displayRows[cell[1]]; - if (!row) return; - // Owner item 10: clicking into the cells is "I am working now" — the frame folds its - // navigation rail to the slim strip (a no-op in the embed; the shell listens). - signal(NAV_MINIMIZE_EVENT); - if (row.kind === "group-header") { - event.preventDefault(); - setCollapsed((current) => { - const next = new Set(current); - if (next.has(row.groupKey)) next.delete(row.groupKey); - else next.add(row.groupKey); - return next; - }); - return; - } - const column = visibleCols[cell[0]]; - const field = column ? fieldByKey.get(column.id!) : undefined; - // A picked field (select / assignee) opens its choices where the cell is. It cannot use - // glide's text overlay — a free-text editor on a constrained column is how you end up - // with "Done", "done" and "DONE" as three different values — and glide's own dropdown - // cell lives in a package we have deliberately not added, so this reuses the same - // AnchoredOverlay the column and view menus already use. Wave-5: a RATING cell rides - // the same picker surface (its choices are 1..max stars); both doors respect the - // permissions verdict. - if (row.kind === "data" && field && canEditField(field) - && (isPickType(field.type) || field.type === "rating")) { - event.preventDefault(); - // Item 6 (2026-07-31) — the click also LANDS the active cell here (preventDefault - // stops glide from committing it), so pick → Enter walks on down the column exactly - // like a typed edit does. - setActiveCell(cell[0], cell[1]); - const b = event.bounds; - setPicker({ - pid: row.record.pid, - fieldKey: field.key, - anchor: { - left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height, - width: b.width, height: b.height, - }, - }); - return; - } - // A linked-record cell is a doorway to the target database, not an opaque id list. - // The large modal mounts the same Grid surface over exactly these pids, so its standard - // search, Filters, Sort, Fields, and column menus keep working for every database kind. - if (row.kind === "data" && field?.type === "link" && field.link?.table) { - if (field.link.table.startsWith("ut_")) { - event.preventDefault(); - setActiveCell(cell[0], cell[1]); - setLinkAt({ pid: row.record.pid, fieldKey: field.key }); - return; - } - } - // ⭐ Wave-23 C7 — a JSON cell opens the big viewer. It is the ONLY door: the cell carries - // `allowOverlay:false`, because glide's overlay is a one-line box and one keystroke in the - // wrong place inside a 32 KB document turns a well-formed payload into an unparseable one, - // saved. Opened for EVERY reader (a document you may not edit is still one you must be - // able to read) — the viewer takes `onSave` only when the permission verdict allows it, - // and the host's write wall is the real one either way. - if (row.kind === "data" && field?.type === "json") { - event.preventDefault(); - setActiveCell(cell[0], cell[1]); - setJsonAt({ pid: row.record.pid, fieldKey: field.key }); - return; - } - // Wave-5 item 11 — a URL cell opens its link on click (scheme-guarded: http/https only, - // a bare domain gets https://). Editing stays with glide's overlay (Enter/double-click). - if (row.kind === "data" && field?.type === "url") { - const raw = String( - overlayEdits[row.record.pid]?.[field.key] ?? row.record[field.key] ?? "" - ).trim(); - if (raw) { - const href = /^https?:\/\//i.test(raw) - ? raw - : /^[\w-]+(\.[\w-]+)+/.test(raw) - ? `https://${raw}` - : null; - if (href) { - event.preventDefault(); - window.open(href, "_blank", "noopener"); - return; - } - } - } - // Owner item 6 (2026-07-31) — CLICKING THE CUSTOMER TICKS THE CHECKBOX. The row marker - // is a ~32px strip; the identity cell is the widest, most natural target on the row, so - // a click there toggles the same pid-anchored set the markers write ("more surface area - // to select individual customers into a Cohort"). No preventDefault: glide still commits - // the cell highlight below, so reading across the row keeps working. - if (row.kind === "data" && field && field.key === lockedKey) { - togglePid(row.record.pid); - } - // Owner item 17 — A SINGLE CLICK HIGHLIGHTS. It used to open the record panel from here, - // which meant a user could not select a cell, read across a row, or copy a value without - // a drawer landing over the table. The highlight is glide's own doing: this handler - // returns without `preventDefault`, so the click commits `gridSelection.current`, the - // accent ring lands on the cell and `getRowThemeOverride` washes the row - // (ACTIVE_ROW_NEUTRAL). Nothing is drawn here. - // - // ⚠ SHIPPED WITH ITEM 19, never alone. Deleting this line is what removes the ONLY way to - // open a record; the hover Expand button below is its replacement, and half of this - // change is a table whose records cannot be opened at all. - }, - [displayRows, visibleCols, fieldByKey, canEditField, overlayEdits, lockedKey, togglePid, - setActiveCell] - ); - /** - * Double-click / Enter. KEPT as a door to the record on purpose (owner item 17 names the - * SINGLE click, and it is the single click that was in the way). - * - * Removing it too would leave the record reachable only by pointer — the hover affordance - * cannot be reached from the keyboard at all — so a keyboard user would lose the panel - * outright. glide routes Enter on an EDITABLE cell to its overlay editor before this fires, - * so the two doors do not collide: this is the activation path for the read-only columns, - * which is most of the table. - */ - const onCellActivated = useCallback( - (cell: Item) => { - const row = displayRows[cell[1]]; - const column = visibleCols[cell[0]]; - const field = column ? fieldByKey.get(column.id!) : undefined; - if (row?.kind === "data" && field?.source !== "overlay") - setDetailPid(row.record.pid); - }, - [displayRows, visibleCols, fieldByKey] - ); - const onCellEdited = useCallback( - (cell: Item, value: EditableGridCell) => { - const row = displayRows[cell[1]]; - const column = visibleCols[cell[0]]; - const field = column ? fieldByKey.get(column.id!) : undefined; - if (!row || row.kind !== "data" || !field || !canEditField(field)) return; - // Wave-5 item 11 — a checkbox toggles straight through glide's BooleanCell (no overlay - // editor); the overlay store keeps its '1'-or-empty contract. - if (value.kind === GridCellKind.Boolean && field.type === "checkbox") { - patchAndRecord(row.record.pid, { [field.key]: value.data ? "1" : "" }, "a tick"); - return; - } - if (value.kind === GridCellKind.Uri) { - patchAndRecord(row.record.pid, { [field.key]: value.data ?? "" }, "an edit"); - return; - } - if (value.kind === GridCellKind.Text || value.kind === GridCellKind.Number) { - patchAndRecord(row.record.pid, { [field.key]: value.data }, "an edit"); - } - }, - [displayRows, visibleCols, fieldByKey, patchAndRecord, canEditField] - ); - const validateCell = useCallback( - (cell: Item): boolean => { - const row = displayRows[cell[1]]; - const column = visibleCols[cell[0]]; - const field = column ? fieldByKey.get(column.id!) : undefined; - // ONE verdict (types.mayEditField): stratum + read-only-by-nature + permissions. The - // picked types and rating never take glide's text overlay — their pickers are the door. - return ( - !!row && - row.kind === "data" && - !!field && - canEditField(field) && - !isPickType(field.type) && - field.type !== "rating" && - field.type !== "status" - ); - }, - [displayRows, visibleCols, fieldByKey, canEditField] - ); - const onGridPaste = useCallback( - (target: Item, values: readonly (readonly string[])[]): boolean => { - const column = visibleCols[target[0]]; - const field = column ? fieldByKey.get(column.id!) : undefined; - if (!field) return false; - // ⭐ Wave-15 item 3 (R8) — THE SELECTION DECIDES HOW FAR THE PASTE REACHES. Until now the - // targets were walked down from the anchor for exactly `values.length` rows, so a - // fifty-row selection and a one-cell clipboard wrote ONE row: the selection was painted - // and obeyed by nothing. `pasteRowCount` is the whole rule and it is pure, so the gate - // can drive it — this handler cannot be reached by any node test in the repo. - const rowCount = pasteRowCount(values.length, gridSelection.current?.range, - { col: target[0], row: target[1] }); - const targetPids = Array.from({ length: rowCount }, (_, offset) => { - const row = displayRows[target[1] + offset]; - return row?.kind === "data" ? row.record.pid : null; - }); - const patches = planFieldPaste({ - field, - sourceFieldKey: copyProvenanceRef.current.fieldKey, - editable: canEditField(field), - values, - targetPids, - allowedChoices: statusValues[field.key] ?? choiceOptions(field), - }); - if (!patches) return false; - // R4 — ONE stack entry for the whole paste. Cell by cell would put forty entries on the - // stack for one Ctrl+V, and undoing a paste one cell at a time is not undoing a paste. - patchManyAndRecord( - patches.map((patch) => ({ pid: patch.pid, updates: { [field.key]: patch.value } })), - "a paste" - ); - // We handled the write ourselves. Returning false tells glide not to run its cell - // renderers' generic paste path (Bubble/rating renderers cannot enforce this contract). - return false; - }, - [visibleCols, fieldByKey, canEditField, statusValues, displayRows, patchManyAndRecord, - gridSelection.current] - ); - - const openHeaderMenu = useCallback( - (column: number, bounds: Rectangle) => { - if (embedded) return; - const definition = visibleCols[column]; - if (!definition?.id) return; - setColumnMenu({ - fieldKey: definition.id, - anchor: { - left: bounds.x, - top: bounds.y, - right: bounds.x + bounds.width, - bottom: bounds.y + bounds.height, - width: bounds.width, - height: bounds.height, - }, - }); - }, - [embedded, visibleCols] - ); - const onHeaderClicked = useCallback( - (column: number, event: HeaderClickedEventArgs) => { - if (event.isEdge) return; - event.preventDefault(); - openHeaderMenu(column, event.bounds); - }, - [openHeaderMenu] - ); - const onGridKeyDown = useCallback( - (event: GridKeyEventArgs) => { - if ( - event.key.toLowerCase() === "c" && - (event.ctrlKey || event.metaKey) && - !event.altKey - ) { - const range = gridSelection.current?.range; - const source = range?.width === 1 ? visibleCols[range.x] : undefined; - copyProvenanceRef.current = markGridCopy(source?.id ?? null, Date.now()); - // Do not prevent default: glide still owns serialization and the OS clipboard write. - return; - } - // Owner item 5 (2026-07-31) — EXCEL-GRADE ENTER. When no editor is open (an open overlay - // editor swallows its own keys before the canvas sees them), Enter moves the active cell - // DOWN one record and Shift+Enter moves UP — never opening the record drawer, which is - // what made keyboard runs down a column "really clunky". Glide's own overlay editor - // already commits-and-moves-down on Enter, so typing → Enter → typing flows like Excel; - // this handles the BETWEEN-edits half. Group headers are skipped in the direction of - // travel. The drawer stays reachable by double-click and the hover Expand button. - if (event.key === "Enter" && !event.ctrlKey && !event.metaKey && !event.altKey) { - const cur = gridSelection.current?.cell; - if (cur) { - event.preventDefault(); - event.stopPropagation(); - event.cancel(); - // Owner item 6 (2026-07-31) — Enter on a PICKED cell (select / multi select / - // assignee / rating) opens its picker: the keyboard door the mouse click already - // had. glide's text overlay cannot serve these types (validateCell refuses them), - // so without this the keyboard run down a column dead-ends at every picked field. - // Shift+Enter stays pure navigation, so walking UP past picked cells still works. - const curRow = displayRows[cur[1]]; - const curColumn = visibleCols[cur[0]]; - const curField = curColumn ? fieldByKey.get(curColumn.id!) : undefined; - if ( - !event.shiftKey && - curRow?.kind === "data" && - curField && - canEditField(curField) && - (isPickType(curField.type) || curField.type === "rating") - ) { - const b = gridRef.current?.getBounds(cur[0], cur[1]); - if (b) { - setPicker({ - pid: curRow.record.pid, - fieldKey: curField.key, - anchor: { - left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height, - width: b.width, height: b.height, - }, - }); - return; - } - } - const dir = event.shiftKey ? -1 : 1; - let row = cur[1] + dir; - while (row >= 0 && row < displayRows.length && displayRows[row]?.kind !== "data") - row += dir; - if (row >= 0 && row < displayRows.length) { - setActiveCell(cur[0], row); - gridRef.current?.scrollTo(cur[0], row, "vertical", 0, 0); - } - return; - } - } - const contextMenu = - event.key === "ContextMenu" || (event.key === "F10" && event.shiftKey); - // DataEditor exposes public controlled selection without the row marker, - // while GridKeyEventArgs.location comes from its internal grid and still - // includes that marker at column zero. - const eventColumn = - event.location && event.location[0] > 0 - ? event.location[0] - 1 - : undefined; - const column = gridSelection.current?.cell[0] ?? eventColumn; - if (!contextMenu || column === undefined || !event.bounds) return; - const gridBounds = gridBoxRef.current?.getBoundingClientRect(); - const headerTop = gridBounds?.top ?? event.bounds.y; - event.preventDefault(); - event.stopPropagation(); - event.cancel(); - openHeaderMenu(column, { - x: event.bounds.x, - y: headerTop, - width: event.bounds.width, - height: 36, - }); - }, - [gridSelection.current, openHeaderMenu, displayRows, setActiveCell, - visibleCols, fieldByKey, canEditField] - ); - - const persistView = useCallback((view: SavedView) => { - setViews((current) => { - const found = current.some((item) => item.id === view.id); - return found - ? current.map((item) => (item.id === view.id ? view : item)) - : [...current, view]; - }); - emitHostEvent({ id: eventId("view"), type: "view_upsert", view }); - }, []); - - /** - * Item 12 (C-LOCK) — set or clear a view's cohort lock, from the rail's menu. - * - * Works on ANY view, not only the active one, which is why it goes through `persistView` - * rather than `updateConfig`: the rail's menu opens on whichever row you clicked. When the - * target IS the active view, the live `config` must move too, or the table keeps showing the - * old row set until the next select and the lock reads as ignored. - * - * Clearing DELETES the key rather than storing null — the no-churn rule every other optional - * config member follows, and the shape `_clean_display`'s sibling validator expects. - */ - const onViewCohortLock = useCallback( - (viewId: string, cohortId: string | null) => { - const target = views.find((v) => v.id === viewId); - if (!target) return; - const nextConfig = { ...target.config }; - if (cohortId) nextConfig.cohortLock = cohortId; - else delete (nextConfig as Record).cohortLock; - persistView({ ...target, config: nextConfig }); - if (viewId === activeViewId) { - setConfig((live) => { - const next = { ...live }; - if (cohortId) next.cohortLock = cohortId; - else delete (next as Record).cohortLock; - return next; - }); - } - }, - [views, activeViewId, persistView] - ); - - const selectView = useCallback( - (id: string) => { - const current = views.find((view) => view.id === activeViewId); - if (current && !sameConfig(current.config, config)) - persistView({ ...current, config }); - const next = views.find((view) => view.id === id); - if (!next) return; - setActiveViewId(id); - setConfig(normalizeConfig(next.config, fields)); - setCollapsed(new Set()); - setSearch(""); - setDetailPid(null); - setDisplayCap(DISPLAY_PAGE); - // Owner item 3 (2026-07-31): tell the host WHERE THE USER IS, so a fresh browser (no - // localStorage copy) resumes on this view instead of the system default. Presentation - // state — the host stores the id and the read side re-validates it. - emitHostEvent({ id: eventId("view"), type: "view_select", viewId: id }); - // Owner item 10: opening a view is "I am working now" — the frame folds its nav rail. - signal(NAV_MINIMIZE_EVENT); - }, - [views, activeViewId, config, persistView, fields] - ); - - /** - * A-S4-3 (item 25) — a notification's click-through. The SHELL routes to the table and fires - * this; the GRID owns view selection, so neither learns the other's state. - * - * ⛔ IGNORE, NEVER THROW, when the view is not ours: an alert can outlive the view it watches - * (deleted, or a share revoked), and a reader who can no longer see it must simply land on the - * table. `selectView` already returns early on an unknown id; the topic check stops one grid - * reacting to another's alert when both are mounted. - */ - useEffect(() => { - const onOpen = (e: Event) => { - const detail = (e as CustomEvent).detail; - if (!detail || detail.topic !== scope) return; - if (!views.some((v) => v.id === detail.viewId)) return; - selectView(detail.viewId); - }; - window.addEventListener(VIEW_OPEN_EVENT, onOpen); - return () => window.removeEventListener(VIEW_OPEN_EVENT, onOpen); - }, [scope, views, selectView]); - const createView = useCallback( - (name: string, mode: DisplayMode, permissions: ViewPermissions) => { - const acceptedName = uniqueDisplayName(name, views.map((view) => view.name)); - /** - * ⭐⭐ WAVE 27 · OWNER ITEM 9 / RULING R4 — **A NEW VIEW IS BLANK. ALL OF IT.** - * - * ⛔ THIS REVERSES THE WAVE-26 COMMENT THAT STOOD HERE, so the reversal is stated rather - * than quietly applied. That comment dropped `cohortLock` from the spread and defended - * keeping the rest: *"their inheritance is a FEATURE (build a view, branch off it) - * precisely because it is visible"*. The owner disagrees, in as many words — R4: - * **"New views ALWAYS start blank — no filters/sorts inherited from anything."** The - * branch-off use it defended gets its own door later (Duplicate view); it is not what the - * "+" button means, and the wave-26 fix was the right diagnosis of the wrong scope — one - * member of the spread was invisible, but ALL of them arrived unasked. - * - * ⛔ AND `defaultViewConfig` IS THE BLANK, not a literal assembled here. It is the same - * function the grid's own initial state and the cohort door already use, so "blank" has - * ONE definition in this file ([[one-evaluator-per-question]]); a second literal beside it - * would be a place for the two to disagree the day a `ViewConfig` member is added — and - * the one that got added last wave is exactly what caused this bug. - * - * WHAT STOPS CARRYING, enumerated because a reader deserves the list and not just the - * ruling: `filters` + `filterConj`, `sorts`, `groupBy`, `colorBy`, `rowHeightMode`, - * `order`/`visible` (hidden columns), `widths`, `memberPids`, `frozenCount` and - * `cohortLock`. `order`/`visible` come back from the FIELD list's own default visibility, - * which is why `fields` replaces `config` in the dependency list below. - * - * ⚠ THE DISPLAY REFS GO TOO, and that is the one deliberate loss. W13's carry rule kept - * the calendar's date field / kanban's stack field / map's lat-lon across a mode switch so - * Grid→Map→Grid→Map did not re-ask; a NEW view is not a mode switch, and inheriting the - * previous view's stack field is inheritance of exactly the kind R4 names. A ref-less - * kanban asks which field to stack by, which is the honest state for a view born empty. - * `cleanDisplay` still collapses a ref-less GRID to absent, so picking "Grid" produces the - * byte-identical shape every pre-wave-9 view has (no churn). - */ - const nextConfig: ViewConfig = { - ...defaultViewConfig(fields), - display: cleanDisplay({ mode }), - }; - const view: SavedView = { - id: nextViewId(), - name: acceptedName, - kind: "custom", - config: nextConfig, - // I17 (C4) — sent EXPLICITLY. Absent on create means 'personal' host-side, so a user - // who chose "Collaborative" would silently get the opposite. `createdBy` is NOT sent: - // the host stamps it and ignores whatever the browser claims. - permissions, - }; - persistView(view); - setActiveViewId(view.id); - // The LIVE config has to move too, not just the stored one. Without this the view is - // created as a Calendar and the user keeps staring at the Grid until they switch views - // and back — the mode would be real in the store and invisible on screen. - setConfig(nextConfig); - setSaveState("saved"); - }, - // `fields`, not `config` (R4): the blank is derived from the COLUMNS, and reading the live - // config here at all is what item 9 deletes. - [fields, persistView, views] - ); - const renameView = useCallback( - (id: string, name: string) => { - const view = views.find((item) => item.id === id); - if (view) { - const acceptedName = uniqueDisplayName( - name, - views.filter((item) => item.id !== id).map((item) => item.name) - ); - persistView({ ...view, name: acceptedName }); - } - }, - [views, persistView] - ); - /** - * I12 (contract C3) — freeze/unfreeze a view's DISPLAY MODE. - * - * The client re-checks the actor before emitting even though the menu entry is already - * gated: an event can be replayed, and "hidden in the client" has never been a permission. - * The host checks it again and that check is the wall — this one only keeps the client from - * showing a change that will not survive the round trip. - */ - const toggleViewLock = useCallback( - (id: string, locked: boolean) => { - const view = views.find((item) => item.id === id); - if (!view || !mayToggleViewLock(view, viewer)) return; - persistView({ ...view, locked }); - }, - [views, persistView, viewer] - ); - // The list description. Template lists ship with prose seeded by the host, but it is - // the USER's text once they touch it — including clearing it. An empty string is - // persisted as an empty string (never coerced back to the seed), because the host - // resolves a saved view OVER its template, so "" is how you delete a description. - const setViewNote = useCallback( - (id: string, note: string) => { - const view = views.find((item) => item.id === id); - if (view) persistView({ ...view, note: note.slice(0, 2000) }); - }, - [views, persistView] - ); - const duplicateView = useCallback( - (id: string) => { - const source = views.find((view) => view.id === id); - if (!source) return; - const copy: SavedView = { - ...source, - id: nextViewId(), - name: uniqueDisplayName( - `${source.name} copy`, - views.map((view) => view.name) - ), - kind: "custom", - locked: false, - // I17 (C4) — a COPY IS THE DUPLICATOR'S OWN, PERSONAL view. Ruled, not inherited: - // - inheriting a 'users' grant would silently re-share the copy with a list the - // person making it never chose; - // - Duplicate is also the escape hatch for someone who may NOT edit the original, - // and 'personal' + the host's fresh createdBy stamp is exactly "mine to work in". - permissions: { edit: "personal" }, - // Never echo the SOURCE's creator: the host stamps and ignores what the browser - // sends, but sending someone else's name is a laundering attempt on its face. - createdBy: undefined, - config: { ...source.config }, - }; - persistView(copy); - setActiveViewId(copy.id); - setConfig(copy.config); - }, - [views, persistView] - ); - const deleteView = useCallback( - (id: string) => { - const view = views.find((item) => item.id === id); - // ⚠ This gate used to read `view.locked`, and the menu entry above it did too. C3 - // redefines `locked` as "the DISPLAY MODE is frozen" on ANY view, so leaving the gate - // here would make a user-frozen Kanban undeletable in the client while the host would - // delete it happily (app.py refuses only `all-customers`, by id). The MENU is just the - // door — this is the gate, and both had to move. - // C4 rides alongside: you may not delete a view you may not edit. - if (!view || isUndeletableView(view) || !mayEditView(view, viewer)) return; - // 2026-08-04 — STAMP BEFORE THE EMIT, the writeLocal-before-emit order every other - // optimistic path here uses. The live adopt below re-reads `payload.workspace.views` - // on every echo, and the queue sends ONE batch at a time — so an echo answering an - // EARLIER batch still lists this view, and without the tombstone the row would come - // back and (since that merge never removes) stay back. See liveWorkspace.ts. - viewTombstonesRef.current = stampTombstone(viewTombstonesRef.current, id, Date.now()); - setViews((current) => current.filter((item) => item.id !== id)); - emitHostEvent({ id: eventId("view-delete"), type: "view_delete", viewId: id }); - if (activeViewId === id) { - const all = views.find((item) => item.id === ALL_VIEW_ID) ?? allRecordsView(fields, scope); - setActiveViewId(all.id); - setConfig(all.config); - } - }, - [views, activeViewId, fields, viewer, scope] - ); - - /** - * Wave-6 item 7 — PER-COHORT view state. Each cohort keeps its own view under the stable id - * `cohort:`; a cohort with no saved view opens CLEAN (defaultViewConfig — no sort, - * no filters), which is exactly what "first created" means. The view is added LOCALLY on - * first open and emitted only when the user actually edits it (the autosave effect already - * compares configs), so "absent = clean" stays true in the store — opening every cohort - * once must not write a store full of empty views. - */ - useEffect(() => { - if (!cohortMode || !workspaceReady || !activeCohortId) return; - const vid = `cohort:${activeCohortId}`; - if (activeViewId === vid) return; - const outgoing = views.find((view) => view.id === activeViewId); - if (outgoing && !sameConfig(outgoing.config, config)) - persistView({ ...outgoing, config }); - const existing = views.find((view) => view.id === vid); - const view: SavedView = existing ?? { - id: vid, - name: lists.find((l) => l.id === activeCohortId)?.name ?? "Cohort", - kind: "custom", - config: defaultViewConfig(fields), - }; - if (!existing) setViews((current) => [...current, view]); - setActiveViewId(vid); - setConfig(normalizeConfig(view.config, fields)); - setCollapsed(new Set()); - setSearch(""); - }, [cohortMode, workspaceReady, activeCohortId, activeViewId, views, config, - fields, lists, persistView]); - - /** - * "Add to list" (owner item 6). Runs the SAME engine over the named view's config to learn - * which customers it matches, then hands those pids to the host. - * - * ⚠ Deliberately re-runs the pipeline for THAT view rather than using what is on screen: the - * menu is available on every view, not only the active one, and "add the rows I can see" would - * quietly mean something different depending on which view happened to be open. - * - * ⚠ Refuses on a windowed table. There the client holds ONE PAGE, so "the customers this view - * matches" is a question it cannot answer — it would add the 200 rows it happens to hold and - * report success. Silence would be worse than the refusal (CG-3, no-unverifiable-aggregates). - * - * ⚠ Refuses for the SAME reason when this view has a measure condition with no answer yet - * (CG-8). A pending condition matches nothing, so the cohort would be built from a filter that - * is currently narrower than the one the user is reading — and a cohort is a FIXED set, so - * that wrong membership would persist long after the answer arrived. - */ - /** - * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — HOW MANY RECORDS EACH ALERTED VIEW MATCHES. - * - * ⛔ ONE EVALUATOR, AND IT IS `runPipeline` RATHER THAN `matchFilterTree`. The question is - * "how many records does this VIEW show", and a view is more than its condition list: a - * `cohortLock` narrows before anything else, `memberPids` are pinned in regardless, and a - * RANK leaf ("top 10") is only answerable against the domain the other conditions leave. - * `matchFilterTree` answers a different question — "does this ROW match this tree" — and - * using it here would report the whole book for a locked view and nonsense for a ranked one. - * `addToList` below already reaches for the pipeline for the same reason - * ([[one-evaluator-per-question]]); this is the same call with the same inputs. - * - * ⛔ THREE CASES WHERE THERE IS NO HONEST NUMBER, and each is ABSENT rather than zero: - * · a WINDOWED table — the client engine is skipped by design (CG-3), so a count here - * would be the size of one PAGE wearing the label of a scope; - * · an unresolved MEASURE condition — the server has not answered it yet, so the view - * matches nothing *yet*, and rendering `0` would state a fact nobody has established; - * · a view the rail no longer holds — an alert outlives its view. - * A missing key means no badge (the prop's own contract), which is the correct rendering of - * "not known" and is distinguishable from `0`, which is a real and interesting answer. - * - * ⚠ MEMOISED ON THE ALERTED SET, not on `views`: this walks the whole book once per alerted - * view, and on a table with no alerts it does nothing at all. - */ - const alertedKey = alerted.join(","); - const alertCounts = useMemo(() => { - const out: Record = {}; - if (serverWindowed || !alerted.length) return out; - for (const id of alerted) { - const view = views.find((v) => v.id === id); - if (!view) continue; - if (pendingMeasures(view.config.filters, measureSets) > 0) continue; - const { pidToIndex } = runPipeline({ - rawRows: computedRows, - fields, - filters: view.config.filters, - search: "", - sorts: [], - groupBy: null, - collapsed: new Set(), - memberPids: view.config.memberPids, - filterConj: view.config.filterConj ?? "and", - measureSets, - cohortSets, - cohortLock: view.config.cohortLock, - today, - }); - // DISTINCT records, which is what `pidToIndex` is — a grouped pipeline can list one - // record under several headings and `visibleRows.length` would then count it twice. - out[id] = pidToIndex.size; - } - return out; - // `alertedKey` is the scalar identity of the list; `alerted` itself is a fresh array on - // every fetch even when the answer has not changed. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [alertedKey, views, computedRows, fields, measureSets, cohortSets, today, serverWindowed]); - - const addToList = useCallback( - (viewId: string, cohortId: string, name: string) => { - if (serverWindowed) return; - const view = views.find((v) => v.id === viewId); - if (!view) return; - if (pendingMeasures(view.config.filters, measureSets) > 0) return; - const { visibleRows: matched } = runPipeline({ - rawRows: computedRows, - fields, - filters: view.config.filters, - search: "", - sorts: view.config.sorts, - groupBy: null, - collapsed: new Set(), - memberPids: view.config.memberPids, - filterConj: view.config.filterConj ?? "and", - measureSets, - cohortSets, - today, - }); - const pids = matched - .filter((r): r is { kind: "data"; record: Row } => r.kind === "data") - .map((r) => r.record.pid); - emitHostEvent({ - id: eventId("addlist"), - type: "add_to_list", - viewId, - cohortId, - name, - pids, - }); - }, - [views, computedRows, fields, serverWindowed, measureSets, cohortSets, today] - ); - - // ---------------------------------------------------------------- I11c (C4) - // Folders for the Views rail. The MODEL (echo reconcile, tombstones, dangling - // refs) lives in folders.ts and is gated by verify_folders.py; this is only the - // wiring: stamps for what this browser just did, and the five events. - const [folderStamps, setFolderStamps] = useState({}); - /** - * Folders THIS browser just created, held until the host echo returns them. - * - * ⚠ Without these, `reconcileFolders` was being called with the host list as - * BOTH arguments, which makes its optimistic-create branch unreachable: a - * folder absent from `host` is equally absent from `local`. The function was - * right and the wiring was wrong, so "+ New folder" painted nothing until the - * round trip landed — seconds, on a cold container, with the user clicking - * again. The gate passed because it called reconcileFolders with two DIFFERENT - * arrays; it now also calls it the way this component does. - */ - const [localFolders, setLocalFolders] = useState([]); - const workspaceFolders = payload?.workspace?.folders; - const folders = useMemo( - () => reconcileFolders(workspaceFolders, localFolders, folderStamps, Date.now()), - [workspaceFolders, localFolders, folderStamps] - ); - const folderIdOf = useCallback( - (viewId: string) => - resolveFolderId( - viewId, - views.find((v) => v.id === viewId)?.folderId ?? null, - folders, - folderStamps, - Date.now() - ), - [views, folders, folderStamps] - ); - const stampFolder = useCallback((patch: (prev: FolderStamps) => FolderStamps) => { - setFolderStamps((prev) => pruneFolderStamps(patch(prev), Date.now())); - }, []); - - /* wave17 R1 / C-LOCKV — the COHORT FOLDER surface is gone. `folders['cohorts']` merged - into `folders['views']` in the host's one-shot migration and `cohortFolders` is no longer - emitted at all, so `reconcileFolders` here would have been reconciling an absent list - against nothing forever. A locked view files into an ORDINARY folder now, which is what R1 - means by "native folders". */ - - /** - * C4 as AMENDED — the folder-level bulk add's PREVIEW. Deliberately re-runs the - * engine per contained view, exactly as `addToList` does, and carries the SAME - * two refusals as named skips rather than dropping them silently: a union that - * quietly omits two views is a wrong cohort that looks right, and a cohort is a - * FIXED set, so it stays wrong ([[no-unverifiable-aggregates]]). - */ - const folderAddPreview = useCallback( - (folderId: string) => { - const pids = new Set(); - const skipped: { name: string; why: string }[] = []; - let counted = 0; - for (const view of views) { - if (folderIdOf(view.id) !== folderId) continue; - if (serverWindowed) { - skipped.push({ name: view.name, why: "this table loads one page at a time" }); - continue; - } - if (pendingMeasures(view.config.filters, measureSets) > 0) { - // C-NAME (item 9) — user-facing, so it says "metric" like every other surface. - skipped.push({ name: view.name, why: "a metric condition has no answer yet" }); - continue; - } - counted += 1; - const { visibleRows: matched } = runPipeline({ - rawRows: computedRows, - fields, - filters: view.config.filters, - search: "", - sorts: view.config.sorts, - groupBy: null, - collapsed: new Set(), - memberPids: view.config.memberPids, - filterConj: view.config.filterConj ?? "and", - measureSets, - cohortSets, - today, - }); - for (const r of matched) if (r.kind === "data") pids.add(r.record.pid); - } - return { pids: [...pids], counted, skipped }; - }, - [views, folderIdOf, serverWindowed, computedRows, fields, measureSets, cohortSets, today] - ); - - const folderAddToList = useCallback( - (folderId: string, cohortId: string, name: string) => { - const { pids } = folderAddPreview(folderId); - if (!pids.length) return; - // ONE emit with the deduped union, through the SAME guarded event a single - // view's "Add to cohort" uses — so the host validates both identically. - emitHostEvent({ - id: eventId("addlist"), - type: "add_to_list", - viewId: "", - cohortId, - name, - pids, - }); - }, - [folderAddPreview] - ); - - /** - * Wave-7 item W2 (contract C2) — export "the current view", CLIENT-side. The same - * engine run addToList uses (that is the point: the file says exactly what the count - * says), over the same pool the named surface scopes to. - * - * ⚠ Same two refusals as addToList, for the same reasons: a server-windowed table - * holds ONE PAGE (the client cannot answer "the view's matches"), and a view with a - * PENDING measure condition currently matches nothing — an export taken then would be - * an empty/narrower file wearing the view's name. Both windows self-resolve; the - * refusal is silent no-op for one round trip. - */ - const exportRows = useCallback( - (name: string, cfg: ViewConfig, poolRows: Row[], format: ExportFormat) => { - if (serverWindowed) return; - if (pendingMeasures(cfg.filters, measureSets) > 0) return; - const { visibleRows: matched } = runPipeline({ - rawRows: poolRows, - fields, - filters: cfg.filters, - search: "", - sorts: cfg.sorts, - groupBy: null, - collapsed: new Set(), - memberPids: cfg.memberPids, - filterConj: cfg.filterConj ?? "and", - measureSets, - cohortSets, - today, - }); - const records = matched - .filter((r): r is { kind: "data"; record: Row } => r.kind === "data") - .map((r) => - overlayEdits[r.record.pid] - ? { ...r.record, ...overlayEdits[r.record.pid] } - : r.record - ); - // Columns = the view's VISIBLE fields in display order (the reconciled order: - // every key present, config order first — the same shape useGridColumns paints). - const inOrder = new Set(cfg.order); - const keys = [ - ...cfg.order, - ...fields.map((f) => f.key).filter((k) => !inOrder.has(k)), - ]; - const vis = new Set(cfg.visible); - const cols = keys - .filter((k) => vis.has(k)) - .map((k) => fieldByKey.get(k)) - .filter((f): f is Field => !!f); - runExport(format, name, today, cols, records); - }, - [serverWindowed, fields, fieldByKey, measureSets, cohortSets, today, overlayEdits] - ); - - /** - * ⭐ owner item 3 (2026-08-03) — THE SHEET THE TIME-SERIES VIEW IS SHOWING, published by the - * panel. A ref rather than state on purpose: nothing renders from it, and setting state on - * every sheet rebuild would re-render the whole grid to feed a menu nobody has opened yet. - */ - const tsSheetRef = useRef(null); - const onTsSheet = useCallback((s: TsSheet) => { tsSheetRef.current = s; }, []); - /** The same channel for a SUMMARY calendar — the other mode whose content is not its rows. - * `null` is CalendarView saying "records mode, export the rows" (see its note). */ - const calSheetRef = useRef<{ fields: Field[]; rows: Row[] } | null>(null); - const onCalSheet = useCallback( - (s: { fields: Field[]; rows: Row[] } | null) => { calSheetRef.current = s; }, - [] - ); - - /** - * W2 — the view menu's Export (Customer page): the named view over the whole pool. - * - * ⭐ owner item 3 (2026-08-03) — AND IT EXPORTS WHAT THE VIEW SHOWS. Every mode used to - * export the same thing: the matched customer rows. For grid / list / kanban / calendar / - * map / chart that is right — those modes ARRANGE rows, so the rows are what they show, and - * a calendar's dates and a kanban's lanes are columns already in the file. - * - * TWO modes are not arrangements of rows, and both were wrong in the direction that matters: - * - * · `timeseries` — metric ROWS over period COLUMNS. Exporting it handed you a customer list - * that shares none of its numbers. It exports the SHEET. - * · `calendar` IN SUMMARY MODE — metric values per DAY (C-DISP item 4). Same problem, and - * the owner named this one by hand. A calendar in RECORDS mode is genuinely an - * arrangement of rows, so it keeps the row export; `CalendarView` says which it is by - * publishing a sheet or publishing null. - * - * ⚠ ONLY FOR THE ACTIVE VIEW, and this is a real limit, not an oversight. Both sheets are - * built by the mounted view — the time series from a server round trip, the calendar from the - * month on screen. A view sitting unopened in the rail has neither, and this component cannot - * conjure one without fetching it. So that case SAYS SO and downloads nothing: the - * alternative is silently handing over the customer rows under that view's name, which is the - * exact substitution this branch exists to stop. - */ - const exportViewData = useCallback( - (viewId: string, format: ExportFormat) => { - const view = views.find((v) => v.id === viewId); - if (!view) return; - const cfg = normalizeConfig(view.config, fields); - const spec = cleanDisplay(cfg.display); - const isActive = viewId === activeViewId; - const refuse = (what: string) => - signal( - TOAST_EVENT, - `Open "${view.name}" first: ${what} is exported from what it draws on screen.` - ); - - if (spec?.mode === "timeseries") { - const sheet = isActive ? tsSheetRef.current : null; - if (!sheet) return refuse("a time series"); - if (sheet.empty) { - signal(TOAST_EVENT, - `"${view.name}" has no metrics on its sheet yet — add one, then export.`); - return; - } - if (format === "csv") { - // The gated builder (verify_timeseries), kept as THE csv path so the file the owner - // downloads is the one the gate proves — footnotes, grouped thousands and all. - triggerDownload( - exportFilename(view.name, today, format), - new Blob(["" + buildTsCsv(sheet)], { type: "text/csv;charset=utf-8" }) - ); - } else { - const { fields: tf, rows: tr } = tsSheetToTable(sheet); - runExport(format, view.name, today, tf, tr); - } - return; - } - - /* ═══ W18-C CATALOG ═══ A catalog is the THIRD mode that is not an arrangement of rows, - and it is the furthest from one: its pages are authored content, and its product codes - are a fraction of the pool in an order the user chose. Handing over the matched rows - under a catalogue's name would be the same substitution the two branches around this - one exist to stop. It exports as a PDF, through the browser's own print dialog (R10), - so this branch refuses the file and names the door that works. */ - if (spec?.mode === "catalog") { - signal( - TOAST_EVENT, - `"${view.name}" is a catalog — open it and use Print to save it as a PDF. ` + - `A spreadsheet of its products is not what it draws.` - ); - return; - } - /* ═══ end W18-C CATALOG ═══ */ - - if (spec?.mode === "calendar" && spec.calendarMode === "summary") { - const sheet = isActive ? calSheetRef.current : null; - if (!sheet) return refuse("a calendar summary"); - runExport(format, view.name, today, sheet.fields, sheet.rows); - return; - } - - exportRows(view.name, cfg, computedRows, format); - }, - [views, fields, computedRows, exportRows, activeViewId, today] - ); - - /** - * The selection bar's "Add to locked view": exactly the CHECKED customers, not a view's - * matches. - * Same guarded `add_to_list` event — the host intersects the pids with the caller's pool - * either way, so a hand-picked set and a view's match set ride one validation path. - */ - const addSelectionToList = useCallback( - (cohortId: string, name: string) => { - const pids = [...selectedPids]; - if (!pids.length || serverWindowed) return; - emitHostEvent({ - id: eventId("addlist"), - type: "add_to_list", - viewId: activeViewId, - cohortId, - name, - pids, - }); - setSelAddOpen(false); - setSelListName(""); - clearSelection(); - }, - [selectedPids, serverWindowed, activeViewId, clearSelection] - ); - - // Item 3c: the local write is the RENDERED truth (the def half of the no-blip contract) — - // stamped so a lagged echo cannot claw it back at the next remount. Item 9c: `scope` rides - // the EVENT at create time only; the def itself carries `scope: 'cohort'` so the echo is - // byte-stable ('global' stays absent — the host normalizes them to one shape). - const scopeChoice = payload?.workspace?.scopeChoice === true; - /** - * Item 12 (C-LOCK) — the Filter banner's copy for a cohort-locked view. - * - * `undefined` when the view is not locked. When it IS locked but the set is not in `lists`, - * this returns an EMPTY object: the banner must still appear (the reader is looking at a - * narrowed table and deserves to know why), but with no name and no count, because a set we - * were given no membership for is a set we must not describe. RECORD's banner reads that - * absence and says "locked to a set you cannot see". - * - * The count is the cohort's OWN size, not the number of rows on screen — the conditions - * narrow within the lock, so the two differ and the banner is stating the lock. - */ - /** - * ⭐ wave17 R1 / C-LOCKV — the lock in force, which is now simply the ACTIVE VIEW's. - * - * Wave 15 had two sources: a saved `config.cohortLock` and an ephemeral pick from the Cohorts - * rail that outranked it while held. The second existed only because a projected cohort was - * not a saved view, so opening one could not be allowed to write anything. Under R1 it IS a - * saved view, so selecting it applies its own stored lock through the ordinary view path — - * one source, and it survives a reload, which the transient one never did. - */ - const effectiveCohortLock = config.cohortLock; - /* wave17 R1 / C-LOCKV — `cohortRows` and the whole `cohortRail` projection are GONE. They - turned `lists` into rail rows for a section that no longer exists; the host projects each - cohort into `views` instead, so the rail renders them through the same path as every other - view and there is nothing left to project. `lists` still arrives and is still load-bearing - — it is the MEMBERSHIP channel that feeds `cohortSets`, which is what the lock resolves - against. Its absence would blank every locked view, so it is not "unused". */ - const cohortLockChip = useMemo(() => { - const id = effectiveCohortLock; - if (!id) return undefined; - const set = lists.find((l) => l.id === id); - if (!set) return {}; - return { name: set.name, count: set.pids?.length ?? 0 }; - }, [effectiveCohortLock, lists]); - // (Setting/clearing the lock lives in `onViewCohortLock` above — the rail's menu operates on - // whichever view you clicked, not necessarily the active one, so it goes through - // `persistView` and only touches the live `config` when the two are the same view.) - /** - * Item 5 / contract C-LAYOUT — this user's record-detail field order, for RECORD's - * `RecordDetail` (they asked for exactly these two props in the wave mailbox). - * - * Read straight off the workspace stratum the host echoes, and emitted back as the - * `record_layout` event HOST landed. Deliberately NOT reconciled or optimistically merged - * here: the event returns False host-side (an autosave hot path like `view_upsert`), so - * there is no rerun to race, and `RecordDetail` holds the live order while a drag is in - * flight. Nothing else in the grid reads it — the order is the MODAL's, never the grid's - * `config.order`. - */ - const recordLayout = payload?.workspace?.recordLayout?.order; - const onRecordLayout = useCallback((order: string[]) => { - emitHostEvent({ id: eventId("rlayout"), type: "record_layout", order }); - }, []); - const saveField = useCallback( - (field: Field, scope?: FieldScope) => { - const requestId = eventId("field"); - const accepted: Field = { - ...field, - label: uniqueDisplayName( - field.label, - fields.filter((item) => item.key !== field.key).map((item) => item.label) - ), - editRequestId: requestId, - }; - delete accepted.labelCorrectedFrom; - delete accepted.labelCorrectionId; - stampFieldEdit(accepted.key); - setFields((current) => - current.some((item) => item.key === accepted.key) - ? current.map((item) => (item.key === accepted.key ? accepted : item)) - : [...current, accepted] - ); - emitHostEvent({ - id: requestId, - type: "field_upsert", - field: accepted, - ...(scope ? { scope } : {}), - }); - }, - [stampFieldEdit, fields] - ); - const createField = useCallback( - ( - label: string, - type: FieldType, - anchorKey: string | null, - side: "left" | "right" | "end", - options?: string[], - measureSpec?: { key: string; window: WindowSpec }, - extra?: FieldBuildExtra - ) => { - /** - * ⚠ RENAMED FROM `scope` (wave 21, item 7). It used to SHADOW the component's own - * `scope` prop — the surface this grid is drawing — with a completely different - * thing: a FIELD's storage stratum (`'cohort'` or nothing). Two unrelated meanings - * under one word, in a function that now has to speak to a per-table endpoint. - * Caught by `tsc` only because `FieldScope` and `SurfaceScope` happen to be - * incompatible unions; had either been a bare `string`, the C2 call below would have - * POSTed to `/tables/undefined/fields` and read as a server bug. - */ - const fieldScope = scopeChoice ? extra?.scope : undefined; - const scoped = fieldScope === "cohort" ? ({ scope: "cohort" } as const) : {}; - // Owner item 7 — a FORMULA-MEASURE column. `measure_` prefixed, source:'odoo' (read-only - // at both ends), derived (values arrive from the host per render), filterable:false (the - // replacement is the measure CONDITION with the same measure+window — the governed path). - // The first render after creation shows blank cells for exactly one round trip: the host - // persists the field, then computes the values into the next payload. - if (measureSpec) { - const field: Field = { - key: `measure_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`, - label, - type, - source: "odoo", - default: true, - custom: true, - derived: true, - filterable: false, - agg: ["currency", "int"].includes(type) ? "sum" : undefined, - measure: measureSpec, - ...scoped, - }; - saveField(field, fieldScope); - insertColumn(field.key, anchorKey, side); - return; - } - const field = { ...buildOverlayField(label, type, options, extra), ...scoped }; - /** - * ⭐ WAVE 21 item 7 (contract C2) — AN AUTOMATION COLUMN ON A USER DATABASE GOES TO - * THE DEFINITION, not to this user's overlay stratum. - * - * The automation editor's column picker reads `user_tables` (the definition); every - * grid-created column went to `_table_workspace` (the per-user overlay). So the - * picker was empty by construction: the user made the column, then could not find it - * in the automation they made it for. C2 routes exactly this one kind through - * `POST /tables/{key}/fields`, and the ROUTE is the wall — `_field_or_refuse` allows - * only the creator or an admin, which is the same answer the grid would have got. - * - * ⚠ NARROW ON PURPOSE, twice over: `type === "automation"` AND a `ut_` scope. The - * connector surfaces have no such endpoint (their schema is their source's), and - * every other kind keeps the overlay path this wave — moving all of them changes who - * can SEE a column, which is a behaviour change three surfaces would feel and is - * booked as debt rather than smuggled in here. - * - * ⚠ AND IT IS ASYNC, where every other create is optimistic. There is no local - * insert first: the server re-slugs the key and can refuse outright (it 400s until - * `UT_FIELD_TYPES` learns `automation` — C2's other half, A's), and a column shown - * under a key the store never took is one that vanishes on the next read with no - * explanation. The wait is one round trip; the alternative is a lie. - */ - /** - * ⭐⭐ 2026-08-09 — THE RELATIONAL PAIR JOINS THAT ROUTE, and for a stronger reason than - * `automation` had. The note above says every other kind "keeps the overlay path this - * wave", weighing it as a question of who can SEE a column. For `link` and `rollup` it is - * not a visibility trade-off at all: their cells are computed SERVER-side by - * `compute_relation_cells`, which walks `user_tables` definitions. A rollup parked in a - * per-user overlay is invisible to the only code that could ever fill it, so it is not a - * narrower feature — it is a column that cannot work, in any account, ever. - * - * ⛔ AND THE BAG TRAVELS. `_clean_field` returns None for a bagless `link`/`rollup`, so - * sending `{key,label,type}` here would trade a silent blank column for a loud 400. The - * bag is on `extra`; `field` already carries it (see `FieldBuildExtra`). - */ - /** - * ⭐⭐ 2026-08-10 — `formula` JOINS THE SAME ROUTE, on the visibility argument the note - * above weighs and the relational note declines to use. - * - * For `link`/`rollup` visibility was not the point (their cells are computed by something - * that reads the definition). For a formula it IS the whole point, and the trade-off falls - * the other way from wave 21's read of it: a `ut_*` database is a SHARED database — the - * owner builds a column on "IG profile" and expects the four people looking at that - * database to see it. In the overlay stratum only its creator ever does. Measured on - * nurilab: `Trimmed reel views`, the trimmed-average column the whole rollup chain exists - * to produce, was visible to exactly one account. - * - * ⛔ THE VALUES STILL DO NOT MOVE, and this must not be sold as more than it is. A formula - * is computed in this browser (`computedRows`) from the row's other cells and is stored - * nowhere; what became shared is the EXPRESSION. An automation, a rollup and an export - * still cannot read this column. Sharing the definition is not materialising the number. - * - * ⚠ `agg: "sum"` rides along so the totals row behaves identically to the overlay path - * (`buildOverlayField` stamps it) — a column that stops totalling because of which door - * created it is [[gate-answers-the-wrong-question]] wearing a create route. - */ - if ((type === "automation" || type === "link" || type === "rollup" - || type === "formula") && isUserTable) { - void addTableField(scope, { - key: field.key, - label: field.label, - type, - ...(extra?.link ? { link: extra.link } : {}), - ...(extra?.rollup ? { rollup: extra.rollup } : {}), - ...(type === "formula" && extra?.formula - ? { formula: extra.formula, agg: "sum" } - : {}), - }).then((made) => { - if (!made) return; // the bridge has already said why - setFields((current) => - current.some((f) => f.key === made.key) ? current : [...current, made] - ); - insertColumn(made.key, anchorKey, side); - /** - * ⛔⛔ 2026-08-10 — THE WORKSPACE RE-READ USED TO FIRE HERE AND IT ATE THE LINE ABOVE. - * - * `insertColumn` adds the new key to the view's `visible` set and autosaves that config; - * `signal(WORKSPACE_STALE_EVENT)` refetches `/workspace`, whose views carry the SERVER's - * config — the one written a moment before the insert. The refetch won, every time, so - * a column created into the shared definition was **stored correctly and rendered - * nowhere**: present in the payload, present in `order` (which is re-derived from the - * field list), and absent from `visible`. MEASURED on live nurilab across all four - * views after migrating `Trimmed reel views`; it has been true for `link`, `rollup` and - * `automation` since that route was widened on 2026-08-09. - * - * ⛔ AND THE COMMENT IT REPLACES WAS THE REASON NOBODY LOOKED. It claimed the signal is - * "what makes the column survive a reload rather than living only in this browser's - * state" — but the SERVER already stored the field; the 201 is what makes it survive. - * The signal only refreshed a payload whose field list `setFields` had just updated by - * hand, and `addTableField` already drops the rows cache. It bought nothing and cost - * the one piece of state the create had just written. - * - * ⚠ The delete and reconfigure paths keep their signals: those change VALUES other - * columns fold (a dropped link takes its reciprocal and every rollup over it), and - * neither writes view config in the same breath, so neither has this race. - */ - }); - return; - } - saveField(field, fieldScope); - insertColumn(field.key, anchorKey, side); - }, - [insertColumn, saveField, scopeChoice, isUserTable, scope] - ); - - /** - * ⭐⭐ 2026-08-07 (D-79's last half) — turn a text column into THE profile column. - * - * The write goes to the DEFINITION (`PATCH /tables/{key}/fields/{fkey}`), never to this - * user's overlay stratum, for the reason wave 25's amendment C3-A1 records: the automation - * engine reads `t['rows']`/`t['fields']`, so a flag parked in one person's overlay is a flag - * the enrich step can never see. Same door, same reasoning, as the automation column above. - * - * ⚠ `profile: null` CLEARS it — `_clean_field` treats an explicit null as "take the flag off" - * rather than "leave it alone", which is what lets a column be un-marked without deleting it. - */ - const setProfileFlag = useCallback( - (key: string, on: boolean) => { - if (!isUserTable) return; - void patchTableField(scope, key, { - profile: on ? { source: "instagram" } : null, - }).then((made) => { - if (!made) return; // the bridge has already said why - setFields((current) => current.map((f) => (f.key === made.key ? made : f))); - signal(WORKSPACE_STALE_EVENT); - }); - }, - [isUserTable, scope] - ); - - /** - * ⭐⭐ 2026-08-09 (owner) — RECONFIGURE a `link`/`rollup` column, in the SHARED definition. - * - * Owner: *"even when i click edit field for rollup, it doesn't actually show me the - * configuration"* and *"No rollup field should be uneditable."* Both land here: the Edit pane - * now renders the bag, and this is where its Save goes. - * - * ⛔ NOT `saveField`, and not `retypeField`. Those write the per-user overlay stratum, which is - * precisely the defect this change exists to close — `compute_relation_cells` reads - * `user_tables`, so a bag parked in one person's workspace can never be computed by anything. - * ⚠ ASYNC with no optimistic insert, like the automation create: the server re-cleans the bag - * and can refuse the whole field (`_clean_rollup` returns None for, say, a `limit` with no - * `sortBy`), and showing a configuration the store never took is the lie the wait avoids. - * The cells arrive on the next read — a rollup is recomputed server-side, never in the browser. - */ - const setFieldConfig = useCallback( - (key: string, patch: { - label?: string; link?: unknown; rollup?: unknown; formula?: string; - }) => { - if (!isUserTable) return; - void patchTableField(scope, key, patch as Record).then((made) => { - if (!made) return; // the bridge has already said why - setFields((current) => current.map((f) => (f.key === made.key ? made : f))); - // The definition changed, so the wire's field list did too — and a rollup's VALUES are - // recomputed host-side on this write, so the ROWS are a beat stale as well. The bridge - // drops its rows cache for this table (it owns that map); these two ask for a re-read. - signal(WORKSPACE_STALE_EVENT); - signal(ROWS_STALE_EVENT); - }); - }, - [isUserTable, scope] - ); - - /** - * Owner item 9 — "Change field": the clicked column starts showing another field, in place. - * The new field takes the old one's slot in `order`; the old field is HIDDEN, not lost — it - * stays in Fields and can be swapped back. The locked primary column is the row identity and - * cannot be changed away (ColumnMenu never offers it the control). - */ - const changeField = useCallback( - (oldKey: string, newKey: string) => { - if (oldKey === lockedKey || oldKey === newKey) return; - if (!fieldByKey.has(newKey)) return; - // The RECONCILED order (every key present, locked first), so the splice is well-defined - // even for a fresh view whose config.order is still the default. - const withoutNew = order.filter((k) => k !== newKey); - const at = withoutNew.indexOf(oldKey); - if (at < 0) return; - withoutNew.splice(at, 0, newKey); - const nextVisible = new Set(visible); - nextVisible.delete(oldKey); - nextVisible.add(newKey); - updateConfig({ ...config, order: withoutNew, visible: [...nextVisible] }); - }, - [config, updateConfig, order, visible, fieldByKey, lockedKey] - ); - - /** - * Delete a USER-CREATED column outright (owner gap closed 2026-07-27). Only the created - * strata qualify — `custom_` overlay fields and `measure_` formula columns, both marked - * `custom` — so a base field can never leave the schema from here (Hide is its only exit). - * The definition leaves local state, the ACTIVE view's config is scrubbed (order / visible / - * widths / sorts / group / color), and the host removes it from the workspace store — other - * views self-heal on their next autosave, the rule every stale colId already rides. - * Two emits leave in one burst (field_delete + the config autosave); the event-log value - * slot exists for exactly this. - */ - const deleteField = useCallback( - (key: string) => { - const f = fieldByKey.get(key); - if (!f?.custom || key === lockedKey) return; - setFields((current) => current.filter((item) => item.key !== key)); - const nextWidths = { ...(config.widths ?? {}) }; - delete nextWidths[key]; - updateConfig({ - ...config, - order: order.filter((k) => k !== key), - visible: [...visible].filter((k) => k !== key), - widths: nextWidths, - sorts: (config.sorts ?? []).filter((s) => s.colId !== key), - groupBy: config.groupBy === key ? null : config.groupBy, - colorBy: config.colorBy === key ? null : config.colorBy, - }); - stampFieldDelete(key); - emitHostEvent({ id: eventId("fielddel"), type: "field_delete", key }); - }, - [config, updateConfig, order, visible, fieldByKey, lockedKey, stampFieldDelete] - ); - - /** - * ⭐⭐ 2026-08-10 — DELETE A COLUMN FROM A USER DATABASE'S SHARED DEFINITION. - * - * `deleteField` above writes the per-user overlay: it emits `field_delete`, which - * `grid_events` gates on the `custom_`/`measure_` prefix and applies to - * `_table_workspace`. A definition column has a `custom_` key too, so that event is - * ACCEPTED and scrubs a bucket the definition never reads — the column disappears for one - * paint and is back on the next fetch. That is why this is a different function rather than a - * branch: two stores, two truths, and the wrong one succeeds quietly. - * - * ⚠ SERVER FIRST, THEN THE SCREEN. `_field_or_refuse` can say no (a pre-set column, a - * non-creator, the last remaining column), and the bridge has already shown its sentence — - * removing the column optimistically would leave the user looking at a grid that disagrees - * with the store until they reload. - * ⚠ The VIEW config is scrubbed on the same terms as the overlay delete; a stale `colId` in - * another view self-heals on its next autosave, the rule every other delete rides. - */ - const deleteDefinitionField = useCallback( - (key: string) => { - if (!isUserTable || key === lockedKey) return; - void deleteTableField(scope, key).then((ok) => { - if (!ok) return; // the bridge has already said why - setFields((current) => current.filter((item) => item.key !== key)); - const nextWidths = { ...(config.widths ?? {}) }; - delete nextWidths[key]; - updateConfig({ - ...config, - order: order.filter((k) => k !== key), - visible: [...visible].filter((k) => k !== key), - widths: nextWidths, - sorts: (config.sorts ?? []).filter((s) => s.colId !== key), - groupBy: config.groupBy === key ? null : config.groupBy, - colorBy: config.colorBy === key ? null : config.colorBy, - }); - signal(WORKSPACE_STALE_EVENT); - signal(ROWS_STALE_EVENT); - }); - }, - [isUserTable, scope, lockedKey, config, updateConfig, order, visible] - ); - - /** - * Change-field's "New field" half (wave-2 item 5): build the overlay field, then swap it into - * the clicked column's slot — one motion, two emits (field_upsert + the config autosave), - * which is exactly the burst the event-log value slot exists for. - */ - const createAndSwapField = useCallback( - ( - oldKey: string, - label: string, - type: FieldType, - options?: string[], - extra?: FieldBuildExtra - ) => { - const scope = scopeChoice ? extra?.scope : undefined; - const field = { - ...buildOverlayField(label, type, options, extra), - ...(scope === "cohort" ? ({ scope: "cohort" } as const) : {}), - }; - saveField(field, scope); - changeField(oldKey, field.key); - }, - [saveField, changeField, scopeChoice] - ); - - /** - * Wave-6 item 2 — Change field on a CREATED (`custom_`) field retypes THAT field in place: - * `field_upsert` with the SAME key and the new type/options; the host rebuilds the def and - * keeps createdBy/scope (this optimistic def carries them too, so the echo is byte-stable). - * Values are never converted — cells re-render per the new type, unreadable values show - * blank (Airtable behavior, disclosed in the pane). A display `format` survives only within - * its own family (number→number); anything else would be junk the host drops anyway. - */ - const retypeField = useCallback( - ( - key: string, - type: FieldType, - options?: string[], - // `label` (owner item 8): rename+retype leave the Edit pane as ONE upsert — two - // sequential emits would each read the stale def and revert the other. - extra?: FieldBuildExtra - ) => { - const old = fieldByKey.get(key); - if (!old?.custom || old.source !== "overlay") return; - if (type === "formula" || type === "created_time") return; // a different stratum - const numberFam = (t: FieldType) => t === "int" || t === "currency"; - const keepFormat = - old.format && ((numberFam(old.type) && numberFam(type)) || old.type === type); - const next: Field = { - key, - label: extra?.label?.trim() || old.label, - type, - source: "overlay", - default: old.default, - custom: true, - agg: ["currency", "int"].includes(type) ? "sum" : undefined, - ...(old.note ? { note: old.note } : {}), - ...(old.description ? { description: old.description } : {}), - ...(old.scope ? { scope: old.scope } : {}), - ...(old.createdBy ? { createdBy: old.createdBy } : {}), - ...(old.permissions ? { permissions: old.permissions } : {}), - ...(keepFormat ? { format: old.format } : {}), - ...((type === "select" || type === "multiselect") && options?.length - ? { options } - : {}), - ...((type === "select" || type === "multiselect") - ? { - colorCodeOptions: extra?.colorCodeOptions !== false, - ...(extra?.optionColors && Object.keys(extra.optionColors).length - ? { optionColors: extra.optionColors } - : {}), - } - : {}), - ...(type === "multiselect" ? { multi: true } : {}), - ...(type === "rating" ? { max: extra?.max ?? 5 } : {}), - }; - saveField(next); - /* ⭐ Owner item 15 / C-RENAME — the VALUES follow the definition. - BESIDE the upsert, never instead of it, and AFTER it: the def is written optimistically - here, while the rename is a host-side migration over overlay values and saved views. The - order matters if the host processes the batch in order — the list must already offer - "Navy" before any cell is moved onto it. */ - const renames = extra?.renames?.filter((r) => r.from && r.to && r.from !== r.to) ?? []; - if (renames.length) { - emitHostEvent({ id: eventId("choicerename"), type: "choice_rename", key, renames }); - // R4 — and it is undoable: Ctrl+Z emits the mapping turned around. The declared LIST is - // restored by the same inverse (the host rewrites values and views back), so undo does - // not need to re-send the definition. - undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { - kind: "choiceRename", fieldKey: key, renames, - }); - } - }, - [fieldByKey, saveField, scope] - ); - - /** - * Wave-5 item 1 — Duplicate field. The CLIENT generates the destination key (same stratum - * prefix as the source — the ~20:20 contract amendment), emits `field_duplicate`, and slots - * an optimistic clone right of its source SYNCHRONOUSLY. The host validates the prefix - * pair, stamps createdBy (the clone's creator is the duplicator — the local copy drops the - * source's), and copies overlay VALUES for `custom_` sources; those values arrive with the - * next payload, so the clone's cells are blank for exactly one round trip. - */ - const duplicateField = useCallback( - (source: Field) => { - const prefix = source.key.startsWith("measure_") ? "measure_" : "custom_"; - const label = uniqueDisplayName( - `${source.label} copy`, - fields.map((field) => field.label) - ); - const key = `${prefix}${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`; - const requestId = eventId("fielddup"); - const clone: Field = { ...source, key, label, editRequestId: requestId }; - delete clone.createdBy; - delete clone.permissions; - delete clone.labelCorrectedFrom; - delete clone.labelCorrectionId; - stampFieldEdit(key); - setFields((current) => [...current, clone]); - emitHostEvent({ - id: requestId, - type: "field_duplicate", - sourceKey: source.key, - key, - label, - // Item 9c: a duplicate INHERITS its source's scope (cohort clone stays cohort-only; - // a global source stays global = absent). Only meaningful on the cohort page. - ...(scopeChoice && source.scope === "cohort" ? { scope: "cohort" as const } : {}), - }); - insertColumn(key, source.key, "right"); - }, - [insertColumn, scopeChoice, stampFieldEdit, fields] - ); - - /** - * Wave-2 item 8c — change the PERIOD of a measure-carrying column (user-created `measure_*` - * and the pre-set measure fields alike). The header auto-renames to - * ` · ` ONLY while it still reads as the auto-name for the CURRENT - * window — a user's own title is never overwritten. Values arrive recomputed from the host on - * the next payload; the ordinary `field_upsert` is the whole protocol. - */ - const changeMeasurePeriod = useCallback( - (key: string, window: WindowSpec) => { - const f = fieldByKey.get(key); - if (!f?.measure) return; - const m = measures.find((item) => item.key === f.measure?.key); - const oldAuto = m ? `${m.label} · ${windowLabel(f.measure.window)}` : null; - const label = - m && oldAuto && f.label === oldAuto - ? `${m.label} · ${windowLabel(window)}` - : f.label; - saveField({ ...f, label, measure: { ...f.measure, window } }); - }, - [fieldByKey, measures, saveField] - ); - - /* wave17 R1 — `selectCohort`, `renameCohort`, `deleteCohort` and `exportCohortData` are - gone. Every one of them was called ONLY by `CohortSidebar`, and every one now has a - better-governed twin: the Views rail renames and deletes a locked view through the - ordinary view path, and the HOST routes those to `cohort_mod.rename` / `.delete` on a - projected id (C-LOCKV guards b and c). One name, one lifecycle, one set of events. */ - - /** - * Cohort mode — the selection bar's "Remove from cohort" (contract event `cohort_remove`). - * The pids are intersected with the cohort's membership client-side as a courtesy; the host - * re-validates ownership and pool anyway (a pid from the browser is untrusted input). - */ - const removeSelectionFromCohort = useCallback(() => { - if (!activeCohortId) return; - const pids = [...selectedPids].filter((p) => cohortMemberSet.has(p)); - if (!pids.length) return; - emitHostEvent({ - id: eventId("cohortrm"), - type: "cohort_remove", - cohortId: activeCohortId, - pids, - }); - clearSelection(); - }, [activeCohortId, selectedPids, cohortMemberSet, clearSelection]); - - - /** - * ⭐ Wave-20 owner item 9 — **REMOVE FROM A COHORT WITHOUT BEING INSIDE IT.** - * - * "Remove from cohort" existed only under `cohortMode` (you had to open the locked view - * first), while "Add to cohort" worked from any view. So the two halves of the same idea - * lived on different screens: you could put a customer in a cohort from wherever you found - * them, and then had to go looking for the cohort to take them out again. - * - * SAME EVENT, same host validation as the cohort-mode button — `cohort_remove` with the pids - * intersected client-side as a courtesy. What is new is only WHICH cohort: the picker names - * it, instead of it being implied by the page you are standing on. - */ - const removeSelectionFromList = useCallback( - (cohortId: string) => { - const members = cohortSets[cohortId]; - if (!members) return; - const pids = [...selectedPids].filter((p) => members.has(p)); - if (!pids.length) return; - emitHostEvent({ id: eventId("cohortrm"), type: "cohort_remove", cohortId, pids }); - setSelRemoveOpen(false); - clearSelection(); - }, - [cohortSets, selectedPids, clearSelection] - ); - - /** - * The cohorts the checked rows can actually be removed FROM: those holding at least one of - * them, and only where this viewer may edit the projected view. - * - * ⚠ A cohort holding NONE of the checked rows is not offered. Airtable's rule and the one R8 - * states for the ghost row: an affordance that can only refuse is a fake affordance — and - * here it would be worse than that, because "Remove from Q3 plan" that removes nothing looks - * exactly like a write that failed. - * - * The edit test is the COURTESY half (`mayEditView`, like every other client-side permission - * check on this surface); the host re-validates ownership and pool on the event regardless. - * A cohort with no projected view is still offered — the host owns that verdict, and hiding - * it here would silently drop sets the reader can see in their own rail. - */ - const removableLists = useMemo(() => { - if (cohortMode || selectedPids.size === 0) return []; - const out: { id: string; name: string; hits: number }[] = []; - for (const l of lists) { - const members = cohortSets[l.id]; - if (!members) continue; - let hits = 0; - for (const pid of selectedPids) if (members.has(pid)) hits += 1; - if (!hits) continue; - const projected = views.find((v) => v.id === l.id); - if (projected && !mayEditView(projected, viewer)) continue; - out.push({ id: l.id, name: l.name, hits }); - } - return out; - }, [cohortMode, selectedPids, lists, cohortSets, views, viewer]); - - /** Cohort mode — confirm the "+ Add customers" picker (contract event `cohort_add`). */ - const addCustomersToCohort = useCallback(() => { - if (!activeCohortId || addCustPicked.size === 0) return; - emitHostEvent({ - id: eventId("cohortadd"), - type: "cohort_add", - cohortId: activeCohortId, - pids: [...addCustPicked], - }); - setAddCustPicked(new Set()); - setAddCustQuery(""); - setAddCustOpen(false); - }, [activeCohortId, addCustPicked]); - - /** - * The picker's candidates: the POOL minus the cohort's current members, narrowed by the - * search. The pool's own order is kept (it leads with the biggest customers, which is the - * useful default for "who am I adding"). - */ - const addCandidates = useMemo(() => { - if (!addCustOpen) return []; - const q = addCustQuery.trim().toLowerCase(); - const out: { pid: number; name: string }[] = []; - for (const r of rawRows) { - if (cohortMemberSet.has(r.pid)) continue; - const name = String(r[lockedKey] ?? ""); - if (q && !name.toLowerCase().includes(q)) continue; - out.push({ pid: r.pid, name }); - } - return out; - }, [addCustOpen, addCustQuery, rawRows, cohortMemberSet, lockedKey]); - - /** - * Wave-6 item 11c — PIN. "Pin up to this field" freezes every visible column through the - * clicked one; when the field sits past the clamp (8), it is REORDERED into the frozen - * prefix first (the contract's wording). Pinning the identity column reads as "just the - * identity" = the legacy 1, stored as ABSENT. - * - * ⚠ CORRECTED wave-14 item 15. This used to read "Grouped mode paints frozenCount 0 (spans - * break under freezing)". **Spans do not break under freezing** — glide splits them - * (`getSpanBounds` → `[frozenRect, contentRect]`) and draws the row's contents from the frozen - * half. What IS true, and is the real cost of the pin, is that those contents are then clipped - * to the frozen strip's width; `fitGroupLabel` is what makes that clip honest. The clamp is - * gone: `freezeColumns` is `frozenN` in every mode. - */ - const visibleKeys = useMemo( - () => order.filter((key) => visible.has(key)), - [order, visible] - ); - const frozenN = frozenCountOf(config); - const pinFieldTo = useCallback( - (key: string) => { - const index = visibleKeys.indexOf(key); - if (index < 0) return; - if (index < MAX_FROZEN) { - updateConfig({ ...config, frozenCount: clampFrozenCount(index + 1) }); - return; - } - const moved = visibleKeys.filter((k) => k !== key); - moved.splice(MAX_FROZEN - 1, 0, key); - let cursor = 0; - const nextOrder = order.map((k) => (visible.has(k) ? moved[cursor++] : k)); - updateConfig({ ...config, order: nextOrder, frozenCount: MAX_FROZEN }); - }, - [visibleKeys, order, visible, config, updateConfig] - ); - const unpinFields = useCallback( - () => updateConfig({ ...config, frozenCount: undefined }), - [config, updateConfig] - ); - - // Wave-6 item 10 — display-mode plumbing. W13 (contract C4 as AMENDED 2026-07-28): - // `config.display` is the ONE home for the field picks, so leaving a mode must NOT - // wipe them — returning to grid keeps the refs in the draft (mode 'grid' + refs; - // cleanDisplay's carry), which is what lets the next Kanban/Calendar restore the - // chosen field instead of feeding the select its default. A view that never picked - // stays byte-identical (ref-less grid normalizes to absent, the legacy rule). - const setDisplayMode = useCallback( - (m: DisplayMode) => { - // I12 (C3) — a locked view's MODE is frozen. The switcher is already hidden for one, - // so reaching here means something other than the switcher asked; refuse rather than - // trust the UI. (The host refuses too — this only stops the local state diverging from - // what the host will store, which would look like a working change that never saved.) - // (Resolved from `views` here rather than the `activeView` binding below — this - // callback is declared above it.) - const av = views.find((v) => v.id === activeViewId); - if (av && isModeFrozen(av)) return; - const spec = cleanDisplay(config.display); - if (m === "grid") { - updateConfig({ - ...config, - display: spec ? cleanDisplay({ ...spec, mode: "grid" }) : undefined, - }); - return; - } - updateConfig({ ...config, display: { ...(spec ?? {}), mode: m } }); - }, - [config, updateConfig, views, activeViewId] - ); - // C4 (amended): the pick is written INTO display and the selects are FED from the - // view def (`display.`), never from component state that dies on unmount. - const setDisplayField = useCallback( - (k: "dateField" | "stackField" | "colorField" | "sizeField", key: string) => { - const spec = cleanDisplay(config.display); - if (!spec) return; - // Wave-8 I3/I5: colour/size are OPTIONAL encodings, so "" is a real choice - // (none) and must delete the ref rather than store an empty key that no - // field can ever match. - const next = { ...spec, [k]: key }; - if (!key) delete (next as Record)[k]; - updateConfig({ ...config, display: next }); - }, - [config, updateConfig] - ); - /** - * Item 3 (C-DISP) — the kanban card clamp. - * - * ⚠ `undefined` DELETES the key rather than storing `true`. Absent means clamped, so a - * stored `true` would be the default wearing a second name — and the host's `_clean_display` - * accepts the literal `false` only, so a `true` would be dropped on save and the toggle - * would look like it does not persist. Delete-to-default keeps both ends agreeing. - */ - const setKanbanClamp = useCallback( - (next: false | undefined) => { - const spec = cleanDisplay(config.display); - if (!spec) return; - const display = { ...spec }; - if (next === false) display.kanbanClamp = false; - else delete (display as Record).kanbanClamp; - updateConfig({ ...config, display }); - }, - [config, updateConfig] - ); - /** - * ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the swipe BINDING. - * - * ⚠ `undefined` DELETES the key, for `setKanbanClamp`'s reason one step further on. A swipe - * binding is a single three-part thing (field + two options); both engines' `_clean_display` - * DROP a half-binding rather than storing it, so writing one back would look like a save that - * silently did not persist. Absent IS the unconfigured state. - */ - const setSwipeSpec = useCallback( - (next: SwipeSpec | undefined) => { - const spec = cleanDisplay(config.display); - if (!spec) return; - const display = { ...spec }; - if (next) display.swipe = next; - else delete (display as Record).swipe; - updateConfig({ ...config, display }); - }, - [config, updateConfig] - ); - /** - * Item 4 (C-DISP) — the per-day totals. An EMPTY list deletes the key rather than storing - * `[]`: the always-on Records count is what an absent list means, so `[]` would be that - * state wearing a second name — and `_clean_display` drops an empty array anyway, which - * would make un-picking the last metric look like it did not save. - */ - const setCalendarMetrics = useCallback( - (next: { id: string; field: string; agg: string }[]) => { - const spec = cleanDisplay(config.display); - if (!spec) return; - const display = { ...spec }; - const capped = next.slice(0, MAX_CALENDAR_METRICS); - if (capped.length) display.calendarMetrics = capped; - else delete (display as Record).calendarMetrics; - updateConfig({ ...config, display }); - }, - [config, updateConfig] - ); - /** Item 4 (C-DISP) — records vs summaries. `records` is the default, so it deletes the key. */ - const setCalendarMode = useCallback( - (next: "records" | "summary") => { - const spec = cleanDisplay(config.display); - if (!spec) return; - const display = { ...spec }; - if (next === "summary") display.calendarMode = "summary"; - else delete (display as Record).calendarMode; - updateConfig({ ...config, display }); - }, - [config, updateConfig] - ); - // The calendar's date field: the view's pick when it is still a real date-family field, - // else `last_order`, else the first date-family field. The kanban's stack field: status or - // single-select ONLY (a multiselect card in two columns at once is a count lie — contract). - const dateFieldChoices = useMemo( - () => fields.filter((f) => isDateFamilyType(f.type)), - [fields] - ); - const stackFieldChoices = useMemo( - () => fields.filter((f) => f.type === "status" || f.type === "select"), - [fields] - ); - const pickBy = (want: string | undefined, choices: Field[], preferred?: string): Field | undefined => { - const wanted = want ? choices.find((f) => f.key === want) : undefined; - if (wanted) return wanted; - const pref = preferred ? choices.find((f) => f.key === preferred) : undefined; - return pref ?? choices[0]; - }; - // W13 (C4 as amended): the VIEW DEF's display feeds the pickers — the refs survive - // every mode switch via cleanDisplay's grid carry. pickBy already falls back to the - // default when the stored key no longer names an offerable field (a dropped ref). - // I3/I5 — the map's encodings. Deliberately NOT resolved through `pickBy`: - // that helper falls back to the first choice, which is right for the calendar - // (a calendar must have a date) and wrong here, where "no encoding" is a - // legitimate, and the DEFAULT, state. An unknown stored key resolves to - // undefined = no encoding, which is also the dropped-ref behaviour. - // I19c (C2) — the chart list lives in `config.display.charts`, so it rides the - // existing view autosave + viewEcho reconcile and needs no new event type. - const charts = useMemo( - () => cleanCharts(displaySpec?.charts) ?? [], - [displaySpec?.charts] - ); - const setCharts = useCallback( - (next: ChartSpec[]) => { - const spec = cleanDisplay(config.display); - if (!spec) return; - const display = { ...spec, charts: next }; - if (!next.length) delete (display as Record).charts; - updateConfig({ ...config, display }); - }, - [config, updateConfig] - ); - const colorFieldChoices = useMemo( - () => fields.filter((f) => f.type === "status" || f.type === "select"), - [fields] - ); - const sizeFieldChoices = useMemo( - () => fields.filter((f) => isNumericFieldType(f.type)), - [fields] - ); - const mapColorField = displaySpec?.colorField - ? colorFieldChoices.find((f) => f.key === displaySpec.colorField) - : undefined; - const mapSizeField = displaySpec?.sizeField - ? sizeFieldChoices.find((f) => f.key === displaySpec.sizeField) - : undefined; - const calendarField = pickBy(displaySpec?.dateField, dateFieldChoices, "last_order"); - const kanbanField = pickBy(displaySpec?.stackField, stackFieldChoices); - /* ⭐ WAVE-27 item 8 (C3) — the BOUND field, resolved by key and NOT through `pickBy`. `pickBy` - falls back to the first choice when the stored key is unknown, which is right for a kanban - (a stack field is a preference) and wrong here: SwipeView SHOWS a rotted binding rather than - guessing at one, so it must receive the loss instead of a silent substitute — the wave-7 trap - [[wrong-parent-not-broken-control]] and the component's own header note both name. */ - const swipeField = displaySpec?.swipe - ? fieldByKey.get(displaySpec.swipe.fieldKey) - : undefined; - // Calendar/kanban project the FULL pipeline result (a month/stack is its own bound), as - // DISTINCT data rows — grouping can repeat a pid — with overlay edits layered so a dragged - // card restacks and an edited title repaints without waiting on any echo (item 3c). - const modeDataRows = useMemo(() => { - if ( - displayMode !== "calendar" && - displayMode !== "kanban" && - displayMode !== "map" && - displayMode !== "chart" && - // C-TS close-out stitch: the panel's pids come from these rows; without this line the - // time-series view asked its question about ZERO customers (caught by the mounted - // judge — the node battery is blind here). - displayMode !== "timeseries" && - /* ═══ W18-C CATALOG ═══ The designer's "add the products this view shows" door reads - these rows — the same rows the toolbar counts, so the two can never disagree about - which products the filter kept. The catalog's PAINT does not: it joins its stored - codes against the whole pool, so a filter narrows what you can ADD and never what a - finished page prints. ═══ end W18-C CATALOG ═══ */ - displayMode !== "catalog" && - /* ⭐ WAVE-27 item 8 (C3) — the swipe deck is these rows filtered to the undecided ones. - Without this line the deck is derived from `[]`, so the view paints its "nothing left - to decide" empty state over a table full of undecided records — the C-TS stitch above, - repeated. */ - displayMode !== "swipe" - ) - return []; - const seen = new Set(); - const out: Row[] = []; - for (const vr of visibleRows) { - if (vr.kind !== "data" || seen.has(vr.record.pid)) continue; - seen.add(vr.record.pid); - const edits = overlayEdits[vr.record.pid]; - out.push(edits ? { ...vr.record, ...edits } : vr.record); - } - return out; - }, [displayMode, visibleRows, overlayEdits]); - // Owner item 1 — the CALL-SITE half of the kanban memo. `KanbanView` is `memo`'d now, but a - // memo whose props are rebuilt every render never bails, and these two were: `cardKeys` is a - // fresh array from `.filter().slice()`, `onMove` a fresh closure. With them stable, a - // `detailPid` change (clicking a card) re-renders the modal and skips the board entirely — - // which is the whole of "laggy record-open". Everything else KanbanView takes was already - // stable: `modeDataRows`/`fieldByKey` are useMemos, `kanbanField` is a reference INTO the - // `stackFieldChoices` useMemo, and `onOpen={setDetailPid}` is a setState identity. - const kanbanCardKeys = useMemo( - () => - kanbanField - ? visibleKeys.filter((k) => k !== lockedKey && k !== kanbanField.key).slice(0, 3) - : [], - [visibleKeys, lockedKey, kanbanField] - ); - const onKanbanMove = useCallback( - (pid: number, value: string) => { - if (!kanbanField) return; - patchAndRecord(pid, { [kanbanField.key]: value }, "a card move"); - }, - [kanbanField, patchAndRecord] - ); - /* ⭐ WAVE-27 item 8 (C3) — a swipe writes through the NORMAL cell door, the same - `patchAndRecord` a card move uses. That is the whole reason the deck needs no permission - logic, no undo entry and no echo handling of its own: it inherits all three. */ - const onSwipeDecide = useCallback( - (pid: number, value: string) => { - if (!swipeField) return; - patchAndRecord(pid, { [swipeField.key]: value }, "a swipe"); - }, - [swipeField, patchAndRecord] - ); - /** - * ⭐ WAVE-26 ITEM 12 (owner ruling R8) — the kanban's "Add option" door. - * - * It opens the ORDINARY ColumnMenu on the stack field, straight into its Edit-field pane. So - * the option is written into the SHARED field definition by the same `onRetype` upsert the - * header menu uses — never a per-view list — and every guarantee that editor already carries - * (permission wall, rename-by-row-identity, option colours, one upsert for name+choices) comes - * along without being re-implemented. R8's "do not invent a second options editor", literally. - * - * ⛔ THE PERMISSION QUESTION IS NOT `canEditField`. Adding an option changes the column's - * DEFINITION; `mayEditField` answers who may type a VALUE into it, and conflating the two is - * the confusion [[schema-role-is-not-a-value-wall]] exists to prevent. The right predicate is - * the one that decides whether `onRetype` is supplied at all — a created overlay field — so it - * is written ONCE here and read at both doors. - * - * `null` (not "absent") when the field cannot take one: a status column is computed by the - * source system, and an Odoo select's vocabulary lives in Odoo. - * - * ⭐ THE PREDICATE ADMITS AN AUTOMATION'S `stage_` COLUMN, and that is INTENDED rather than - * incidental — worth stating because it was arrived at by matching `onRetype`'s condition, and - * a coincidence and a decision look identical in code. It is exactly what R6 and R8 compose - * to: R6 deleted the board's built-in terminals, so the lanes a review stage offers are now - * only the ones a user defined — and R8 is the control that lets them define one. Landed - * without this the pair is worse than either alone (SESSION A's A-7 says the same thing from - * the engine side: `humanMoves` had to learn to admit user-added options, or the move door - * would refuse the lane the product just invited you to create). - * ⚠ CARRIED CONSEQUENCE, pre-existing and not introduced here: the pane this opens is the - * whole Edit-field window, so it also offers the TYPE picker — one wrong click retypes a stage - * column. The header menu has offered exactly that on the same column since wave 6; this door - * adds a second way in, not a new hazard. Written down rather than left to be rediscovered. - */ - const kanbanCanAddOption = - !!kanbanField && kanbanField.type === "select" && - !!kanbanField.custom && kanbanField.source === "overlay"; - const onKanbanAddOption = useCallback( - (anchor: AnchorRect) => { - if (!kanbanField) return; - setColumnMenu({ fieldKey: kanbanField.key, anchor, pane: "edit" }); - }, - [kanbanField] - ); - const onListToggleGroup = useCallback((groupKey: string) => { - setCollapsed((current) => { - const next = new Set(current); - if (next.has(groupKey)) next.delete(groupKey); - else next.add(groupKey); - return next; - }); - }, []); - /** - * Item 7 (C-TS) — the pid set the time series asks about: exactly the rows the current - * filter kept, so the panel and the toolbar count can never disagree about "who". - * - * `modeDataRows` is already the de-duplicated data rows for a non-grid mode, so this is a - * projection of it and NOT a second pipeline. The server intersects with the caller's book - * anyway — this narrows the question, it can never widen the answer. - */ - const timeseriesPids = useMemo( - () => (displayMode === "timeseries" ? modeDataRows.map((r) => r.pid) : []), - [displayMode, modeDataRows] - ); - /* ═══ W18-C CATALOG ═══ (owner item 4, contract C6) - The catalogs this view holds, and the door that persists an edit to them. - - `displaySpec.catalogs` is ALREADY validated — `cleanDisplay` ran `cleanCatalogs` over it on - read — so there is no second normalisation here and no chance of the two disagreeing. - - ⚠ The write goes through `updateConfig` exactly like `onTimeseriesDisplay` below, which - means a catalog edit is a VIEW-CONFIG save. That is the point: a catalogue is authored - content and has to survive a reload, and the display spec is the only per-view store the - grid has. It also means the 500-code / 40-page / 12-catalog caps are enforced twice on the - round trip (here on read, and by the host's `_clean_catalogs` on write) — the designer - states its budget in the toolbar so the cap is never the user's first news of it. */ - const catalogs = useMemo( - () => (displayMode === "catalog" ? (displaySpec?.catalogs ?? []) : []), - [displayMode, displaySpec] - ); - /** The codes the current filter kept — the designer's "add what the view shows" shortcut. - * Derived from `modeDataRows`, so it counts what the toolbar counts. */ - const catalogFilteredCodes = useMemo( - () => - displayMode === "catalog" - ? modeDataRows - .map((r) => r[CATALOG_CODE_FIELD]) - .filter((c): c is string => typeof c === "string" && !!c) - : [], - [displayMode, modeDataRows] - ); - const onCatalogs = useCallback( - (next: CatalogSpec[]) => { - const spec = cleanDisplay(config.display) ?? { mode: "catalog" as const }; - updateConfig({ ...config, display: { ...spec, catalogs: next } }); - }, - [config, updateConfig] - ); - /* ═══ end W18-C CATALOG ═══ */ - /** Persist a bucket/span/metric pick onto the view, the way every other display key rides. */ - const onTimeseriesDisplay = useCallback( - (next: Partial) => { - const spec = cleanDisplay(config.display) ?? { mode: "timeseries" as const }; - updateConfig({ ...config, display: { ...spec, ...next } }); - }, - [config, updateConfig] - ); - const kanbanCanMove = - !!kanbanField && kanbanField.type === "select" && canEditField(kanbanField); - const kanbanReason = !kanbanField - ? null - : kanbanField.type === "status" - ? `Stacked by ${kanbanField.label} — read-only, computed from the source system.` - : !canEditField(kanbanField) - ? "You do not have permission to edit this field, so cards cannot be moved." - : null; - /* ⭐ WAVE-27 item 8 (C3) — the kanban pair, said again for the deck. Same wall, same words - shaped for the gesture: a swipe writes a VALUE, so `canEditField` is the right predicate. */ - const swipeCanWrite = !!swipeField && swipeField.type === "select" && canEditField(swipeField); - const swipeReason = !swipeField - ? null - : swipeField.type === "status" - ? `Bound to ${swipeField.label} — read-only, computed from the source system.` - : !canEditField(swipeField) - ? "You do not have permission to edit this field, so records cannot be decided." - : null; - - if (loading || !workspaceReady) { - /* wave17 GRID — item 3 / owner R6: one small bare rotating icon, NO WORDS. `aria-label` - is not a word on screen and stays: `.lp-spin` is an empty span, so without it the wait - is announced to a screen reader as nothing at all. (C-SPIN, SHELL defines the class.) - `--lg` because this is the case C-SPIN sizes it for — the whole surface is this mark and - nothing else, where the 14px version "reads as dust". */ - return ( -
- -
- ); - } - - const activeView = views.find((view) => view.id === activeViewId); - // Cohort mode's toolbar control (rendered by Toolbar via the `cohortAction` slot; the popover - // it opens is with the other overlays at the bottom). Disabled without an active cohort — - // zero cohorts is the host page's near-empty state, not this button's error to explain. - const cohortAction = cohortMode ? ( - - ) : undefined; - // Wave-7 item W9 — the Fields panel's permanent delete rides the SAME wall as the - // column menu's Delete: created strata only (`custom`), never the locked identity - // column. One predicate, two doors. - const deletableKeys = new Set( - fields.filter((f) => f.custom && f.key !== lockedKey).map((f) => f.key) - ); - const menuField = columnMenu ? fieldByKey.get(columnMenu.fieldKey) : undefined; - // Wave-5 item 3 — what the CURRENT VIEW does with the menu's field, so the conditional - // "Don't sort/filter/group" entries render exactly when they apply. - const menuSortedDir = menuField - ? config.sorts.find((s) => s.colId === menuField.key)?.dir ?? null - : null; - const menuIsFiltered = menuField - ? treeNamesField(config.filters, menuField.key, measureCols) - : false; - // Item 11c — is the menu's field the current frozen boundary (its Pin would be a no-op)? - const menuVisIndex = menuField ? visibleKeys.indexOf(menuField.key) : -1; - const menuPinnedTo = menuVisIndex >= 0 && frozenN > 1 && menuVisIndex + 1 === frozenN; - // Item 10 — the toolbar's mode switcher (grid-only on windowed tables, see displayMode). - const modeControl = serverWindowed || embedded ? undefined : ( - - ); - // ⭐ Wave-23 C7 — the open document, resolved exactly like the picker's value below it: - // the OPTIMISTIC edit wins over the raw record, so a save the server has not echoed yet is - // what re-opening the cell shows (the echo-suppression law — reading the raw row here would - // make a just-saved document appear to revert). - const jsonField = jsonAt ? fieldByKey.get(jsonAt.fieldKey) : undefined; - const jsonRow = jsonAt ? rawRows.find((r) => r.pid === jsonAt.pid) : undefined; - const jsonValue = String( - (jsonAt && overlayEdits[jsonAt.pid]?.[jsonAt.fieldKey]) - ?? (jsonField && jsonRow ? jsonRow[jsonField.key] : "") - ?? "" - ); - const linkField = linkAt ? fieldByKey.get(linkAt.fieldKey) : undefined; - const linkRow = linkAt ? rawRows.find((row) => row.pid === linkAt.pid) : undefined; - const linkValue = String( - (linkAt && overlayEdits[linkAt.pid]?.[linkAt.fieldKey]) - ?? (linkField && linkRow ? linkRow[linkField.key] : "") - ?? "" - ); - const linkedRecordIds = [...new Set( - linkValue.split(",").map((part) => Number(part.trim())).filter( - (pid) => Number.isInteger(pid) && pid > 0 - ) - )]; - const pickerField = picker ? fieldByKey.get(picker.fieldKey) : undefined; - // A `user` field's choices come from the HOST's real user list, a `select`'s from its own - // definition — so an assignee is always someone who can log in, and a status is always one of - // the choices the column was created with. - const pickerChoices = !pickerField - ? [] - : pickerField.type === "user" - ? payload?.userOptions ?? [] - : choiceOptions(pickerField); - const pickerRow = picker ? rawRows.find((r) => r.pid === picker.pid) : undefined; - const pickerValue = String( - (picker && overlayEdits[picker.pid]?.[picker.fieldKey]) - ?? (pickerField && pickerRow ? pickerRow[pickerField.key] : "") - ?? "" - ); - // A multiselect cell is a SET: choices TOGGLE and the picker stays open for the next pick - // (item 5). Single select/user keep pick-and-close. - const pickerMulti = pickerField?.type === "multiselect"; - const pickerParts = pickerMulti ? splitMulti(pickerValue) : []; - /** Item 6 — which choice receives focus when the picker opens: the current value where it - * still exists in the list, else the first choice. Keyboard-only editing starts HERE. */ - const pickerFocusChoice = !pickerChoices.length - ? null - : pickerMulti - ? pickerParts.find((p) => pickerChoices.includes(p)) ?? pickerChoices[0] - : pickerChoices.includes(pickerValue) - ? pickerValue - : pickerChoices[0]; - - return ( - // `data-today` is the TENANT'S day, exactly as the payload delivered it. It is here so the - // date every relative condition resolves against is OBSERVABLE rather than inferred: a live - // QA that reads its own clock instead compares two engines on "now" and fails whenever a run - // straddles midnight — which is the flake harness/windows.py takes `today` as a parameter to - // avoid, reintroduced one layer up. It cost a false failure (57 vs 55, both correct, one - // computed either side of a date change) to notice. - // `data-measure-rules` / `data-measure-sets` are BUG-1's handshake, published the same way - // and for the same reason as `data-today`: a measure condition stuck on "Calculating…" - // renders identically whether the rule carries no id, the host resolved nothing, or the two - // sides name the rule differently — and only the first of those is ours. A console.log - // cannot serve this: the grid runs in an iframe that is CROSS-ORIGIN on the Space, and every - // Streamlit rerun replaces it. An attribute survives the remount and is one `get_attribute` - // away from any harness, local or deployed. -
- {/* ⭐ wave17 R1 / C-LOCKV — the COHORT SIDEBAR is gone, and with it the last surface that - treated a cohort as its own kind of object. `CohortSidebar` was the retired `#/cohort` - page's left panel: a second rail, with its own rename, delete, export and folder - machinery, over `workspace.lists`. Under R1 a cohort IS a saved view, so the Views rail - below renders every one of them and the second rail has nothing to switch between. - ⚠ `lists` is NOT gone with it — it stays the membership channel that feeds `cohortSets` - (C-LOCKV point 2), which is what the lock resolves against. */} - {!hideViews && ( - setSelectFromFile(true)} - onAddToList={serverWindowed ? undefined : addToList} - today={today} - onExport={serverWindowed ? undefined : exportViewData} - folders={folders} - folderIdOf={folderIdOf} - onFolderCreate={(name, icon) => { - const id = newFolderId(); - // Bounded: reconcileFolders drops a local entry once the echo - // carries it or its create stamp ages out, so this only has to not - // grow without limit. - // C5 (I15): the icon rides the LOCAL entry too, or the folder the user just - // styled renders grey for one round trip and then changes colour under them. - setLocalFolders((prev) => [...prev.slice(-31), { id, name, order: 9_999, icon }]); - stampFolder((p) => ({ ...p, created: { ...p.created, [id]: Date.now() } })); - emitHostEvent({ - id: eventId("foldnew"), - type: "folder_create", - surface: "views", - folderId: id, - name, - icon, - }); - }} - onFolderRename={(folderId, name) => { - stampFolder((p) => ({ - ...p, - renamed: { ...p.renamed, [folderId]: { at: Date.now(), name } }, - })); - emitHostEvent({ - id: eventId("foldren"), - type: "folder_rename", - surface: "views", - folderId, - name, - }); - }} - onFolderDelete={(folderId) => { - // The tombstone is stamped BEFORE the emit: between the click and the - // echo the payload still lists the folder, and rendering it again for - // one round trip is the delete blip (folders.ts). - stampFolder((p) => ({ ...p, deleted: { ...p.deleted, [folderId]: Date.now() } })); - emitHostEvent({ - id: eventId("folddel"), - type: "folder_delete", - surface: "views", - folderId, - }); - }} - onFolderDuplicate={(folderId) => { - const newId = newFolderId(); - const src = folders.find((f) => f.id === folderId); - setLocalFolders((prev) => [ - ...prev.slice(-31), - { id: newId, name: `${src?.name ?? "Folder"} copy`, order: 9_999 }, - ]); - stampFolder((p) => ({ ...p, created: { ...p.created, [newId]: Date.now() } })); - emitHostEvent({ - id: eventId("folddup"), - type: "folder_duplicate", - surface: "views", - folderId, - newId, - }); - }} - onFolderReorder={(order) => { - // A-S4-1 (item 19). S4 built the drag and the reconcile behind an OPTIONAL prop, so - // its absence silently disabled the whole feature rather than breaking anything — - // which is why every gate stayed green while folders would not drag at all. The - // stamp is what stops the rail snapping back for one round trip (the echo window). - stampFolder((p) => ({ ...p, ordered: { at: Date.now(), order } })); - emitHostEvent({ - id: eventId("foldord"), - type: "folder_reorder", - surface: "views", - order, - }); - }} - /** - * ⭐ WAVE 27 · OWNER ITEM 5 (contract C7) — the rail's new VIEW order. - * - * Three things happen and each one is needed: - * · the LOCAL list is reordered, so the row lands where it was dropped and stays - * there for this render; - * · a STAMP is written, so the next workspace echo — which is one round trip - * behind and still carries the server's order — does not snap it back - * (`applyViewOrder` in folders.ts holds it for the echo window and no longer); - * · ONE event carries the WHOLE order, never a delta. A partial list cannot say - * where an UNNAMED view went, so "moved B after C" leaves every other position - * to be re-derived by two sides that can disagree. The server's own - * `folder_reorder` note states this rule; the same rule, same door. - * - * ⚠ AMENDMENT C7-A1 (2026-08-08, SESSION C): the event is - * `{type: "view_reorder", surface: "views", order}` — `surface`, not `workspace`. - * The contract said `{workspace, order}`; the door `grid_events.py:1299` actually - * opens on `surface ∈ FOLDER_SURFACES`, and the TABLE is resolved from the request - * context rather than the payload (`_store_of(ctx)`). Emitting a key the handler - * does not read would have shipped a drag that persists nothing, behind a green - * gate on each side. - */ - onViewReorder={(order) => { - setViews((cur) => { - const byId = new Map(cur.map((v) => [v.id, v])); - const out: SavedView[] = []; - const placed = new Set(); - for (const id of order) { - const v = byId.get(id); - if (!v || placed.has(id)) continue; - placed.add(id); - out.push(v); - } - // Anything the order does not name keeps its current place, after the named - // ones — never dropped. A view created in another tab is not evidence the - // drop was wrong. - for (const v of cur) if (!placed.has(v.id)) out.push(v); - return out; - }); - stampFolder((p) => ({ ...p, orderedViews: { at: Date.now(), order } })); - // ⛔ ONE CAST, AT ONE SITE, WITH A RATCHET BEHIND IT (ASK ->D, wave-27 mailbox - // C-6). `HostEvent`'s union lives in `types.ts`, which is SESSION D's file, and - // D owns C7's persistence half — so the member `{ id; type: "view_reorder"; - // surface: FolderSurface; order: string[] }` is theirs to add. Until it lands - // this emit cannot be typed from here. - // ⚠ The gap is DECLARED, not hidden: `verify_wiring.py`'s - // `PENDING_UNION_MEMBERS` carries `view_reorder`, prints the ask, and goes RED - // the moment `types.ts` gains the member — which forces this cast to be deleted - // in the same change. That is the D-70 ratchet's shape, and it fired on its own - // this wave, which is the argument for reusing it rather than a comment. - emitHostEvent({ - id: eventId("vieword"), - type: "view_reorder", - surface: "views", - order, - } as unknown as HostEvent); - }} - onItemMove={(viewId, folderId) => { - stampFolder((p) => ({ - ...p, - moved: { ...p.moved, [viewId]: { at: Date.now(), folderId } }, - })); - emitHostEvent({ - id: eventId("itemmove"), - type: "item_move", - surface: "views", - itemId: viewId, - folderId, - }); - }} - folderAddPreview={serverWindowed ? undefined : folderAddPreview} - onFolderAddToList={serverWindowed ? undefined : folderAddToList} - viewer={viewer} - onToggleLock={toggleViewLock} - userOptions={payload?.userOptions} - /> - )} -
- v.id !== activeViewId)} - onCopyConfig={(patch) => updateConfig({ ...config, ...patch })} - fields={fields} - visible={visible} - lockedKey={lockedKey} - onColumnVisible={setColumnVisible} - onColumnsVisible={(keys) => - updateConfig({ - ...config, - visible: [...new Set([lockedKey, ...keys])], - }) - } - // Owner item 23 — the Hide-fields panel lists and reorders COLUMN order, so it is - // handed the ordered projection, not the definition-ordered `fields` above. - // `order` is `useGridColumns`' reconciled copy (locked key first, unknown keys - // dropped, new keys appended), so the list can never show a field the grid does not - // have or omit one it does. - orderedFields={orderedFields} - onFieldOrder={(keys) => updateConfig({ ...config, order: keys })} - deletableKeys={deletableKeys} - onDeleteField={deleteField} - colorOn={config.colorBy !== null} - onToggleColor={(on) => - updateConfig({ - ...config, - colorBy: on - ? fields.find((field) => field.type === "status")?.key ?? null - : null, - }) - } - rowHeightMode={config.rowHeightMode} - onRowHeightMode={(rowHeightMode) => updateConfig({ ...config, rowHeightMode })} - filters={config.filters} - onFilters={(filters) => updateConfig({ ...config, filters })} - filterConj={config.filterConj ?? "and"} - onFilterConj={(filterConj) => updateConfig({ ...config, filterConj })} - sorts={config.sorts} - onSorts={(sorts) => updateConfig({ ...config, sorts })} - // the toolbar must not advertise a grouping the pipeline is not applying - groupBy={serverWindowed ? null : config.groupBy} - onGroupBy={(groupBy) => updateConfig({ ...config, groupBy })} - statusValues={statusValues} - measures={measures} - unresolvedCount={unresolvedCount} - lists={lists} - pendingMeasureCount={pendingMeasureCount} - search={search} - onSearch={setSearch} - recordCount={recordCount} - scopeCounts={payload?.counts} - pool={payload?.workspace?.pool} - serverWindowed={serverWindowed} - cohortAction={cohortAction} - filterSeed={filterSeed} - modeControl={modeControl} - /> - {/* Wave-10 item 9: the view's description used to render HERE, as a full-width band - between the toolbar and the grid — it cost every row a strip of vertical space and - read as an alert rather than as a label. It now sits under the view's name in the - rail (ViewSidebar), where it annotates the thing it describes. */} -
setExpandAt((prev) => (prev === null ? prev : null))} - > - {displayMode === "grid" && ( - <> - openHeaderMenu(column, bounds)} - onHeaderClicked={onHeaderClicked} - onHeaderContextMenu={onHeaderClicked} - onKeyDown={onGridKeyDown} - maxColumnWidth={600} - getCellsForSelection={true} - rowSelectionMode="multi" - smoothScrollX - smoothScrollY - width={gridSize.width} - height={gridSize.height} - customRenderers={[ratingCellRenderer, userCellRenderer, imageCellRenderer]} - headerIcons={HEADER_ICONS} - rightElement={embedded ? undefined : ( - // Wave-6 item 4 — the "+" of the header row: the create-field form, - // insert-at-end. - // - // ⚠ Wave-20 owner item 5 UNPINNED it. It used to be `sticky: true` ("so it - // stays visible however far the columns scroll"), which parked it against the - // right edge of the WINDOW — on a four-column table that is half a screen of - // white between the last field and the control that adds the next one, and it - // reads as page chrome rather than as part of the table. It now sits - // immediately after the last column, where Airtable puts it. - // - // TWO changes are needed and neither works alone: `sticky:false` stops glide - // pinning the wrapper (`infinite-scroller.js:204` sets `right` only when - // sticky), and the CSS region kills `.dvn-spacer`'s `flex-grow: 1`, which - // would otherwise expand to fill the scroller and push the button right back - // to the edge. Consequence, accepted: on a table wider than the viewport the - // "+" is off-screen until you scroll to the end of the columns — which is - // where the new column is going to land anyway. - - )} - rightElementProps={{ sticky: false, fill: false }} - // Owner item 4 (R8) — the trailing "+" row, USER DATABASES ONLY. Passing - // `onRowAppended` is what makes glide paint the ghost row at all, so the - // affordance exists exactly where a POST can succeed. See `appendRow`. - onRowAppended={canMutateRecords ? appendRow : undefined} - trailingRowOptions={ - canMutateRecords - ? { - // The hint sits in the identity column (targetColumn 0), which is the - // one a new record is named in — the same cell the cursor lands on. - hint: "New record", - sticky: false, - targetColumn: 0, - } - : undefined - } - /> - {/* ═══ W18-B VOID ═══ (wave 18, owner item 1b / ruling R12) — the two rectangles. - - They are SIBLINGS of the canvas, immediately after it and before everything else - `.cg-grid-box` holds, which is the whole of their stacking story: `z-index: auto` - plus tree order puts them over the canvas, and `.cg-selbar` / the popovers carry - explicit `--cg-z-*` and stay over them. `pointer-events: none` keeps glide's - scroller — which is UNDER this and owns every mouse event the grid reacts to — - hit-testable straight through them. - - The right-hand one starts BELOW the header rather than at 0: in `Airtable 7.png` - the header band continues past the last column in its own near-white (#FBFCFE at - (1250,141)), and it is where the add-field "+" lives. The two rectangles overlap - in the bottom-right corner; they are the same flat colour, so the overlap is not - visible and neither one needs to know about the other. */} - {gridVoid.below !== null && ( -
- )} - {gridVoid.right !== null && ( -
- )} - {/* ═══ end W18-B VOID ═══ */} - - )} - {displayMode === "list" && ( - - )} - {displayMode === "calendar" && - (calendarField ? ( - - ) : ( -
- This table has no date field to place records by. -
- ))} - {displayMode === "kanban" && - (kanbanField ? ( - - ) : ( -
- This table has no status or single-select field to stack by. -
- ))} - {/* ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the SWIPE deck. - ⚠ No `field ? ... : empty-state` fork, and the asymmetry with kanban above is - deliberate. A kanban with no select field cannot exist, so the host answers for it. - A swipe deck with no binding is the ORDINARY state of a view somebody just created, - and SwipeView owns that state along with the three ways a stored binding rots — - each one re-opening the picker rather than being repaired by a guess. Forking here - would put a second, dumber explanation in front of the good one. */} - {displayMode === "swipe" && ( - - )} - {/* Wave-8 I19c → wave-9 I10 (C2) — CHART is a VIEW like the rest: the charts - are computed from `modeDataRows`, the very rows the grid counts, so a filter - change moves the charts and the toolbar count together. - ⚠ Only 'chart' is tested, never 'dashboard': `cleanDisplay` normalises the - legacy value on READ, so a stored 'dashboard' has already become 'chart' by - the time it reaches here. Testing both would hide a broken normalisation. */} - {displayMode === "chart" && ( - - )} - {/* Owner item 7 (C-TS) — Time series is a VIEW: metric rows x bucket columns over - the pids the current filter kept, so the table and the series answer the same - question. The panel is HOST-AGNOSTIC by contract (RECORD re-mounts it inside the - record modal for item 13), which is why it takes pids and fields rather than - reaching into anything here. */} - {displayMode === "timeseries" && ( - - )} - {/* ═══ W18-C CATALOG ═══ Owner item 4 (C6) — Catalog is a VIEW, and the only one - whose payload is authored CONTENT rather than an arrangement of the rows. It - joins its stored product codes against the WHOLE pool (`rawRows`), not the - filtered rows: a finished page must keep printing the products it names after - somebody narrows the view. The filter still has a job — `catalogFilteredCodes` - is the "add what this view shows" door — but it narrows what you can ADD, never - what a page prints. - - ⚠ Gated on the product surface. A catalog joins products to their images by - `code`, which is `product_data`'s business key AND the C2-ASSET key; on the - customer table the primary field is a customer NAME, and joining on it would ask - the asset route for a picture of a person. The refusal says so in words rather - than rendering an empty designer. */} - {displayMode === "catalog" && - (scope === "product" ? ( - - ) : ( -
- A catalog prints products. Open the Products table to build one. -
- ))} - {/* ═══ end W18-C CATALOG ═══ */} - {/* Wave-7 item W11c (C3) — Map is a VIEW: same pipeline rows, pins for rows - with host-supplied coords, click-through to the same record detail, and - an honest no-location chip. */} - {displayMode === "map" && fieldByKey.get(lockedKey) && ( - - )} - {/* Owner item 19 — the hover-only Expand. The COUNTERPART to item 17: the single - click that used to open a record now highlights it, and this is where opening - moved to. `position: fixed` because glide reports viewport coordinates (the same - space `.cg-header-tip` is placed in). - - ⚠ `onMouseDown` preventDefault, not just onClick: mousedown would otherwise reach - the canvas underneath, move the selection to the primary cell, and repaint the - row a frame before the panel opens — a visible flicker on every expand. - - Rendered only in `grid` mode: list / calendar / kanban / map have their own - open affordances and glide is not mounted to report bounds for them. */} - {!embedded && displayMode === "grid" && expandAt && ( - - )} - {/* Wave-5 item 6 — the header description tip. pointer-events:none + aria-hidden: - it can never become a click target, so the control under it stays clickable - (the tooltip-swallows-the-next-click trap, paid for once already). */} - {headerTip && ( -
- {headerTip.text} -
- )} - {/* I2 — the cell tip. `pointer-events: none` + `aria-hidden` are not styling here, - they are the contract: the control under the tip must stay clickable, and a - screen reader already has the cell's own value. */} - {cellTip && ( -
- {cellTip.text} -
- )} - {/* Owner item 2's point: checked customers accumulate, and the selection DOES - something. FLOATED over the grid, never in the flow: a bar that reflows the grid - on first selection shifts every row under the user's cursor mid-click-run (and - broke this gate's own coordinate math before it broke a user). It states the - count and offers "Add to cohort" over exactly the checked pids — the same guarded - add_to_list event the view menu uses, so the host treats both alike. */} - {!embedded && selectedPids.size > 0 && !serverWindowed && - (displayMode === "grid" || displayMode === "map") && ( -
- - {selectedPids.size.toLocaleString()} selected - - {/* ⭐ 2026-08-06 (owner) — *"right now it only shows 'Add to cohort'"*. On a user - database the checked rows are RECORDS the reader owns, and there was no door to - remove one; the whole delete path existed server-side with nothing calling it. - ⚠ ONLY on a `ut_` database: the customer and product grids are projections of - Odoo, where a "delete" would be a write to somebody else's system of record — - the server refuses it, and a button that must be refused is worse than none. */} - {canMutateRecords && ( - - )} - {cohortMode ? ( - // Item 2c: on the Cohort page the selection EDITS MEMBERSHIP of the fixed set — - // "Remove from cohort" replaces "Add to cohort". Direct (no popover): the - // checked set is explicit, the button says what it does, and re-adding is one - // picker away. - - ) : ( - - )} - {/* ⭐ Owner item 9 — the counterpart, and it renders ONLY when it can do - something: the checked rows have to be in a cohort this viewer may edit for - "Remove from cohort…" to mean anything. Same bar, same selection, opposite - direction — the ellipsis says a picker follows, because unlike cohort mode - the target is not implied by where you are standing. */} - {!cohortMode && removableLists.length > 0 && ( - - )} - -
- )} -
- {/* Owner item 8 — the cap is VISIBLE, beside the control that lifts it. The toolbar - count above still states the full matched count; this bar says what is painted. - Grid + list only: calendar/kanban project the FULL set with their own stated - bounds (a month, a stack's Show-all). */} - {capped && (displayMode === "grid" || displayMode === "list") && ( -
- - Showing first {shownRecords.toLocaleString()} of{" "} - {recordCount.toLocaleString()} - - - -
- )} -
- - {menuField && columnMenu && ( - - * t.key !== scope)` and said so honestly: "a legal relation in Airtable and a confusing - * one to explain in a one-line picker … left out until somebody asks". It was a taste - * call, not a constraint, and nothing below the picker ever needed it — `_clean_link` - * only requires a `ut_*` key, and `sync_reciprocal_link` puts the backlink on the target - * whichever table that is. "This lead is a duplicate of that lead" and "this record's - * parent" are the ordinary cases, and both were unbuildable. - * - * ⚠ THE UNFILTERED LIST ALSO REPAIRS THE ROLLUP EDITOR, which is the half a one-line - * change hides: it resolves the fold's source columns with - * `linkTargets.find(t => t.key === chosenLink.link.table)`, so a rollup over a self-link - * would have offered an EMPTY column picker and looked like the rollup was broken. - * ⚠ Both `` renders take the same list — the full menu and the "+" create - * menu. One of the two is the half-landing this repo keeps paying for. - */ - linkTargets={linkTargets} - rollupSourceOffer={rollupSourceOffer} - locked={menuField.key === lockedKey} - schemaLocked={isSchemaLocked(menuField)} - viewer={viewer} - sortedDir={menuSortedDir} - isFiltered={menuIsFiltered} - isGrouped={config.groupBy === menuField.key} - groupable={isGroupableField(menuField, lockedKey)} - // Item 12 (R8) — the kanban's "Add option" opens THIS menu straight into its Edit-field - // pane. Absent for every other opening, which is the action list as before. - initialPane={columnMenu.pane} - onClose={() => setColumnMenu(null)} - onNote={(note) => saveField({ ...menuField, note })} - onHide={() => { - setColumnVisible(menuField.key, false); - setColumnMenu(null); - }} - onCreate={createField} - onChangeField={(newKey) => changeField(menuField.key, newKey)} - onCreateAndSwap={(label, type, options, extra) => - createAndSwapField(menuField.key, label, type, options, extra) - } - onProfileFlag={isUserTable ? setProfileFlag : undefined} - onFieldConfig={isUserTable ? (patch) => setFieldConfig(menuField.key, patch) : undefined} - onRetype={ - // Item 2 — a CREATED (`custom_`) field retypes IN PLACE from the pane. - menuField.custom && menuField.source === "overlay" - ? (type, options, extra) => - retypeField(menuField.key, type, options, extra) - : undefined - } - onRename={ - // Item 5 — rename is a def edit on created strata (host accepts the label - // change via the ordinary field_upsert). - menuField.custom - ? (label) => saveField({ ...menuField, label }) - : undefined - } - onFormula={ - // Owner item 8 — edit a created FORMULA field's source in place; the optional - // label rides the SAME upsert (see ColumnMenu's one-upsert rule). - // - // ⭐⭐ 2026-08-10 — TWO STRATA, TWO DOORS, and the gate is `custom` rather than the - // type. A formula column on a `ut_*` database now lives in the SHARED definition, and - // definition fields carry no `custom` flag (`_clean_field` never stamps one) — so the - // old single condition made the expression editor DISAPPEAR for exactly the columns - // this change creates: creatable and then permanently frozen, which is a worse state - // than the one it replaces. `setFieldConfig` PATCHes the definition; `saveField` - // writes this user's overlay, and sending a definition field down that path would - // fork the column into a private second copy of itself. - menuField.type !== "formula" - ? undefined - : isUserTable && !menuField.custom - ? (formula, label) => - setFieldConfig(menuField.key, { - formula, - ...(label ? { label } : {}), - }) - : menuField.custom - ? (formula, label) => - saveField({ - ...menuField, - formula, - ...(label ? { label } : {}), - }) - : undefined - } - onPeriod={ - menuField.measure - ? (w) => changeMeasurePeriod(menuField.key, w) - : undefined - } - onDelete={ - menuField.custom && menuField.key !== lockedKey - ? () => { - deleteField(menuField.key); - setColumnMenu(null); - } - : /** - * ⭐⭐ 2026-08-10 — the DEFINITION stratum's own delete. - * - * ⛔ NARROWED TO THE THREE KINDS THAT HOLD NO DATA OF THEIR OWN, which is - * `preset_editable`'s argument reused rather than a new one: a link, a rollup and - * a formula are all QUESTIONS asked of other cells, so removing one destroys no - * measurement and can be undone by asking it again. Every other definition column - * on a `ut_*` database holds real values, and offering a type-blind Delete over - * those is a destructive widening this change has no mandate for (booked as - * D-114). It is also exactly the three kinds `createField` now routes to the - * definition — so what a user can make here, a user can unmake. - * ⚠ `schemaLocked` still gates the button inside ColumnMenu, so a pre-set link - * never shows it; a pre-set ROLLUP does, by the same owner ruling that made it - * editable, and the confirm says it will come back. - */ - isUserTable && - !menuField.custom && - menuField.key !== lockedKey && - (menuField.type === "link" || menuField.type === "rollup" || - menuField.type === "formula") - ? () => { - deleteDefinitionField(menuField.key); - setColumnMenu(null); - } - : undefined - } - onDuplicate={ - // Creatable strata only — a base Odoo field offers no Duplicate (contract). - menuField.custom ? () => duplicateField(menuField) : undefined - } - onPermissions={ - // Creatable strata only; ColumnMenu further gates on the viewer (creator/admin). - menuField.custom - ? (edit) => saveField({ ...menuField, permissions: { edit } }) - : undefined - } - onFormat={(format) => saveField({ ...menuField, format })} - onSort={(dir) => - updateConfig({ ...config, sorts: [{ colId: menuField.key, dir }] }) - } - onClearSort={() => - updateConfig({ - ...config, - sorts: config.sorts.filter((s) => s.colId !== menuField.key), - }) - } - onFilterBy={() => - // Item 6c — a MEASURE-carrying column seeds the equivalent MEASURE condition - // (same measure key + window, value empty); every other filterable column seeds - // an ordinary column condition. Same signal, one consumer (the Toolbar). - setFilterSeed((prev) => ({ - n: (prev?.n ?? 0) + 1, - ...(menuField.measure - ? { measure: { key: menuField.measure.key, window: menuField.measure.window } } - : { key: menuField.key }), - })) - } - pinnedTo={menuPinnedTo} - onPinTo={ - // Item 11c — pinning the identity column is the same as unpinning; skip it. - menuVisIndex > 0 && !menuPinnedTo - ? () => pinFieldTo(menuField.key) - : undefined - } - onUnpin={frozenN > 1 ? unpinFields : undefined} - scopeChoice={scopeChoice} - onClearFilter={() => - updateConfig({ - ...config, - filters: dropFieldFromTree(config.filters, menuField.key, measureCols), - }) - } - onGroupByField={() => updateConfig({ ...config, groupBy: menuField.key })} - onClearGroup={() => updateConfig({ ...config, groupBy: null })} - userOptions={payload?.userOptions ?? []} - measures={measures} - /> - )} - - {/* Wave-6 item 4 — the header "+": the SAME ColumnMenu, opened straight into its - create form in insert-at-end mode. Anchored on the button; the nominal field is the - locked identity column (only its position matters — the create form never shows it). - The action/pane handlers below are unreachable in this mode (create-only; Cancel - closes), supplied inert to satisfy the surface. */} - {plusMenu && fieldByKey.get(lockedKey) && ( - setPlusMenu(null)} - onNote={() => undefined} - onHide={() => undefined} - onCreate={createField} - onChangeField={() => undefined} - onCreateAndSwap={() => undefined} - onSort={() => undefined} - onClearSort={() => undefined} - onFilterBy={() => undefined} - onClearFilter={() => undefined} - onGroupByField={() => undefined} - onClearGroup={() => undefined} - scopeChoice={scopeChoice} - userOptions={payload?.userOptions ?? []} - measures={measures} - // ⛔⛔ 2026-08-09 — THE SECOND HALF OF THE SAME DEFECT. The sibling instance above has - // passed these since the relational types shipped; this one never did, so a rollup - // created through "+" had no databases to resolve its link against and no source offer - // to switch modes with — the Column picker held its placeholder and nothing else. - // ⚠ TWO THINGS WERE BOTH WRONG (the fetch was gated on `columnMenu` and these props - // were absent), so fixing either alone would still have shipped an empty picker and - // read as "the fix did not work" — [[defects-that-mask-each-other]]. - // ⚠ 2026-08-10 — THE SELF-EXCLUSION IS GONE FROM BOTH SITES (D-113). The line this - // replaces said the two call sites "must not disagree about that", and it was right for - // a reason it did not anticipate: the exclusion was removed from the sibling first and - // this copy kept it, so for one edit the "+" menu and the column menu offered DIFFERENT - // databases — caught by grepping the prop rather than by the build, which is happy - // either way. See the sibling's note for why self-links are offered now. - linkTargets={linkTargets} - rollupSourceOffer={rollupSourceOffer} - /> - )} - - {/* ⭐ WAVE 21 item 11 (R10 / contract C5) — "Select records from a list". - ⚠ `inView` is `visibleRows`, NOT `displayRows`. `displayRows` is a capped slice - (`DISPLAY_PAGE`) of what the view matched; matching over it would report every - record past the cap as "not found" — a lie that grows with the table. The whole - pool goes in beside it so a value the FILTER hides is reported as filtered, not - as missing. */} - {selectFromFile && ( - (vr.kind === "data" ? [vr.record] : []))} - wholeTable={computedRows} - viewName={activeView?.name ?? "this view"} - onSelect={(pids) => selectPids(pids, "replace")} - onClose={() => setSelectFromFile(false)} - /> - )} - - {/* The selection bar's cohort picker: existing cohorts, or a new one by name. Mirrors the - view menu's Add-to-list surface so the two ways of building a cohort read identically. */} - {selAddOpen && selAddRef.current && selectedPids.size > 0 && ( - setSelAddOpen(false)} - dataKind="selection-add-to-list" - > -
Add to locked view
-
- Adds the {selectedPids.size.toLocaleString()} checked customer - {selectedPids.size === 1 ? "" : "s"}. The cohort stays fixed as the data changes. -
- {lists.map((l) => ( - - ))} -
- - setSelListName(event.target.value)} - onKeyDown={(event) => { - if (event.key === "Enter" && selListName.trim()) - addSelectionToList("", selListName.trim()); - }} - /> -
- -
-
-
- )} - - {/* ⭐ Owner item 9 — the remove picker. Every row states how many of the CHECKED rows it - would take out ("3 of 4"), because the selection and the cohort are two different sets - and the difference is the whole reason this needs a picker rather than a button. */} - {selRemoveOpen && selRemoveRef.current && removableLists.length > 0 && ( - setSelRemoveOpen(false)} - dataKind="selection-remove-from-list" - > -
Remove from locked view
-
- Only the locked views holding one of the {selectedPids.size.toLocaleString()} checked - customer{selectedPids.size === 1 ? "" : "s"} are listed. The records themselves are - not touched. -
- {removableLists.map((l) => ( - - ))} -
- )} - - {/* Cohort mode's "+ Add customers" picker (item 2c): search the POOL, tick many, confirm - once. Candidates exclude current members; picks ACCUMULATE across searches, so the - confirm button carries the running total. The 50-row render cap is stated beside the - list — nothing silently truncated. */} - {cohortMode && addCustOpen && addCustRef.current && activeCohortId && ( - setAddCustOpen(false)} - dataKind="cohort-add-customers" - > -
Add customers
-
- Search your customer pool and tick who to add to{" "} - {activeCohort?.name ?? "this locked view"}. -
- setAddCustQuery(e.target.value)} - /> -
- {addCandidates.slice(0, 50).map((c) => ( - - ))} - {addCandidates.length === 0 && ( -
- {addCustQuery.trim() - ? "No customer outside this locked view matches your search." - : "Every customer in your pool is already in this locked view."} -
- )} -
- {addCandidates.length > 50 && ( -
- Showing the first 50 of {addCandidates.length.toLocaleString()} matches — keep - typing to narrow. -
- )} -
- - -
-
- )} - - {/* ⭐ Wave-23 C7 — the document viewer. `onSave` is OMITTED, not disabled, when the reader - may not edit: an absent handler is what makes the raw tab a `
` instead of a
-          textarea, so there is no editor to be refused by. */}
-      {jsonField && jsonAt && (
-         patchAndRecord(jsonAt.pid, { [jsonField.key]: next }, "a document")
-              : undefined
-          }
-          onClose={closeJson}
-        />
-      )}
-      {linkField?.type === "link" && linkField.link?.table.startsWith("ut_") && linkAt && (
-         patchAndRecord(
-            linkAt.pid,
-            { [linkField.key]: ids.join(",") },
-            "linked records"
-          )}
-          onClose={closeLink}
-        />
-      )}
-      {pickerField && picker && (
-         {
-            const fwd = e.key === "ArrowDown" || e.key === "ArrowRight";
-            const back = e.key === "ArrowUp" || e.key === "ArrowLeft";
-            if (!fwd && !back) return;
-            const items = Array.from(
-              e.currentTarget.querySelectorAll(
-                ".cg-pick-row, .cg-star-btn"
-              )
-            );
-            if (!items.length) return;
-            e.preventDefault();
-            const at = items.indexOf(document.activeElement as HTMLButtonElement);
-            const next = fwd
-              ? (at + 1) % items.length
-              : (at - 1 + items.length) % items.length;
-            items[next]?.focus();
-          }}
-        >
-          {/* Wave-5 item 11 — a RATING cell's picker is the star row itself. */}
-          {pickerField.type === "rating" && (
-            
- {Array.from({ length: ratingMax(pickerField) }, (_, i) => i + 1).map( - (n) => ( - - ) - )} -
- )} - {pickerField.type !== "rating" && pickerChoices.length === 0 && ( -
- {pickerField.type === "user" - ? "No assignable people were supplied for this workspace." - : "This field has no choices yet. Add them from the column menu."} -
- )} -
- {pickerChoices.map((choice) => { - const tint = optionTint(pickerField, choice); - const on = pickerMulti - ? pickerParts.includes(choice) - : pickerValue === choice; - return ( - - ); - })} -
- {pickerMulti && ( -
- -
- )} - {/* Clearing must be possible, and it must write "" rather than delete the key — - the overlay patch contract is a value per field, and a missing key would leave - the previous value standing on the next render. */} - {pickerValue !== "" && ( - - )} -
- )} - - {!embedded && detailRecord && detailPid !== null && ( - go(-1)} - onNext={() => go(1)} - onClose={() => setDetailPid(null)} - // Item 5 / C-LAYOUT — see `recordLayout` above. Both optional on RECORD's side, so - // this compiles whether or not their reorder UI has landed yet. - recordLayout={recordLayout} - onRecordLayout={onRecordLayout} - // Item 13 / C-EMBED — the host names WHICH question Insights asks. Without this the - // tab deliberately does not render (RECORD's no-safe-default rule: a wrong scope - // succeeds with plausible wrong numbers, so absence must mean absent, not guessed). - scope={scope} - docs={payload?.docs?.[String(detailPid)] ?? []} - docPayload={payload?.docPayload} - // C5 — documents ride the ordinary event log. The host answers a fetch - // on the NEXT render via `payload.docPayload`; Documents.tsx matches it - // to its own pending request before touching it. - onDocAdd={ - payload?.docs - ? (file) => - emitHostEvent({ - id: eventId("docadd"), - type: "doc_add", - pid: detailPid, - docId: `doc_${eventId("d").slice(2, 14).replace(/[^a-z0-9]/gi, "")}`, - ...file, - }) - : undefined - } - onDocFetch={ - payload?.docs - ? (docId) => - emitHostEvent({ - id: eventId("docget"), - type: "doc_fetch", - pid: detailPid, - docId, - }) - : undefined - } - onDocDelete={ - payload?.docs - ? (docId) => - emitHostEvent({ - id: eventId("docdel"), - type: "doc_delete", - pid: detailPid, - docId, - }) - : undefined - } - onNotesChange={(key, value) => - setOverlayEdits((current) => ({ - ...current, - [detailPid]: { ...current[detailPid], [key]: value }, - })) - } - onNotesCommit={(key, value) => patchAndRecord(detailPid, { [key]: value })} - viewer={viewer} - // Wave-14 close-out stitch (HOST): TSREC's two light-switch props, unwired when - // the GRID session ended — item 9's picker vocabulary + item 11's photos. Same - // values the grid's own surfaces already read (the wave-13 sixth-prop precedent). - userOptions={payload?.userOptions ?? []} - userAvatars={userAvatars} - /> - )} -
- ); -} - -export default memo(CustomerGrid); +import { + memo, + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from "react"; +import { DataEditor, GridCellKind } from "@glideapps/glide-data-grid"; +import type { + CellClickedEventArgs, + DataEditorRef, + EditableGridCell, + GridKeyEventArgs, + GridMouseEventArgs, + GridSelection, + DrawHeaderCallback, + HeaderClickedEventArgs, + Item, + Rectangle, + Theme, +} from "@glideapps/glide-data-grid"; +import "@glideapps/glide-data-grid/dist/index.css"; + +import { useCustomerData } from "./useCustomerData"; +import type { SurfaceScope } from "./apiBridge"; +import { defaultViewConfig, useGridColumns } from "./useGridColumns"; +import { activeMeasureRuleIds, pendingMeasures, runPipeline, sliceForDisplay, + unresolvedConditions, useVisibleRows } from "./useVisibleRows"; +import { computeAggs } from "./aggregations"; +import type { CohortSets } from "./useVisibleRows"; +import type { MeasureSets } from "./useVisibleRows"; +import { useGetCellContent } from "./useGetCellContent"; +import { useGridSelection } from "./useGridSelection"; +import Toolbar from "./Toolbar"; +import JsonViewer from "./JsonViewer"; +import RecordDetail from "./RecordDetail"; +import ViewSidebar from "./ViewSidebar"; +import ColumnMenu from "./ColumnMenu"; +import type { ColumnMenuState } from "./ColumnMenu"; +import { HEADER_ICONS } from "./iconShapes"; +import { emitHostEvent, eventId } from "./hostBridge"; +// ⭐ WAVE 27 item 21 (R14) — which of this table's views carry an alert. The SERVER's +// list is the only answer; the rail's alert door is a one-way create. +import { fetchAlerts } from "../alerts/alertsApi"; +import { NAV_MINIMIZE_EVENT, ROWS_STALE_EVENT, TOAST_EVENT, VIEW_OPEN_EVENT, + WORKSPACE_STALE_EVENT, signal } + from "../apiContract"; +import type { ViewOpenDetail } from "../apiContract"; +import { addTableField, addTableRow, deleteTableField, deleteTableRow, fetchLinkTargets, + fetchRollupSources, patchTableField } from "./apiBridge"; +import type { LinkTarget, RollupSourceOffer } from "./apiBridge"; +import SelectFromFile from "./SelectFromFile"; +import ImportDialog from "./ImportDialog"; +import { FormInterface } from "./FormInterface"; +import type { FormSpec } from "./FormInterface"; +// Owner item 16 / R4 / C-UNDO — the stack and every inverse. Pure, so a node gate can run it. +import { describe, directed, popRedo, popUndo, pushUndo, stackFor } from "./undoStack"; +import type { CellChange, UndoBook, UndoEntry, UndoRow, UndoValue } from "./undoStack"; +import { exportFilename, runExport, triggerDownload } from "./export"; +import type { ExportFormat } from "./export"; +// owner item 3 (2026-08-03) — a time-series view exports its SHEET, not the rows under it. +import { buildTsCsv, tsSheetToTable } from "./timeSeriesData"; +import type { TsSheet } from "./timeSeriesData"; +import { echoReemit, reconcileEchoView } from "./viewEcho"; +import { pruneStamps, reconcileFields } from "./optimism"; +import { applyViewOrder, newFolderId, pruneFolderStamps, reconcileFolders, + resolveFolderId } from "./folders"; +import type { FolderStamps } from "./folders"; +// ⭐ WAVE 27 item 22 — the cells the active view's filter forces on a record added under it. +import { filterSeedValues } from "./filterSeed"; +import { adoptNewFields, adoptNewViews, pruneTombstones, seedLocalViews, stampTombstone } + from "./liveWorkspace"; +import type { Tombstones } from "./liveWorkspace"; +import type { GridFolder, ViewPermissions } from "./types"; +import type { FieldStamps } from "./optimism"; +import { evalFormula, orderFormulas, parseFormula } from "./formulaEngine"; +import type { FormulaAst } from "./formulaEngine"; +import { CalendarView, KanbanView, ListView, ModeSwitch } from "./viewModes"; +import { SwipeView } from "./SwipeView"; +import type { SwipeSpec } from "./SwipeView"; +// The chart engine lives in `viz/` since EXIT wave 2 (W2-5/Y3) — the grid is now +// one of its two callers, the Y1 page envelope being the other. +import { DashboardView } from "../viz/DashboardView"; +import { cleanCharts } from "../viz/chartData"; +import type { ChartSpec } from "../viz/chartData"; +import { MapView } from "./MapView"; +/* ═══ W18-C CATALOG ═══ (owner item 4, contract C6) */ +import { CatalogView } from "./CatalogView"; +import { CATALOG_CODE_FIELD } from "./catalogData"; +import type { CatalogSpec } from "./types"; +/* ═══ end W18-C CATALOG ═══ */ +import { lightTheme, STATUS_ROW_THEME, HOVER_ROW_THEME, HOVER_NEUTRAL, + ACTIVE_ROW_NEUTRAL, CUSTOM_FIELD_MARK } from "./theme"; +import { avatarInitials, formatDisplay, imageCellRenderer, ratingCellRenderer, setAvatarRepaint, + userCellRenderer } from "./cells"; +import { optionTint, pickTint } from "./choiceColors"; +import { + EMPTY_GRID_COPY_PROVENANCE, + markGridCopy, + observeCopyEvent, + planFieldPaste, + pasteRowCount, +} from "./clipboard"; +import { cellTipText, expandButtonRect, GROUP_HEADER_FONT, GROUP_LABEL_PAD, headerMarkLayout, + headerMarkSizes, tipLeft } from "./overlayPlacement"; +import { AnchoredOverlay, BodyPortal, useOverlayLayer } from "./OverlaySurface"; +import type { AnchorRect } from "./OverlaySurface"; +import { StarIcon } from "./Stars"; +import { ALL_VIEW_ID, MAX_CALENDAR_METRICS, allViewName, + MAX_FROZEN, choiceOptions, choiceVocabulary, clampFrozenCount, cleanDisplay, formulaOf, + topicForScope, + isDateFamilyType, isFilterGroup, isGroupableField, isMachineOwned, + isDerivedLink, isUserSchemaField, TOTAL_GROUP_KEY, + isNumericFieldType, + machineFoundRows, reFindConsequences, + isPickType, mayEditField, + isModeFrozen, isUndeletableView, mayEditView, mayToggleViewLock, + measureColumnIndex, ratingMax, ruleColumnKeys, + tableMode, uniqueDisplayName } from "./types"; +import type { + DisplayMode, + DisplaySpec, + Field, + HostEvent, + FieldScope, + FieldType, + FilterNode, + FilterRule, + Row, + RowHeightMode, + SavedView, + ViewConfig, +} from "./types"; +// Item 7 (C-TS). Default-exported because it is a leaf VIEW like MapView, and because RECORD +// imports the same default for the record-detail Insights tab (C-EMBED). +import TimeSeriesPanel from "./TimeSeriesPanel"; +import type { WindowSpec } from "./windows"; +import { windowLabel } from "./windows"; + +const ROW_PX: Record = { short: 28, medium: 34, tall: 48 }; + +/** + * Wave-14 R7 — **a button label is ONE LINE.** Every button row in this file is a flex row with + * no width reservation, so on a narrow grid box the items shrink and their labels wrap: + * "Remove from cohort" becomes two lines, the row grows, and the bar stops being a bar. It only + * happens below a width no fixed-size screenshot is taken at, which is why it survived. + * + * ⚠ Applied INLINE rather than on `.cg-btn`, where it belongs: `index.css` is PANEL's fence this + * wave. A mailbox line asks them to hoist it onto `.cg-btn` globally — after which this is + * redundant, and harmless, because it says exactly the same thing. + */ +const ONE_LINE = { whiteSpace: "nowrap" } as const; + +/** + * How many VISIBLE columns glide freezes. Module-level and used twice on purpose (item 15): the + * `freezeColumns` prop and the group bar's label fit must read ONE answer, because the label is + * clipped to exactly this strip and a second copy of the clamp would let the two disagree by a + * column — which shows up as a label cut early for no visible reason. + */ +function frozenCountOf(config: ViewConfig): number { + return Math.min(MAX_FROZEN, Math.max(1, config.frozenCount ?? 1)); +} +/* WAVE 21 item 3 (R6): the id and the NAME both moved to `types.ts` — the id because a second + copy of a pinned literal is the drift class this repo gates against, the name because it is + now topic-derived and the host mints the same string. */ + +/** + * owner item 2 (2026-08-03) — the measure-cell skeleton's pulse, and its ceiling. + * + * 140ms against the 4-step colour ramp is a ~0.6s cycle: a wait, not a strobe. The ceiling is + * ~60s, after which the cells fall back to ordinary blanks — a measure that has not resolved in + * a minute is not "still loading", and a shimmer that never ends promises a number that is not + * coming. Shared constant so the two are read in one place rather than tuned apart. + */ +const PULSE_MS = 140; +const PULSE_MAX_TICKS = Math.round(60_000 / PULSE_MS); +/** Referentially stable, so the fallback does not change `getCellContent`'s identity per render + * and repaint the canvas forever. */ +const NO_PENDING_KEYS: ReadonlySet = new Set(); + +/** + * I2 — measure a cell's text in GLIDE'S OWN font, so "is this cut off?" is a fact rather than + * a character-count guess (a guess is wrong in both directions: "IIIII" is narrow, "WWWWW" is + * wide, and being wrong means either a missing tip or a tip over text you can already read). + * One lazily-built offscreen context for the whole module — `measureText` is cheap, but + * creating a canvas per mouse-move would not be. + */ +let _tipCtx: CanvasRenderingContext2D | null | undefined; +function measureCellText(text: string): number { + if (_tipCtx === undefined) { + _tipCtx = document.createElement("canvas").getContext("2d"); + // glide's default `baseFontStyle` is 13px; the family comes from our own theme. + if (_tipCtx) _tipCtx.font = `13px ${lightTheme.fontFamily ?? "Inter, sans-serif"}`; + } + return _tipCtx ? _tipCtx.measureText(text).width : 0; +} +/** + * Item 15 — the GROUP BAR's measurer, and it is a SECOND context on purpose. + * + * `measureCellText` above is set to plain "13px", which is right for ordinary cells and WRONG for + * a group bar: that row carries `baseFontStyle: GROUP_HEADER_FONT` (semibold), and glide paints a + * cell with the merged theme's font. Measuring the fit with the lighter weight under-truncates + * and the label overflows into the frozen strip's hard clip — the exact failure `fitGroupLabel` + * exists to prevent. Font string built from the SAME constant the cell's themeOverride uses, so + * the two cannot drift. + */ +let _groupCtx: CanvasRenderingContext2D | null | undefined; +function measureGroupText(text: string): number { + if (_groupCtx === undefined) { + _groupCtx = document.createElement("canvas").getContext("2d"); + if (_groupCtx) + _groupCtx.font = `${GROUP_HEADER_FONT} ${lightTheme.fontFamily ?? "Inter, sans-serif"}`; + } + return _groupCtx ? _groupCtx.measureText(text).width : 0; +} +/** + * Owner item 8 (2026-07-27): the first render shows this many rows; "See more" reveals the rest. + * A DISPLAY cap only — the pipeline still runs over the whole book, the toolbar count is still + * the full matched count, and the cap is stated beside the control that lifts it, which is what + * keeps [[no-unverifiable-aggregates]] satisfied: nothing is silently truncated. + */ +const DISPLAY_PAGE = 50; +const DISPLAY_STEP = 250; + +/* ═══════════════════════════════════════════════════════════════════════════════════════════ + ═══ W18-B VOID ═══ (wave 18, owner item 1b) — the two constants the void geometry needs + that belong to the BROWSER and to GLIDE rather than to us. Both are mirrors, and both say + here what they are mirroring, because a mirror that does not name its original is how the + two copies stop agreeing. + ═══════════════════════════════════════════════════════════════════════════════════════════ */ + +/** The header band. `headerHeight={36}` at the DataEditor mount is the original. + * Not `+ groupHeaderHeight`: glide adds that only when `enableGroups` is on, which it turns on + * when a COLUMN carries a `group` — `useGridColumns` never sets one (scrolling-data-grid.js:13). */ +const HEADER_PX = 36; + +/** glide's own row-marker width ladder, `data-editor.js:103`, verbatim. + * + * ⚠ It is passed BACK to glide as `rowMarkerWidth` rather than merely predicted here. The + * marker column is part of the content width the void's left edge is measured from, so a + * four-pixel disagreement between what glide draws and what we compute is a four-pixel seam of + * white against the tint — visible, and invisible to every gate. Pinning makes the two the same + * number by construction; mirroring the ladder (rather than pinning one constant) is what keeps + * the pin from changing the marker column's width on tables of more than 100 rows. */ +function rowMarkerPx(rows: number): number { + return rows > 10_000 ? 48 : rows > 1000 ? 44 : rows > 100 ? 36 : 32; +} + +/** The scrollbar gutter this browser steals from a scrollable box, measured once. + * + * ⛔ WHY THE VOID HAS TO KNOW. The void rectangles sit ON TOP of glide's scroller, so a + * rectangle drawn to the box's own edge paints over the scrollbar and hides it. The two cases + * are CROSSED, which is the part that is easy to get backwards: the BOTTOM void has to stop + * short when the columns overflow (a horizontal scrollbar), and the RIGHT void has to stop + * short when the rows overflow (a vertical one). Both happen in ordinary use — twelve columns + * filtered to five rows is exactly the owner's case. + * + * 0 on overlay-scrollbar platforms (macOS, touch), which is correct: nothing is stolen there. */ +let scrollbarGutterPx: number | null = null; +function scrollbarGutter(): number { + if (scrollbarGutterPx !== null) return scrollbarGutterPx; + if (typeof document === "undefined") return 0; + const probe = document.createElement("div"); + probe.style.cssText = + "position:absolute;top:-9999px;width:100px;height:100px;overflow:scroll"; + document.body.append(probe); + scrollbarGutterPx = probe.offsetWidth - probe.clientWidth; + probe.remove(); + return scrollbarGutterPx; +} +/* ═══ end W18-B VOID (module scope) ═══ */ + +interface LocalWorkspace { + fields: Field[]; + views: SavedView[]; + activeViewId: string; + /** When this copy was written (browser clock — same-machine freshness only, + * never date semantics). Absent on copies from before the echo reconcile. */ + savedAt?: number; + /** Wave-6 item 3c — per-key freshness for this browser's own field-def + * writes (edits + tombstones), consumed by reconcileFields at init. */ + fieldStamps?: FieldStamps; + /** BUG-1 (wave 11) — viewId → the filter tree already pushed back to the host, so the + * echo re-emit happens once per user edit rather than once per remount. NOT a timestamp: + * `savedAt` beside it is rewritten on every init, so any time-based bound is refreshed by + * the remount it is meant to bound. See viewEcho.echoReemit. */ + reemitted?: Record; + /** 2026-08-04 — viewId → when THIS browser deleted it. The views half of the tombstone + * rule fields and folders already had, needed the moment the rail adopts host views + * after mount (liveWorkspace.ts). Persisted, not a ref, because a remount inside the + * echo window would otherwise re-adopt a view the user deleted a second ago. */ + viewTombstones?: Tombstones; + /** D-19 (wave 20) — viewId → when THIS browser last WROTE it. The counterpart of the + * tombstones above: they stop a deleted view coming back, this stops a just-created one + * being dropped by the host-list rule at init, before its upsert has round-tripped. + * Same window, same prune, same browser-clock caveat. */ + viewWrites?: Tombstones; +} + +function sameConfig(a: ViewConfig, b: ViewConfig): boolean { + return JSON.stringify(a) === JSON.stringify(b); +} + +function normalizeConfig(config: Partial | undefined, fields: Field[]): ViewConfig { + const base = defaultViewConfig(fields); + return { + ...base, + ...config, + filters: Array.isArray(config?.filters) ? config.filters : [], + // Legacy views persisted a flat rule array with an implicit AND — anything + // other than an explicit "or" normalizes to "and", so old views are unchanged. + filterConj: config?.filterConj === "or" ? "or" : "and", + sorts: Array.isArray(config?.sorts) ? config.sorts : [], + order: Array.isArray(config?.order) && config.order.length ? config.order : base.order, + visible: + Array.isArray(config?.visible) && config.visible.length + ? config.visible + : base.visible, + widths: config?.widths && typeof config.widths === "object" ? config.widths : {}, + memberPids: Array.isArray(config?.memberPids) ? config.memberPids : [], + // Wave-6 items 10/11. Both normalize junk to ABSENT (grid / 1 frozen), so + // a legacy view is byte-identical to itself after a round trip. W13 (C4 as + // amended): cleanDisplay keeps grid-carried field picks. + display: cleanDisplay(config?.display), + frozenCount: clampFrozenCount(config?.frozenCount), + }; +} + +/** + * The pinned system view, as this browser mints it when no host copy has arrived. + * + * WAVE 21 item 3 (R6) — the NAME is the scope's now (`allViewName`), not the literal "All + * customers" this minted on every topic including `ut_*` databases of anything at all. The + * ID is unchanged and deliberately so (see `ALL_VIEW_ID`). + */ +function allRecordsView(fields: Field[], scope: string): SavedView { + return { + id: ALL_VIEW_ID, + name: allViewName(scope), + kind: "system", + locked: true, + config: defaultViewConfig(fields), + }; +} + +function readLocal(storageKey: string): LocalWorkspace | null { + try { + const raw = localStorage.getItem(`aios-grid:${storageKey}`); + return raw ? (JSON.parse(raw) as LocalWorkspace) : null; + } catch { + return null; + } +} + +function writeLocal(storageKey: string, value: LocalWorkspace): void { + try { + localStorage.setItem(`aios-grid:${storageKey}`, JSON.stringify(value)); + } catch { + // Storage can be blocked in hardened/opaque iframes. The host bridge remains durable. + } +} + +function nextViewId(): string { + return `view_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`; +} + +/** A multiselect cell's SET, out of its comma-joined string (the `multi` contract). */ +function splitMulti(v: string): string[] { + return v + .split(",") + .map((s) => s.trim()) + .filter((s) => s !== ""); +} + +/** Wave-5 item 3 — does any leaf of the filter tree name this column? + * + * ⚠ Wave-20 item 2 RETARGETED the note that stood here ("measure leaves carry measure keys, + * which are never field keys, so they cannot false-positive"). True about false POSITIVES, + * and precisely why it was a false NEGATIVE: a measure column's own condition names the + * MEASURE, so this answered "not filtered" for a column the user had visibly filtered, and + * the menu never offered "Don't filter by this field" on it. Both doors now go through + * `ruleColumnKeys` — the same resolution the tint uses, so the menu and the colour can never + * disagree about which columns a filter is about. */ +function treeNamesField( + nodes: FilterNode[], + key: string, + measureCols: Map +): boolean { + for (const n of nodes) { + if (isFilterGroup(n)) { + if (treeNamesField(n.children, key, measureCols)) return true; + } else if (ruleColumnKeys(n as FilterRule, measureCols).includes(key)) return true; + } + return false; +} + +/** Wave-5 item 3 — "Don't filter by this field": drop every leaf naming the column, prune + * groups that end up empty. Returns new arrays throughout (the config is state). */ +function dropFieldFromTree( + nodes: FilterNode[], + key: string, + measureCols: Map +): FilterNode[] { + const out: FilterNode[] = []; + for (const n of nodes) { + if (isFilterGroup(n)) { + const children = dropFieldFromTree(n.children, key, measureCols); + if (children.length) out.push({ ...n, children }); + } else if (!ruleColumnKeys(n as FilterRule, measureCols).includes(key)) { + out.push(n); + } + } + return out; +} + +// The header sprite map moved to iconShapes.ts (wave-8 I20/I21) — it is data + +// SVG-source builders with no React in it, which is what lets verify_icons.py +// import it under node and assert the (i) actually contrasts. + +function slugify(label: string): string { + return ( + label + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_|_$/g, "") + .slice(0, 28) || "field" + ); +} + +interface FieldBuildExtra { + formula?: string; + max?: number; + scope?: FieldScope; + label?: string; + colorCodeOptions?: boolean; + optionColors?: Record; + /** + * ⭐⭐ 2026-08-09 — THE RELATIONAL BAGS, WHICH THIS INTERFACE USED TO DROP ON THE FLOOR. + * + * `ColumnMenu.FieldConfigExtra` has carried them since 2026-08-07 and `extraFor` builds a + * complete bag; this type — the receiving end of the same `extra` argument — never declared + * either key, so `buildOverlayField` could not copy what it could not see. The measured result + * on nurilab: a column stored as + * `{key:'custom_video_views_90j26', label:'Video Views', type:'rollup', source:'overlay'}` + * with NO bag, in one user's workspace stratum, which `compute_relation_cells` never reads. + * Named, configured, rendering, and permanently blank — the owner's *"the Rollup doesn't + * work"*. ⛔ Both halves were needed: this type, and the `createField` route below. + */ + link?: Record; + rollup?: Record; + /** Item 15 (C-RENAME) — the option renames this save carries, by row identity. Consumed by + * `retypeField` (which emits `choice_rename`) and ignored by every create path: a field + * being CREATED has no values to migrate. */ + renames?: { from: string; to: string }[]; +} + +/** + * One constructor for a user-created field, shared by "insert field" and Change-field's + * "New field" half (wave-2 item 5) so the two doors cannot drift. A `multiselect` carries its + * declared options AND `multi: true` — the cell is a comma-joined SET, and the flag is what the + * grouping contract keys on (belt to the type-derived brace in groupRows). + * + * Wave-5: `formula` and `created_time` are built in the HOST's OWN emission shape (source + * 'odoo' + derived — aios_grid.READONLY_CUSTOM_TYPES), so the optimistic local def and the + * next payload's echo are byte-identical and nothing restyles on the round trip. `rating` + * carries top-level `max`, `formula` carries top-level `formula` (the ~20:20 contract + * amendment: options stays the select-family list). Wave-6 item 6: both types are now + * `filterable: true` — the host flipped its emission, and conditions on them evaluate in the + * client engine over computedRows (this table's counts are client-mode; a windowed table's + * columns come from the semantic model, so the SQL engine never sees these types). + */ +/** + * ⭐⭐ 2026-08-09 (owner ruling) — DOES THE PRE-SET LOCK CLOSE THIS COLUMN'S SCHEMA? + * + * Owner: *"No rollup field should be uneditable, everything is custom and changeable always."* + * A ROLLUP holds no data of its own — it is a question asked of other rows, re-askable at any + * time, and re-asking it costs nothing because the answer is recomputed from the authoritative + * store on the next pass. `preset` still closes every column that HOLDS something: retyping a + * pre-set `followers` column would strand real measurements in a column that can no longer read + * them. + * + * ⛔ THE CLIENT MIRROR OF `core.user_tables.preset_editable`, and `verify_rollup_editor.py` + * holds the two in step. This check has THREE enforcement points (here, `may_edit_field`, and + * `routes_tables._field_or_refuse`'s sentence); a client that kept hiding the editor while the + * route allowed the PATCH would read as "you fixed nothing", which is the failure mode a fix in + * one of three places always wears. + */ +function isSchemaLocked(field: Field): boolean { + return field.automation?.preset === true && field.type !== "rollup"; +} + +function buildOverlayField( + label: string, + type: FieldType, + options?: string[], + extra?: FieldBuildExtra +): Field { + const key = `custom_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`; + if (type === "formula" || type === "created_time") { + return { + key, + label, + type, + source: "odoo", + derived: true, + filterable: true, + default: true, + custom: true, + ...(type === "formula" + ? { agg: "sum", ...(extra?.formula ? { formula: extra.formula } : {}) } + : {}), + }; + } + return { + key, + label, + type, + source: "overlay", + default: true, + custom: true, + agg: ["currency", "int"].includes(type) ? "sum" : undefined, + // The relational bags ride the local definition too, so the optimistic field and the + // server's echo describe the same column. Without them the grid held a bagless twin of a + // field the store had configured, and `mayEditField`/`isDerivedLink` read the twin. + ...(extra?.link ? { link: extra.link as Field["link"] } : {}), + ...(extra?.rollup ? { rollup: extra.rollup as Field["rollup"] } : {}), + ...((type === "select" || type === "multiselect") && options?.length ? { options } : {}), + ...((type === "select" || type === "multiselect") + ? { + colorCodeOptions: extra?.colorCodeOptions !== false, + ...(extra?.optionColors && Object.keys(extra.optionColors).length + ? { optionColors: extra.optionColors } + : {}), + } + : {}), + ...(type === "multiselect" ? { multi: true } : {}), + ...(type === "rating" ? { max: extra?.max ?? 5 } : {}), + }; +} + +/** The surface this grid draws. `cohort` is the same table over hand-curated SETS — the server + * confirms it by stamping `workspace.cohortMode`, which is the only thing the body below reads. + * + * MEMOIZED (owner item 1, 2026-07-31): the shell re-renders on every chrome state flip — nav + * collapse, toast, settings — and an unmemoized grid re-rendered its whole 3,400-line tree + * each time. Measured cost: 200ms-5s of main-thread block, which ate the 240ms rail-fold + * transition whole (the fold froze, then SNAPPED — the exact "static" the owner named). The + * props surface is one stable string, so memo makes chrome state changes free; the grid still + * re-renders for its own state (edits, resize) and remounts on route change via `key`. */ +interface CustomerGridProps { + scope?: SurfaceScope; + /** Read-only Grid view embedded in a linked-record modal. It keeps the standard + * filter/sort/search toolbar while withdrawing schema and row mutations. */ + embedded?: boolean; + /** Exact linked pids to project from the target database. */ + embeddedRecordIds?: readonly number[]; + /** Selection mode used by an editable ordinary-Link modal. */ + embeddedSelectable?: boolean; + embeddedSelectedIds?: readonly number[]; + onEmbeddedSelectionChange?: (recordIds: number[]) => void; +} + +interface LinkGridModalProps { + label: string; + table: SurfaceScope; + recordIds: readonly number[]; + editable?: boolean; + single?: boolean; + onSave?: (recordIds: number[]) => void; + onClose: () => void; +} + +function LinkGridModal({ label, table, recordIds, editable = false, single = false, + onSave, onClose }: LinkGridModalProps) { + const panelRef = useRef(null); + const [selectedIds, setSelectedIds] = useState(() => [...recordIds]); + const changeSelectedIds = useCallback((ids: number[]) => { + const next = single ? ids.slice(-1) : ids; + setSelectedIds((current) => + current.join(",") === next.join(",") ? current : next + ); + }, [single]); + useOverlayLayer({ + panelRef, + onDismiss: onClose, + dismissOnOutside: true, + initialFocus: "[data-overlay-autofocus]", + trapFocus: true, + }); + return ( + +
+
+
+
+
{label}
+
+ {(editable ? selectedIds.length : recordIds.length).toLocaleString()} linked{ + single ? " (one allowed)" : "" + } {(editable ? selectedIds.length : recordIds.length) === 1 ? "record" : "records"} +
+
+ +
+
+ +
+ {editable ? ( +
+ + +
+ ) : null} +
+
+
+ ); +} + +function CustomerGrid({ + scope = "customer", + embedded = false, + embeddedRecordIds, + embeddedSelectable = false, + embeddedSelectedIds = [], + onEmbeddedSelectionChange, +}: CustomerGridProps = {}) { + // Wave 16 C-TOPIC: which TABLE this tree is drawing, derived from the one scope prop. + const topic = topicForScope(scope); + const { + fields: payloadFields, + rawRows: fetchedRows, + payload, + loading, + overlayEdits, + setOverlayEdits, + patchOverlay, + } = useCustomerData(scope, { + bindSurface: !embedded, + includeWorkspace: !embedded, + writable: !embedded, + }); + const embeddedIdsKey = embeddedRecordIds?.join(",") ?? ""; + /** + * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — **ROWS THIS BROWSER JUST CREATED**, held locally + * until the server's own copy comes back. + * + * ⛔ THE OWNER'S WORDS: *"adding a new record visually takes too long, I need to be able to + * spam it."* The old path awaited the POST, then fired `ROWS_STALE_EVENT` and re-read the + * WHOLE table — so every "+" cost a round trip plus a full refetch before anything moved, and + * pressing it five times queued five full refetches of a table that grew by five rows. + * + * ⚠ ADD-ONLY AND SELF-PRUNING, which is `liveWorkspace.ts`'s discipline applied to rows: a + * pending row is merged in only while the fetched payload does NOT carry its pid, and the + * effect below drops it the moment the server's copy arrives. There is no third state and no + * merge of values — the server's row wins outright, because by then it IS this row. + */ + const [pendingRows, setPendingRows] = useState([]); + const rawRows = useMemo(() => { + const base = embeddedRecordIds + ? fetchedRows.filter((row) => new Set(embeddedRecordIds).has(row.pid)) + : fetchedRows; + if (!pendingRows.length) return base; + const known = new Set(base.map((r) => r.pid)); + const extra = pendingRows.filter((r) => !known.has(r.pid)); + return extra.length ? [...base, ...extra] : base; + // The scalar key keeps this stable when a caller reconstructs the same id list. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [fetchedRows, embeddedIdsKey, pendingRows]); + useEffect(() => { + // ⚠ PRUNED AGAINST THE FETCHED PAYLOAD, not against `rawRows` — `rawRows` contains the + // pending rows, so it can never report that one has been absorbed. Pruning against the + // memo's own output is the loop that keeps a placeholder alive forever. + if (!pendingRows.length) return; + const known = new Set(fetchedRows.map((r) => r.pid)); + if (!pendingRows.some((r) => known.has(r.pid))) return; + setPendingRows((cur) => cur.filter((r) => !known.has(r.pid))); + }, [fetchedRows, pendingRows]); + /** + * The rows as of the last render, for callbacks that must not be rebuilt per payload. + * `appendRow` reads it to mint the next row id the way the server does (`max + 1`); putting + * `rawRows` in that callback's dependency list would give the trailing "+" a new identity on + * every refetch, which is a re-render of the grid for a value only used at click time. + */ + const rowsRef = useRef(rawRows); + rowsRef.current = rawRows; + /** + * ⭐ WAVE 27 · ITEM 2 — THE NEXT ROW ID, ADVANCED SYNCHRONOUSLY AT MINT TIME. + * + * ⛔ A RENDER-TIME SNAPSHOT IS NOT ENOUGH FOR THE GESTURE THIS ITEM IS ABOUT. `rowsRef.current` + * is assigned during RENDER, so two "+" presses that land before React commits the first + * `setPendingRows` both read the same list and both mint the same id. The owner's words were + * *"I need to be able to spam it"* — that IS the failing case, and the damage is not a + * duplicate row: `minted` is the identity key the failure and re-anchor branches filter and + * map on, so ONE refused write would withdraw BOTH rows and one taken id would rewrite both. + * + * ⚠ SEEDED FROM THE ROWS ON EVERY MINT, then advanced: `Math.max(seen, storeMax) + 1`. Reading + * the store's max each time is what keeps the client in step after a refetch brings rows in + * from elsewhere; the ref is what stops a burst from colliding with itself in between. + */ + const nextRidRef = useRef(0); + const mintRid = useCallback(() => { + const storeMax = rowsRef.current.reduce((m, r) => Math.max(m, r.pid), 0); + nextRidRef.current = Math.max(nextRidRef.current, storeMax) + 1; + return nextRidRef.current; + }, []); + /** + * Withdraw rows this browser is still holding locally, because they have been DELETED. + * + * ⛔ THE PRUNE EFFECT CANNOT DO THIS AND THAT IS THE WHOLE BUG IT FIXES. Pruning is "the + * server's payload now CONTAINS this row, so drop my copy" — a deleted row will never appear + * in a payload again, so a pending row that is deleted before its first refetch stays merged + * into `rawRows` for the life of the mount. Add a record, press Ctrl+Z, and the row sits there + * looking undeleted while the store has already lost it. + * ⚠ CALLED AT EVERY `deleteTableRow` SITE (the selection delete and undo's `rowDelete` + * inverse), because those are the only two doors that remove a row — and the undo one is + * exactly the path that produces a still-pending victim. + */ + const withdrawPending = useCallback((rids: (string | number)[]) => { + if (!rids.length) return; + const gone = new Set(rids.map((r) => Number(r))); + setPendingRows((cur) => (cur.some((r) => gone.has(r.pid)) + ? cur.filter((r) => !gone.has(r.pid)) + : cur)); + }, []); + /** + * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — WHICH VIEWS ON THIS TABLE CARRY AN ALERT. + * + * ⛔ THE SERVER'S LIST IS THE ONLY ANSWER. The rail's "Alert me about new records…" is a + * one-way CREATE (it emits an event the shell turns into a POST) and nothing in the grid + * hears the outcome — a `no_filter` refusal is a real answer, so an optimistic local set + * would paint a badge on a view that has no alert. `GET /alerts` is the state. + * + * ⚠ RE-READ ON FOCUS, the same idiom the shell's own inbox uses. An alert created or deleted + * in another surface (the Alerts pane owns the delete door) reaches this rail on the next + * focus rather than instantly; the alternative was for the grid to guess at the outcome of a + * request another component made. + * ⚠ EMBEDDED GRIDS DO NOT ASK. A linked-record grid inside a modal is not a table anyone + * alerts on, and a second fetch per relation cell would be a workspace read per click. + */ + const [alerted, setAlerted] = useState([]); + useEffect(() => { + if (embedded) return; + let live = true; + const pull = () => { + void fetchAlerts().then((r) => { + if (!live || !r.ok) return; // a failed read leaves the badges as they were + setAlerted(r.value.filter((a) => a.topic === scope).map((a) => a.viewId)); + }); + }; + pull(); + window.addEventListener("focus", pull); + return () => { + live = false; + window.removeEventListener("focus", pull); + }; + }, [embedded, scope]); + + const [fields, setFields] = useState([]); + const [views, setViews] = useState([]); + const [activeViewId, setActiveViewId] = useState(ALL_VIEW_ID); + const [config, setConfig] = useState(() => defaultViewConfig([])); + const [workspaceReady, setWorkspaceReady] = useState(false); + const [saveState, setSaveState] = useState<"saved" | "saving">("saved"); + const [search, setSearch] = useState(""); + const [collapsed, setCollapsed] = useState>(new Set()); + const [hoverRow, setHoverRow] = useState(undefined); + /** + * Owner item 19 — the hover-only Expand button: which record it opens and where it sits, in + * VIEWPORT coordinates (glide's `getBounds` space, the same one `.cg-header-tip` uses). + * + * A real DOM button rather than a mark painted into the canvas, and that is the whole design: + * a drawn affordance needs a second hit-test that drifts out of step with the drawing, and + * neither `tsc` nor a screenshot can see the drift — the button is painted, and clicking it + * does nothing ([[ui-invisible-to-assertions]]). An element IS its own hit test. + */ + const [expandAt, setExpandAt] = useState< + { pid: number; x: number; y: number; size: number } | null + >(null); + const [detailPid, setDetailPid] = useState(null); + const [columnMenu, setColumnMenu] = useState(null); + /** Open choice list for a `select` / `user` cell — see onCellClicked. */ + const [picker, setPicker] = useState<{ + pid: number; + fieldKey: string; + anchor: AnchorRect; + } | null>(null); + /** Owner item 5 (2026-07-31) — closing a picker RETURNS FOCUS TO THE GRID, so Enter keeps + * navigating (pick → Enter → next record) instead of stranding focus on a dead overlay. */ + const closePicker = useCallback(() => { + setPicker(null); + requestAnimationFrame(() => gridRef.current?.focus()); + }, []); + /** ⭐ Wave-23 C7 — which `json` cell the big viewer is open on. NOT anchored like the picker + * above: a document is not a choice list, so it opens as a centred modal (the record drawer's + * surface) rather than a popover the size of the cell it came from. */ + const [jsonAt, setJsonAt] = useState<{ pid: number; fieldKey: string } | null>(null); + const closeJson = useCallback(() => { + setJsonAt(null); + requestAnimationFrame(() => gridRef.current?.focus()); + }, []); + /** A relation opens as the target database's real Grid view, projected to the linked ids. + * This is intentionally distinct from JsonViewer: filters, sorts, search, column visibility, + * and the standard cell renderers all remain available inside the large modal. */ + const [linkAt, setLinkAt] = useState<{ pid: number; fieldKey: string } | null>(null); + const closeLink = useCallback(() => { + setLinkAt(null); + requestAnimationFrame(() => gridRef.current?.focus()); + }, []); + /** WAVE 21 item 11 (R10) — is "Select records from a list" open? Opened from the view + * rail's "…" and closed by the dialog; the SELECTION it produces outlives it. */ + const [selectFromFile, setSelectFromFile] = useState(false); + const [importOpen, setImportOpen] = useState(false); + /** The selection bar's "Add to cohort" popover (owner item 2's purpose for the checkboxes). */ + const [selAddOpen, setSelAddOpen] = useState(false); + const [selListName, setSelListName] = useState(""); + const selAddRef = useRef(null); + /** Owner item 9 (wave 20) — its counterpart: "Remove from cohort…", in ORDINARY views. */ + const [selRemoveOpen, setSelRemoveOpen] = useState(false); + const selRemoveRef = useRef(null); + /** Owner item 4 / C-ADDROW — the pid the trailing "+" just created, so the cursor can land on + * it once the re-read actually brings it back (the row does not exist locally before that). */ + const [newRowPid, setNewRowPid] = useState(null); + /** + * ⭐ WARN-THEN-ALLOW on delete (owner, 2026-08-06) — the SIGNATURE of the selection currently + * armed, or `""`. + * + * A signature and not a boolean, deliberately: it is the pid list itself, so changing the + * selection DISARMS automatically. A boolean would stay true while you re-selected, and the + * second Delete would then destroy a set nobody had been warned about — which is worse than no + * warning at all, because the warning is what made it feel checked. + */ + const [delArmed, setDelArmed] = useState(""); + /** Wave-2 item 2c — cohort mode: the active cohort and the "+ Add customers" picker. Picks + * ACCUMULATE across searches (search, tick, search again, confirm once). */ + const [activeCohortId, setActiveCohortId] = useState(null); + const [addCustOpen, setAddCustOpen] = useState(false); + const [addCustQuery, setAddCustQuery] = useState(""); + const [addCustPicked, setAddCustPicked] = useState>(new Set()); + const addCustRef = useRef(null); + /** Wave-5 item 1 — "Filter by this field": a monotonic signal the Toolbar consumes (it + * appends the seeded condition and opens the builder). Wave-6 item 6c: a MEASURE-carrying + * column seeds the equivalent measure condition instead (same measure key + window). */ + const [filterSeed, setFilterSeed] = useState< + | { key?: string; measure?: { key: string; window: WindowSpec }; n: number } + | null + >(null); + /** Wave-6 item 4 — the header-row "+" (insert at end): anchors the create-only menu. */ + const [plusMenu, setPlusMenu] = useState(null); + /** + * Wave-6 item 3c — per-key freshness of this browser's own field-def writes, persisted with + * the workspace copy and consumed by reconcileFields at init. A ref, not state: stamps ride + * along with the setFields that caused them; nothing renders from a stamp. + */ + const fieldStampsRef = useRef<{ edited: Record; deleted: Record }>( + { edited: {}, deleted: {} } + ); + /** Wave-5 item 6 — the header description tip. Floated, pointer-events:none — a tooltip + * that can receive the pointer swallows the NEXT click ([[ui-invisible-to-assertions]]). */ + const [headerTip, setHeaderTip] = useState<{ text: string; x: number; y: number } | null>( + null + ); + /** Wave-9 I2 — the CELL tip: hovering a cell whose text is cut off reveals it in full. + * Same contract as the header tip above (floated, pointer-events:none, aria-hidden) for + * the same reason — a tooltip that can receive the pointer swallows the NEXT click, which + * this surface has already paid for once ([[ui-invisible-to-assertions]]). */ + const [cellTip, setCellTip] = useState<{ text: string; x: number; y: number } | null>(null); + + /** BUG-1 — viewId → the filter tree we have already pushed back to the host. Persisted with + * the local workspace because the remount it guards against destroys any ref. */ + const reemittedRef = useRef>({}); + /** 2026-08-04 — views THIS browser deleted, by id. Read by the live adopt below AND by + * init, so the mount path and the after-mount path answer "was this deleted here?" the + * same way. Persisted with the local workspace for the same reason `reemitted` is. */ + const viewTombstonesRef = useRef({}); + /** D-19 — viewId → when this browser last wrote it (see LocalWorkspace.viewWrites). */ + const viewWritesRef = useRef({}); + const initializedKey = useRef(null); + const saveTimer = useRef(null); + const gridRef = useRef(null); + const gridBoxRef = useRef(null); + /** Internal clipboard provenance. The browser exposes pasted text, not its source column; + * recording the grid's own copy keeps cross-field paste fail-closed without blocking + * ordinary external text pasted into an editable field. */ + const copyProvenanceRef = useRef(EMPTY_GRID_COPY_PROVENANCE); + useEffect(() => { + const onCopy = () => { + // The native copy event follows our grid key event. Preserve that provenance once; + // every later copy (address bar, another app surface, another input) invalidates it. + copyProvenanceRef.current = observeCopyEvent(copyProvenanceRef.current, Date.now()); + }; + const onWindowBlur = () => { + copyProvenanceRef.current = EMPTY_GRID_COPY_PROVENANCE; + }; + document.addEventListener("copy", onCopy, true); + window.addEventListener("blur", onWindowBlur); + return () => { + document.removeEventListener("copy", onCopy, true); + window.removeEventListener("blur", onWindowBlur); + }; + }, []); + const [gridSize, setGridSize] = useState({ width: 800, height: 600 }); + + /** + * ⛔ WAVE 21 item 3 (3c) — THE FALLBACK KEY IS THE SCOPE'S, and the literal it replaces is + * the cross-table bleed the owner reported as "RI fields on a new database". + * + * MEASURED, not theorised. `fetchWorkspace` collapses 403 / 5xx / network / unreadable into + * one silent `null` — deliberately, because the workspace is an ENHANCEMENT of the table and + * an older host simply does not serve one. But `null` also meant "no `storageKey`", and the + * fallback was the LITERAL `"customer-grid-standalone"` — ONE localStorage bucket for every + * scope this component can be mounted at. So on a tenant whose `/workspace` 403s for `ut_*` + * (routes_grid gated the route on `module_gate("customer_data")` — C4 fixes that half), the + * FIRST user table to be opened writes its views and custom fields into that bucket and the + * SECOND one reads them back as its own. Two databases with nothing to do with each other, + * sharing a schema, with no error anywhere. + * + * Scoping the key makes that structurally impossible: a bucket belongs to one surface, so + * the worst a failed workspace read can now do is show THIS table its own last-known local + * copy. The prefix marks it as the local-only stratum — a key the server never issues, so it + * can never collide with a real `storageKey` (those are `_table_workspace`-shaped). + * + * ⚠ Existing browsers lose the contents of the old shared bucket. That is the point: every + * byte in it is a workspace some other surface persisted, and there is no way to tell whose. + */ + const storageKey = payload?.workspace?.storageKey ?? `local:${scope}`; + + // Initialize once per permission/data scope. Host state wins; local state + // fills only missing objects and keeps the standalone path useful. + useEffect(() => { + if (!payload || payloadFields.length === 0 || initializedKey.current === storageKey) return; + const local = embedded ? null : readLocal(storageKey); + // Item 3c: the def half of the no-blip layer. A RECENT local stamp beats a + // lagged host echo (rename survives, retype holds, a delete stays deleted); + // a caught-up echo returns host objects byte-identical (see optimism.ts). + const stamps = pruneStamps(local?.fieldStamps, Date.now()); + fieldStampsRef.current = { + edited: { ...(stamps.edited ?? {}) }, + deleted: { ...(stamps.deleted ?? {}) }, + }; + const initialFields = reconcileFields( + payloadFields, + local?.fields ?? [], + stamps, + Date.now(), + payload.workspace != null + ); + const hostViews = embedded ? [] : payload.workspace?.views ?? []; + const byId = new Map(); + byId.set(ALL_VIEW_ID, allRecordsView(initialFields, scope)); + // D-19 — the HOST'S LIST DECIDES WHICH VIEWS EXIST. A local copy the host no longer + // names is a ghost (deleted elsewhere, share revoked, store moved) and is dropped here, + // guarded by this browser's own recent writes so an optimistic create survives. + viewWritesRef.current = pruneTombstones(local?.viewWrites, Date.now()); + for (const view of seedLocalViews(local?.views ?? [], hostViews, viewWritesRef.current, + Date.now(), payload.workspace != null)) + byId.set(view.id, view); + // Host state wins — EXCEPT when it is a lagged echo of this browser's own + // in-flight edit, where taking it would turn a just-completed measure rule + // valueless: inactive, no pending marker, whole-book count. A rerun replaces + // the iframe, so this init runs after every host round trip; the reconcile + // is what keeps a rule with a typed rhs PENDING rather than inactive across + // that remount (see viewEcho.ts). + // + // BUG-1 (wave 11) — which views carry a FILTER TREE the host copy does not have, and have + // not already been told about. `echoReemit` owns both guards (see viewEcho.ts): identity + // alone would also fire on a display-only reconcile, and a time-based bound cannot work + // because the local-workspace effect below restamps `savedAt` on every init. + const reemit = new Map(); + // 2026-08-04 — the views tombstone map, seeded from the persisted copy so a remount + // INSIDE the echo window cannot walk back a view this browser just deleted. Init had + // no such guard before (fields and folders both did), which was survivable only while + // every host round trip was itself a remount. + const tombstones = pruneTombstones(local?.viewTombstones, Date.now()); + viewTombstonesRef.current = tombstones; + for (const view of hostViews) { + // Both doors, the `reconcileFields` rule: a tombstone drops the host copy AND any + // local copy, so a delete that raced localStorage cannot re-enter through either. + if (tombstones[view.id] !== undefined) { + byId.delete(view.id); + continue; + } + const merged = reconcileEchoView( + view, byId.get(view.id), local?.savedAt, Date.now() + ); + const key = echoReemit(merged, view, local?.reemitted?.[view.id]); + if (key !== null) reemit.set(view.id, key); + byId.set(view.id, merged); + } + const initialViews = [...byId.values()].map((view) => ({ + ...view, + config: normalizeConfig(view.config, initialFields), + })); + const requested = + local?.activeViewId ?? payload.workspace?.activeViewId ?? ALL_VIEW_ID; + const active = initialViews.find((view) => view.id === requested) ?? initialViews[0]; + setFields(initialFields); + setViews(initialViews); + setActiveViewId(active.id); + setConfig(active.config); + setWorkspaceReady(true); + initializedKey.current = storageKey; + + /** + * BUG-1 (wave 11) — TELL THE HOST. The reconcile above was right about the state and silent + * about the consequence: it restored a rule this browser completed and the host copy does + * not have, and then nothing sent it. The autosave effect cannot: init seeds `views` AND + * `config` from the same reconciled object, so `sameConfig` reports "saved" and no upsert + * is ever emitted. The rule stays ACTIVE client-side, PENDING forever, and unknown to + * `_cl_resolve_measures` — which is exactly W11-HOST's live trace, a pending view carrying + * no measure rule at all while the saved-view path resolves correctly. + * + * ⚠ THIS TERMINATES, and NOT because the host will store what we send. It might not: a + * measure key outside this user's BU offer is stripped by `clean_filter_tree`, so the host + * copy never gains the rule and `lagged` stays true forever. Termination comes from the + * IDEMPOTENCE KEY — the filter tree itself, persisted per view — so we tell the host about + * a given tree exactly once and only speak again when the USER edits the conditions. + * ⚠ A ref cannot hold that: the remount this exists to survive destroys it. It rides in + * localStorage beside the views, and `writeLocal` below carries it forward. + * + * Emitted from the NORMALIZED copy (`initialViews`), never from the raw host view: that is + * the config the client is actually filtering with, and sending anything else would ask the + * host to resolve a question nobody on screen is asking. + */ + if (!embedded && reemit.size > 0) { + const stamped = { ...(local?.reemitted ?? {}) }; + for (const view of initialViews) { + const key = reemit.get(view.id); + if (key === undefined) continue; + viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id, + Date.now()); + viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id, Date.now()); + emitHostEvent({ id: eventId("view"), type: "view_upsert", view }); + stamped[view.id] = key; + } + reemittedRef.current = stamped; + } else { + reemittedRef.current = { ...(local?.reemitted ?? {}) }; + } + }, [payload, payloadFields, storageKey, embedded, scope]); + + useEffect(() => { + if (!workspaceReady || embedded) return; + writeLocal(storageKey, { + fields, + views, + activeViewId, + savedAt: Date.now(), + fieldStamps: pruneStamps(fieldStampsRef.current, Date.now()), + // BUG-1 — carried forward, never recomputed here. Dropping it would restore the loop: + // the next init would find no record of what it already told the host and say it again. + reemitted: reemittedRef.current, + // Pruned on the way out, like every other stamp map here: the persisted blob is a + // recent window, never an archive of everything this tab ever deleted. + viewTombstones: pruneTombstones(viewTombstonesRef.current, Date.now()), + // D-19's other half — without persisting these, a reload inside the echo window would + // drop a view this browser created seconds ago. + viewWrites: pruneTombstones(viewWritesRef.current, Date.now()), + }); + }, [workspaceReady, storageKey, fields, views, activeViewId, embedded]); + + /** + * ⭐ THE LIVE WORKSPACE (owner report, 2026-08-04) — what has APPEARED since we mounted. + * + * The init effect above runs ONCE per mount (`initializedKey`), which under Streamlit was + * indistinguishable from "once per host round trip" because a rerun replaces the iframe. + * The standalone shell has no rerun: this component stays mounted until `key={active.key}` + * changes in Shell.tsx — switching modules. So a cohort created here reached the store, + * came back in the very next `/workspace` re-read, sat in `payload.workspace.views`, and + * never reached the rail. That is the owner's report, exactly: it appears after you visit + * another module and come back, because that is the only thing that remounts this tree. + * + * ⚠ FIELDS FIRST, AND THE ORDER IS LOAD-BEARING. `fields_from_workspace(ws, + * cohorts=bool(cohort_lists))` emits the derived "Locked views" column only once a cohort + * EXISTS — so a user's FIRST cohort changes the field contract in the same payload that + * carries the new view. The adopted views are normalized against `nextFields`, not against + * the `fields` state (which React has not committed yet), or a projected `config.order` + * naming the new column would be normalized against a list that does not have it. + * + * ⚠ Add-only, tombstoned, and identity-stable — see liveWorkspace.ts for why each of those + * is a correctness property rather than a nicety. + */ + const hostWorkspaceViews = payload?.workspace?.views; + useEffect(() => { + if (!workspaceReady || embedded) return; + const now = Date.now(); + const nextFields = adoptNewFields( + fields, payloadFields, fieldStampsRef.current.deleted, now + ); + if (nextFields !== fields) setFields(nextFields); + setViews((current) => + adoptNewViews(current, hostWorkspaceViews, viewTombstonesRef.current, now, + (config) => normalizeConfig(config, nextFields)) + ); + }, [workspaceReady, hostWorkspaceViews, payloadFields, fields, embedded]); + + /** Item 3c — stamp a def write / a delete. Pruned at every touch so the persisted blob + * stays a recent window, never an archive. */ + const stampFieldEdit = useCallback((key: string) => { + const s = pruneStamps(fieldStampsRef.current, Date.now()); + fieldStampsRef.current = { + edited: { ...(s.edited ?? {}), [key]: Date.now() }, + deleted: { ...(s.deleted ?? {}) }, + }; + }, []); + const stampFieldDelete = useCallback((key: string) => { + const s = pruneStamps(fieldStampsRef.current, Date.now()); + const edited = { ...(s.edited ?? {}) }; + delete edited[key]; + fieldStampsRef.current = { + edited, + deleted: { ...(s.deleted ?? {}), [key]: Date.now() }, + }; + }, []); + + const updateConfig = useCallback( + (next: ViewConfig) => { + if (next.groupBy !== config.groupBy) setCollapsed(new Set()); + setConfig(next); + }, + [config.groupBy] + ); + + const { + visibleCols, + fieldByKey, + order, + visible, + lockedKey, + onColumnResize, + onColumnMoved, + onColumnProposeMove, + setColumnVisible, + insertColumn, + } = useGridColumns(fields, config, updateConfig); + + /** + * ⭐ Wave-20 owner item 4 (ruling R8, contract C-ADDROW) — **THE GHOST ROW.** + * + * A trailing "+" row at the bottom of the grid, the way Airtable grows a table, replacing the + * "Add record" button that used to sit in the shell's header bar (S4 deleted it this wave). + * + * ⛔ USER DATABASES ONLY. A connector's rows are read-synced from Odoo; a "+" there could only + * refuse, and R8 names that a fake affordance. The test is the SCOPE (`ut_*`), which is also + * the only scope with a rows endpoint to POST to — so the affordance and the capability come + * from the same fact rather than from two lists that can drift. + * + * ⚠ THE ROW IS NOT ADDED LOCALLY. `rawRows` is the server's answer, and a client-invented row + * would have no rid, no defaults and no place in anyone else's copy. The POST clears the rows + * cache and fires `ROWS_STALE_EVENT`, `useCustomerData` re-reads, and the row arrives with the + * id the store gave it. The cursor then follows it (`newRowPid` + the effect below) — which is + * why the pid is remembered instead of glide's `"bottom"` being returned here: at the moment + * this resolves, the appended row does not exist yet, so "bottom" would land on the last OLD + * row. + */ + const isUserTable = !embedded && scope.startsWith("ut_"); + const recordsMutable = payload?.recordsMutable !== false; + const canMutateRecords = isUserTable && recordsMutable; + /** + * ⭐ 2026-08-07 — the databases a `link` column may point at, fetched when the column menu + * OPENS rather than on every render of the grid. + * + * ⚠ ON DEMAND IS THE WHOLE DESIGN. `GET /tables` walks every table in the tenant and returns + * their full field lists; hanging that off the grid's mount would put a workspace-wide read + * behind every page view for a picker almost nobody opens. The column menu is the only surface + * that needs it, so it is the surface that asks. + * ⚠ The list is left standing once fetched — a database created in another tab mid-session is + * a staleness a menu re-open corrects, and re-fetching per open would spend the same read + * repeatedly for a list that changes about once a week. + */ + const [linkTargets, setLinkTargets] = useState([]); + useEffect(() => { + // ⛔⛔ 2026-08-09 — `plusMenu` TOO, AND THAT OMISSION WAS THE WHOLE BUG. Owner: *"I am not + // able to see Choose Column for Post rows link"*. There are TWO doors into this editor — + // a column's own menu (`columnMenu`) and the "+" Add-field button (`plusMenu`) — and this + // effect knew about one. Via "+", `linkTargets` stayed `[]`, so `target` never resolved and + // the rollup Column picker held nothing but its placeholder, permanently. Editing an + // EXISTING column worked, which is what made it look like a rollup bug rather than a fetch + // that never fired. MEASURED on the live app: 5 selects rendered, "Column to roll up" with + // exactly 1 option while `GET /tables` returns `ut_ig_posts` with 22 fields. + // ⚠ A picker that is EMPTY and a picker whose data never loaded look identical, which is + // why this survived: both render "Choose a column…" over nothing. + if ((!columnMenu && !plusMenu) || linkTargets.length) return; + let live = true; + void fetchLinkTargets().then((t) => { + if (live) setLinkTargets(t); + }); + return () => { + live = false; + }; + }, [columnMenu, plusMenu, linkTargets.length]); + /** + * ⭐⭐ 2026-08-09 — the READ-THROUGH rollup's offer, fetched on the same terms as the link + * targets above: only when the column menu opens, and once per session. + * + * ⚠ THE GUARD IS A SEPARATE `asked` FLAG, NOT `topics.length`. An empty offer is the CORRECT + * and common answer (a tenant with nothing connected), so guarding on the length would re-ask + * on every single menu open for exactly the workspaces where the answer can never change — + * the one case the "fetch once" rule exists for. + */ + const [rollupSourceOffer, setRollupSourceOffer] = + useState({ topics: [], windows: [] }); + const rollupOfferAsked = useRef(false); + useEffect(() => { + // ⛔ SAME TWO DOORS as the link targets above. This one is less visible because an empty + // offer is the correct answer for a tenant with nothing connected — so a source offer that + // never loaded is indistinguishable from one that is legitimately empty, and the mode switch + // simply never appears. Fixing only the link half would have left that asymmetry in place. + if ((!columnMenu && !plusMenu) || rollupOfferAsked.current) return; + rollupOfferAsked.current = true; + let live = true; + void fetchRollupSources().then((o) => { + if (live) setRollupSourceOffer(o); + }); + return () => { + live = false; + }; + }, [columnMenu, plusMenu]); + /** + * ⭐ WAVE 27 · OWNER ITEM 22 — the values the ACTIVE VIEW forces on a new record. + * + * ⛔ THE DEFECT: "+" posted `{values: {}}` on every view, so a record added while a filter was + * on was created, was real, and was invisible — the button read as broken and the row the user + * then typed into did not exist as far as they could see. `filterSeed.ts` owns the derivation + * and its whole design is how much it REFUSES to derive (negations, ranges, `or` branches, + * contradictions, machine columns — each with its reason at the clause). + * + * ⚠ ONE FUNCTION, TWO CALLERS, deliberately: item 2's optimistic add must insert the row + * carrying these same cells, or the row it paints locally fails the filter it was added under + * and disappears on the next read — the exact blip C2 exists to prevent. + */ + const seedValues = useCallback( + () => filterSeedValues(config.filters, config.filterConj, fields, payload?.viewer), + [config.filters, config.filterConj, fields, payload?.viewer] + ); + /** + * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — the "+" paints its row NOW. + * + * ⛔ THE ID IS MINTED THE WAY THE SERVER MINTS IT — `max(numeric row id) + 1`, which is + * `core.user_tables.add_row`'s own rule — and then POSTed as `{rid, values}` through the door + * the undo path already opened (C-ADDROW). That is what makes the optimism honest rather than + * hopeful: on the overwhelmingly common single-writer path the server stores exactly the id + * this browser drew, so the row on screen and the row in the store are the same record from + * the first frame. + * + * ⚠ AND WHEN THEY ARE NOT, IT RE-ANCHORS. Two people adding at once means the second POST + * finds the id taken; `add_row` falls back to `max+1` and ANSWERS with what it actually wrote + * (its own note says so), so the local row is re-keyed to the returned pid. Assuming the + * requested id came back is the one way this could leave a row on screen that no longer + * matches anything in the store. + * + * ⛔ NO `ROWS_STALE_EVENT` ON THE HAPPY PATH. That event re-reads the whole table, which is + * the cost item 2 exists to remove — and re-reading would also un-paint the row for the + * duration of the fetch, which is the NO-BLIP law's exact subject. The pending row is pruned + * by the effect above when a payload that already contains it arrives, from whatever cause. + * + * ⚠ THE NARROW RACE, STATED: a cell PATCHed within the POST's round trip could reach the + * server before the row exists. The window is now ~20ms (A moved the relation refresh off + * this path), it needs a keystroke inside it, and the PATCH's failure is a rolled-back + * overlay edit rather than lost data — the alternative, blocking the paint until the POST + * resolves, is the defect being fixed. + */ + const appendRow = useCallback(async (): Promise => { + if (!canMutateRecords) return undefined; + const seeded = seedValues(); + const minted = mintRid(); + const optimistic: Row = { ...seeded, pid: minted }; + setPendingRows((cur) => [...cur, optimistic]); + setNewRowPid(minted); + const made = await addTableRow(scope, seeded, minted); + if (!made) { + // `addTableRow` already said WHY, in the server's own words (a row cap, a refused profile + // cell, an unreachable host). The row is withdrawn rather than left standing: a row that + // survives its own failed write is the lie this whole path has to avoid. + setPendingRows((cur) => cur.filter((r) => r.pid !== minted)); + setNewRowPid(null); + return undefined; + } + if (made.pid !== minted) { + setPendingRows((cur) => + cur.map((r) => (r.pid === minted ? { ...r, pid: made.pid } : r)) + ); + setNewRowPid(made.pid); + } + // R4 — the append is undoable: Ctrl+Z deletes the row it just created, and a redo restores + // it under the SAME rid (the server's `{rid}` passthrough, C-ADDROW). + // ⛔ ITEM 22 MOVED THIS LINE. It read `values: {}` with a comment saying "an empty new record + // carries no values" — true until this wave, and now false: a row added on a filtered view is + // born holding the cells that filter forces. `redo` replays `r.values` verbatim + // (`:1569`), so leaving the literal would have made Ctrl+Z → Ctrl+Y restore the row under the + // right id with its seeded cells silently dropped — [[undo-capture-before-the-write]] in its + // quietest form, since the id and the row count would both be right. + undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { + kind: "rowAdd", table: scope, rows: [{ rid: made.rid, values: seeded }], + }); + // ⛔ NO `signal(ROWS_STALE_EVENT)` HERE ANY MORE (item 2 / C2). It re-read the whole table + // on every "+", which is the whole cost the owner reported — and every OTHER caller of it + // still fires, so a row created by an automation, another tab or an undo still arrives. + // The one thing that used to depend on this refetch was the cursor landing on the new row, + // and the row is now on screen before the POST resolves, so it lands immediately. + // `undefined` is returned rather than glide's "bottom": "bottom" is resolved against the + // rows glide knew about when the gesture started, which is the row BEFORE this one. + return undefined; + }, [canMutateRecords, scope, seedValues, mintRid]); + + /* wave20 item 2 — measure key + window -> the columns that display it. Built ONCE per field + list and handed to every consumer of "which column is this rule about", so the tint + (`columnTones`, inside the hook above) and the column menu's filter doors resolve a + measure condition the same way. See `types.ruleColumnKeys`. */ + const measureCols = useMemo(() => measureColumnIndex(fields), [fields]); + + // Airtable behavior: configuration changes to the active view autosave. + useEffect(() => { + if (!workspaceReady || embedded) return; + const active = views.find((view) => view.id === activeViewId); + if (!active || sameConfig(active.config, config)) { + setSaveState("saved"); + return; + } + setSaveState("saving"); + if (saveTimer.current !== null) window.clearTimeout(saveTimer.current); + saveTimer.current = window.setTimeout(() => { + const updated = { ...active, config }; + setViews((current) => + current.map((view) => (view.id === updated.id ? updated : view)) + ); + viewWritesRef.current = stampTombstone(viewWritesRef.current, updated.id, + Date.now()); + emitHostEvent({ id: eventId("view"), type: "view_upsert", view: updated }); + setSaveState("saved"); + saveTimer.current = null; + }, 420); + return () => { + if (saveTimer.current !== null) window.clearTimeout(saveTimer.current); + }; + }, [workspaceReady, activeViewId, config, views, embedded]); + + // Numeric dimensions force a glide relayout when either the component frame + // or Streamlit's main column changes width (notably sidebar collapse). + useLayoutEffect(() => { + const element = gridBoxRef.current; + if (!element) return; + const measure = () => { + const rect = element.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + setGridSize({ + width: Math.floor(rect.width), + height: Math.floor(rect.height), + }); + } + }; + measure(); + const observer = new ResizeObserver(measure); + observer.observe(element); + window.addEventListener("resize", measure); + return () => { + observer.disconnect(); + window.removeEventListener("resize", measure); + }; + }, [workspaceReady]); + + // CG-3: one mode per table, decided by whether the payload is a window. See types.ts + // `TableMode` for why both engines running at once is the failure this prevents. + const mode = tableMode(payload?.counts); + const serverWindowed = mode === "server-windowed"; + // Owner items 4+6 — the Cohort page's grid renders without the Views sidebar. + const hideViews = embedded || payload?.workspace?.hideViews === true; + // Wave-6 item 10 — how this view displays. A WINDOWED table is always the grid: list/ + // calendar/kanban compute over the whole matched set, and one page is not it (CG-3's rule, + // the same reason grouping is off there). + const displaySpec = cleanDisplay(config.display); + const displayMode: DisplayMode = serverWindowed ? "grid" : displaySpec?.mode ?? "grid"; + + // CG-8 — measure conditions, answered by the host as `{ruleId: pid[]}`. Converted to Sets + // once here rather than per row: the pipeline runs this over every row of the book. + // + // ⚠ REPLACED, not merged. Merging looks like the safer choice — "hold the previous result" + // is the design — but it holds it FOREVER: the host omits a rule's id when it could not + // resolve it (a warming store, a scope change mid-flight), and a merged map keeps yesterday's + // answer under today's question with `pendingMeasures` reporting zero, so the marker never + // shows. That is the widening sin wearing a confident count. + // + // Holding the previous result across a LOCAL edit does not need the merge and never did: the + // memo is keyed on `payload.measureSets` identity, which does not change while the user is + // typing, so the ref already carries the last answers through to the next payload. Merge and + // replace therefore differ only in the dangerous case. + const measures = useMemo(() => payload?.measures ?? [], [payload?.measures]); + const measureSetsRef = useRef({}); + const measureSets = useMemo(() => { + const incoming = payload?.measureSets; + if (incoming) { + const next: MeasureSets = {}; + for (const [ruleId, pids] of Object.entries(incoming)) next[ruleId] = new Set(pids); + measureSetsRef.current = next; + } + return measureSetsRef.current; + }, [payload?.measureSets]); + const pendingMeasureCount = useMemo( + () => pendingMeasures(config.filters, measureSets), + [config.filters, measureSets] + ); + // BUG-1 (wave 11) — the two key sets, published for the handshake. See the `.cg-shell` + // attributes below for why they are DOM attributes and not console logs. + const measureRuleKeys = useMemo( + () => activeMeasureRuleIds(config.filters).join(","), + [config.filters] + ); + const measureSetKeys = useMemo(() => Object.keys(measureSets).join(","), [measureSets]); + + // Owner item 5. Cohort membership rides `workspace.lists`, the same array the "Add to list" + // picker already reads, so the names offered and the sets tested come from one place. A + // cohort with no `pids` is a target you can add to but cannot filter on; the engine treats it + // as unanswerable (matches nothing) rather than absent (matches everything). + const lists = useMemo(() => payload?.workspace?.lists ?? [], [payload?.workspace?.lists]); + /* wave17 R1 / C-LOCKV — `railCohortId` is gone. It held the TRANSIENT lock the retired + Cohorts section applied on click: a lock that lived in component state and was never + persisted. A locked view now carries its own `config.cohortLock`, so SELECTING the view + is the lock and the saved config is the only source. One mechanism, and it survives a + reload — which the transient one never did. */ + const cohortSets = useMemo(() => { + const out: CohortSets = {}; + for (const l of lists) if (l.pids) out[l.id] = new Set(l.pids); + return out; + }, [lists]); + // The TENANT'S today. Never `new Date()`: a browser a day ahead of the server would resolve + // "the past month" to a different month than the host summed, and the count would deny the + // rows with nothing erroring. + const today = payload?.today; + // Wave-5 item 1 — who is looking (host-computed). ONE editability verdict for every edit + // door (cells, pickers, drawer): types.mayEditField — stratum + read-only-by-nature + + // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown. + const viewer = payload?.viewer; + const canEditField = useCallback( + (f: Field): boolean => !embedded && recordsMutable && mayEditField(f, viewer), + [embedded, recordsMutable, viewer] + ); + const unresolvedCount = useMemo( + () => unresolvedConditions(config.filters, { cohortSets, today }), + [config.filters, cohortSets, today] + ); + + // Wave-5 items 9/11 — CLIENT-COMPUTED cells, injected AT their field keys so the whole + // pipeline (filter/search/sort/group), the cells and the drawer read them like any other + // value. Formulas parse ONCE per definition change (never per row); evaluation reads the + // row WITH this session's overlay edits layered, so editing a referenced field recomputes + // live. A formula that does not parse, or errors on a row, yields BLANK — never a wrong + // number (formulaEngine.ts). `created_time` copies the row's `_created`. + // 2026-07-31 (owner item 2): formulas may reference OTHER formulas now, so parse order is + // TOPOLOGICAL (orderFormulas) — a formula runs after the formulas it reads, cycle members + // never run (blank, never a stale number), and each row's results feed the next formula + // through a per-row scope. + const formulaAsts = useMemo(() => { + const sources = new Map(); + for (const f of fields) { + if (f.type !== "formula") continue; + const src = formulaOf(f); + if (src) sources.set(f.key, src); + } + const { order, cyclic } = orderFormulas(sources); + const out: { key: string; ast: FormulaAst }[] = []; + for (const key of order) { + if (cyclic.has(key)) continue; + const p = parseFormula(sources.get(key)!); + if (p.ok) out.push({ key, ast: p.ast }); + } + return out; + }, [fields]); + const createdTimeKeys = useMemo( + () => fields.filter((f) => f.type === "created_time").map((f) => f.key), + [fields] + ); + const computedRows = useMemo(() => { + if (formulaAsts.length === 0 && createdTimeKeys.length === 0) return rawRows; + const env = { today: payload?.today }; + return rawRows.map((r) => { + const edits = overlayEdits[r.pid]; + const scope: Row = edits ? { ...r, ...edits } : { ...r }; + const out: Row = { ...r }; + for (const k of createdTimeKeys) { + out[k] = (r._created as string | undefined) ?? null; + scope[k] = out[k]; + } + for (const { key, ast } of formulaAsts) { + const v = evalFormula(ast, (k) => scope[k], env); + out[key] = v; + scope[key] = v; // later formulas read this one's result — the topo order above + } + return out; + }); + }, [rawRows, overlayEdits, formulaAsts, createdTimeKeys, payload?.today]); + + // Wave-2 item 2c — COHORT MODE (the Cohort page). The host serves the WHOLE pool (rows + + // derived values over the pool); the ACTIVE cohort scopes the table to its pids CLIENT-side. + const cohortMode = payload?.workspace?.cohortMode === true; + useEffect(() => { + if (!cohortMode) return; + if (activeCohortId && lists.some((l) => l.id === activeCohortId)) return; + setActiveCohortId(lists[0]?.id ?? null); + }, [cohortMode, lists, activeCohortId]); + const activeCohort = cohortMode + ? lists.find((l) => l.id === activeCohortId) ?? null + : null; + const cohortMemberSet = useMemo( + () => new Set(activeCohort?.pids ?? []), + [activeCohort] + ); + // The FIXED subset is the scope. A cohort with no membership shows nothing — falling back to + // the whole pool would put 1,500 rows under a rail entry that says 12, which is the widening + // sin wearing a sidebar. Everything downstream (pipeline, counts, selection, detail) runs + // over this scope, so the toolbar count is the cohort's matched count by construction. + const scopedRows = useMemo( + () => + cohortMode + ? computedRows.filter((r) => cohortMemberSet.has(r.pid)) + : computedRows, + [cohortMode, computedRows, cohortMemberSet] + ); + + const { visibleRows, pidToIndex } = useVisibleRows( + scopedRows, + fields, + config.filters, + search, + config.sorts, + // grouping a WINDOW would headline a page's count/subtotals as the group's + serverWindowed ? null : config.groupBy, + collapsed, + config.memberPids, + config.filterConj, + serverWindowed, + measureSets, + cohortSets, + today, + // Item 12 (C-LOCK) — the 14th positional, RECORD's engine input. The lock intersects + // FIRST, so everything downstream (conditions, memberPids, the ranking domain) operates + // inside the cohort. Absent = today's behaviour exactly. + // wave17 R1 / C-LOCKV: the view's SAVED lock is now the only source. A locked view is a + // saved view whose `config.cohortLock` names its own id, so opening it IS applying the + // lock — the wave-15 transient rail pick that used to outrank this is gone with its rail. + config.cohortLock + ); + + // Owner item 8 — the DISPLAY cap. The pipeline above still ran over the whole book (counts, + // "Add to list" and selection-by-pid all depend on that); only what glide PAINTS is sliced. + // pidToIndex is rebuilt over the slice because every index consumer (selection, detail + // navigation, hover) is positional against what is actually on screen. + const [displayCap, setDisplayCap] = useState(DISPLAY_PAGE); + const capped = visibleRows.length > displayCap; + /** + * ⭐⭐ WAVE-29 T33 (owner item 17) — THE TOTALS ROW, and the cap fix it forced. + * + * ⛔ THE SUMMARY IS OVER ALL M MATCHED ROWS, NEVER THE N PAINTED. The pipeline above already + * ran over the whole book; only what glide paints is sliced. A total computed from the slice + * would silently describe the first `DISPLAY_PAGE` records while sitting under a bar that says + * "Showing first N of M" — an answer to a question nobody asked, wearing the right label. + * + * ⛔ AND THE SLICE ITSELF WAS ALREADY WRONG FOR GROUPS. `visibleRows` is FLATTENED (header, + * rows, footer, header, …), so a straight `slice(0, cap)` could cut a group between its last + * row and its footer — the subtotal simply vanished, for the groups furthest down, with no + * marker of any kind. `sliceForDisplay` counts DATA rows toward the cap and keeps the + * structural rows of every group it admits, so a group is whole or absent. + */ + const totalsAggs = useMemo( + () => computeAggs( + visibleRows.flatMap((vr) => (vr.kind === "data" ? [vr.record] : [])), + fields + ), + [visibleRows, fields] + ); + const showTotals = useMemo( + () => !config.groupBy && Object.keys(totalsAggs).length > 0, + [config.groupBy, totalsAggs] + ); + const displayRows = useMemo(() => { + const shown = capped ? sliceForDisplay(visibleRows, displayCap) : visibleRows; + // ⚠ APPENDED AFTER THE SLICE, so the cap can never eat the totals row itself — and it is the + // LAST row, which is what `freezeTrailingRows={1}` pins. + return showTotals + ? [...shown, + { kind: "group-footer" as const, groupKey: TOTAL_GROUP_KEY, aggs: totalsAggs }] + : shown; + }, [capped, visibleRows, displayCap, showTotals, totalsAggs]); + const displayPidToIndex = useMemo(() => { + if (!capped) return pidToIndex; + const m = new Map(); + displayRows.forEach((vr, i) => { + if (vr.kind === "data" && !m.has(vr.record.pid)) m.set(vr.record.pid, i); + }); + return m; + }, [capped, displayRows, pidToIndex]); + + // C-AVATAR (wave-14 item 11) — profile photos, absent until HOST serves them. The tick is the + // async half: `cells.setAvatarRepaint` fires it when an Image finishes decoding, and it is a + // dep of `getCellContent`, which is the only thing glide watches. See useGetCellContent. + const [avatarTick, setAvatarTick] = useState(0); + useEffect(() => { + setAvatarRepaint(() => setAvatarTick((t) => t + 1)); + return () => setAvatarRepaint(undefined); + }, []); + const userAvatars = payload?.workspace?.userAvatars; + /** + * Item 15 — the frozen strip's usable width, which is what a GROUP BAR's label is clipped to + * now that the first column is pinned in every mode. `null` when nothing is grouped (no bar to + * clip) — see `fitGroupLabel` for why the clip exists and why the count outranks the name. + */ + const groupLabelSpace = useMemo(() => { + if (!config.groupBy) return null; + const n = Math.min(frozenCountOf(config), visibleCols.length); + let px = 0; + // `GridColumn` is a union — only its SIZED member declares `width`, and `useGridColumns` + // always builds that one. Narrowed rather than asserted so an auto-sized column (which we + // do not create) would read 0 and simply not be counted, instead of throwing in a memo. + for (let i = 0; i < n; i += 1) + px += (visibleCols[i] as { width?: number } | undefined)?.width ?? 0; + return Math.max(0, px - GROUP_LABEL_PAD); + }, [config, visibleCols]); + /** + * ⭐ owner item 2 (2026-08-03) — WHICH MEASURE COLUMNS ARE STILL BEING CALCULATED. + * + * A measure column's numbers are resolved by the server (one aggregate over the whole book) + * and reach the browser on the workspace re-read that follows the create — seconds later. + * In between, the column is on screen with nothing in it, which the owner correctly read as + * an error rather than as a wait. + * + * ⛔ `editRequestId` IS THE TEST, and it is the honest one. It is set optimistically at the + * moment the create is emitted and cleared by `reconcileFields` when the host's own copy of + * that field comes back — and the host's copy travels in the SAME `/workspace` response as + * `derived`, which is where the values are. So the flag is true across exactly the window + * where the column exists and its numbers do not, and false the instant they land. + * + * The alternative — "no value anywhere in the column" — cannot tell a pending column from + * one that permanently failed to resolve (a BU-scoped caller on a company-level measure, + * say), and would spin forever on the second. This one resolves either way: when the echo + * arrives with no values, the flag clears and the cells go honestly blank. + */ + const pendingMeasureKeys = useMemo(() => { + const out = new Set(); + for (const f of fields) + if (f.editRequestId && f.key.startsWith("measure_")) out.add(f.key); + return out; + }, [fields]); + /** + * The skeleton's pulse. A canvas cell cannot hold a CSS animation, so the motion is repaints: + * this counter is a dependency of `getCellContent`, which is the only thing glide watches. + * + * ⚠ IT RUNS ONLY WHILE SOMETHING IS PENDING, and the effect's own guard is what stops it — + * an interval left running would repaint the whole canvas ~7×/s forever, on every grid, to + * animate nothing. 140ms × the 4-step ramp is a ~0.6s cycle: a wait, not a strobe. + */ + const [pulse, setPulse] = useState(0); + /** + * ⛔ AND IT GIVES UP. A skeleton that never resolves is worse than the blank it replaced: a + * blank cell is at least honest about having no number, while a permanent shimmer promises + * one that is never coming. + * + * The window it guards is narrow but real. `editRequestId` clears when `reconcileFields` takes + * the host's copy — and that function has a branch (`hostAuthoritative: false`, i.e. a payload + * with no workspace) whose `{...local, ...host}` spread would PRESERVE the flag forever. Today + * that branch cannot be reached with a measure column on screen (measures are offered through + * the workspace, so a payload without one cannot have produced this field), which is an + * argument about the current call graph and not a property of the code. This bound holds + * whether or not the argument stays true, and it costs one boolean. + */ + const [pendingGaveUp, setPendingGaveUp] = useState(false); + useEffect(() => { + setPendingGaveUp(false); + if (pendingMeasureKeys.size === 0) return; + let n = 0; + const id = window.setInterval(() => { + n += 1; + if (n > PULSE_MAX_TICKS) { + window.clearInterval(id); + setPendingGaveUp(true); // fall back to ordinary blank cells + return; + } + setPulse((p) => p + 1); + }, PULSE_MS); + return () => window.clearInterval(id); + }, [pendingMeasureKeys]); + const activePendingKeys = pendingGaveUp ? NO_PENDING_KEYS : pendingMeasureKeys; + const getCellContent = useGetCellContent( + displayRows, + visibleCols, + fieldByKey, + overlayEdits, + canEditField, + userAvatars, + avatarTick, + groupLabelSpace, + measureGroupText, + activePendingKeys, + pulse + ); + const { gridSelection, selectedPids, onGridSelectionChange, selectPids, togglePid, + setActiveCell, clearSelection } = + useGridSelection( + displayRows, + displayPidToIndex, + visibleCols.length, + embeddedSelectable ? embeddedSelectedIds : [] + ); + const embeddedSelectedKey = embeddedSelectedIds.join(","); + const selectedPidKey = [...selectedPids].join(","); + useEffect(() => { + if (!embeddedSelectable) return; + if (selectedPidKey !== embeddedSelectedKey) + selectPids(embeddedSelectedIds as number[], "replace"); + }, [embeddedSelectable, embeddedSelectedKey, embeddedSelectedIds, selectedPidKey, selectPids]); + useEffect(() => { + if (!embeddedSelectable) return; + onEmbeddedSelectionChange?.([...selectedPids]); + }, [embeddedSelectable, onEmbeddedSelectionChange, selectedPids]); + /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════ + THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one + thing it cannot: reading the value a cell held BEFORE the write, which only exists at the + call site. Everything below funnels through `patchAndRecord` / `patchManyAndRecord`, so a + write path that forgets to record is a write path that does not reach the store either. + + ⚠ A REF, NOT STATE. Nothing on screen depends on the stack in v1 (no undo button), so + keeping it in state would repaint the grid on every keystroke of a paste for no pixels. + Per SCOPE (a Ctrl+Z on the Customer grid must never rewrite a user table's cell) and per + tab (nothing persists it — a stack restored into a session that did not make those edits + would undo somebody else's work). */ + const undoBook = useRef({}); + const applyingUndo = useRef(false); + + /** The value a cell holds RIGHT NOW: the overlay stratum wins over the payload row, exactly + * as `useGetCellContent` renders it — so what undo restores is what was on screen. */ + const rowByPid = useMemo(() => { + const map = new Map(); + for (const r of rawRows) map.set(r.pid, r); + return map; + }, [rawRows]); + const currentValue = useCallback( + (pid: number, key: string): UndoValue => { + const edited = overlayEdits[pid]?.[key]; + const raw = edited !== undefined ? edited : rowByPid.get(pid)?.[key]; + return raw == null ? null : (raw as UndoValue); + }, + [overlayEdits, rowByPid] + ); + + const recordCells = useCallback( + (changes: CellChange[], label: string) => { + if (applyingUndo.current || changes.length === 0) return; + undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { + kind: "cells", label, changes, + }); + }, + [scope] + ); + + /** ONE cell write, recorded. Every editor, picker and drag goes through this. */ + const patchAndRecord = useCallback( + (pid: number, updates: Partial, label = "an edit") => { + const changes: CellChange[] = Object.entries(updates).map(([key, value]) => ({ + pid, key, + before: currentValue(pid, key), + after: (value ?? null) as UndoValue, + })); + patchOverlay(pid, updates); + recordCells(changes, label); + }, + [currentValue, patchOverlay, recordCells] + ); + + /** MANY cells, ONE stack entry — a paste and a bulk clear are each one user action (R4). */ + const patchManyAndRecord = useCallback( + (writes: { pid: number; updates: Partial }[], label: string) => { + const changes: CellChange[] = []; + for (const w of writes) + for (const [key, value] of Object.entries(w.updates)) + changes.push({ + pid: w.pid, key, + before: currentValue(w.pid, key), + after: (value ?? null) as UndoValue, + }); + for (const w of writes) patchOverlay(w.pid, w.updates); + recordCells(changes, label); + }, + [currentValue, patchOverlay, recordCells] + ); + + /** + * Apply one stack entry in one direction. The INVERSE lives in `undoStack.directed` — this + * only knows how to WRITE each op, and it writes through the same doors the user does + * (`patchOverlay`, the rows endpoint), so an undone edit is persisted exactly like the edit + * was. Nothing here is optimistic-only: a Ctrl+Z that reverted the screen and not the store + * would come back on the next reload. + */ + const applyEntry = useCallback( + async (entry: UndoEntry, dir: "back" | "forward") => { + const op = directed(entry, dir); + if (op.kind === "cells") { + const byPid = new Map>(); + for (const c of op.changes) { + const at = byPid.get(c.pid) ?? {}; + at[c.key] = (c.after ?? "") as Row[string]; + byPid.set(c.pid, at); + } + applyingUndo.current = true; + try { + for (const [pid, updates] of byPid) patchOverlay(pid, updates); + } finally { + applyingUndo.current = false; + } + } else if (op.kind === "rowAdd") { + // Restore under the OLD id where the store still has it free; the server answers with + // what it actually wrote and the re-read is what puts the rows back on screen. + // + // ⚠ SEQUENTIAL, not `Promise.all`. `add_row` picks `max(id)+1` when the requested id is + // taken, and it reads the store to do it — firing ten restores concurrently is ten + // readers racing one counter, which is how two rows end up sharing an id. + let any = false; + let moved = 0; + for (const r of op.rows) { + const made = await addTableRow(op.table, r.values as Record, r.rid); + any = any || !!made; // a refusal already said why, in the server's words + // ⚠ THE ID IS PART OF WHAT IS BEING UNDONE, and `add_row` falls back to `max(id)+1` + // when the one it was asked for is no longer free — which happens if anything created + // a row in the gap. The restore is still the right thing to do, but it is no longer + // the SAME record to anything that named the old id (a cohort, a comment, a filter), + // and a redo would then look for an id that is not there. Said out loud rather than + // discovered later. + if (made && String(made.rid) !== String(r.rid)) moved += 1; + } + if (!any) return; + if (moved) + signal( + TOAST_EVENT, + `${moved} restored record${moved === 1 ? "" : "s"} came back under a new id — ` + + `something had taken the original while it was gone.` + ); + signal(ROWS_STALE_EVENT); + } else if (op.kind === "rowDelete") { + let any = false; + for (const r of op.rows) any = (await deleteTableRow(op.table, r.rid)) || any; + if (!any) return; + // ⭐ ITEM 2 — the row may still be one of THIS browser's pending copies (add, then + // Ctrl+Z). A deleted row never reappears in a payload, so the prune effect can never + // absorb it and it would sit on screen looking undeleted. See `withdrawPending`. + withdrawPending(op.rows.map((r) => r.rid)); + signal(ROWS_STALE_EVENT); + } else if (op.kind === "choiceRename") { + // Item 15 — the inverse mapping. The host rewrites the values and the saved views that + // name them, exactly as it did on the way out; `directed` already turned the pairs + // around, so this emits what it is given. + emitHostEvent({ + id: eventId("choicerename"), + type: "choice_rename", + key: op.fieldKey, + renames: op.renames, + }); + } + signal(TOAST_EVENT, describe(entry, dir)); + }, + [patchOverlay, withdrawPending] + ); + + /** + * Ctrl+Z / Ctrl+Shift+Z (and Ctrl+Y, which is the same request on Windows). + * + * ⚠ ON `window`, IN CAPTURE, and it steps aside for real text fields. The grid is a canvas — + * glide's key handling only fires while the canvas has focus, so a Ctrl+Z after clicking the + * toolbar would do nothing, which is exactly the "sometimes it works" the owner would report + * next. But an with a cursor in it has its OWN undo that belongs to the browser, and + * stealing that would be worse than not having ours: the search box, the rename field and + * glide's own cell editor are all inputs, so `activeElement` decides. + * + * Nothing to undo says so out loud rather than silently ignoring the key — an undo that + * appears to do nothing is indistinguishable from one that is broken. + */ + useEffect(() => { + const onKey = (event: KeyboardEvent) => { + if (!(event.ctrlKey || event.metaKey) || event.altKey) return; + const key = event.key.toLowerCase(); + if (key !== "z" && key !== "y") return; + const el = document.activeElement as HTMLElement | null; + if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable)) + return; + const forward = key === "y" || event.shiftKey; + event.preventDefault(); + const book = undoBook.current; + const state = stackFor(book, scope); + const { state: next, entry } = forward ? popRedo(state) : popUndo(state); + if (!entry) { + signal(TOAST_EVENT, forward ? "Nothing to redo." : "Nothing to undo."); + return; + } + book[scope] = next; + void applyEntry(entry, forward ? "forward" : "back"); + }; + window.addEventListener("keydown", onKey, true); + return () => window.removeEventListener("keydown", onKey, true); + }, [scope, applyEntry]); + + + /** + * R4 — **BULK BACKSPACE/DELETE IS ONE ACTION.** glide's own delete walks the selection and + * calls `onCellEdited` per cell, which would put forty entries on the stack for one keypress; + * returning `false` takes the whole operation over so it lands as one. + * + * ⚠ PRESETS ARE NEVER ATTEMPTED (the contract says so, and it is also the only honest + * behaviour): `canEditField` is the same verdict the editor and the paste path use, so a + * selection spanning read-only columns clears the editable ones and leaves the rest exactly + * as they were — rather than firing writes the server will refuse one by one. + */ + /** + * ⭐ 2026-08-06 (owner) — DELETE RECORDS, with Ctrl+Z. + * + * Owner, verbatim: *"I should always be able to delete records that I created myself manually + * (not from automation), because right now it only shows 'Add to cohort'… make sure I can use + * 'Delete/Backspace' to delete the record and use Ctrl Z if i want to undo."* + * + * Everything under this was already built and had no door: `DELETE /tables/{key}/rows/{rid}`, + * `deleteTableRow`, and `undoStack`'s `rowDelete` — whose inverse (`rowAdd` under the SAME rid) + * is exactly "put it back". The only missing piece was something that calls them. + * + * ⛔ THE VALUES ARE CAPTURED BEFORE THE DELETE, and this is the line the whole feature rests on. + * `deleteTableRow` answers a boolean; it does not hand the row back. Push the undo entry after + * the round trip and there is nothing left to read, so Ctrl+Z would faithfully restore an EMPTY + * row under the right id — a silent data loss wearing the costume of a working undo. + * + * ⚠ ONE ENTRY FOR THE WHOLE GESTURE (R4's "bulk Backspace = ONE grouped stack entry"), which is + * why `UndoEntry` carries a row LIST now. Ten rows deleted must be one Ctrl+Z, not ten. + * + * ⚠ NO CONFIRM DIALOG, deliberately. The owner named Ctrl+Z as the safety net in the same + * sentence as the delete; a modal on top of a working undo is friction that teaches people to + * dismiss modals. + */ + const deleteRecords = useCallback( + async (pids: number[]): Promise => { + if (!canMutateRecords || !pids.length) return false; + // The row as it stands NOW, straight off the rendered records — the same values the + // reader can see, so a restore puts back what they watched disappear. + const byPid = new Map(); + for (const vr of displayRows) + if (vr.kind === "data") byPid.set(vr.record.pid, vr.record); + const captured: UndoRow[] = []; + /* + * ⛔ A SKIPPED ROW IS COUNTED AND NAMED, never dropped quietly ([[no-unverifiable-aggregates]]). + * `displayRows` is FILTERED and CAPPED, and the selection is not: Select-from-file matches + * on a value and can tick pids the current view does not render — `fileSelect` has a whole + * bucket for exactly that ("not in this view"). Capturing only what is on screen and + * `continue`ing past the rest would delete two rows and report three, which is the silent-cap + * defect in the one place it is least forgivable. + * + * REFUSED WHOLE rather than partially applied: a delete that half-happened leaves the reader + * reconciling a count against a table, and the fix (scroll or clear the filter, then select + * again) is one sentence away. + */ + const offscreen: number[] = []; + for (const pid of pids) { + const rec = byPid.get(pid); + if (!rec) { + offscreen.push(pid); + continue; + } + const values: Record = {}; + for (const f of fields) { + const v = rec[f.key]; + if (v !== undefined && v !== null && v !== "") + values[f.key] = v as UndoValue; + } + captured.push({ rid: pid, values }); + } + if (offscreen.length) { + signal( + TOAST_EVENT, + `Nothing was deleted — ${offscreen.length} of the ${pids.length} selected ` + + `record${pids.length === 1 ? " is" : "s are"} not shown in this view. Clear the ` + + `filter (or scroll them into view) and select again.` + ); + return false; + } + if (!captured.length) return false; + + /* + * ⭐ WARN THEN ALLOW (owner ruling, 2026-08-06) — the arm fires ONLY where something is + * actually at stake, so deleting your own scratch rows stays one keypress. + * + * ⛔ THE PREDICATE IS "WHAT WOULD BE LOST", NOT "WHO CREATED THIS". There is no stored + * creator on a row, and deriving one from a stage column would flip meaning as the + * automation runs. But the question the warning answers is a different and answerable one: + * *has an automation written to this row* — because that is exactly what a re-find cannot + * give back. MEASURED in `run_discover_instagram`: the seen-before test reads the rows + * CURRENTLY IN THE TABLE, so a deleted candidate returns as NEW — `found_count` back to 1, + * `first_found` re-stamped, and its stage reset to Review. Deleting a candidate somebody + * had already judged throws that judgement away. + * + * ⚠ This paragraph used to name a "Declined" stage. WAVE 26 / R6 DELETED THE BOARD'S + * BUILT-IN TERMINALS — the lanes are whatever the user defined now — so the warning names + * the mechanism (the stage resets to Review) and no longer a stage that does not ship. + */ + /** + * ⭐⭐ WAVE-29 T23 (owner item 2b) — **THE QUESTION IS AUTHORSHIP, AND IT HAS AN ANSWER.** + * + * The note above this used to argue there is no stored creator on a row, so the warning + * asked "has a machine written here" instead. There IS one: `automation_engine` stamps + * `created_by` ("Found by") on every candidate row it discovers, and `add_row` stamps + * nothing — so the two authors are distinguishable, and `machineFoundRows` (types.ts) is + * that distinction. Everything the old predicate got wrong followed from asking the + * answerable-but-different question: every preset column is machine-tagged, a hand-added + * row is born holding materialised rollups, and the STAGE cell a human drives was + * force-included — so pressing "+" and then Delete produced a loud warning about losing + * an automation's work on a row the person had just made themselves. + * + * ⚠ The wave-27 note that lived here (`stageField` survives the stage deletion, so a + * legacy stage column stays read-only) is about the READ-ONLY readers — `isMachineWritten` + * / `mayEditField` in types.ts, which are untouched. It never described this predicate; the + * risk test is the one place where including the stage column is the defect rather than the + * protection. + * + * ⛔ AND THE SENTENCE IS BUILT FROM THE COLUMNS THIS TABLE HAS. It used to name first-found, + * times-found and a stage reset unconditionally while all three live in `CANDIDATE_FIELDS` + * — so on a user-named IG Profile database it named the loss of three columns that are not + * there. `reFindConsequences` reads the field list. + */ + const riskyIds = new Set(machineFoundRows(fields, captured)); + const risky = captured.filter((r) => riskyIds.has(r.rid)); + const sig = captured.map((r) => r.rid).join(","); + if (risky.length && delArmed !== sig) { + setDelArmed(sig); + const losses = reFindConsequences(fields); + const one = risky.length === 1; + signal( + TOAST_EVENT, + `${risky.length} of these ${one ? "records was" : "records were"} FOUND by an ` + + `automation. Deleting ${one ? "it" : "them"} is not undone by a re-find` + + (losses.length + ? `: ${losses.join(", ")} — so a card somebody had already moved on comes back ` + + `undecided` + : ` — ${one ? "it" : "they"} would come back as a new record with no history`) + + `. Press Delete again (or click again) to confirm.` + ); + return false; + } + setDelArmed(""); + // SEQUENTIAL, matching the restore path: the store is one document and ten concurrent + // read-modify-writes against it is how a delete silently misses a row. + const gone: UndoRow[] = []; + for (const r of captured) + if (await deleteTableRow(scope, r.rid)) gone.push(r); + if (!gone.length) { + signal(TOAST_EVENT, "Nothing was deleted — the server refused."); + return false; + } + undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { + kind: "rowDelete", table: scope, rows: gone, + }); + // ⭐ ITEM 2 — same reason as the undo path: a row added and then deleted in one session + // is still a PENDING copy here, and no future payload will ever prune it. + withdrawPending(gone.map((r) => r.rid)); + clearSelection(); + signal(ROWS_STALE_EVENT); + // ⚠ THE CONSEQUENCE IS NAMED IN FULL, and the first version of this sentence undersold it. + // It said only that a matching record "will return", which is true and reassuring and + // leaves out the part that actually costs something. + // + // MEASURED in `run_discover_instagram`: the seen-before test is built from the rows + // CURRENTLY IN THE TABLE (`seen` comes from `existing2`). A deleted profile is therefore + // not "seen again" on the next run — it is NEW. `found_count` resets to 1, `first_found` + // is re-stamped, and the stage goes back to Review, because a new candidate's card starts + // at the human gate. So deleting a candidate somebody had already moved out of Review puts + // it back there undecided — the human judgement is the thing the re-find cannot give back. + // (W26/R6 deleted the board's built-in terminals, so there is no named stage to cite here.) + // + // (The follower SNAPSHOT history is safe either way — it lives in the platform master keyed + // by handle, a different store, which only `purge_handle` touches.) + // Same survival, same reason as `machineKeys` above (item 12 / R3, B's ASK ->C): the flag + // is no longer WRITTEN, and a table that has not been migrated yet still carries it. + const fedByAutomation = fields.some((f) => f.automation?.stageField); + signal( + TOAST_EVENT, + `Deleted ${gone.length} record${gone.length === 1 ? "" : "s"}. Ctrl+Z to undo.` + + (fedByAutomation + ? " If this automation finds one again it comes back as a NEW candidate — first-found" + + " and times-found reset, and its stage back to Review. Undo keeps all of that." + : "") + ); + return true; + }, + [canMutateRecords, displayRows, fields, scope, clearSelection, delArmed, + withdrawPending] + ); + + const onGridDelete = useCallback( + (sel: GridSelection): boolean => { + /* + * ⭐ 2026-08-06 (owner) — A ROW SELECTION + Delete/Backspace DELETES THE RECORDS. + * + * Checking rows and pressing Delete used to CLEAR every editable cell in them, which on the + * owner's Instagram table now clears nothing at all: every column but one is machine-written + * and refuses the write. So the key appeared to do nothing, which is what they reported. + * + * ⚠ ROWS ONLY, AND ONLY WHEN NO **MULTI-CELL** RANGE IS DRAWN. Checking rows is an explicit + * gesture about RECORDS; DRAGGING a range is a gesture about CELLS, and a Delete that + * destroyed records because some rows happened to be checked would feel unrecoverable even + * with an undo behind it. When both are present, the narrower reading (clear the cells) wins. + * + * ⛔ A SINGLE ACTIVE CELL IS NOT A RANGE, and the first version of this got it wrong. + * `sel.current.range` is ALWAYS set when a cell is merely focused — a 1×1 rect — so + * `!sel.current?.range` meant the feature worked only if you had never clicked a cell. + * Caught by driving it live rather than by the gate: click a cell, then check a row, then + * press Delete, and the key silently went back to clearing cells. The gesture the rule is + * actually about is a DRAG, which is width or height greater than one. + */ + const dragged = !!sel.current + && (sel.current.range.width > 1 || sel.current.range.height > 1); + if (canMutateRecords && sel.rows.length > 0 && !dragged) { + const pids: number[] = []; + for (const rowIndex of sel.rows) { + const vr = displayRows[rowIndex]; + if (vr && vr.kind === "data") pids.push(vr.record.pid); + } + // ⛔ `true`, NOT `false`, WHEN THIS BRANCH DOES NOT DO THE WORK — the scar the comment at + // the bottom of this handler already records, pointed at a second case. `false` cancels + // glide's own delete, so bailing with it would leave Delete doing nothing at all on a + // selection this branch declined. `deleteRecords` is async and the keypress cannot wait + // for it, so the branch commits here and reports the outcome through its own toast. + if (!pids.length) return true; + void deleteRecords(pids); + return false; + } + const byPid = new Map>(); + const clear = (rowIndex: number, colIndex: number) => { + const vr = displayRows[rowIndex]; + if (!vr || vr.kind !== "data") return; + const column = visibleCols[colIndex]; + const field = column ? fieldByKey.get(column.id!) : undefined; + if (!field || !canEditField(field)) return; + const at = byPid.get(vr.record.pid) ?? {}; + at[field.key] = ""; + byPid.set(vr.record.pid, at); + }; + for (const rowIndex of sel.rows) + for (let c = 0; c < visibleCols.length; c++) clear(rowIndex, c); + const range = sel.current?.range; + if (range) + for (let y = range.y; y < range.y + range.height; y++) + for (let x = range.x; x < range.x + range.width; x++) clear(y, x); + const writes = [...byPid.entries()].map(([pid, updates]) => ({ pid, updates })); + if (!writes.length) return true; + patchManyAndRecord(writes, "clearing cells"); + // ⚠ `false` ONLY WHEN THIS ACTUALLY DID THE WORK. Returning it unconditionally cancels + // glide's own delete for cases this handler does not cover — a COLUMN selection, which + // glide deletes from `toDelete.columns` and the loops above never look at — so Delete + // would silently clear nothing. Handing the keypress back when there is nothing to + // group is strictly safer than swallowing it: the per-cell path still refuses read-only + // fields (`onCellEdited`'s own `canEditField`), it just does not arrive as one entry. + return false; + }, + [canMutateRecords, deleteRecords, displayRows, visibleCols, fieldByKey, canEditField, + patchManyAndRecord] + ); + + /** + * Owner item 4 / C-ADDROW — the cursor FOLLOWS the appended row, once it exists. + * + * The append is a server round trip, so at click time there is nothing to focus; this waits + * for the re-read to bring the pid back and then lands the active cell on its first column, + * scrolled into view. `newRowPid` is cleared either way — a row the re-read never produced + * (a refused write, a filter that excludes it) must not leave a cursor waiting forever. + */ + useEffect(() => { + if (newRowPid === null) return; + const index = displayPidToIndex.get(newRowPid); + if (index === undefined) return; + setActiveCell(0, index); + gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" }); + setNewRowPid(null); + }, [newRowPid, displayPidToIndex, setActiveCell]); + + /** Owner item 23 — the fields in the view's column order, for the Hide-fields panel. Built + * from `order` (already reconciled by useGridColumns) rather than from `config.order` so the + * panel and the grid can never disagree about which fields exist or where they sit. */ + const orderedFields = useMemo( + () => order.map((key) => fieldByKey.get(key)).filter((f): f is Field => !!f), + [order, fieldByKey] + ); + /** Owner item 17 — the row of the ACTIVE cell, i.e. what the last click highlighted. Read off + * `gridSelection` rather than tracked separately so keyboard navigation moves the wash too; + * `useGridSelection` already guards this index against a grid that shrank this render. */ + const activeRow = gridSelection.current?.cell[1]; + + const statusValues = useMemo(() => { + const map: Record = {}; + for (const field of fields) { + // A `user`'s people come from the HOST, never from the rows — an assignee nobody has been + // given yet is still assignable. + if (field.type === "user") { + map[field.key] = payload?.userOptions ?? []; + continue; + } + // ⭐ Owner item 24 — every other choice column goes through ONE rule (`choiceVocabulary`): + // the DECLARED list when the field has one, the values seen in the data when it does not. + // This used to branch on the type, which sent `stock_bucket` — a `select` whose vocabulary + // is computed server-side and declares no `options` — down the declared path to an empty + // list, and an empty supplied list WINS in the filter panel. See the function's own note; + // it is pure so the gate can run it, which nothing inside this file can be. + if (field.type === "select" || field.type === "multiselect" || field.type === "status") + map[field.key] = choiceVocabulary(field, rawRows); + } + return map; + }, [fields, rawRows, payload?.userOptions]); + + const rowPx = ROW_PX[config.rowHeightMode]; + // DISTINCT customers, not painted data rows. Grouping a MULTI field (Cohorts) puts the same + // customer under every group it belongs to, so counting rows would report more records than + // there are customers — a number nobody could reconcile against the book. `pidToIndex` is + // first-wins per pid, so its size IS the distinct count, and it equals the row count for every + // non-multi grouping and for no grouping at all. + // + // ⚠ The FULL pipeline's map, never the display slice's: the toolbar count must state what the + // view MATCHES. What is painted is the "Showing first N" bar's job (owner item 8). + const recordCount = pidToIndex.size; + const shownRecords = displayPidToIndex.size; + /* wave17 GRID — item 2 / owner R5. The whole-table row band is GONE, and with it the + `bandTone` / `bandMask` pair that used to sit here: the winning control's tone, and the + alternating-row mask built from the DATA-ROW ORDINAL so group headers could not flip the + stripe at a boundary. What follows is now three row states, not four. + ⛔ wave-29 R8 (2026-08-11) — R5 also kept an involved-COLUMN cell wash; that is now gone too, + so `COLUMN_TONE_THEME` carries HEADER keys only. This callback and `cells.AUTOMATION_TINT` + are therefore the ONLY writers of a body `bgCell` on this canvas: a sorted or filtered column + can no longer layer anything over a status wash, which is exactly what R8 asked for. */ + const getRowThemeOverride = useCallback( + (row: number): Partial | undefined => { + const visibleRow = displayRows[row]; + if (!visibleRow || visibleRow.kind !== "data") return undefined; + // Owner item 17 — the row holding the ACTIVE cell stays washed after the pointer moves + // on. Ordered AFTER hover on purpose: the pointer is the more immediate signal, and a + // hovered-and-active row reading as merely active would make hover look broken. + if (row === hoverRow) { + if (!config.colorBy) return HOVER_NEUTRAL; + return ( + HOVER_ROW_THEME[String(visibleRow.record[config.colorBy] ?? "").toLowerCase()] ?? + HOVER_NEUTRAL + ); + } + // A colour-by wash already marks this row with meaning the user chose; overlaying the + // active tint on top would blend two hues into a third that means neither. There, glide's + // accent ring is the active marker and this stays out of the way. + if (!config.colorBy) { + if (row === activeRow) return ACTIVE_ROW_NEUTRAL; + return undefined; + } + return STATUS_ROW_THEME[ + String(visibleRow.record[config.colorBy] ?? "").toLowerCase() + ]; + }, + [displayRows, config.colorBy, hoverRow, activeRow] + ); + const onItemHovered = useCallback( + (args: GridMouseEventArgs) => { + const row = args.kind === "cell" ? args.location[1] : undefined; + setHoverRow((previous) => (previous === row ? previous : row)); + // Owner item 19 — the hover-only Expand. Placed from glide's OWN bounds for the primary + // cell of this row, so freeze, horizontal scroll and row-height mode are handled by the + // component that owns them rather than re-derived here. + // + // ⚠ Only ever CLEARED from here for a different row, never for "the pointer left the + // canvas". Moving the pointer ONTO the button leaves the canvas, so clearing on + // out-of-bounds would unmount the control between the mouse arriving and the click + // landing — an affordance that vanishes exactly when you reach for it, and one that + // every assertion still sees because it exists in every state except the one that + // matters. `.cg-grid-box`'s own onMouseLeave is what dismisses it. + const vrow = row !== undefined ? displayRows[row] : undefined; + if (args.kind === "cell" && vrow?.kind === "data" && row !== undefined) { + const pid = vrow.record.pid; + const b = gridRef.current?.getBounds(0, row); + // Clamped to the grid's own box. ⚠ RETARGETED wave-14 item 15: this used to say "with a + // grouping active `freezeColumns` is 0, so the primary column scrolls away". It no + // longer does — the first column is now frozen in EVERY mode. The clamp stays because + // its other half is still live: glide's bounds are VIEWPORT coordinates, so a row + // scrolled under the header or past the bottom still reports a rect outside the grid, + // and a `position: fixed` button with no clamp would paint over the chrome on a row + // nobody can see. The horizontal legs are now defence in depth (a column drag, a box + // narrower than the frozen strip) rather than the everyday case. + const at = b + ? expandButtonRect(b, gridBoxRef.current?.getBoundingClientRect()) + : null; + // Recomputed EVERY move, not memoised on the pid: a scroll can leave the pointer over + // the same record at a new y, and a button that keeps the pid but not the position + // floats over the wrong row. Referential stability is preserved by comparing values, + // which is the cheap half — `getBounds` is arithmetic over glide's own layout. + setExpandAt((prev) => + !at + ? null + : prev && prev.pid === pid && prev.x === at.x && prev.y === at.y + ? prev + : { pid, x: at.x, y: at.y, size: at.size } + ); + } + // Wave-5 item 6 — hovering a header whose field carries a description floats the text + // under the header. The tip element is pointer-events:none and aria-hidden: it can + // NEVER become the click target, which is the tooltip trap this page has already paid + // for once ([[ui-invisible-to-assertions]]). + if (args.kind === "header") { + const colDef = visibleCols[args.location[0]]; + const field = colDef ? fieldByKey.get(colDef.id!) : undefined; + const note = field ? field.note || field.description : undefined; + // Owner item 16 — a title the fit SHORTENED must still be readable somewhere, and the + // tip is where. `colDef.title !== field.label` IS the truncation test: `fitHeaderTitle` + // returns the label by identity when it fits, so this asks the renderer what it did + // rather than re-running the measurement and hoping the two agree. + const cut = !!field && !!colDef && colDef.title !== field.label; + const text = cut && field ? (note ? `${field.label} — ${note}` : field.label) : note; + const bounds = args.bounds; + if (text && bounds) { + setHeaderTip((prev) => + prev && prev.text === text && prev.x === bounds.x + ? prev + : { text, x: bounds.x, y: bounds.y + bounds.height + 4 } + ); + return; + } + } + setHeaderTip((prev) => (prev === null ? prev : null)); + + // I2 — the cell half. Only text that is genuinely CUT OFF gets a tip: a tooltip on + // every cell just repeats what is already legible and covers the row under it. + // Truncation is MEASURED with glide's own font rather than guessed from a character + // count, which is what makes "e.g. a Note" work and a short currency cell stay quiet. + if (args.kind === "cell" && row !== undefined) { + const vr = displayRows[row]; + const cellCol = visibleCols[args.location[0]]; + // Group headers and footers carry no cell text of their own — a tip over the group + // bar would repeat the label glide has already drawn across it. + if (vr?.kind === "data" && cellCol) { + const field = fieldByKey.get(cellCol.id!); + const text = field ? formatDisplay(field, vr.record[field.key]) : ""; + const bounds = args.bounds; + const tip = bounds ? cellTipText(text, measureCellText(text), bounds.width) : null; + if (tip && bounds) { + const x = tipLeft(bounds.x, window.innerWidth, measureCellText(tip) + 20); + const y = bounds.y + bounds.height + 4; + setCellTip((prev) => + prev && prev.text === tip && prev.x === x && prev.y === y + ? prev + : { text: tip, x, y } + ); + return; + } + } + } + setCellTip((prev) => (prev === null ? prev : null)); + }, + [visibleCols, fieldByKey, displayRows] + ); + const rowHeight = useMemo( + () => + config.groupBy + ? (row: number) => (displayRows[row]?.kind === "group-header" ? 32 : rowPx) + : rowPx, + [config.groupBy, rowPx, displayRows] + ); + + /* ═══ W18-B VOID ═══ (wave 18, owner item 1b) — WHERE THE TABLE ENDS, in canvas pixels. + Consumed by the two `.cg-grid-void` rectangles at the DataEditor mount; see the CSS region + of the same name for why the void is painted in the DOM rather than in the glide theme. + + ⭐ THE PROPERTY THAT MAKES THIS ARITHMETIC AND NOT SCROLL-TRACKING, and the reason there is + no `onVisibleRegionChanged` handler anywhere near it: **a void can only exist in an axis the + content does not overflow.** glide's scroll extent is EXACTLY the content — `scrollWidth = + nonGrowWidth (+ overscrollX)`, `scrollHeight = header + rows (+ overscrollY)`, and we pass + neither overscroll prop (scrolling-data-grid.js:12-24) — so at maximum scroll the last + row/column lands flush on the client edge. Wherever there is something to paint, the scroll + offset in that axis is 0 and cannot become anything else. The rectangles therefore depend on + the columns, the rows and the box, and on nothing that moves while the user drags. + + Every column carries an explicit `width` (`useGridColumns` sets `config.widths[key] ?? + DEFAULT_WIDTH`) and none carries `grow`, so glide's column sizer passes them through + untouched and this sum is the width it actually lays out. */ + const gridVoid = useMemo(() => { + const gutter = scrollbarGutter(); + let colsPx = rowMarkerPx(displayRows.length); + for (const c of visibleCols) colsPx += (c as { width?: number }).width ?? 0; + let rowsPx = HEADER_PX; + if (typeof rowHeight === "number") rowsPx += displayRows.length * rowHeight; + else for (let i = 0; i < displayRows.length; i++) rowsPx += rowHeight(i); + // ⚠ Owner item 4 — THE GHOST ROW IS PART OF THE TABLE, and this line is what makes it + // visible. glide draws its trailing row AFTER the last data row, but `displayRows` (our + // rows) does not contain it, so the void started exactly where the ghost row does and + // painted flat #F6F8FC straight over it. The "+" was still clickable the whole time + // (`.cg-grid-void` is `pointer-events: none`), which is the worst version of this bug: + // the gate went green on an affordance nobody could see. Found by READING THE SCREENSHOT + // ([[ui-invisible-to-assertions]], [[finalize-visual-review-sop]]). + if (canMutateRecords) + rowsPx += typeof rowHeight === "number" ? rowHeight : rowHeight(displayRows.length); + + /* Fit is tested against the client box the OTHER axis's scrollbar leaves behind — the same + `clientWidth`/`clientHeight` glide's own scroll handler reads (infinite-scroller.js: + 110-116), so this branch and glide's cannot disagree about whether a bar is there. The + two `if`s resolve the circularity in the only direction it can run: a bar in one axis can + CREATE one in the other, but two bars can never un-create each other. */ + let vBar = rowsPx > gridSize.height; + let hBar = colsPx > gridSize.width; + if (vBar && !hBar) hBar = colsPx > gridSize.width - gutter; + if (hBar && !vBar) vBar = rowsPx > gridSize.height - gutter; + const clientW = gridSize.width - (vBar ? gutter : 0); + const clientH = gridSize.height - (hBar ? gutter : 0); + + /* `null` = the content reaches that edge, so there is no void and no rectangle. The + comparison is strict: a table ending exactly on the edge has nothing past it. */ + return { + clientW, + clientH, + below: rowsPx < clientH ? rowsPx : null, + right: colsPx < clientW ? colsPx : null, + }; + }, [visibleCols, displayRows, rowHeight, gridSize, canMutateRecords]); + /* ═══ end W18-B VOID (geometry) ═══ */ + + // The record drawer resolves positions against what the MODE paints: the display slice for + // grid/list (the cap is real there), the FULL pipeline for calendar/kanban/map (a month, a + // stack or a pin reaches past the cap by design — a drawer that refused those pids would + // close itself on a card the user can plainly see). + // ⭐ WAVE-27 item 8 (C3) — `swipe` belongs here for calendar/kanban/map's own reason: the deck + // is derived from `modeDataRows` (the FULL pipeline), so a card past the display cap is one + // the user can plainly see, and a drawer that refused its pid would close itself on open. + const fullSetMode = + displayMode === "calendar" || displayMode === "kanban" || displayMode === "map" || + displayMode === "swipe"; + const detailRows = fullSetMode ? visibleRows : displayRows; + const detailIdxMap = fullSetMode ? pidToIndex : displayPidToIndex; + const detailIndex = detailPid !== null ? detailIdxMap.get(detailPid) : undefined; + useEffect(() => { + if (detailPid !== null && detailIndex === undefined) setDetailPid(null); + }, [detailPid, detailIndex]); + const detailRecord = useMemo(() => { + if (detailIndex === undefined) return null; + const row = detailRows[detailIndex]; + if (!row || row.kind !== "data") return null; + return overlayEdits[row.record.pid] + ? { ...row.record, ...overlayEdits[row.record.pid] } + : row.record; + }, [detailIndex, detailRows, overlayEdits]); + const dataPosition = useMemo(() => { + if (detailIndex === undefined) return 0; + return detailRows + .slice(0, detailIndex + 1) + .filter((row) => row.kind === "data").length; + }, [detailIndex, detailRows]); + const neighborExists = useCallback( + (delta: -1 | 1) => { + if (detailIndex === undefined) return false; + for ( + let index = detailIndex + delta; + index >= 0 && index < detailRows.length; + index += delta + ) + if (detailRows[index]?.kind === "data") return true; + return false; + }, + [detailIndex, detailRows] + ); + const go = useCallback( + (delta: -1 | 1) => { + if (detailIndex === undefined) return; + for ( + let index = detailIndex + delta; + index >= 0 && index < detailRows.length; + index += delta + ) { + const row = detailRows[index]; + if (row?.kind !== "data") continue; + setDetailPid(row.record.pid); + if (displayMode === "grid") + gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" }); + return; + } + }, + [detailIndex, detailRows, displayMode] + ); + + /** + * Wave-9 I3 — the description "(i)", drawn RIGHT-ALIGNED and vertically centred with the + * field name. + * + * ⚠ This is the header-draw alternative, taken because glide's `overlayIcon` PROVABLY + * cannot do it: `drawHeaderInner` paints an overlay at a hard-coded `drawX + 9` / + * `(height - 18) / 2 + 6` — a badge on the bottom-right corner of the TYPE mark at the far + * LEFT of the header — and no prop moves it. `drawHeader` is glide's supported escape + * hatch: it hands over the ctx, the rect, the menu bounds and the sprite manager, plus a + * `drawContent()` that runs its own rendering first. So glide still draws the header it + * always drew (type mark, title, menu); only the (i) is ours, and `overlayIcon` is no + * longer set on any column. + * + * The geometry is `infoMarkRect()` in overlayPlacement.ts, so "right-aligned and centred" + * is asserted numerically by a gate rather than judged from one screenshot at one width. + */ + const drawGridHeader = useCallback( + (args, drawContent) => { + const field = args.column.id ? fieldByKey.get(args.column.id) : undefined; + // ⚠⚠ WAVE-14 ITEM 2 — THE ONE LINE THAT MAKES AN INVOLVED HEADER BOLD, and it is not a + // font setting anywhere near where you would look for one. + // + // glide's `drawGridHeaders` does, verbatim: + // if (theme !== outerTheme) ctx.font = theme.baseFontFull; // :38-40 + // i.e. the moment a column carries ANY `themeOverride` its header title is painted in the + // CELL font ("13px") instead of the header font ("600 13px"). That is why the owner's + // filtered/sorted/grouped headers had already lost their weight — nothing in our code + // asked for it, and setting `headerFontStyle` on the involved theme alone would have + // changed precisely nothing, because glide never reads it on that path. Re-asserting the + // MERGED theme's header font on the ctx here is what puts it back, for every column, + // uniformly: on a column with no override this is a no-op (glide set the same value at + // :21), and glide's own save/restore around each column stops it leaking to the next. + // + // Built from `headerFontStyle` + `fontFamily` rather than reading `headerFontFull`, + // which is real at runtime but is declared on `FullTheme`, not the public `Theme` the + // callback is typed with — same string, no cast, and it is exactly how + // `mergeAndRealizeTheme` composes it. + // + // Bonus fix, worth knowing: `drawHeaderInner` passes `theme.headerFontFull` to + // `getMiddleCenterBias`, which measures with the CURRENT ctx font but CACHES under the + // string it was handed. Before this line, an overridden column measured its baseline bias + // in the 13px font and filed it under the "600 13px" key, so whichever column drew first + // decided the vertical centring for all of them. + args.ctx.font = `${args.theme.headerFontStyle} ${args.theme.fontFamily}`; + drawContent(); + if (!field) return; + const hasInfo = !!(field.note || field.description); + // ⭐ OWNER ITEM 4 (2026-08-06): *"why is the Field still have the Dot at the header to + // mark its a custom editable field?"* Because the predicate was `source === "overlay"` + // alone — true of columns a person added BESIDE the Odoo ones, and true of literally + // every column in a `ut_*` database, including the ones an automation spawns and fills. + // So on the owner's Instagram table the dot marked all 23 columns as "yours", which is + // both meaningless (it never varies) and wrong (they are not yours to edit). + const isCustom = field.source === "overlay" && !isMachineOwned(field); + // ONE layout for both marks, from the same `headerMarkSizes` order `useGridColumns` used + // to reserve the label's room — so the number of marks drawn and the number reserved for + // cannot disagree. null = the column is too narrow to carry them without covering its own + // label, and then NOTHING is drawn (all-or-nothing; see headerMarkLayout). + const marks = headerMarkLayout( + args.rect, + args.menuBounds.width, + headerMarkSizes(hasInfo, isCustom) + ); + if (!marks) return; + let slot = 0; + if (hasInfo) { + const at = marks[slot++]; + args.spriteManager.drawSprite( + "aiosInfo", + "normal", + args.ctx, + at.x, + at.y, + at.size, + args.theme + ); + } + if (isCustom) { + // Wave-14 item 1 / R11 — the user-created-field marker that REPLACES the yellow header + // wash. A muted dot: quiet enough to ignore while reading, present enough to answer + // "which of these columns are mine?" at a glance. Never a background — the owner killed + // the wash, and a paler wash would have been the same answer in a lower voice. + const at = marks[slot++]; + const ctx = args.ctx; + ctx.save(); + ctx.beginPath(); + ctx.arc(at.x + at.size / 2, at.y + at.size / 2, at.size / 2, 0, Math.PI * 2); + ctx.fillStyle = CUSTOM_FIELD_MARK; + ctx.fill(); + ctx.restore(); + } + }, + [fieldByKey] + ); + + const onCellClicked = useCallback( + (cell: Item, event: CellClickedEventArgs) => { + const row = displayRows[cell[1]]; + if (!row) return; + // Owner item 10: clicking into the cells is "I am working now" — the frame folds its + // navigation rail to the slim strip (a no-op in the embed; the shell listens). + signal(NAV_MINIMIZE_EVENT); + if (row.kind === "group-header") { + event.preventDefault(); + setCollapsed((current) => { + const next = new Set(current); + if (next.has(row.groupKey)) next.delete(row.groupKey); + else next.add(row.groupKey); + return next; + }); + return; + } + const column = visibleCols[cell[0]]; + const field = column ? fieldByKey.get(column.id!) : undefined; + // A picked field (select / assignee) opens its choices where the cell is. It cannot use + // glide's text overlay — a free-text editor on a constrained column is how you end up + // with "Done", "done" and "DONE" as three different values — and glide's own dropdown + // cell lives in a package we have deliberately not added, so this reuses the same + // AnchoredOverlay the column and view menus already use. Wave-5: a RATING cell rides + // the same picker surface (its choices are 1..max stars); both doors respect the + // permissions verdict. + if (row.kind === "data" && field && canEditField(field) + && (isPickType(field.type) || field.type === "rating")) { + event.preventDefault(); + // Item 6 (2026-07-31) — the click also LANDS the active cell here (preventDefault + // stops glide from committing it), so pick → Enter walks on down the column exactly + // like a typed edit does. + setActiveCell(cell[0], cell[1]); + const b = event.bounds; + setPicker({ + pid: row.record.pid, + fieldKey: field.key, + anchor: { + left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height, + width: b.width, height: b.height, + }, + }); + return; + } + // A linked-record cell is a doorway to the target database, not an opaque id list. + // The large modal mounts the same Grid surface over exactly these pids, so its standard + // search, Filters, Sort, Fields, and column menus keep working for every database kind. + if (row.kind === "data" && field?.type === "link" && field.link?.table) { + if (field.link.table.startsWith("ut_")) { + event.preventDefault(); + setActiveCell(cell[0], cell[1]); + setLinkAt({ pid: row.record.pid, fieldKey: field.key }); + return; + } + } + // ⭐ Wave-23 C7 — a JSON cell opens the big viewer. It is the ONLY door: the cell carries + // `allowOverlay:false`, because glide's overlay is a one-line box and one keystroke in the + // wrong place inside a 32 KB document turns a well-formed payload into an unparseable one, + // saved. Opened for EVERY reader (a document you may not edit is still one you must be + // able to read) — the viewer takes `onSave` only when the permission verdict allows it, + // and the host's write wall is the real one either way. + if (row.kind === "data" && field?.type === "json") { + event.preventDefault(); + setActiveCell(cell[0], cell[1]); + setJsonAt({ pid: row.record.pid, fieldKey: field.key }); + return; + } + // Wave-5 item 11 — a URL cell opens its link on click (scheme-guarded: http/https only, + // a bare domain gets https://). Editing stays with glide's overlay (Enter/double-click). + if (row.kind === "data" && field?.type === "url") { + const raw = String( + overlayEdits[row.record.pid]?.[field.key] ?? row.record[field.key] ?? "" + ).trim(); + if (raw) { + const href = /^https?:\/\//i.test(raw) + ? raw + : /^[\w-]+(\.[\w-]+)+/.test(raw) + ? `https://${raw}` + : null; + if (href) { + event.preventDefault(); + window.open(href, "_blank", "noopener"); + return; + } + } + } + // Owner item 6 (2026-07-31) — CLICKING THE CUSTOMER TICKS THE CHECKBOX. The row marker + // is a ~32px strip; the identity cell is the widest, most natural target on the row, so + // a click there toggles the same pid-anchored set the markers write ("more surface area + // to select individual customers into a Cohort"). No preventDefault: glide still commits + // the cell highlight below, so reading across the row keeps working. + if (row.kind === "data" && field && field.key === lockedKey) { + togglePid(row.record.pid); + } + // Owner item 17 — A SINGLE CLICK HIGHLIGHTS. It used to open the record panel from here, + // which meant a user could not select a cell, read across a row, or copy a value without + // a drawer landing over the table. The highlight is glide's own doing: this handler + // returns without `preventDefault`, so the click commits `gridSelection.current`, the + // accent ring lands on the cell and `getRowThemeOverride` washes the row + // (ACTIVE_ROW_NEUTRAL). Nothing is drawn here. + // + // ⚠ SHIPPED WITH ITEM 19, never alone. Deleting this line is what removes the ONLY way to + // open a record; the hover Expand button below is its replacement, and half of this + // change is a table whose records cannot be opened at all. + }, + [displayRows, visibleCols, fieldByKey, canEditField, overlayEdits, lockedKey, togglePid, + setActiveCell] + ); + /** + * Double-click / Enter. KEPT as a door to the record on purpose (owner item 17 names the + * SINGLE click, and it is the single click that was in the way). + * + * Removing it too would leave the record reachable only by pointer — the hover affordance + * cannot be reached from the keyboard at all — so a keyboard user would lose the panel + * outright. glide routes Enter on an EDITABLE cell to its overlay editor before this fires, + * so the two doors do not collide: this is the activation path for the read-only columns, + * which is most of the table. + */ + const onCellActivated = useCallback( + (cell: Item) => { + const row = displayRows[cell[1]]; + const column = visibleCols[cell[0]]; + const field = column ? fieldByKey.get(column.id!) : undefined; + if (row?.kind === "data" && field?.source !== "overlay") + setDetailPid(row.record.pid); + }, + [displayRows, visibleCols, fieldByKey] + ); + const onCellEdited = useCallback( + (cell: Item, value: EditableGridCell) => { + const row = displayRows[cell[1]]; + const column = visibleCols[cell[0]]; + const field = column ? fieldByKey.get(column.id!) : undefined; + if (!row || row.kind !== "data" || !field || !canEditField(field)) return; + // Wave-5 item 11 — a checkbox toggles straight through glide's BooleanCell (no overlay + // editor); the overlay store keeps its '1'-or-empty contract. + if (value.kind === GridCellKind.Boolean && field.type === "checkbox") { + patchAndRecord(row.record.pid, { [field.key]: value.data ? "1" : "" }, "a tick"); + return; + } + if (value.kind === GridCellKind.Uri) { + patchAndRecord(row.record.pid, { [field.key]: value.data ?? "" }, "an edit"); + return; + } + if (value.kind === GridCellKind.Text || value.kind === GridCellKind.Number) { + patchAndRecord(row.record.pid, { [field.key]: value.data }, "an edit"); + } + }, + [displayRows, visibleCols, fieldByKey, patchAndRecord, canEditField] + ); + const validateCell = useCallback( + (cell: Item): boolean => { + const row = displayRows[cell[1]]; + const column = visibleCols[cell[0]]; + const field = column ? fieldByKey.get(column.id!) : undefined; + // ONE verdict (types.mayEditField): stratum + read-only-by-nature + permissions. The + // picked types and rating never take glide's text overlay — their pickers are the door. + return ( + !!row && + row.kind === "data" && + !!field && + canEditField(field) && + !isPickType(field.type) && + field.type !== "rating" && + field.type !== "status" + ); + }, + [displayRows, visibleCols, fieldByKey, canEditField] + ); + const onGridPaste = useCallback( + (target: Item, values: readonly (readonly string[])[]): boolean => { + const column = visibleCols[target[0]]; + const field = column ? fieldByKey.get(column.id!) : undefined; + if (!field) return false; + // ⭐ Wave-15 item 3 (R8) — THE SELECTION DECIDES HOW FAR THE PASTE REACHES. Until now the + // targets were walked down from the anchor for exactly `values.length` rows, so a + // fifty-row selection and a one-cell clipboard wrote ONE row: the selection was painted + // and obeyed by nothing. `pasteRowCount` is the whole rule and it is pure, so the gate + // can drive it — this handler cannot be reached by any node test in the repo. + const rowCount = pasteRowCount(values.length, gridSelection.current?.range, + { col: target[0], row: target[1] }); + const targetPids = Array.from({ length: rowCount }, (_, offset) => { + const row = displayRows[target[1] + offset]; + return row?.kind === "data" ? row.record.pid : null; + }); + const patches = planFieldPaste({ + field, + sourceFieldKey: copyProvenanceRef.current.fieldKey, + editable: canEditField(field), + values, + targetPids, + allowedChoices: statusValues[field.key] ?? choiceOptions(field), + }); + if (!patches) return false; + // R4 — ONE stack entry for the whole paste. Cell by cell would put forty entries on the + // stack for one Ctrl+V, and undoing a paste one cell at a time is not undoing a paste. + patchManyAndRecord( + patches.map((patch) => ({ pid: patch.pid, updates: { [field.key]: patch.value } })), + "a paste" + ); + // We handled the write ourselves. Returning false tells glide not to run its cell + // renderers' generic paste path (Bubble/rating renderers cannot enforce this contract). + return false; + }, + [visibleCols, fieldByKey, canEditField, statusValues, displayRows, patchManyAndRecord, + gridSelection.current] + ); + + const openHeaderMenu = useCallback( + (column: number, bounds: Rectangle) => { + if (embedded) return; + const definition = visibleCols[column]; + if (!definition?.id) return; + setColumnMenu({ + fieldKey: definition.id, + anchor: { + left: bounds.x, + top: bounds.y, + right: bounds.x + bounds.width, + bottom: bounds.y + bounds.height, + width: bounds.width, + height: bounds.height, + }, + }); + }, + [embedded, visibleCols] + ); + const onHeaderClicked = useCallback( + (column: number, event: HeaderClickedEventArgs) => { + if (event.isEdge) return; + event.preventDefault(); + openHeaderMenu(column, event.bounds); + }, + [openHeaderMenu] + ); + const onGridKeyDown = useCallback( + (event: GridKeyEventArgs) => { + if ( + event.key.toLowerCase() === "c" && + (event.ctrlKey || event.metaKey) && + !event.altKey + ) { + const range = gridSelection.current?.range; + const source = range?.width === 1 ? visibleCols[range.x] : undefined; + copyProvenanceRef.current = markGridCopy(source?.id ?? null, Date.now()); + // Do not prevent default: glide still owns serialization and the OS clipboard write. + return; + } + // Owner item 5 (2026-07-31) — EXCEL-GRADE ENTER. When no editor is open (an open overlay + // editor swallows its own keys before the canvas sees them), Enter moves the active cell + // DOWN one record and Shift+Enter moves UP — never opening the record drawer, which is + // what made keyboard runs down a column "really clunky". Glide's own overlay editor + // already commits-and-moves-down on Enter, so typing → Enter → typing flows like Excel; + // this handles the BETWEEN-edits half. Group headers are skipped in the direction of + // travel. The drawer stays reachable by double-click and the hover Expand button. + if (event.key === "Enter" && !event.ctrlKey && !event.metaKey && !event.altKey) { + const cur = gridSelection.current?.cell; + if (cur) { + event.preventDefault(); + event.stopPropagation(); + event.cancel(); + // Owner item 6 (2026-07-31) — Enter on a PICKED cell (select / multi select / + // assignee / rating) opens its picker: the keyboard door the mouse click already + // had. glide's text overlay cannot serve these types (validateCell refuses them), + // so without this the keyboard run down a column dead-ends at every picked field. + // Shift+Enter stays pure navigation, so walking UP past picked cells still works. + const curRow = displayRows[cur[1]]; + const curColumn = visibleCols[cur[0]]; + const curField = curColumn ? fieldByKey.get(curColumn.id!) : undefined; + if ( + !event.shiftKey && + curRow?.kind === "data" && + curField && + canEditField(curField) && + (isPickType(curField.type) || curField.type === "rating") + ) { + const b = gridRef.current?.getBounds(cur[0], cur[1]); + if (b) { + setPicker({ + pid: curRow.record.pid, + fieldKey: curField.key, + anchor: { + left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height, + width: b.width, height: b.height, + }, + }); + return; + } + } + const dir = event.shiftKey ? -1 : 1; + let row = cur[1] + dir; + while (row >= 0 && row < displayRows.length && displayRows[row]?.kind !== "data") + row += dir; + if (row >= 0 && row < displayRows.length) { + setActiveCell(cur[0], row); + gridRef.current?.scrollTo(cur[0], row, "vertical", 0, 0); + } + return; + } + } + const contextMenu = + event.key === "ContextMenu" || (event.key === "F10" && event.shiftKey); + // DataEditor exposes public controlled selection without the row marker, + // while GridKeyEventArgs.location comes from its internal grid and still + // includes that marker at column zero. + const eventColumn = + event.location && event.location[0] > 0 + ? event.location[0] - 1 + : undefined; + const column = gridSelection.current?.cell[0] ?? eventColumn; + if (!contextMenu || column === undefined || !event.bounds) return; + const gridBounds = gridBoxRef.current?.getBoundingClientRect(); + const headerTop = gridBounds?.top ?? event.bounds.y; + event.preventDefault(); + event.stopPropagation(); + event.cancel(); + openHeaderMenu(column, { + x: event.bounds.x, + y: headerTop, + width: event.bounds.width, + height: 36, + }); + }, + [gridSelection.current, openHeaderMenu, displayRows, setActiveCell, + visibleCols, fieldByKey, canEditField] + ); + + const persistView = useCallback((view: SavedView) => { + setViews((current) => { + const found = current.some((item) => item.id === view.id); + return found + ? current.map((item) => (item.id === view.id ? view : item)) + : [...current, view]; + }); + emitHostEvent({ id: eventId("view"), type: "view_upsert", view }); + }, []); + + /** + * Item 12 (C-LOCK) — set or clear a view's cohort lock, from the rail's menu. + * + * Works on ANY view, not only the active one, which is why it goes through `persistView` + * rather than `updateConfig`: the rail's menu opens on whichever row you clicked. When the + * target IS the active view, the live `config` must move too, or the table keeps showing the + * old row set until the next select and the lock reads as ignored. + * + * Clearing DELETES the key rather than storing null — the no-churn rule every other optional + * config member follows, and the shape `_clean_display`'s sibling validator expects. + */ + const onViewCohortLock = useCallback( + (viewId: string, cohortId: string | null) => { + const target = views.find((v) => v.id === viewId); + if (!target) return; + const nextConfig = { ...target.config }; + if (cohortId) nextConfig.cohortLock = cohortId; + else delete (nextConfig as Record).cohortLock; + persistView({ ...target, config: nextConfig }); + if (viewId === activeViewId) { + setConfig((live) => { + const next = { ...live }; + if (cohortId) next.cohortLock = cohortId; + else delete (next as Record).cohortLock; + return next; + }); + } + }, + [views, activeViewId, persistView] + ); + + const selectView = useCallback( + (id: string) => { + const current = views.find((view) => view.id === activeViewId); + if (current && !sameConfig(current.config, config)) + persistView({ ...current, config }); + const next = views.find((view) => view.id === id); + if (!next) return; + setActiveViewId(id); + setConfig(normalizeConfig(next.config, fields)); + setCollapsed(new Set()); + setSearch(""); + setDetailPid(null); + setDisplayCap(DISPLAY_PAGE); + // Owner item 3 (2026-07-31): tell the host WHERE THE USER IS, so a fresh browser (no + // localStorage copy) resumes on this view instead of the system default. Presentation + // state — the host stores the id and the read side re-validates it. + emitHostEvent({ id: eventId("view"), type: "view_select", viewId: id }); + // Owner item 10: opening a view is "I am working now" — the frame folds its nav rail. + signal(NAV_MINIMIZE_EVENT); + }, + [views, activeViewId, config, persistView, fields] + ); + + /** + * A-S4-3 (item 25) — a notification's click-through. The SHELL routes to the table and fires + * this; the GRID owns view selection, so neither learns the other's state. + * + * ⛔ IGNORE, NEVER THROW, when the view is not ours: an alert can outlive the view it watches + * (deleted, or a share revoked), and a reader who can no longer see it must simply land on the + * table. `selectView` already returns early on an unknown id; the topic check stops one grid + * reacting to another's alert when both are mounted. + */ + useEffect(() => { + const onOpen = (e: Event) => { + const detail = (e as CustomEvent).detail; + if (!detail || detail.topic !== scope) return; + if (!views.some((v) => v.id === detail.viewId)) return; + selectView(detail.viewId); + }; + window.addEventListener(VIEW_OPEN_EVENT, onOpen); + return () => window.removeEventListener(VIEW_OPEN_EVENT, onOpen); + }, [scope, views, selectView]); + const createView = useCallback( + (name: string, mode: DisplayMode, permissions: ViewPermissions) => { + const acceptedName = uniqueDisplayName(name, views.map((view) => view.name)); + /** + * ⭐⭐ WAVE 27 · OWNER ITEM 9 / RULING R4 — **A NEW VIEW IS BLANK. ALL OF IT.** + * + * ⛔ THIS REVERSES THE WAVE-26 COMMENT THAT STOOD HERE, so the reversal is stated rather + * than quietly applied. That comment dropped `cohortLock` from the spread and defended + * keeping the rest: *"their inheritance is a FEATURE (build a view, branch off it) + * precisely because it is visible"*. The owner disagrees, in as many words — R4: + * **"New views ALWAYS start blank — no filters/sorts inherited from anything."** The + * branch-off use it defended gets its own door later (Duplicate view); it is not what the + * "+" button means, and the wave-26 fix was the right diagnosis of the wrong scope — one + * member of the spread was invisible, but ALL of them arrived unasked. + * + * ⛔ AND `defaultViewConfig` IS THE BLANK, not a literal assembled here. It is the same + * function the grid's own initial state and the cohort door already use, so "blank" has + * ONE definition in this file ([[one-evaluator-per-question]]); a second literal beside it + * would be a place for the two to disagree the day a `ViewConfig` member is added — and + * the one that got added last wave is exactly what caused this bug. + * + * WHAT STOPS CARRYING, enumerated because a reader deserves the list and not just the + * ruling: `filters` + `filterConj`, `sorts`, `groupBy`, `colorBy`, `rowHeightMode`, + * `order`/`visible` (hidden columns), `widths`, `memberPids`, `frozenCount` and + * `cohortLock`. `order`/`visible` come back from the FIELD list's own default visibility, + * which is why `fields` replaces `config` in the dependency list below. + * + * ⚠ THE DISPLAY REFS GO TOO, and that is the one deliberate loss. W13's carry rule kept + * the calendar's date field / kanban's stack field / map's lat-lon across a mode switch so + * Grid→Map→Grid→Map did not re-ask; a NEW view is not a mode switch, and inheriting the + * previous view's stack field is inheritance of exactly the kind R4 names. A ref-less + * kanban asks which field to stack by, which is the honest state for a view born empty. + * `cleanDisplay` still collapses a ref-less GRID to absent, so picking "Grid" produces the + * byte-identical shape every pre-wave-9 view has (no churn). + */ + const nextConfig: ViewConfig = { + ...defaultViewConfig(fields), + display: cleanDisplay({ mode }), + }; + const view: SavedView = { + id: nextViewId(), + name: acceptedName, + kind: "custom", + config: nextConfig, + // I17 (C4) — sent EXPLICITLY. Absent on create means 'personal' host-side, so a user + // who chose "Collaborative" would silently get the opposite. `createdBy` is NOT sent: + // the host stamps it and ignores whatever the browser claims. + permissions, + }; + persistView(view); + setActiveViewId(view.id); + // The LIVE config has to move too, not just the stored one. Without this the view is + // created as a Calendar and the user keeps staring at the Grid until they switch views + // and back — the mode would be real in the store and invisible on screen. + setConfig(nextConfig); + setSaveState("saved"); + }, + // `fields`, not `config` (R4): the blank is derived from the COLUMNS, and reading the live + // config here at all is what item 9 deletes. + [fields, persistView, views] + ); + const renameView = useCallback( + (id: string, name: string) => { + const view = views.find((item) => item.id === id); + if (view) { + const acceptedName = uniqueDisplayName( + name, + views.filter((item) => item.id !== id).map((item) => item.name) + ); + persistView({ ...view, name: acceptedName }); + } + }, + [views, persistView] + ); + /** + * I12 (contract C3) — freeze/unfreeze a view's DISPLAY MODE. + * + * The client re-checks the actor before emitting even though the menu entry is already + * gated: an event can be replayed, and "hidden in the client" has never been a permission. + * The host checks it again and that check is the wall — this one only keeps the client from + * showing a change that will not survive the round trip. + */ + const toggleViewLock = useCallback( + (id: string, locked: boolean) => { + const view = views.find((item) => item.id === id); + if (!view || !mayToggleViewLock(view, viewer)) return; + persistView({ ...view, locked }); + }, + [views, persistView, viewer] + ); + // The list description. Template lists ship with prose seeded by the host, but it is + // the USER's text once they touch it — including clearing it. An empty string is + // persisted as an empty string (never coerced back to the seed), because the host + // resolves a saved view OVER its template, so "" is how you delete a description. + const setViewNote = useCallback( + (id: string, note: string) => { + const view = views.find((item) => item.id === id); + if (view) persistView({ ...view, note: note.slice(0, 2000) }); + }, + [views, persistView] + ); + const duplicateView = useCallback( + (id: string) => { + const source = views.find((view) => view.id === id); + if (!source) return; + const copy: SavedView = { + ...source, + id: nextViewId(), + name: uniqueDisplayName( + `${source.name} copy`, + views.map((view) => view.name) + ), + kind: "custom", + locked: false, + // I17 (C4) — a COPY IS THE DUPLICATOR'S OWN, PERSONAL view. Ruled, not inherited: + // - inheriting a 'users' grant would silently re-share the copy with a list the + // person making it never chose; + // - Duplicate is also the escape hatch for someone who may NOT edit the original, + // and 'personal' + the host's fresh createdBy stamp is exactly "mine to work in". + permissions: { edit: "personal" }, + // Never echo the SOURCE's creator: the host stamps and ignores what the browser + // sends, but sending someone else's name is a laundering attempt on its face. + createdBy: undefined, + config: { ...source.config }, + }; + persistView(copy); + setActiveViewId(copy.id); + setConfig(copy.config); + }, + [views, persistView] + ); + const deleteView = useCallback( + (id: string) => { + const view = views.find((item) => item.id === id); + // ⚠ This gate used to read `view.locked`, and the menu entry above it did too. C3 + // redefines `locked` as "the DISPLAY MODE is frozen" on ANY view, so leaving the gate + // here would make a user-frozen Kanban undeletable in the client while the host would + // delete it happily (app.py refuses only `all-customers`, by id). The MENU is just the + // door — this is the gate, and both had to move. + // C4 rides alongside: you may not delete a view you may not edit. + if (!view || isUndeletableView(view) || !mayEditView(view, viewer)) return; + // 2026-08-04 — STAMP BEFORE THE EMIT, the writeLocal-before-emit order every other + // optimistic path here uses. The live adopt below re-reads `payload.workspace.views` + // on every echo, and the queue sends ONE batch at a time — so an echo answering an + // EARLIER batch still lists this view, and without the tombstone the row would come + // back and (since that merge never removes) stay back. See liveWorkspace.ts. + viewTombstonesRef.current = stampTombstone(viewTombstonesRef.current, id, Date.now()); + setViews((current) => current.filter((item) => item.id !== id)); + emitHostEvent({ id: eventId("view-delete"), type: "view_delete", viewId: id }); + if (activeViewId === id) { + const all = views.find((item) => item.id === ALL_VIEW_ID) ?? allRecordsView(fields, scope); + setActiveViewId(all.id); + setConfig(all.config); + } + }, + [views, activeViewId, fields, viewer, scope] + ); + + /** + * Wave-6 item 7 — PER-COHORT view state. Each cohort keeps its own view under the stable id + * `cohort:`; a cohort with no saved view opens CLEAN (defaultViewConfig — no sort, + * no filters), which is exactly what "first created" means. The view is added LOCALLY on + * first open and emitted only when the user actually edits it (the autosave effect already + * compares configs), so "absent = clean" stays true in the store — opening every cohort + * once must not write a store full of empty views. + */ + useEffect(() => { + if (!cohortMode || !workspaceReady || !activeCohortId) return; + const vid = `cohort:${activeCohortId}`; + if (activeViewId === vid) return; + const outgoing = views.find((view) => view.id === activeViewId); + if (outgoing && !sameConfig(outgoing.config, config)) + persistView({ ...outgoing, config }); + const existing = views.find((view) => view.id === vid); + const view: SavedView = existing ?? { + id: vid, + name: lists.find((l) => l.id === activeCohortId)?.name ?? "Cohort", + kind: "custom", + config: defaultViewConfig(fields), + }; + if (!existing) setViews((current) => [...current, view]); + setActiveViewId(vid); + setConfig(normalizeConfig(view.config, fields)); + setCollapsed(new Set()); + setSearch(""); + }, [cohortMode, workspaceReady, activeCohortId, activeViewId, views, config, + fields, lists, persistView]); + + /** + * "Add to list" (owner item 6). Runs the SAME engine over the named view's config to learn + * which customers it matches, then hands those pids to the host. + * + * ⚠ Deliberately re-runs the pipeline for THAT view rather than using what is on screen: the + * menu is available on every view, not only the active one, and "add the rows I can see" would + * quietly mean something different depending on which view happened to be open. + * + * ⚠ Refuses on a windowed table. There the client holds ONE PAGE, so "the customers this view + * matches" is a question it cannot answer — it would add the 200 rows it happens to hold and + * report success. Silence would be worse than the refusal (CG-3, no-unverifiable-aggregates). + * + * ⚠ Refuses for the SAME reason when this view has a measure condition with no answer yet + * (CG-8). A pending condition matches nothing, so the cohort would be built from a filter that + * is currently narrower than the one the user is reading — and a cohort is a FIXED set, so + * that wrong membership would persist long after the answer arrived. + */ + /** + * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — HOW MANY RECORDS EACH ALERTED VIEW MATCHES. + * + * ⛔ ONE EVALUATOR, AND IT IS `runPipeline` RATHER THAN `matchFilterTree`. The question is + * "how many records does this VIEW show", and a view is more than its condition list: a + * `cohortLock` narrows before anything else, `memberPids` are pinned in regardless, and a + * RANK leaf ("top 10") is only answerable against the domain the other conditions leave. + * `matchFilterTree` answers a different question — "does this ROW match this tree" — and + * using it here would report the whole book for a locked view and nonsense for a ranked one. + * `addToList` below already reaches for the pipeline for the same reason + * ([[one-evaluator-per-question]]); this is the same call with the same inputs. + * + * ⛔ THREE CASES WHERE THERE IS NO HONEST NUMBER, and each is ABSENT rather than zero: + * · a WINDOWED table — the client engine is skipped by design (CG-3), so a count here + * would be the size of one PAGE wearing the label of a scope; + * · an unresolved MEASURE condition — the server has not answered it yet, so the view + * matches nothing *yet*, and rendering `0` would state a fact nobody has established; + * · a view the rail no longer holds — an alert outlives its view. + * A missing key means no badge (the prop's own contract), which is the correct rendering of + * "not known" and is distinguishable from `0`, which is a real and interesting answer. + * + * ⚠ MEMOISED ON THE ALERTED SET, not on `views`: this walks the whole book once per alerted + * view, and on a table with no alerts it does nothing at all. + */ + const alertedKey = alerted.join(","); + const alertCounts = useMemo(() => { + const out: Record = {}; + if (serverWindowed || !alerted.length) return out; + for (const id of alerted) { + const view = views.find((v) => v.id === id); + if (!view) continue; + if (pendingMeasures(view.config.filters, measureSets) > 0) continue; + const { pidToIndex } = runPipeline({ + rawRows: computedRows, + fields, + filters: view.config.filters, + search: "", + sorts: [], + groupBy: null, + collapsed: new Set(), + memberPids: view.config.memberPids, + filterConj: view.config.filterConj ?? "and", + measureSets, + cohortSets, + cohortLock: view.config.cohortLock, + today, + }); + // DISTINCT records, which is what `pidToIndex` is — a grouped pipeline can list one + // record under several headings and `visibleRows.length` would then count it twice. + out[id] = pidToIndex.size; + } + return out; + // `alertedKey` is the scalar identity of the list; `alerted` itself is a fresh array on + // every fetch even when the answer has not changed. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [alertedKey, views, computedRows, fields, measureSets, cohortSets, today, serverWindowed]); + + const addToList = useCallback( + (viewId: string, cohortId: string, name: string) => { + if (serverWindowed) return; + const view = views.find((v) => v.id === viewId); + if (!view) return; + if (pendingMeasures(view.config.filters, measureSets) > 0) return; + const { visibleRows: matched } = runPipeline({ + rawRows: computedRows, + fields, + filters: view.config.filters, + search: "", + sorts: view.config.sorts, + groupBy: null, + collapsed: new Set(), + memberPids: view.config.memberPids, + filterConj: view.config.filterConj ?? "and", + measureSets, + cohortSets, + today, + }); + const pids = matched + .filter((r): r is { kind: "data"; record: Row } => r.kind === "data") + .map((r) => r.record.pid); + emitHostEvent({ + id: eventId("addlist"), + type: "add_to_list", + viewId, + cohortId, + name, + pids, + }); + }, + [views, computedRows, fields, serverWindowed, measureSets, cohortSets, today] + ); + + // ---------------------------------------------------------------- I11c (C4) + // Folders for the Views rail. The MODEL (echo reconcile, tombstones, dangling + // refs) lives in folders.ts and is gated by verify_folders.py; this is only the + // wiring: stamps for what this browser just did, and the five events. + const [folderStamps, setFolderStamps] = useState({}); + /** + * Folders THIS browser just created, held until the host echo returns them. + * + * ⚠ Without these, `reconcileFolders` was being called with the host list as + * BOTH arguments, which makes its optimistic-create branch unreachable: a + * folder absent from `host` is equally absent from `local`. The function was + * right and the wiring was wrong, so "+ New folder" painted nothing until the + * round trip landed — seconds, on a cold container, with the user clicking + * again. The gate passed because it called reconcileFolders with two DIFFERENT + * arrays; it now also calls it the way this component does. + */ + const [localFolders, setLocalFolders] = useState([]); + const workspaceFolders = payload?.workspace?.folders; + const folders = useMemo( + () => reconcileFolders(workspaceFolders, localFolders, folderStamps, Date.now()), + [workspaceFolders, localFolders, folderStamps] + ); + const folderIdOf = useCallback( + (viewId: string) => + resolveFolderId( + viewId, + views.find((v) => v.id === viewId)?.folderId ?? null, + folders, + folderStamps, + Date.now() + ), + [views, folders, folderStamps] + ); + const stampFolder = useCallback((patch: (prev: FolderStamps) => FolderStamps) => { + setFolderStamps((prev) => pruneFolderStamps(patch(prev), Date.now())); + }, []); + + /* wave17 R1 / C-LOCKV — the COHORT FOLDER surface is gone. `folders['cohorts']` merged + into `folders['views']` in the host's one-shot migration and `cohortFolders` is no longer + emitted at all, so `reconcileFolders` here would have been reconciling an absent list + against nothing forever. A locked view files into an ORDINARY folder now, which is what R1 + means by "native folders". */ + + /** + * C4 as AMENDED — the folder-level bulk add's PREVIEW. Deliberately re-runs the + * engine per contained view, exactly as `addToList` does, and carries the SAME + * two refusals as named skips rather than dropping them silently: a union that + * quietly omits two views is a wrong cohort that looks right, and a cohort is a + * FIXED set, so it stays wrong ([[no-unverifiable-aggregates]]). + */ + const folderAddPreview = useCallback( + (folderId: string) => { + const pids = new Set(); + const skipped: { name: string; why: string }[] = []; + let counted = 0; + for (const view of views) { + if (folderIdOf(view.id) !== folderId) continue; + if (serverWindowed) { + skipped.push({ name: view.name, why: "this table loads one page at a time" }); + continue; + } + if (pendingMeasures(view.config.filters, measureSets) > 0) { + // C-NAME (item 9) — user-facing, so it says "metric" like every other surface. + skipped.push({ name: view.name, why: "a metric condition has no answer yet" }); + continue; + } + counted += 1; + const { visibleRows: matched } = runPipeline({ + rawRows: computedRows, + fields, + filters: view.config.filters, + search: "", + sorts: view.config.sorts, + groupBy: null, + collapsed: new Set(), + memberPids: view.config.memberPids, + filterConj: view.config.filterConj ?? "and", + measureSets, + cohortSets, + today, + }); + for (const r of matched) if (r.kind === "data") pids.add(r.record.pid); + } + return { pids: [...pids], counted, skipped }; + }, + [views, folderIdOf, serverWindowed, computedRows, fields, measureSets, cohortSets, today] + ); + + const folderAddToList = useCallback( + (folderId: string, cohortId: string, name: string) => { + const { pids } = folderAddPreview(folderId); + if (!pids.length) return; + // ONE emit with the deduped union, through the SAME guarded event a single + // view's "Add to cohort" uses — so the host validates both identically. + emitHostEvent({ + id: eventId("addlist"), + type: "add_to_list", + viewId: "", + cohortId, + name, + pids, + }); + }, + [folderAddPreview] + ); + + /** + * Wave-7 item W2 (contract C2) — export "the current view", CLIENT-side. The same + * engine run addToList uses (that is the point: the file says exactly what the count + * says), over the same pool the named surface scopes to. + * + * ⚠ Same two refusals as addToList, for the same reasons: a server-windowed table + * holds ONE PAGE (the client cannot answer "the view's matches"), and a view with a + * PENDING measure condition currently matches nothing — an export taken then would be + * an empty/narrower file wearing the view's name. Both windows self-resolve; the + * refusal is silent no-op for one round trip. + */ + const exportRows = useCallback( + (name: string, cfg: ViewConfig, poolRows: Row[], format: ExportFormat) => { + if (serverWindowed) return; + if (pendingMeasures(cfg.filters, measureSets) > 0) return; + const { visibleRows: matched } = runPipeline({ + rawRows: poolRows, + fields, + filters: cfg.filters, + search: "", + sorts: cfg.sorts, + groupBy: null, + collapsed: new Set(), + memberPids: cfg.memberPids, + filterConj: cfg.filterConj ?? "and", + measureSets, + cohortSets, + today, + }); + const records = matched + .filter((r): r is { kind: "data"; record: Row } => r.kind === "data") + .map((r) => + overlayEdits[r.record.pid] + ? { ...r.record, ...overlayEdits[r.record.pid] } + : r.record + ); + // Columns = the view's VISIBLE fields in display order (the reconciled order: + // every key present, config order first — the same shape useGridColumns paints). + const inOrder = new Set(cfg.order); + const keys = [ + ...cfg.order, + ...fields.map((f) => f.key).filter((k) => !inOrder.has(k)), + ]; + const vis = new Set(cfg.visible); + const cols = keys + .filter((k) => vis.has(k)) + .map((k) => fieldByKey.get(k)) + .filter((f): f is Field => !!f); + runExport(format, name, today, cols, records); + }, + [serverWindowed, fields, fieldByKey, measureSets, cohortSets, today, overlayEdits] + ); + + /** + * ⭐ owner item 3 (2026-08-03) — THE SHEET THE TIME-SERIES VIEW IS SHOWING, published by the + * panel. A ref rather than state on purpose: nothing renders from it, and setting state on + * every sheet rebuild would re-render the whole grid to feed a menu nobody has opened yet. + */ + const tsSheetRef = useRef(null); + const onTsSheet = useCallback((s: TsSheet) => { tsSheetRef.current = s; }, []); + /** The same channel for a SUMMARY calendar — the other mode whose content is not its rows. + * `null` is CalendarView saying "records mode, export the rows" (see its note). */ + const calSheetRef = useRef<{ fields: Field[]; rows: Row[] } | null>(null); + const onCalSheet = useCallback( + (s: { fields: Field[]; rows: Row[] } | null) => { calSheetRef.current = s; }, + [] + ); + + /** + * W2 — the view menu's Export (Customer page): the named view over the whole pool. + * + * ⭐ owner item 3 (2026-08-03) — AND IT EXPORTS WHAT THE VIEW SHOWS. Every mode used to + * export the same thing: the matched customer rows. For grid / list / kanban / calendar / + * map / chart that is right — those modes ARRANGE rows, so the rows are what they show, and + * a calendar's dates and a kanban's lanes are columns already in the file. + * + * TWO modes are not arrangements of rows, and both were wrong in the direction that matters: + * + * · `timeseries` — metric ROWS over period COLUMNS. Exporting it handed you a customer list + * that shares none of its numbers. It exports the SHEET. + * · `calendar` IN SUMMARY MODE — metric values per DAY (C-DISP item 4). Same problem, and + * the owner named this one by hand. A calendar in RECORDS mode is genuinely an + * arrangement of rows, so it keeps the row export; `CalendarView` says which it is by + * publishing a sheet or publishing null. + * + * ⚠ ONLY FOR THE ACTIVE VIEW, and this is a real limit, not an oversight. Both sheets are + * built by the mounted view — the time series from a server round trip, the calendar from the + * month on screen. A view sitting unopened in the rail has neither, and this component cannot + * conjure one without fetching it. So that case SAYS SO and downloads nothing: the + * alternative is silently handing over the customer rows under that view's name, which is the + * exact substitution this branch exists to stop. + */ + const exportViewData = useCallback( + (viewId: string, format: ExportFormat) => { + const view = views.find((v) => v.id === viewId); + if (!view) return; + const cfg = normalizeConfig(view.config, fields); + const spec = cleanDisplay(cfg.display); + const isActive = viewId === activeViewId; + const refuse = (what: string) => + signal( + TOAST_EVENT, + `Open "${view.name}" first: ${what} is exported from what it draws on screen.` + ); + + if (spec?.mode === "timeseries") { + const sheet = isActive ? tsSheetRef.current : null; + if (!sheet) return refuse("a time series"); + if (sheet.empty) { + signal(TOAST_EVENT, + `"${view.name}" has no metrics on its sheet yet — add one, then export.`); + return; + } + if (format === "csv") { + // The gated builder (verify_timeseries), kept as THE csv path so the file the owner + // downloads is the one the gate proves — footnotes, grouped thousands and all. + triggerDownload( + exportFilename(view.name, today, format), + new Blob(["" + buildTsCsv(sheet)], { type: "text/csv;charset=utf-8" }) + ); + } else { + const { fields: tf, rows: tr } = tsSheetToTable(sheet); + runExport(format, view.name, today, tf, tr); + } + return; + } + + /* ═══ W18-C CATALOG ═══ A catalog is the THIRD mode that is not an arrangement of rows, + and it is the furthest from one: its pages are authored content, and its product codes + are a fraction of the pool in an order the user chose. Handing over the matched rows + under a catalogue's name would be the same substitution the two branches around this + one exist to stop. It exports as a PDF, through the browser's own print dialog (R10), + so this branch refuses the file and names the door that works. */ + if (spec?.mode === "catalog") { + signal( + TOAST_EVENT, + `"${view.name}" is a catalog — open it and use Print to save it as a PDF. ` + + `A spreadsheet of its products is not what it draws.` + ); + return; + } + /* ═══ end W18-C CATALOG ═══ */ + + if (spec?.mode === "calendar" && spec.calendarMode === "summary") { + const sheet = isActive ? calSheetRef.current : null; + if (!sheet) return refuse("a calendar summary"); + runExport(format, view.name, today, sheet.fields, sheet.rows); + return; + } + + exportRows(view.name, cfg, computedRows, format); + }, + [views, fields, computedRows, exportRows, activeViewId, today] + ); + + /** + * The selection bar's "Add to locked view": exactly the CHECKED customers, not a view's + * matches. + * Same guarded `add_to_list` event — the host intersects the pids with the caller's pool + * either way, so a hand-picked set and a view's match set ride one validation path. + */ + const addSelectionToList = useCallback( + (cohortId: string, name: string) => { + const pids = [...selectedPids]; + if (!pids.length || serverWindowed) return; + emitHostEvent({ + id: eventId("addlist"), + type: "add_to_list", + viewId: activeViewId, + cohortId, + name, + pids, + }); + setSelAddOpen(false); + setSelListName(""); + clearSelection(); + }, + [selectedPids, serverWindowed, activeViewId, clearSelection] + ); + + // Item 3c: the local write is the RENDERED truth (the def half of the no-blip contract) — + // stamped so a lagged echo cannot claw it back at the next remount. Item 9c: `scope` rides + // the EVENT at create time only; the def itself carries `scope: 'cohort'` so the echo is + // byte-stable ('global' stays absent — the host normalizes them to one shape). + const scopeChoice = payload?.workspace?.scopeChoice === true; + /** + * Item 12 (C-LOCK) — the Filter banner's copy for a cohort-locked view. + * + * `undefined` when the view is not locked. When it IS locked but the set is not in `lists`, + * this returns an EMPTY object: the banner must still appear (the reader is looking at a + * narrowed table and deserves to know why), but with no name and no count, because a set we + * were given no membership for is a set we must not describe. RECORD's banner reads that + * absence and says "locked to a set you cannot see". + * + * The count is the cohort's OWN size, not the number of rows on screen — the conditions + * narrow within the lock, so the two differ and the banner is stating the lock. + */ + /** + * ⭐ wave17 R1 / C-LOCKV — the lock in force, which is now simply the ACTIVE VIEW's. + * + * Wave 15 had two sources: a saved `config.cohortLock` and an ephemeral pick from the Cohorts + * rail that outranked it while held. The second existed only because a projected cohort was + * not a saved view, so opening one could not be allowed to write anything. Under R1 it IS a + * saved view, so selecting it applies its own stored lock through the ordinary view path — + * one source, and it survives a reload, which the transient one never did. + */ + const effectiveCohortLock = config.cohortLock; + /* wave17 R1 / C-LOCKV — `cohortRows` and the whole `cohortRail` projection are GONE. They + turned `lists` into rail rows for a section that no longer exists; the host projects each + cohort into `views` instead, so the rail renders them through the same path as every other + view and there is nothing left to project. `lists` still arrives and is still load-bearing + — it is the MEMBERSHIP channel that feeds `cohortSets`, which is what the lock resolves + against. Its absence would blank every locked view, so it is not "unused". */ + const cohortLockChip = useMemo(() => { + const id = effectiveCohortLock; + if (!id) return undefined; + const set = lists.find((l) => l.id === id); + if (!set) return {}; + return { name: set.name, count: set.pids?.length ?? 0 }; + }, [effectiveCohortLock, lists]); + // (Setting/clearing the lock lives in `onViewCohortLock` above — the rail's menu operates on + // whichever view you clicked, not necessarily the active one, so it goes through + // `persistView` and only touches the live `config` when the two are the same view.) + /** + * Item 5 / contract C-LAYOUT — this user's record-detail field order, for RECORD's + * `RecordDetail` (they asked for exactly these two props in the wave mailbox). + * + * Read straight off the workspace stratum the host echoes, and emitted back as the + * `record_layout` event HOST landed. Deliberately NOT reconciled or optimistically merged + * here: the event returns False host-side (an autosave hot path like `view_upsert`), so + * there is no rerun to race, and `RecordDetail` holds the live order while a drag is in + * flight. Nothing else in the grid reads it — the order is the MODAL's, never the grid's + * `config.order`. + */ + const recordLayout = payload?.workspace?.recordLayout?.order; + const onRecordLayout = useCallback((order: string[]) => { + emitHostEvent({ id: eventId("rlayout"), type: "record_layout", order }); + }, []); + const saveField = useCallback( + (field: Field, scope?: FieldScope) => { + const requestId = eventId("field"); + const accepted: Field = { + ...field, + label: uniqueDisplayName( + field.label, + fields.filter((item) => item.key !== field.key).map((item) => item.label) + ), + editRequestId: requestId, + }; + delete accepted.labelCorrectedFrom; + delete accepted.labelCorrectionId; + stampFieldEdit(accepted.key); + setFields((current) => + current.some((item) => item.key === accepted.key) + ? current.map((item) => (item.key === accepted.key ? accepted : item)) + : [...current, accepted] + ); + emitHostEvent({ + id: requestId, + type: "field_upsert", + field: accepted, + ...(scope ? { scope } : {}), + }); + }, + [stampFieldEdit, fields] + ); + const createField = useCallback( + ( + label: string, + type: FieldType, + anchorKey: string | null, + side: "left" | "right" | "end", + options?: string[], + measureSpec?: { key: string; window: WindowSpec }, + extra?: FieldBuildExtra + ) => { + /** + * ⚠ RENAMED FROM `scope` (wave 21, item 7). It used to SHADOW the component's own + * `scope` prop — the surface this grid is drawing — with a completely different + * thing: a FIELD's storage stratum (`'cohort'` or nothing). Two unrelated meanings + * under one word, in a function that now has to speak to a per-table endpoint. + * Caught by `tsc` only because `FieldScope` and `SurfaceScope` happen to be + * incompatible unions; had either been a bare `string`, the C2 call below would have + * POSTed to `/tables/undefined/fields` and read as a server bug. + */ + const fieldScope = scopeChoice ? extra?.scope : undefined; + const scoped = fieldScope === "cohort" ? ({ scope: "cohort" } as const) : {}; + // Owner item 7 — a FORMULA-MEASURE column. `measure_` prefixed, source:'odoo' (read-only + // at both ends), derived (values arrive from the host per render), filterable:false (the + // replacement is the measure CONDITION with the same measure+window — the governed path). + // The first render after creation shows blank cells for exactly one round trip: the host + // persists the field, then computes the values into the next payload. + if (measureSpec) { + const field: Field = { + key: `measure_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`, + label, + type, + source: "odoo", + default: true, + custom: true, + derived: true, + filterable: false, + agg: ["currency", "int"].includes(type) ? "sum" : undefined, + measure: measureSpec, + ...scoped, + }; + saveField(field, fieldScope); + insertColumn(field.key, anchorKey, side); + return; + } + const field = { ...buildOverlayField(label, type, options, extra), ...scoped }; + /** + * ⭐ WAVE 21 item 7 (contract C2) — AN AUTOMATION COLUMN ON A USER DATABASE GOES TO + * THE DEFINITION, not to this user's overlay stratum. + * + * The automation editor's column picker reads `user_tables` (the definition); every + * grid-created column went to `_table_workspace` (the per-user overlay). So the + * picker was empty by construction: the user made the column, then could not find it + * in the automation they made it for. C2 routes exactly this one kind through + * `POST /tables/{key}/fields`, and the ROUTE is the wall — `_field_or_refuse` allows + * only the creator or an admin, which is the same answer the grid would have got. + * + * ⚠ NARROW ON PURPOSE, twice over: `type === "automation"` AND a `ut_` scope. The + * connector surfaces have no such endpoint (their schema is their source's), and + * every other kind keeps the overlay path this wave — moving all of them changes who + * can SEE a column, which is a behaviour change three surfaces would feel and is + * booked as debt rather than smuggled in here. + * + * ⚠ AND IT IS ASYNC, where every other create is optimistic. There is no local + * insert first: the server re-slugs the key and can refuse outright (it 400s until + * `UT_FIELD_TYPES` learns `automation` — C2's other half, A's), and a column shown + * under a key the store never took is one that vanishes on the next read with no + * explanation. The wait is one round trip; the alternative is a lie. + */ + /** + * ⭐⭐ 2026-08-09 — THE RELATIONAL PAIR JOINS THAT ROUTE, and for a stronger reason than + * `automation` had. The note above says every other kind "keeps the overlay path this + * wave", weighing it as a question of who can SEE a column. For `link` and `rollup` it is + * not a visibility trade-off at all: their cells are computed SERVER-side by + * `compute_relation_cells`, which walks `user_tables` definitions. A rollup parked in a + * per-user overlay is invisible to the only code that could ever fill it, so it is not a + * narrower feature — it is a column that cannot work, in any account, ever. + * + * ⛔ AND THE BAG TRAVELS. `_clean_field` returns None for a bagless `link`/`rollup`, so + * sending `{key,label,type}` here would trade a silent blank column for a loud 400. The + * bag is on `extra`; `field` already carries it (see `FieldBuildExtra`). + */ + /** + * ⭐⭐ 2026-08-10 — `formula` JOINS THE SAME ROUTE, on the visibility argument the note + * above weighs and the relational note declines to use. + * + * For `link`/`rollup` visibility was not the point (their cells are computed by something + * that reads the definition). For a formula it IS the whole point, and the trade-off falls + * the other way from wave 21's read of it: a `ut_*` database is a SHARED database — the + * owner builds a column on "IG profile" and expects the four people looking at that + * database to see it. In the overlay stratum only its creator ever does. Measured on + * nurilab: `Trimmed reel views`, the trimmed-average column the whole rollup chain exists + * to produce, was visible to exactly one account. + * + * ⛔ THE VALUES STILL DO NOT MOVE, and this must not be sold as more than it is. A formula + * is computed in this browser (`computedRows`) from the row's other cells and is stored + * nowhere; what became shared is the EXPRESSION. An automation, a rollup and an export + * still cannot read this column. Sharing the definition is not materialising the number. + * + * ⚠ `agg: "sum"` rides along so the totals row behaves identically to the overlay path + * (`buildOverlayField` stamps it) — a column that stops totalling because of which door + * created it is [[gate-answers-the-wrong-question]] wearing a create route. + */ + /** + * ⭐⭐ WAVE-29 T22 (owner item 2a) — **EVERY grid-created column on a `ut_*` database goes + * to the DEFINITION.** The three notes above widened this set one kind at a time + * (`automation`, then `link`/`rollup`, then `formula`), each ending by asking whoever needed + * a fifth to come back here and decide. This is that decision, and it goes to ALL of them + * rather than to `select` alone, because the defect the owner reported is not about select: + * + * ⛔ **ON A `ut_` SCOPE THERE IS NO PER-USER CELL STRATUM LEFT TO WRITE TO.** Every accepted + * cell edit is routed to `user_tables.patch_cells` — the shared DEFINITION rows + * (`grid_events.py:1979-1984`) — while both `ut_` row doors project rows through the + * definition's own field keys (`routes_tables.scoped_pool:176`, `user_tables.add_row:678`). + * So a column living only in `
_table_workspace` accepts a value, stores it under a + * key the projection does not know, and reads back BLANK. **The write is not lost; the read + * cannot see it** — which is why the server reports the write as accepted and the cell is + * empty after any refetch. That is item 2a, and it applied to `text`, `int`, `date`, `user`, + * `image` and `checkbox` exactly as much as to `select`. The owner reported the one they + * happened to build. + * + * ⛔ **`created_time` is the ONE exclusion, and it is the SERVER's, not a preference:** + * `UT_FIELD_TYPES` does not contain it, so this POST would 400 (measured — CREATABLE_TYPES + * minus UT_FIELD_TYPES is exactly `{created_time}`). It keeps the overlay path, where it is + * harmless: it is `derived`, so it has no cell of its own to lose. + * + * ⚠ **The body is a PROJECTION of the field the constructor already built**, not a + * per-kind list of keys. The old spelling enumerated what each kind needed, which is how + * `options` came to be missing for the kind that needed it most — a select POSTed without + * its options is degraded to `text` on read (`aios_grid.py:664-668`). `buildOverlayField` + * decides the shape; this sends what it decided, so a new kind cannot arrive half-described. + * (`_clean_field` ignores what it does not know, so `custom`/`derived`/`multi` staying + * client-side costs nothing; `max` on a `rating` is the one key it drops — booked, not lost.) + */ + if (isUserTable && type !== "created_time") { + void addTableField(scope, { + key: field.key, + label: field.label, + type, + ...(field.options?.length ? { options: field.options } : {}), + ...(field.colorCodeOptions !== undefined + ? { colorCodeOptions: field.colorCodeOptions } + : {}), + ...(field.optionColors ? { optionColors: field.optionColors } : {}), + ...(extra?.link ? { link: extra.link } : {}), + ...(extra?.rollup ? { rollup: extra.rollup } : {}), + ...(type === "formula" && extra?.formula ? { formula: extra.formula } : {}), + ...(field.agg ? { agg: field.agg } : {}), + }).then((made) => { + if (!made) return; // the bridge has already said why + setFields((current) => + current.some((f) => f.key === made.key) ? current : [...current, made] + ); + insertColumn(made.key, anchorKey, side); + /** + * ⛔⛔ 2026-08-10 — THE WORKSPACE RE-READ USED TO FIRE HERE AND IT ATE THE LINE ABOVE. + * + * `insertColumn` adds the new key to the view's `visible` set and autosaves that config; + * `signal(WORKSPACE_STALE_EVENT)` refetches `/workspace`, whose views carry the SERVER's + * config — the one written a moment before the insert. The refetch won, every time, so + * a column created into the shared definition was **stored correctly and rendered + * nowhere**: present in the payload, present in `order` (which is re-derived from the + * field list), and absent from `visible`. MEASURED on live nurilab across all four + * views after migrating `Trimmed reel views`; it has been true for `link`, `rollup` and + * `automation` since that route was widened on 2026-08-09. + * + * ⛔ AND THE COMMENT IT REPLACES WAS THE REASON NOBODY LOOKED. It claimed the signal is + * "what makes the column survive a reload rather than living only in this browser's + * state" — but the SERVER already stored the field; the 201 is what makes it survive. + * The signal only refreshed a payload whose field list `setFields` had just updated by + * hand, and `addTableField` already drops the rows cache. It bought nothing and cost + * the one piece of state the create had just written. + * + * ⚠ The delete and reconfigure paths keep their signals: those change VALUES other + * columns fold (a dropped link takes its reciprocal and every rollup over it), and + * neither writes view config in the same breath, so neither has this race. + */ + }); + return; + } + saveField(field, fieldScope); + insertColumn(field.key, anchorKey, side); + }, + [insertColumn, saveField, scopeChoice, isUserTable, scope] + ); + + /** + * ⭐⭐ 2026-08-07 (D-79's last half) — turn a text column into THE profile column. + * + * The write goes to the DEFINITION (`PATCH /tables/{key}/fields/{fkey}`), never to this + * user's overlay stratum, for the reason wave 25's amendment C3-A1 records: the automation + * engine reads `t['rows']`/`t['fields']`, so a flag parked in one person's overlay is a flag + * the enrich step can never see. Same door, same reasoning, as the automation column above. + * + * ⚠ `profile: null` CLEARS it — `_clean_field` treats an explicit null as "take the flag off" + * rather than "leave it alone", which is what lets a column be un-marked without deleting it. + */ + const setProfileFlag = useCallback( + (key: string, on: boolean) => { + if (!isUserTable) return; + void patchTableField(scope, key, { + profile: on ? { source: "instagram" } : null, + }).then((made) => { + if (!made) return; // the bridge has already said why + setFields((current) => current.map((f) => (f.key === made.key ? made : f))); + signal(WORKSPACE_STALE_EVENT); + }); + }, + [isUserTable, scope] + ); + + /** + * ⭐⭐ 2026-08-09 (owner) — RECONFIGURE a `link`/`rollup` column, in the SHARED definition. + * + * Owner: *"even when i click edit field for rollup, it doesn't actually show me the + * configuration"* and *"No rollup field should be uneditable."* Both land here: the Edit pane + * now renders the bag, and this is where its Save goes. + * + * ⛔ NOT `saveField`, and not `retypeField`. Those write the per-user overlay stratum, which is + * precisely the defect this change exists to close — `compute_relation_cells` reads + * `user_tables`, so a bag parked in one person's workspace can never be computed by anything. + * ⚠ ASYNC with no optimistic insert, like the automation create: the server re-cleans the bag + * and can refuse the whole field (`_clean_rollup` returns None for, say, a `limit` with no + * `sortBy`), and showing a configuration the store never took is the lie the wait avoids. + * The cells arrive on the next read — a rollup is recomputed server-side, never in the browser. + */ + const setFieldConfig = useCallback( + (key: string, patch: { + label?: string; link?: unknown; rollup?: unknown; formula?: string; agg?: string; + }) => { + if (!isUserTable) return; + void patchTableField(scope, key, patch as Record).then((made) => { + if (!made) return; // the bridge has already said why + setFields((current) => current.map((f) => (f.key === made.key ? made : f))); + // The definition changed, so the wire's field list did too — and a rollup's VALUES are + // recomputed host-side on this write, so the ROWS are a beat stale as well. The bridge + // drops its rows cache for this table (it owns that map); these two ask for a re-read. + signal(WORKSPACE_STALE_EVENT); + signal(ROWS_STALE_EVENT); + }); + }, + [isUserTable, scope] + ); + + /** + * Owner item 9 — "Change field": the clicked column starts showing another field, in place. + * The new field takes the old one's slot in `order`; the old field is HIDDEN, not lost — it + * stays in Fields and can be swapped back. The locked primary column is the row identity and + * cannot be changed away (ColumnMenu never offers it the control). + */ + const changeField = useCallback( + (oldKey: string, newKey: string) => { + if (oldKey === lockedKey || oldKey === newKey) return; + if (!fieldByKey.has(newKey)) return; + // The RECONCILED order (every key present, locked first), so the splice is well-defined + // even for a fresh view whose config.order is still the default. + const withoutNew = order.filter((k) => k !== newKey); + const at = withoutNew.indexOf(oldKey); + if (at < 0) return; + withoutNew.splice(at, 0, newKey); + const nextVisible = new Set(visible); + nextVisible.delete(oldKey); + nextVisible.add(newKey); + updateConfig({ ...config, order: withoutNew, visible: [...nextVisible] }); + }, + [config, updateConfig, order, visible, fieldByKey, lockedKey] + ); + + /** + * Delete a USER-CREATED column outright (owner gap closed 2026-07-27). Only the created + * strata qualify — `custom_` overlay fields and `measure_` formula columns, both marked + * `custom` — so a base field can never leave the schema from here (Hide is its only exit). + * The definition leaves local state, the ACTIVE view's config is scrubbed (order / visible / + * widths / sorts / group / color), and the host removes it from the workspace store — other + * views self-heal on their next autosave, the rule every stale colId already rides. + * Two emits leave in one burst (field_delete + the config autosave); the event-log value + * slot exists for exactly this. + */ + const deleteField = useCallback( + (key: string) => { + const f = fieldByKey.get(key); + if (!f?.custom || key === lockedKey) return; + setFields((current) => current.filter((item) => item.key !== key)); + const nextWidths = { ...(config.widths ?? {}) }; + delete nextWidths[key]; + updateConfig({ + ...config, + order: order.filter((k) => k !== key), + visible: [...visible].filter((k) => k !== key), + widths: nextWidths, + sorts: (config.sorts ?? []).filter((s) => s.colId !== key), + groupBy: config.groupBy === key ? null : config.groupBy, + colorBy: config.colorBy === key ? null : config.colorBy, + }); + stampFieldDelete(key); + emitHostEvent({ id: eventId("fielddel"), type: "field_delete", key }); + }, + [config, updateConfig, order, visible, fieldByKey, lockedKey, stampFieldDelete] + ); + + /** + * ⭐⭐ 2026-08-10 — DELETE A COLUMN FROM A USER DATABASE'S SHARED DEFINITION. + * + * `deleteField` above writes the per-user overlay: it emits `field_delete`, which + * `grid_events` gates on the `custom_`/`measure_` prefix and applies to + * `
_table_workspace`. A definition column has a `custom_` key too, so that event is + * ACCEPTED and scrubs a bucket the definition never reads — the column disappears for one + * paint and is back on the next fetch. That is why this is a different function rather than a + * branch: two stores, two truths, and the wrong one succeeds quietly. + * + * ⚠ SERVER FIRST, THEN THE SCREEN. `_field_or_refuse` can say no (a pre-set column, a + * non-creator, the last remaining column), and the bridge has already shown its sentence — + * removing the column optimistically would leave the user looking at a grid that disagrees + * with the store until they reload. + * ⚠ The VIEW config is scrubbed on the same terms as the overlay delete; a stale `colId` in + * another view self-heals on its next autosave, the rule every other delete rides. + */ + const deleteDefinitionField = useCallback( + (key: string) => { + if (!isUserTable || key === lockedKey) return; + void deleteTableField(scope, key).then((ok) => { + if (!ok) return; // the bridge has already said why + setFields((current) => current.filter((item) => item.key !== key)); + const nextWidths = { ...(config.widths ?? {}) }; + delete nextWidths[key]; + updateConfig({ + ...config, + order: order.filter((k) => k !== key), + visible: [...visible].filter((k) => k !== key), + widths: nextWidths, + sorts: (config.sorts ?? []).filter((s) => s.colId !== key), + groupBy: config.groupBy === key ? null : config.groupBy, + colorBy: config.colorBy === key ? null : config.colorBy, + }); + signal(WORKSPACE_STALE_EVENT); + signal(ROWS_STALE_EVENT); + }); + }, + [isUserTable, scope, lockedKey, config, updateConfig, order, visible] + ); + + /** + * Change-field's "New field" half (wave-2 item 5): build the overlay field, then swap it into + * the clicked column's slot — one motion, two emits (field_upsert + the config autosave), + * which is exactly the burst the event-log value slot exists for. + */ + const createAndSwapField = useCallback( + ( + oldKey: string, + label: string, + type: FieldType, + options?: string[], + extra?: FieldBuildExtra + ) => { + const scope = scopeChoice ? extra?.scope : undefined; + const field = { + ...buildOverlayField(label, type, options, extra), + ...(scope === "cohort" ? ({ scope: "cohort" } as const) : {}), + }; + saveField(field, scope); + changeField(oldKey, field.key); + }, + [saveField, changeField, scopeChoice] + ); + + /** + * Wave-6 item 2 — Change field on a CREATED (`custom_`) field retypes THAT field in place: + * `field_upsert` with the SAME key and the new type/options; the host rebuilds the def and + * keeps createdBy/scope (this optimistic def carries them too, so the echo is byte-stable). + * Values are never converted — cells re-render per the new type, unreadable values show + * blank (Airtable behavior, disclosed in the pane). A display `format` survives only within + * its own family (number→number); anything else would be junk the host drops anyway. + */ + const retypeField = useCallback( + ( + key: string, + type: FieldType, + options?: string[], + // `label` (owner item 8): rename+retype leave the Edit pane as ONE upsert — two + // sequential emits would each read the stale def and revert the other. + extra?: FieldBuildExtra + ) => { + const old = fieldByKey.get(key); + /** + * ⭐⭐ WAVE-29 T27 — the same widening as the pane above, and it is load-bearing rather + * than cosmetic: supplying `onRetype` while this function still bailed would render a + * Field-type picker that silently does nothing, which is worse than the missing control + * the owner reported. + * + * ⭐ AND IT IS ALSO T24's SECOND HALF. Option colours have a validator now, but the EDIT + * path for them is this function: on a `ut_*` database the field lives in the shared + * DEFINITION, so `saveField` (this user's overlay stratum) would fork the column into a + * private copy of itself and the colours would never reach the door that stores them. + */ + if (!old || !isUserSchemaField(old, isUserTable)) return; + if (type === "formula" || type === "created_time") return; // a different stratum + const numberFam = (t: FieldType) => t === "int" || t === "currency"; + const keepFormat = + old.format && ((numberFam(old.type) && numberFam(type)) || old.type === type); + const next: Field = { + key, + label: extra?.label?.trim() || old.label, + type, + source: "overlay", + default: old.default, + custom: true, + agg: ["currency", "int"].includes(type) ? "sum" : undefined, + ...(old.note ? { note: old.note } : {}), + ...(old.description ? { description: old.description } : {}), + ...(old.scope ? { scope: old.scope } : {}), + ...(old.createdBy ? { createdBy: old.createdBy } : {}), + ...(old.permissions ? { permissions: old.permissions } : {}), + ...(keepFormat ? { format: old.format } : {}), + ...((type === "select" || type === "multiselect") && options?.length + ? { options } + : {}), + ...((type === "select" || type === "multiselect") + ? { + colorCodeOptions: extra?.colorCodeOptions !== false, + ...(extra?.optionColors && Object.keys(extra.optionColors).length + ? { optionColors: extra.optionColors } + : {}), + } + : {}), + ...(type === "multiselect" ? { multi: true } : {}), + ...(type === "rating" ? { max: extra?.max ?? 5 } : {}), + }; + if (isUserTable && !old.custom) { + /* THE DEFINITION DOOR. `patch_field` merges the body through the same `_clean_field` the + create used, so the type, the options, the colours and a rating's max all land in the + one validator — and the SERVER's echo replaces the local field, exactly as the create + path does, because it re-cleans what it was given and may refuse. */ + void patchTableField(scope, key, { + label: next.label, + type, + ...(next.options ? { options: next.options } : {}), + ...(next.optionColors ? { optionColors: next.optionColors } : {}), + ...(next.colorCodeOptions !== undefined + ? { colorCodeOptions: next.colorCodeOptions } + : {}), + ...(next.max !== undefined ? { max: next.max } : {}), + ...(next.agg ? { agg: next.agg } : {}), + ...(next.format ? { format: next.format } : {}), + }).then((made) => { + if (!made) return; // the bridge has already said why + setFields((current) => current.map((f) => (f.key === made.key ? made : f))); + signal(WORKSPACE_STALE_EVENT); + signal(ROWS_STALE_EVENT); + }); + } else { + saveField(next); + } + /* ⭐ Owner item 15 / C-RENAME — the VALUES follow the definition. + BESIDE the upsert, never instead of it, and AFTER it: the def is written optimistically + here, while the rename is a host-side migration over overlay values and saved views. The + order matters if the host processes the batch in order — the list must already offer + "Navy" before any cell is moved onto it. */ + const renames = extra?.renames?.filter((r) => r.from && r.to && r.from !== r.to) ?? []; + if (renames.length) { + emitHostEvent({ id: eventId("choicerename"), type: "choice_rename", key, renames }); + // R4 — and it is undoable: Ctrl+Z emits the mapping turned around. The declared LIST is + // restored by the same inverse (the host rewrites values and views back), so undo does + // not need to re-send the definition. + undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), { + kind: "choiceRename", fieldKey: key, renames, + }); + } + }, + // ⚠ `isUserTable` is in the list because T27 made this function BRANCH on it. A callback that + // closed over a stale `isUserTable` would route a definition retype to the overlay on the + // first render after a scope change — the exact defect this ticket removes, reintroduced by + // a memo rather than by a predicate. + [fieldByKey, saveField, scope, isUserTable] + ); + + /** + * Wave-5 item 1 — Duplicate field. The CLIENT generates the destination key (same stratum + * prefix as the source — the ~20:20 contract amendment), emits `field_duplicate`, and slots + * an optimistic clone right of its source SYNCHRONOUSLY. The host validates the prefix + * pair, stamps createdBy (the clone's creator is the duplicator — the local copy drops the + * source's), and copies overlay VALUES for `custom_` sources; those values arrive with the + * next payload, so the clone's cells are blank for exactly one round trip. + */ + const duplicateField = useCallback( + (source: Field) => { + const prefix = source.key.startsWith("measure_") ? "measure_" : "custom_"; + const label = uniqueDisplayName( + `${source.label} copy`, + fields.map((field) => field.label) + ); + const key = `${prefix}${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`; + const requestId = eventId("fielddup"); + const clone: Field = { ...source, key, label, editRequestId: requestId }; + delete clone.createdBy; + delete clone.permissions; + delete clone.labelCorrectedFrom; + delete clone.labelCorrectionId; + stampFieldEdit(key); + setFields((current) => [...current, clone]); + emitHostEvent({ + id: requestId, + type: "field_duplicate", + sourceKey: source.key, + key, + label, + // Item 9c: a duplicate INHERITS its source's scope (cohort clone stays cohort-only; + // a global source stays global = absent). Only meaningful on the cohort page. + ...(scopeChoice && source.scope === "cohort" ? { scope: "cohort" as const } : {}), + }); + insertColumn(key, source.key, "right"); + }, + [insertColumn, scopeChoice, stampFieldEdit, fields] + ); + + /** + * Wave-2 item 8c — change the PERIOD of a measure-carrying column (user-created `measure_*` + * and the pre-set measure fields alike). The header auto-renames to + * ` · ` ONLY while it still reads as the auto-name for the CURRENT + * window — a user's own title is never overwritten. Values arrive recomputed from the host on + * the next payload; the ordinary `field_upsert` is the whole protocol. + */ + const changeMeasurePeriod = useCallback( + (key: string, window: WindowSpec) => { + const f = fieldByKey.get(key); + if (!f?.measure) return; + const m = measures.find((item) => item.key === f.measure?.key); + const oldAuto = m ? `${m.label} · ${windowLabel(f.measure.window)}` : null; + const label = + m && oldAuto && f.label === oldAuto + ? `${m.label} · ${windowLabel(window)}` + : f.label; + saveField({ ...f, label, measure: { ...f.measure, window } }); + }, + [fieldByKey, measures, saveField] + ); + + /* wave17 R1 — `selectCohort`, `renameCohort`, `deleteCohort` and `exportCohortData` are + gone. Every one of them was called ONLY by `CohortSidebar`, and every one now has a + better-governed twin: the Views rail renames and deletes a locked view through the + ordinary view path, and the HOST routes those to `cohort_mod.rename` / `.delete` on a + projected id (C-LOCKV guards b and c). One name, one lifecycle, one set of events. */ + + /** + * Cohort mode — the selection bar's "Remove from cohort" (contract event `cohort_remove`). + * The pids are intersected with the cohort's membership client-side as a courtesy; the host + * re-validates ownership and pool anyway (a pid from the browser is untrusted input). + */ + const removeSelectionFromCohort = useCallback(() => { + if (!activeCohortId) return; + const pids = [...selectedPids].filter((p) => cohortMemberSet.has(p)); + if (!pids.length) return; + emitHostEvent({ + id: eventId("cohortrm"), + type: "cohort_remove", + cohortId: activeCohortId, + pids, + }); + clearSelection(); + }, [activeCohortId, selectedPids, cohortMemberSet, clearSelection]); + + + /** + * ⭐ Wave-20 owner item 9 — **REMOVE FROM A COHORT WITHOUT BEING INSIDE IT.** + * + * "Remove from cohort" existed only under `cohortMode` (you had to open the locked view + * first), while "Add to cohort" worked from any view. So the two halves of the same idea + * lived on different screens: you could put a customer in a cohort from wherever you found + * them, and then had to go looking for the cohort to take them out again. + * + * SAME EVENT, same host validation as the cohort-mode button — `cohort_remove` with the pids + * intersected client-side as a courtesy. What is new is only WHICH cohort: the picker names + * it, instead of it being implied by the page you are standing on. + */ + const removeSelectionFromList = useCallback( + (cohortId: string) => { + const members = cohortSets[cohortId]; + if (!members) return; + const pids = [...selectedPids].filter((p) => members.has(p)); + if (!pids.length) return; + emitHostEvent({ id: eventId("cohortrm"), type: "cohort_remove", cohortId, pids }); + setSelRemoveOpen(false); + clearSelection(); + }, + [cohortSets, selectedPids, clearSelection] + ); + + /** + * The cohorts the checked rows can actually be removed FROM: those holding at least one of + * them, and only where this viewer may edit the projected view. + * + * ⚠ A cohort holding NONE of the checked rows is not offered. Airtable's rule and the one R8 + * states for the ghost row: an affordance that can only refuse is a fake affordance — and + * here it would be worse than that, because "Remove from Q3 plan" that removes nothing looks + * exactly like a write that failed. + * + * The edit test is the COURTESY half (`mayEditView`, like every other client-side permission + * check on this surface); the host re-validates ownership and pool on the event regardless. + * A cohort with no projected view is still offered — the host owns that verdict, and hiding + * it here would silently drop sets the reader can see in their own rail. + */ + const removableLists = useMemo(() => { + if (cohortMode || selectedPids.size === 0) return []; + const out: { id: string; name: string; hits: number }[] = []; + for (const l of lists) { + const members = cohortSets[l.id]; + if (!members) continue; + let hits = 0; + for (const pid of selectedPids) if (members.has(pid)) hits += 1; + if (!hits) continue; + const projected = views.find((v) => v.id === l.id); + if (projected && !mayEditView(projected, viewer)) continue; + out.push({ id: l.id, name: l.name, hits }); + } + return out; + }, [cohortMode, selectedPids, lists, cohortSets, views, viewer]); + + /** Cohort mode — confirm the "+ Add customers" picker (contract event `cohort_add`). */ + const addCustomersToCohort = useCallback(() => { + if (!activeCohortId || addCustPicked.size === 0) return; + emitHostEvent({ + id: eventId("cohortadd"), + type: "cohort_add", + cohortId: activeCohortId, + pids: [...addCustPicked], + }); + setAddCustPicked(new Set()); + setAddCustQuery(""); + setAddCustOpen(false); + }, [activeCohortId, addCustPicked]); + + /** + * The picker's candidates: the POOL minus the cohort's current members, narrowed by the + * search. The pool's own order is kept (it leads with the biggest customers, which is the + * useful default for "who am I adding"). + */ + const addCandidates = useMemo(() => { + if (!addCustOpen) return []; + const q = addCustQuery.trim().toLowerCase(); + const out: { pid: number; name: string }[] = []; + for (const r of rawRows) { + if (cohortMemberSet.has(r.pid)) continue; + const name = String(r[lockedKey] ?? ""); + if (q && !name.toLowerCase().includes(q)) continue; + out.push({ pid: r.pid, name }); + } + return out; + }, [addCustOpen, addCustQuery, rawRows, cohortMemberSet, lockedKey]); + + /** + * Wave-6 item 11c — PIN. "Pin up to this field" freezes every visible column through the + * clicked one; when the field sits past the clamp (8), it is REORDERED into the frozen + * prefix first (the contract's wording). Pinning the identity column reads as "just the + * identity" = the legacy 1, stored as ABSENT. + * + * ⚠ CORRECTED wave-14 item 15. This used to read "Grouped mode paints frozenCount 0 (spans + * break under freezing)". **Spans do not break under freezing** — glide splits them + * (`getSpanBounds` → `[frozenRect, contentRect]`) and draws the row's contents from the frozen + * half. What IS true, and is the real cost of the pin, is that those contents are then clipped + * to the frozen strip's width; `fitGroupLabel` is what makes that clip honest. The clamp is + * gone: `freezeColumns` is `frozenN` in every mode. + */ + const visibleKeys = useMemo( + () => order.filter((key) => visible.has(key)), + [order, visible] + ); + const frozenN = frozenCountOf(config); + const pinFieldTo = useCallback( + (key: string) => { + const index = visibleKeys.indexOf(key); + if (index < 0) return; + if (index < MAX_FROZEN) { + updateConfig({ ...config, frozenCount: clampFrozenCount(index + 1) }); + return; + } + const moved = visibleKeys.filter((k) => k !== key); + moved.splice(MAX_FROZEN - 1, 0, key); + let cursor = 0; + const nextOrder = order.map((k) => (visible.has(k) ? moved[cursor++] : k)); + updateConfig({ ...config, order: nextOrder, frozenCount: MAX_FROZEN }); + }, + [visibleKeys, order, visible, config, updateConfig] + ); + const unpinFields = useCallback( + () => updateConfig({ ...config, frozenCount: undefined }), + [config, updateConfig] + ); + + // Wave-6 item 10 — display-mode plumbing. W13 (contract C4 as AMENDED 2026-07-28): + // `config.display` is the ONE home for the field picks, so leaving a mode must NOT + // wipe them — returning to grid keeps the refs in the draft (mode 'grid' + refs; + // cleanDisplay's carry), which is what lets the next Kanban/Calendar restore the + // chosen field instead of feeding the select its default. A view that never picked + // stays byte-identical (ref-less grid normalizes to absent, the legacy rule). + const setDisplayMode = useCallback( + (m: DisplayMode) => { + // I12 (C3) — a locked view's MODE is frozen. The switcher is already hidden for one, + // so reaching here means something other than the switcher asked; refuse rather than + // trust the UI. (The host refuses too — this only stops the local state diverging from + // what the host will store, which would look like a working change that never saved.) + // (Resolved from `views` here rather than the `activeView` binding below — this + // callback is declared above it.) + const av = views.find((v) => v.id === activeViewId); + if (av && isModeFrozen(av)) return; + const spec = cleanDisplay(config.display); + if (m === "grid") { + updateConfig({ + ...config, + display: spec ? cleanDisplay({ ...spec, mode: "grid" }) : undefined, + }); + return; + } + updateConfig({ ...config, display: { ...(spec ?? {}), mode: m } }); + }, + [config, updateConfig, views, activeViewId] + ); + // C4 (amended): the pick is written INTO display and the selects are FED from the + // view def (`display.`), never from component state that dies on unmount. + const setDisplayField = useCallback( + (k: "dateField" | "stackField" | "colorField" | "sizeField", key: string) => { + const spec = cleanDisplay(config.display); + if (!spec) return; + // Wave-8 I3/I5: colour/size are OPTIONAL encodings, so "" is a real choice + // (none) and must delete the ref rather than store an empty key that no + // field can ever match. + const next = { ...spec, [k]: key }; + if (!key) delete (next as Record)[k]; + updateConfig({ ...config, display: next }); + }, + [config, updateConfig] + ); + /** + * Item 3 (C-DISP) — the kanban card clamp. + * + * ⚠ `undefined` DELETES the key rather than storing `true`. Absent means clamped, so a + * stored `true` would be the default wearing a second name — and the host's `_clean_display` + * accepts the literal `false` only, so a `true` would be dropped on save and the toggle + * would look like it does not persist. Delete-to-default keeps both ends agreeing. + */ + const setKanbanClamp = useCallback( + (next: false | undefined) => { + const spec = cleanDisplay(config.display); + if (!spec) return; + const display = { ...spec }; + if (next === false) display.kanbanClamp = false; + else delete (display as Record).kanbanClamp; + updateConfig({ ...config, display }); + }, + [config, updateConfig] + ); + /** + * ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the swipe BINDING. + * + * ⚠ `undefined` DELETES the key, for `setKanbanClamp`'s reason one step further on. A swipe + * binding is a single three-part thing (field + two options); both engines' `_clean_display` + * DROP a half-binding rather than storing it, so writing one back would look like a save that + * silently did not persist. Absent IS the unconfigured state. + */ + const setSwipeSpec = useCallback( + (next: SwipeSpec | undefined) => { + const spec = cleanDisplay(config.display); + if (!spec) return; + const display = { ...spec }; + if (next) display.swipe = next; + else delete (display as Record).swipe; + updateConfig({ ...config, display }); + }, + [config, updateConfig] + ); + /** + * ⭐⭐ WAVE-29 C4 / T29 (owner R7) — the FORM spec's writer, shaped exactly like `setSwipeSpec` + * above it: `null` DELETES the key rather than storing an empty object, because "this view has + * no form" and "this view has a form with nothing in it" are different states and only one of + * them should survive a reload. `_clean_display` drops an empty bag anyway, so storing one + * would make clearing a form look like it did not save. + */ + const setFormSpec = useCallback( + (next: FormSpec | null) => { + const spec = cleanDisplay(config.display); + if (!spec) return; + const display = { ...spec }; + if (next) display.form = next; + else delete (display as Record).form; + updateConfig({ ...config, display }); + }, + [config, updateConfig] + ); + /** + * Item 4 (C-DISP) — the per-day totals. An EMPTY list deletes the key rather than storing + * `[]`: the always-on Records count is what an absent list means, so `[]` would be that + * state wearing a second name — and `_clean_display` drops an empty array anyway, which + * would make un-picking the last metric look like it did not save. + */ + const setCalendarMetrics = useCallback( + (next: { id: string; field: string; agg: string }[]) => { + const spec = cleanDisplay(config.display); + if (!spec) return; + const display = { ...spec }; + const capped = next.slice(0, MAX_CALENDAR_METRICS); + if (capped.length) display.calendarMetrics = capped; + else delete (display as Record).calendarMetrics; + updateConfig({ ...config, display }); + }, + [config, updateConfig] + ); + /** Item 4 (C-DISP) — records vs summaries. `records` is the default, so it deletes the key. */ + const setCalendarMode = useCallback( + (next: "records" | "summary") => { + const spec = cleanDisplay(config.display); + if (!spec) return; + const display = { ...spec }; + if (next === "summary") display.calendarMode = "summary"; + else delete (display as Record).calendarMode; + updateConfig({ ...config, display }); + }, + [config, updateConfig] + ); + // The calendar's date field: the view's pick when it is still a real date-family field, + // else `last_order`, else the first date-family field. The kanban's stack field: status or + // single-select ONLY (a multiselect card in two columns at once is a count lie — contract). + const dateFieldChoices = useMemo( + () => fields.filter((f) => isDateFamilyType(f.type)), + [fields] + ); + const stackFieldChoices = useMemo( + () => fields.filter((f) => f.type === "status" || f.type === "select"), + [fields] + ); + const pickBy = (want: string | undefined, choices: Field[], preferred?: string): Field | undefined => { + const wanted = want ? choices.find((f) => f.key === want) : undefined; + if (wanted) return wanted; + const pref = preferred ? choices.find((f) => f.key === preferred) : undefined; + return pref ?? choices[0]; + }; + // W13 (C4 as amended): the VIEW DEF's display feeds the pickers — the refs survive + // every mode switch via cleanDisplay's grid carry. pickBy already falls back to the + // default when the stored key no longer names an offerable field (a dropped ref). + // I3/I5 — the map's encodings. Deliberately NOT resolved through `pickBy`: + // that helper falls back to the first choice, which is right for the calendar + // (a calendar must have a date) and wrong here, where "no encoding" is a + // legitimate, and the DEFAULT, state. An unknown stored key resolves to + // undefined = no encoding, which is also the dropped-ref behaviour. + // I19c (C2) — the chart list lives in `config.display.charts`, so it rides the + // existing view autosave + viewEcho reconcile and needs no new event type. + const charts = useMemo( + () => cleanCharts(displaySpec?.charts) ?? [], + [displaySpec?.charts] + ); + const setCharts = useCallback( + (next: ChartSpec[]) => { + const spec = cleanDisplay(config.display); + if (!spec) return; + const display = { ...spec, charts: next }; + if (!next.length) delete (display as Record).charts; + updateConfig({ ...config, display }); + }, + [config, updateConfig] + ); + const colorFieldChoices = useMemo( + () => fields.filter((f) => f.type === "status" || f.type === "select"), + [fields] + ); + const sizeFieldChoices = useMemo( + () => fields.filter((f) => isNumericFieldType(f.type)), + [fields] + ); + const mapColorField = displaySpec?.colorField + ? colorFieldChoices.find((f) => f.key === displaySpec.colorField) + : undefined; + const mapSizeField = displaySpec?.sizeField + ? sizeFieldChoices.find((f) => f.key === displaySpec.sizeField) + : undefined; + const calendarField = pickBy(displaySpec?.dateField, dateFieldChoices, "last_order"); + const kanbanField = pickBy(displaySpec?.stackField, stackFieldChoices); + /* ⭐ WAVE-27 item 8 (C3) — the BOUND field, resolved by key and NOT through `pickBy`. `pickBy` + falls back to the first choice when the stored key is unknown, which is right for a kanban + (a stack field is a preference) and wrong here: SwipeView SHOWS a rotted binding rather than + guessing at one, so it must receive the loss instead of a silent substitute — the wave-7 trap + [[wrong-parent-not-broken-control]] and the component's own header note both name. */ + const swipeField = displaySpec?.swipe + ? fieldByKey.get(displaySpec.swipe.fieldKey) + : undefined; + // Calendar/kanban project the FULL pipeline result (a month/stack is its own bound), as + // DISTINCT data rows — grouping can repeat a pid — with overlay edits layered so a dragged + // card restacks and an edited title repaints without waiting on any echo (item 3c). + const modeDataRows = useMemo(() => { + if ( + displayMode !== "calendar" && + displayMode !== "kanban" && + displayMode !== "map" && + displayMode !== "chart" && + // C-TS close-out stitch: the panel's pids come from these rows; without this line the + // time-series view asked its question about ZERO customers (caught by the mounted + // judge — the node battery is blind here). + displayMode !== "timeseries" && + /* ═══ W18-C CATALOG ═══ The designer's "add the products this view shows" door reads + these rows — the same rows the toolbar counts, so the two can never disagree about + which products the filter kept. The catalog's PAINT does not: it joins its stored + codes against the whole pool, so a filter narrows what you can ADD and never what a + finished page prints. ═══ end W18-C CATALOG ═══ */ + displayMode !== "catalog" && + /* ⭐ WAVE-27 item 8 (C3) — the swipe deck is these rows filtered to the undecided ones. + Without this line the deck is derived from `[]`, so the view paints its "nothing left + to decide" empty state over a table full of undecided records — the C-TS stitch above, + repeated. */ + displayMode !== "swipe" + ) + return []; + const seen = new Set(); + const out: Row[] = []; + for (const vr of visibleRows) { + if (vr.kind !== "data" || seen.has(vr.record.pid)) continue; + seen.add(vr.record.pid); + const edits = overlayEdits[vr.record.pid]; + out.push(edits ? { ...vr.record, ...edits } : vr.record); + } + return out; + }, [displayMode, visibleRows, overlayEdits]); + // Owner item 1 — the CALL-SITE half of the kanban memo. `KanbanView` is `memo`'d now, but a + // memo whose props are rebuilt every render never bails, and these two were: `cardKeys` is a + // fresh array from `.filter().slice()`, `onMove` a fresh closure. With them stable, a + // `detailPid` change (clicking a card) re-renders the modal and skips the board entirely — + // which is the whole of "laggy record-open". Everything else KanbanView takes was already + // stable: `modeDataRows`/`fieldByKey` are useMemos, `kanbanField` is a reference INTO the + // `stackFieldChoices` useMemo, and `onOpen={setDetailPid}` is a setState identity. + const kanbanCardKeys = useMemo( + () => + kanbanField + ? visibleKeys.filter((k) => k !== lockedKey && k !== kanbanField.key).slice(0, 3) + : [], + [visibleKeys, lockedKey, kanbanField] + ); + const onKanbanMove = useCallback( + (pid: number, value: string) => { + if (!kanbanField) return; + patchAndRecord(pid, { [kanbanField.key]: value }, "a card move"); + }, + [kanbanField, patchAndRecord] + ); + /* ⭐ WAVE-27 item 8 (C3) — a swipe writes through the NORMAL cell door, the same + `patchAndRecord` a card move uses. That is the whole reason the deck needs no permission + logic, no undo entry and no echo handling of its own: it inherits all three. */ + const onSwipeDecide = useCallback( + (pid: number, value: string) => { + if (!swipeField) return; + patchAndRecord(pid, { [swipeField.key]: value }, "a swipe"); + }, + [swipeField, patchAndRecord] + ); + /** + * ⭐ WAVE-26 ITEM 12 (owner ruling R8) — the kanban's "Add option" door. + * + * It opens the ORDINARY ColumnMenu on the stack field, straight into its Edit-field pane. So + * the option is written into the SHARED field definition by the same `onRetype` upsert the + * header menu uses — never a per-view list — and every guarantee that editor already carries + * (permission wall, rename-by-row-identity, option colours, one upsert for name+choices) comes + * along without being re-implemented. R8's "do not invent a second options editor", literally. + * + * ⛔ THE PERMISSION QUESTION IS NOT `canEditField`. Adding an option changes the column's + * DEFINITION; `mayEditField` answers who may type a VALUE into it, and conflating the two is + * the confusion [[schema-role-is-not-a-value-wall]] exists to prevent. The right predicate is + * the one that decides whether `onRetype` is supplied at all — a created overlay field — so it + * is written ONCE here and read at both doors. + * + * `null` (not "absent") when the field cannot take one: a status column is computed by the + * source system, and an Odoo select's vocabulary lives in Odoo. + * + * ⭐ THE PREDICATE ADMITS AN AUTOMATION'S `stage_` COLUMN, and that is INTENDED rather than + * incidental — worth stating because it was arrived at by matching `onRetype`'s condition, and + * a coincidence and a decision look identical in code. It is exactly what R6 and R8 compose + * to: R6 deleted the board's built-in terminals, so the lanes a review stage offers are now + * only the ones a user defined — and R8 is the control that lets them define one. Landed + * without this the pair is worse than either alone (SESSION A's A-7 says the same thing from + * the engine side: `humanMoves` had to learn to admit user-added options, or the move door + * would refuse the lane the product just invited you to create). + * ⚠ CARRIED CONSEQUENCE, pre-existing and not introduced here: the pane this opens is the + * whole Edit-field window, so it also offers the TYPE picker — one wrong click retypes a stage + * column. The header menu has offered exactly that on the same column since wave 6; this door + * adds a second way in, not a new hazard. Written down rather than left to be rediscovered. + */ + /* ⭐ WAVE-29 T27: the same stratum predicate as the Edit-field pane, for the same reason — this + door OPENS that pane, so gating the two differently means offering an add-option control that + leads to a pane with no editor in it. On a `ut_*` kanban (every automation board) the old + `custom` test was false for every column, so the lane header offered nothing. */ + const kanbanCanAddOption = + !!kanbanField && kanbanField.type === "select" && + isUserSchemaField(kanbanField, isUserTable); + const onKanbanAddOption = useCallback( + (anchor: AnchorRect) => { + if (!kanbanField) return; + setColumnMenu({ fieldKey: kanbanField.key, anchor, pane: "edit" }); + }, + [kanbanField] + ); + const onListToggleGroup = useCallback((groupKey: string) => { + setCollapsed((current) => { + const next = new Set(current); + if (next.has(groupKey)) next.delete(groupKey); + else next.add(groupKey); + return next; + }); + }, []); + /** + * Item 7 (C-TS) — the pid set the time series asks about: exactly the rows the current + * filter kept, so the panel and the toolbar count can never disagree about "who". + * + * `modeDataRows` is already the de-duplicated data rows for a non-grid mode, so this is a + * projection of it and NOT a second pipeline. The server intersects with the caller's book + * anyway — this narrows the question, it can never widen the answer. + */ + const timeseriesPids = useMemo( + () => (displayMode === "timeseries" ? modeDataRows.map((r) => r.pid) : []), + [displayMode, modeDataRows] + ); + /* ═══ W18-C CATALOG ═══ (owner item 4, contract C6) + The catalogs this view holds, and the door that persists an edit to them. + + `displaySpec.catalogs` is ALREADY validated — `cleanDisplay` ran `cleanCatalogs` over it on + read — so there is no second normalisation here and no chance of the two disagreeing. + + ⚠ The write goes through `updateConfig` exactly like `onTimeseriesDisplay` below, which + means a catalog edit is a VIEW-CONFIG save. That is the point: a catalogue is authored + content and has to survive a reload, and the display spec is the only per-view store the + grid has. It also means the 500-code / 40-page / 12-catalog caps are enforced twice on the + round trip (here on read, and by the host's `_clean_catalogs` on write) — the designer + states its budget in the toolbar so the cap is never the user's first news of it. */ + const catalogs = useMemo( + () => (displayMode === "catalog" ? (displaySpec?.catalogs ?? []) : []), + [displayMode, displaySpec] + ); + /** The codes the current filter kept — the designer's "add what the view shows" shortcut. + * Derived from `modeDataRows`, so it counts what the toolbar counts. */ + const catalogFilteredCodes = useMemo( + () => + displayMode === "catalog" + ? modeDataRows + .map((r) => r[CATALOG_CODE_FIELD]) + .filter((c): c is string => typeof c === "string" && !!c) + : [], + [displayMode, modeDataRows] + ); + const onCatalogs = useCallback( + (next: CatalogSpec[]) => { + const spec = cleanDisplay(config.display) ?? { mode: "catalog" as const }; + updateConfig({ ...config, display: { ...spec, catalogs: next } }); + }, + [config, updateConfig] + ); + /* ═══ end W18-C CATALOG ═══ */ + /** Persist a bucket/span/metric pick onto the view, the way every other display key rides. */ + const onTimeseriesDisplay = useCallback( + (next: Partial) => { + const spec = cleanDisplay(config.display) ?? { mode: "timeseries" as const }; + updateConfig({ ...config, display: { ...spec, ...next } }); + }, + [config, updateConfig] + ); + const kanbanCanMove = + !!kanbanField && kanbanField.type === "select" && canEditField(kanbanField); + const kanbanReason = !kanbanField + ? null + : kanbanField.type === "status" + ? `Stacked by ${kanbanField.label} — read-only, computed from the source system.` + : !canEditField(kanbanField) + ? "You do not have permission to edit this field, so cards cannot be moved." + : null; + /* ⭐ WAVE-27 item 8 (C3) — the kanban pair, said again for the deck. Same wall, same words + shaped for the gesture: a swipe writes a VALUE, so `canEditField` is the right predicate. */ + const swipeCanWrite = !!swipeField && swipeField.type === "select" && canEditField(swipeField); + const swipeReason = !swipeField + ? null + : swipeField.type === "status" + ? `Bound to ${swipeField.label} — read-only, computed from the source system.` + : !canEditField(swipeField) + ? "You do not have permission to edit this field, so records cannot be decided." + : null; + + if (loading || !workspaceReady) { + /* wave17 GRID — item 3 / owner R6: one small bare rotating icon, NO WORDS. `aria-label` + is not a word on screen and stays: `.lp-spin` is an empty span, so without it the wait + is announced to a screen reader as nothing at all. (C-SPIN, SHELL defines the class.) + `--lg` because this is the case C-SPIN sizes it for — the whole surface is this mark and + nothing else, where the 14px version "reads as dust". */ + return ( +
+ +
+ ); + } + + const activeView = views.find((view) => view.id === activeViewId); + // Cohort mode's toolbar control (rendered by Toolbar via the `cohortAction` slot; the popover + // it opens is with the other overlays at the bottom). Disabled without an active cohort — + // zero cohorts is the host page's near-empty state, not this button's error to explain. + const cohortAction = cohortMode ? ( + + ) : undefined; + // Wave-7 item W9 — the Fields panel's permanent delete rides the SAME wall as the + // column menu's Delete: created strata only (`custom`), never the locked identity + // column. One predicate, two doors. + // + // ⚠ WAVE-29 T22 NARROWED WHAT THIS SET CONTAINS, and the narrowing is CORRECT rather than an + // oversight — said out loud because it looks like a regression. A grid-created column on a + // `ut_*` database now lives in the shared DEFINITION and carries no `custom` flag, so it drops + // out of this set. It has to: this door emits `field_delete`, which scrubs the per-user + // workspace overlay — a bucket the definition never reads — so the column would vanish for one + // paint and be back on the next fetch (`deleteDefinitionField`'s note says the same thing from + // the other side). ⛔ The consequence is real and is NOT silently widened here: a definition + // column can only be deleted through the column menu, which offers it for `link`/`rollup`/ + // `formula` alone (D-114's deliberate narrowness — every other kind holds real values). Booked + // as pending work rather than fixed by handing a type-blind delete to a panel. + const deletableKeys = new Set( + fields.filter((f) => f.custom && f.key !== lockedKey).map((f) => f.key) + ); + const menuField = columnMenu ? fieldByKey.get(columnMenu.fieldKey) : undefined; + // Wave-5 item 3 — what the CURRENT VIEW does with the menu's field, so the conditional + // "Don't sort/filter/group" entries render exactly when they apply. + const menuSortedDir = menuField + ? config.sorts.find((s) => s.colId === menuField.key)?.dir ?? null + : null; + const menuIsFiltered = menuField + ? treeNamesField(config.filters, menuField.key, measureCols) + : false; + // Item 11c — is the menu's field the current frozen boundary (its Pin would be a no-op)? + const menuVisIndex = menuField ? visibleKeys.indexOf(menuField.key) : -1; + const menuPinnedTo = menuVisIndex >= 0 && frozenN > 1 && menuVisIndex + 1 === frozenN; + // Item 10 — the toolbar's mode switcher (grid-only on windowed tables, see displayMode). + const modeControl = serverWindowed || embedded ? undefined : ( + + ); + // ⭐ Wave-23 C7 — the open document, resolved exactly like the picker's value below it: + // the OPTIMISTIC edit wins over the raw record, so a save the server has not echoed yet is + // what re-opening the cell shows (the echo-suppression law — reading the raw row here would + // make a just-saved document appear to revert). + const jsonField = jsonAt ? fieldByKey.get(jsonAt.fieldKey) : undefined; + const jsonRow = jsonAt ? rawRows.find((r) => r.pid === jsonAt.pid) : undefined; + const jsonValue = String( + (jsonAt && overlayEdits[jsonAt.pid]?.[jsonAt.fieldKey]) + ?? (jsonField && jsonRow ? jsonRow[jsonField.key] : "") + ?? "" + ); + const linkField = linkAt ? fieldByKey.get(linkAt.fieldKey) : undefined; + const linkRow = linkAt ? rawRows.find((row) => row.pid === linkAt.pid) : undefined; + const linkValue = String( + (linkAt && overlayEdits[linkAt.pid]?.[linkAt.fieldKey]) + ?? (linkField && linkRow ? linkRow[linkField.key] : "") + ?? "" + ); + const linkedRecordIds = [...new Set( + linkValue.split(",").map((part) => Number(part.trim())).filter( + (pid) => Number.isInteger(pid) && pid > 0 + ) + )]; + const pickerField = picker ? fieldByKey.get(picker.fieldKey) : undefined; + // A `user` field's choices come from the HOST's real user list, a `select`'s from its own + // definition — so an assignee is always someone who can log in, and a status is always one of + // the choices the column was created with. + const pickerChoices = !pickerField + ? [] + : pickerField.type === "user" + ? payload?.userOptions ?? [] + : choiceOptions(pickerField); + const pickerRow = picker ? rawRows.find((r) => r.pid === picker.pid) : undefined; + const pickerValue = String( + (picker && overlayEdits[picker.pid]?.[picker.fieldKey]) + ?? (pickerField && pickerRow ? pickerRow[pickerField.key] : "") + ?? "" + ); + // A multiselect cell is a SET: choices TOGGLE and the picker stays open for the next pick + // (item 5). Single select/user keep pick-and-close. + const pickerMulti = pickerField?.type === "multiselect"; + const pickerParts = pickerMulti ? splitMulti(pickerValue) : []; + /** Item 6 — which choice receives focus when the picker opens: the current value where it + * still exists in the list, else the first choice. Keyboard-only editing starts HERE. */ + const pickerFocusChoice = !pickerChoices.length + ? null + : pickerMulti + ? pickerParts.find((p) => pickerChoices.includes(p)) ?? pickerChoices[0] + : pickerChoices.includes(pickerValue) + ? pickerValue + : pickerChoices[0]; + + return ( + // `data-today` is the TENANT'S day, exactly as the payload delivered it. It is here so the + // date every relative condition resolves against is OBSERVABLE rather than inferred: a live + // QA that reads its own clock instead compares two engines on "now" and fails whenever a run + // straddles midnight — which is the flake harness/windows.py takes `today` as a parameter to + // avoid, reintroduced one layer up. It cost a false failure (57 vs 55, both correct, one + // computed either side of a date change) to notice. + // `data-measure-rules` / `data-measure-sets` are BUG-1's handshake, published the same way + // and for the same reason as `data-today`: a measure condition stuck on "Calculating…" + // renders identically whether the rule carries no id, the host resolved nothing, or the two + // sides name the rule differently — and only the first of those is ours. A console.log + // cannot serve this: the grid runs in an iframe that is CROSS-ORIGIN on the Space, and every + // Streamlit rerun replaces it. An attribute survives the remount and is one `get_attribute` + // away from any harness, local or deployed. +
+ {/* ⭐ wave17 R1 / C-LOCKV — the COHORT SIDEBAR is gone, and with it the last surface that + treated a cohort as its own kind of object. `CohortSidebar` was the retired `#/cohort` + page's left panel: a second rail, with its own rename, delete, export and folder + machinery, over `workspace.lists`. Under R1 a cohort IS a saved view, so the Views rail + below renders every one of them and the second rail has nothing to switch between. + ⚠ `lists` is NOT gone with it — it stays the membership channel that feeds `cohortSets` + (C-LOCKV point 2), which is what the lock resolves against. */} + {!hideViews && ( + setSelectFromFile(true)} + /* ⭐ WAVE-29 T25 (owner item 6) — the Import door, offered ONLY where records can be + added. `canMutateRecords` is `isUserTable && recordsMutable`: the same predicate the + "+" row and the delete path read, so a locked database (an Odoo mirror, an + automation-owned child table) has no Import row at all rather than an Import that + fails at the server. */ + onImport={canMutateRecords ? () => setImportOpen(true) : undefined} + onAddToList={serverWindowed ? undefined : addToList} + today={today} + onExport={serverWindowed ? undefined : exportViewData} + folders={folders} + folderIdOf={folderIdOf} + onFolderCreate={(name, icon) => { + const id = newFolderId(); + // Bounded: reconcileFolders drops a local entry once the echo + // carries it or its create stamp ages out, so this only has to not + // grow without limit. + // C5 (I15): the icon rides the LOCAL entry too, or the folder the user just + // styled renders grey for one round trip and then changes colour under them. + setLocalFolders((prev) => [...prev.slice(-31), { id, name, order: 9_999, icon }]); + stampFolder((p) => ({ ...p, created: { ...p.created, [id]: Date.now() } })); + emitHostEvent({ + id: eventId("foldnew"), + type: "folder_create", + surface: "views", + folderId: id, + name, + icon, + }); + }} + onFolderRename={(folderId, name) => { + stampFolder((p) => ({ + ...p, + renamed: { ...p.renamed, [folderId]: { at: Date.now(), name } }, + })); + emitHostEvent({ + id: eventId("foldren"), + type: "folder_rename", + surface: "views", + folderId, + name, + }); + }} + onFolderDelete={(folderId) => { + // The tombstone is stamped BEFORE the emit: between the click and the + // echo the payload still lists the folder, and rendering it again for + // one round trip is the delete blip (folders.ts). + stampFolder((p) => ({ ...p, deleted: { ...p.deleted, [folderId]: Date.now() } })); + emitHostEvent({ + id: eventId("folddel"), + type: "folder_delete", + surface: "views", + folderId, + }); + }} + onFolderDuplicate={(folderId) => { + const newId = newFolderId(); + const src = folders.find((f) => f.id === folderId); + setLocalFolders((prev) => [ + ...prev.slice(-31), + { id: newId, name: `${src?.name ?? "Folder"} copy`, order: 9_999 }, + ]); + stampFolder((p) => ({ ...p, created: { ...p.created, [newId]: Date.now() } })); + emitHostEvent({ + id: eventId("folddup"), + type: "folder_duplicate", + surface: "views", + folderId, + newId, + }); + }} + onFolderReorder={(order) => { + // A-S4-1 (item 19). S4 built the drag and the reconcile behind an OPTIONAL prop, so + // its absence silently disabled the whole feature rather than breaking anything — + // which is why every gate stayed green while folders would not drag at all. The + // stamp is what stops the rail snapping back for one round trip (the echo window). + stampFolder((p) => ({ ...p, ordered: { at: Date.now(), order } })); + emitHostEvent({ + id: eventId("foldord"), + type: "folder_reorder", + surface: "views", + order, + }); + }} + /** + * ⭐ WAVE 27 · OWNER ITEM 5 (contract C7) — the rail's new VIEW order. + * + * Three things happen and each one is needed: + * · the LOCAL list is reordered, so the row lands where it was dropped and stays + * there for this render; + * · a STAMP is written, so the next workspace echo — which is one round trip + * behind and still carries the server's order — does not snap it back + * (`applyViewOrder` in folders.ts holds it for the echo window and no longer); + * · ONE event carries the WHOLE order, never a delta. A partial list cannot say + * where an UNNAMED view went, so "moved B after C" leaves every other position + * to be re-derived by two sides that can disagree. The server's own + * `folder_reorder` note states this rule; the same rule, same door. + * + * ⚠ AMENDMENT C7-A1 (2026-08-08, SESSION C): the event is + * `{type: "view_reorder", surface: "views", order}` — `surface`, not `workspace`. + * The contract said `{workspace, order}`; the door `grid_events.py:1299` actually + * opens on `surface ∈ FOLDER_SURFACES`, and the TABLE is resolved from the request + * context rather than the payload (`_store_of(ctx)`). Emitting a key the handler + * does not read would have shipped a drag that persists nothing, behind a green + * gate on each side. + */ + onViewReorder={(order) => { + setViews((cur) => { + const byId = new Map(cur.map((v) => [v.id, v])); + const out: SavedView[] = []; + const placed = new Set(); + for (const id of order) { + const v = byId.get(id); + if (!v || placed.has(id)) continue; + placed.add(id); + out.push(v); + } + // Anything the order does not name keeps its current place, after the named + // ones — never dropped. A view created in another tab is not evidence the + // drop was wrong. + for (const v of cur) if (!placed.has(v.id)) out.push(v); + return out; + }); + stampFolder((p) => ({ ...p, orderedViews: { at: Date.now(), order } })); + // ⛔ ONE CAST, AT ONE SITE, WITH A RATCHET BEHIND IT (ASK ->D, wave-27 mailbox + // C-6). `HostEvent`'s union lives in `types.ts`, which is SESSION D's file, and + // D owns C7's persistence half — so the member `{ id; type: "view_reorder"; + // surface: FolderSurface; order: string[] }` is theirs to add. Until it lands + // this emit cannot be typed from here. + // ⚠ The gap is DECLARED, not hidden: `verify_wiring.py`'s + // `PENDING_UNION_MEMBERS` carries `view_reorder`, prints the ask, and goes RED + // the moment `types.ts` gains the member — which forces this cast to be deleted + // in the same change. That is the D-70 ratchet's shape, and it fired on its own + // this wave, which is the argument for reusing it rather than a comment. + emitHostEvent({ + id: eventId("vieword"), + type: "view_reorder", + surface: "views", + order, + } as unknown as HostEvent); + }} + onItemMove={(viewId, folderId) => { + stampFolder((p) => ({ + ...p, + moved: { ...p.moved, [viewId]: { at: Date.now(), folderId } }, + })); + emitHostEvent({ + id: eventId("itemmove"), + type: "item_move", + surface: "views", + itemId: viewId, + folderId, + }); + }} + folderAddPreview={serverWindowed ? undefined : folderAddPreview} + onFolderAddToList={serverWindowed ? undefined : folderAddToList} + viewer={viewer} + onToggleLock={toggleViewLock} + userOptions={payload?.userOptions} + /> + )} +
+ v.id !== activeViewId)} + onCopyConfig={(patch) => updateConfig({ ...config, ...patch })} + fields={fields} + visible={visible} + lockedKey={lockedKey} + onColumnVisible={setColumnVisible} + onColumnsVisible={(keys) => + updateConfig({ + ...config, + visible: [...new Set([lockedKey, ...keys])], + }) + } + // Owner item 23 — the Hide-fields panel lists and reorders COLUMN order, so it is + // handed the ordered projection, not the definition-ordered `fields` above. + // `order` is `useGridColumns`' reconciled copy (locked key first, unknown keys + // dropped, new keys appended), so the list can never show a field the grid does not + // have or omit one it does. + orderedFields={orderedFields} + onFieldOrder={(keys) => updateConfig({ ...config, order: keys })} + deletableKeys={deletableKeys} + onDeleteField={deleteField} + colorOn={config.colorBy !== null} + onToggleColor={(on) => + updateConfig({ + ...config, + colorBy: on + ? fields.find((field) => field.type === "status")?.key ?? null + : null, + }) + } + rowHeightMode={config.rowHeightMode} + onRowHeightMode={(rowHeightMode) => updateConfig({ ...config, rowHeightMode })} + filters={config.filters} + onFilters={(filters) => updateConfig({ ...config, filters })} + filterConj={config.filterConj ?? "and"} + onFilterConj={(filterConj) => updateConfig({ ...config, filterConj })} + sorts={config.sorts} + onSorts={(sorts) => updateConfig({ ...config, sorts })} + // the toolbar must not advertise a grouping the pipeline is not applying + groupBy={serverWindowed ? null : config.groupBy} + onGroupBy={(groupBy) => updateConfig({ ...config, groupBy })} + statusValues={statusValues} + measures={measures} + unresolvedCount={unresolvedCount} + lists={lists} + pendingMeasureCount={pendingMeasureCount} + search={search} + onSearch={setSearch} + recordCount={recordCount} + scopeCounts={payload?.counts} + pool={payload?.workspace?.pool} + serverWindowed={serverWindowed} + cohortAction={cohortAction} + filterSeed={filterSeed} + modeControl={modeControl} + /> + {/* Wave-10 item 9: the view's description used to render HERE, as a full-width band + between the toolbar and the grid — it cost every row a strip of vertical space and + read as an alert rather than as a label. It now sits under the view's name in the + rail (ViewSidebar), where it annotates the thing it describes. */} +
setExpandAt((prev) => (prev === null ? prev : null))} + > + {displayMode === "grid" && ( + <> + openHeaderMenu(column, bounds)} + onHeaderClicked={onHeaderClicked} + onHeaderContextMenu={onHeaderClicked} + onKeyDown={onGridKeyDown} + maxColumnWidth={600} + getCellsForSelection={true} + rowSelectionMode="multi" + smoothScrollX + smoothScrollY + width={gridSize.width} + height={gridSize.height} + customRenderers={[ratingCellRenderer, userCellRenderer, imageCellRenderer]} + headerIcons={HEADER_ICONS} + rightElement={embedded ? undefined : ( + // Wave-6 item 4 — the "+" of the header row: the create-field form, + // insert-at-end. + // + // ⚠ Wave-20 owner item 5 UNPINNED it. It used to be `sticky: true` ("so it + // stays visible however far the columns scroll"), which parked it against the + // right edge of the WINDOW — on a four-column table that is half a screen of + // white between the last field and the control that adds the next one, and it + // reads as page chrome rather than as part of the table. It now sits + // immediately after the last column, where Airtable puts it. + // + // TWO changes are needed and neither works alone: `sticky:false` stops glide + // pinning the wrapper (`infinite-scroller.js:204` sets `right` only when + // sticky), and the CSS region kills `.dvn-spacer`'s `flex-grow: 1`, which + // would otherwise expand to fill the scroller and push the button right back + // to the edge. Consequence, accepted: on a table wider than the viewport the + // "+" is off-screen until you scroll to the end of the columns — which is + // where the new column is going to land anyway. + + )} + rightElementProps={{ sticky: false, fill: false }} + // Owner item 4 (R8) — the trailing "+" row, USER DATABASES ONLY. Passing + // `onRowAppended` is what makes glide paint the ghost row at all, so the + // affordance exists exactly where a POST can succeed. See `appendRow`. + onRowAppended={canMutateRecords ? appendRow : undefined} + trailingRowOptions={ + canMutateRecords + ? { + // The hint sits in the identity column (targetColumn 0), which is the + // one a new record is named in — the same cell the cursor lands on. + hint: "New record", + sticky: false, + targetColumn: 0, + } + : undefined + } + /> + {/* ═══ W18-B VOID ═══ (wave 18, owner item 1b / ruling R12) — the two rectangles. + + They are SIBLINGS of the canvas, immediately after it and before everything else + `.cg-grid-box` holds, which is the whole of their stacking story: `z-index: auto` + plus tree order puts them over the canvas, and `.cg-selbar` / the popovers carry + explicit `--cg-z-*` and stay over them. `pointer-events: none` keeps glide's + scroller — which is UNDER this and owns every mouse event the grid reacts to — + hit-testable straight through them. + + The right-hand one starts BELOW the header rather than at 0: in `Airtable 7.png` + the header band continues past the last column in its own near-white (#FBFCFE at + (1250,141)), and it is where the add-field "+" lives. The two rectangles overlap + in the bottom-right corner; they are the same flat colour, so the overlap is not + visible and neither one needs to know about the other. */} + {gridVoid.below !== null && ( +
+ )} + {gridVoid.right !== null && ( +
+ )} + {/* ═══ end W18-B VOID ═══ */} + + )} + {displayMode === "list" && ( + + )} + {displayMode === "calendar" && + (calendarField ? ( + + ) : ( +
+ This table has no date field to place records by. +
+ ))} + {displayMode === "kanban" && + (kanbanField ? ( + + ) : ( +
+ This table has no status or single-select field to stack by. +
+ ))} + {/* ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the SWIPE deck. + ⚠ No `field ? ... : empty-state` fork, and the asymmetry with kanban above is + deliberate. A kanban with no select field cannot exist, so the host answers for it. + A swipe deck with no binding is the ORDINARY state of a view somebody just created, + and SwipeView owns that state along with the three ways a stored binding rots — + each one re-opening the picker rather than being repaired by a guess. Forking here + would put a second, dumber explanation in front of the good one. */} + {displayMode === "swipe" && ( + k !== lockedKey)} + scope={scope} + viewer={viewer} + docs={payload?.docs} + docPayload={payload?.docPayload} + onDocAdd={ + payload?.docs + ? (pid, file) => + emitHostEvent({ + id: eventId("docadd"), + type: "doc_add", + pid, + docId: `doc_${eventId("d").slice(2, 14).replace(/[^a-z0-9]/gi, "")}`, + ...file, + }) + : undefined + } + onDocFetch={ + payload?.docs + ? (pid, docId) => + emitHostEvent({ id: eventId("docget"), type: "doc_fetch", pid, docId }) + : undefined + } + onDocDelete={ + payload?.docs + ? (pid, docId) => + emitHostEvent({ id: eventId("docdel"), type: "doc_delete", pid, docId }) + : undefined + } + /> + )} + {/** + * ⭐⭐ WAVE-29 T29 / CONTRACT C4 (owner R7) — THE FORM INTERFACE, mounted. + * + * ⛔ THE WAVE'S ONLY CROSS-FENCE WIRING, and the single most reliably-dropped artifact + * in this protocol: four consecutive waves lost one, and one wave shipped four dead + * features behind 43 green gates. C declared it, A carries the `verify_wiring` row, + * and this is the mount — three parties, so the session that REQUESTS a wiring cannot + * be the one that certifies it. + * + * ⛔ EIGHT PROPS, ALL REQUIRED, and none of them guessed — C published the exact names + * and the exact expressions in its mailbox before this ticket started. `spec` is + * required-but-NULLABLE and the `?? null` is the contract: `displaySpec?.form` is + * `undefined`, and widening the prop to accept it would be the optional-prop + * degradation wearing another hat. + * ⚠ `readOnlyReason` carries the `canEdit` reason ONLY. A locked database is a + * different sentence and `FormInterface` owns it, because it is a property of the + * database rather than of this viewer's grant. + */} + {displayMode === "form" && ( + + )} + {/* Wave-8 I19c → wave-9 I10 (C2) — CHART is a VIEW like the rest: the charts + are computed from `modeDataRows`, the very rows the grid counts, so a filter + change moves the charts and the toolbar count together. + ⚠ Only 'chart' is tested, never 'dashboard': `cleanDisplay` normalises the + legacy value on READ, so a stored 'dashboard' has already become 'chart' by + the time it reaches here. Testing both would hide a broken normalisation. */} + {displayMode === "chart" && ( + + )} + {/* Owner item 7 (C-TS) — Time series is a VIEW: metric rows x bucket columns over + the pids the current filter kept, so the table and the series answer the same + question. The panel is HOST-AGNOSTIC by contract (RECORD re-mounts it inside the + record modal for item 13), which is why it takes pids and fields rather than + reaching into anything here. */} + {displayMode === "timeseries" && ( + + )} + {/* ═══ W18-C CATALOG ═══ Owner item 4 (C6) — Catalog is a VIEW, and the only one + whose payload is authored CONTENT rather than an arrangement of the rows. It + joins its stored product codes against the WHOLE pool (`rawRows`), not the + filtered rows: a finished page must keep printing the products it names after + somebody narrows the view. The filter still has a job — `catalogFilteredCodes` + is the "add what this view shows" door — but it narrows what you can ADD, never + what a page prints. + + ⚠ Gated on the product surface. A catalog joins products to their images by + `code`, which is `product_data`'s business key AND the C2-ASSET key; on the + customer table the primary field is a customer NAME, and joining on it would ask + the asset route for a picture of a person. The refusal says so in words rather + than rendering an empty designer. */} + {displayMode === "catalog" && + (scope === "product" ? ( + + ) : ( +
+ A catalog prints products. Open the Products table to build one. +
+ ))} + {/* ═══ end W18-C CATALOG ═══ */} + {/* Wave-7 item W11c (C3) — Map is a VIEW: same pipeline rows, pins for rows + with host-supplied coords, click-through to the same record detail, and + an honest no-location chip. */} + {displayMode === "map" && fieldByKey.get(lockedKey) && ( + + )} + {/* Owner item 19 — the hover-only Expand. The COUNTERPART to item 17: the single + click that used to open a record now highlights it, and this is where opening + moved to. `position: fixed` because glide reports viewport coordinates (the same + space `.cg-header-tip` is placed in). + + ⚠ `onMouseDown` preventDefault, not just onClick: mousedown would otherwise reach + the canvas underneath, move the selection to the primary cell, and repaint the + row a frame before the panel opens — a visible flicker on every expand. + + Rendered only in `grid` mode: list / calendar / kanban / map have their own + open affordances and glide is not mounted to report bounds for them. */} + {!embedded && displayMode === "grid" && expandAt && ( + + )} + {/* Wave-5 item 6 — the header description tip. pointer-events:none + aria-hidden: + it can never become a click target, so the control under it stays clickable + (the tooltip-swallows-the-next-click trap, paid for once already). */} + {headerTip && ( +
+ {headerTip.text} +
+ )} + {/* I2 — the cell tip. `pointer-events: none` + `aria-hidden` are not styling here, + they are the contract: the control under the tip must stay clickable, and a + screen reader already has the cell's own value. */} + {cellTip && ( +
+ {cellTip.text} +
+ )} + {/* Owner item 2's point: checked customers accumulate, and the selection DOES + something. FLOATED over the grid, never in the flow: a bar that reflows the grid + on first selection shifts every row under the user's cursor mid-click-run (and + broke this gate's own coordinate math before it broke a user). It states the + count and offers "Add to cohort" over exactly the checked pids — the same guarded + add_to_list event the view menu uses, so the host treats both alike. */} + {!embedded && selectedPids.size > 0 && !serverWindowed && + (displayMode === "grid" || displayMode === "map") && ( +
+ + {selectedPids.size.toLocaleString()} selected + + {/* ⭐ 2026-08-06 (owner) — *"right now it only shows 'Add to cohort'"*. On a user + database the checked rows are RECORDS the reader owns, and there was no door to + remove one; the whole delete path existed server-side with nothing calling it. + ⚠ ONLY on a `ut_` database: the customer and product grids are projections of + Odoo, where a "delete" would be a write to somebody else's system of record — + the server refuses it, and a button that must be refused is worse than none. */} + {canMutateRecords && ( + + )} + {cohortMode ? ( + // Item 2c: on the Cohort page the selection EDITS MEMBERSHIP of the fixed set — + // "Remove from cohort" replaces "Add to cohort". Direct (no popover): the + // checked set is explicit, the button says what it does, and re-adding is one + // picker away. + + ) : ( + + )} + {/* ⭐ Owner item 9 — the counterpart, and it renders ONLY when it can do + something: the checked rows have to be in a cohort this viewer may edit for + "Remove from cohort…" to mean anything. Same bar, same selection, opposite + direction — the ellipsis says a picker follows, because unlike cohort mode + the target is not implied by where you are standing. */} + {!cohortMode && removableLists.length > 0 && ( + + )} + +
+ )} +
+ {/* Owner item 8 — the cap is VISIBLE, beside the control that lifts it. The toolbar + count above still states the full matched count; this bar says what is painted. + Grid + list only: calendar/kanban project the FULL set with their own stated + bounds (a month, a stack's Show-all). */} + {capped && (displayMode === "grid" || displayMode === "list") && ( +
+ + Showing first {shownRecords.toLocaleString()} of{" "} + {recordCount.toLocaleString()} + + + +
+ )} +
+ + {menuField && columnMenu && ( + + * t.key !== scope)` and said so honestly: "a legal relation in Airtable and a confusing + * one to explain in a one-line picker … left out until somebody asks". It was a taste + * call, not a constraint, and nothing below the picker ever needed it — `_clean_link` + * only requires a `ut_*` key, and `sync_reciprocal_link` puts the backlink on the target + * whichever table that is. "This lead is a duplicate of that lead" and "this record's + * parent" are the ordinary cases, and both were unbuildable. + * + * ⚠ THE UNFILTERED LIST ALSO REPAIRS THE ROLLUP EDITOR, which is the half a one-line + * change hides: it resolves the fold's source columns with + * `linkTargets.find(t => t.key === chosenLink.link.table)`, so a rollup over a self-link + * would have offered an EMPTY column picker and looked like the rollup was broken. + * ⚠ Both `` renders take the same list — the full menu and the "+" create + * menu. One of the two is the half-landing this repo keeps paying for. + */ + linkTargets={linkTargets} + rollupSourceOffer={rollupSourceOffer} + locked={menuField.key === lockedKey} + schemaLocked={isSchemaLocked(menuField)} + viewer={viewer} + sortedDir={menuSortedDir} + isFiltered={menuIsFiltered} + isGrouped={config.groupBy === menuField.key} + groupable={isGroupableField(menuField, lockedKey)} + // Item 12 (R8) — the kanban's "Add option" opens THIS menu straight into its Edit-field + // pane. Absent for every other opening, which is the action list as before. + initialPane={columnMenu.pane} + onClose={() => setColumnMenu(null)} + onNote={(note) => saveField({ ...menuField, note })} + onHide={() => { + setColumnVisible(menuField.key, false); + setColumnMenu(null); + }} + onCreate={createField} + onChangeField={(newKey) => changeField(menuField.key, newKey)} + onCreateAndSwap={(label, type, options, extra) => + createAndSwapField(menuField.key, label, type, options, extra) + } + onProfileFlag={isUserTable ? setProfileFlag : undefined} + onFieldConfig={isUserTable ? (patch) => setFieldConfig(menuField.key, patch) : undefined} + onRetype={ + // Item 2 — a CREATED field retypes IN PLACE from the pane. + // + // ⭐⭐ WAVE-29 T27 (owner item 14): the predicate is `isUserSchemaField`, not the raw + // `custom` flag. `user_tables._clean_field` never stamps `custom`, so EVERY column on + // a `ut_*` database failed the old test — the Field-type picker vanished, the pane was + // left showing only the "Change field" swap picker (a different feature entirely, + // holding the autofocus), and the owner reported the innocent control as broken. + isUserSchemaField(menuField, isUserTable) + ? (type, options, extra) => + retypeField(menuField.key, type, options, extra) + : undefined + } + onRename={ + // Item 5 — rename is a def edit on created strata. + // + // ⛔ T27: withheld by the SAME flag, which is why the Name box went disabled saying + // *"A source field keeps its name and type from the data source"* about a column the + // user created themselves. A `ut_*` definition field renames through the DEFINITION + // door; `saveField` would fork it into a private overlay copy of itself (the reason + // `onFormula` below already branches this way). + !isUserSchemaField(menuField, isUserTable) + ? undefined + : menuField.custom + ? (label) => saveField({ ...menuField, label }) + : (label) => setFieldConfig(menuField.key, { label }) + } + onFormula={ + // Owner item 8 — edit a created FORMULA field's source in place; the optional + // label rides the SAME upsert (see ColumnMenu's one-upsert rule). + // + // ⭐⭐ 2026-08-10 — TWO STRATA, TWO DOORS, and the gate is `custom` rather than the + // type. A formula column on a `ut_*` database now lives in the SHARED definition, and + // definition fields carry no `custom` flag (`_clean_field` never stamps one) — so the + // old single condition made the expression editor DISAPPEAR for exactly the columns + // this change creates: creatable and then permanently frozen, which is a worse state + // than the one it replaces. `setFieldConfig` PATCHes the definition; `saveField` + // writes this user's overlay, and sending a definition field down that path would + // fork the column into a private second copy of itself. + menuField.type !== "formula" + ? undefined + : isUserTable && !menuField.custom + ? (formula, label) => + setFieldConfig(menuField.key, { + formula, + ...(label ? { label } : {}), + }) + : menuField.custom + ? (formula, label) => + saveField({ + ...menuField, + formula, + ...(label ? { label } : {}), + }) + : undefined + } + onPeriod={ + menuField.measure + ? (w) => changeMeasurePeriod(menuField.key, w) + : undefined + } + onDelete={ + menuField.custom && menuField.key !== lockedKey + ? () => { + deleteField(menuField.key); + setColumnMenu(null); + } + : /** + * ⭐⭐ 2026-08-10 — the DEFINITION stratum's own delete. + * + * ⛔ NARROWED TO THE THREE KINDS THAT HOLD NO DATA OF THEIR OWN, which is + * `preset_editable`'s argument reused rather than a new one: a link, a rollup and + * a formula are all QUESTIONS asked of other cells, so removing one destroys no + * measurement and can be undone by asking it again. Every other definition column + * on a `ut_*` database holds real values, and offering a type-blind Delete over + * those is a destructive widening this change has no mandate for (booked as + * D-114). It is also exactly the three kinds `createField` now routes to the + * definition — so what a user can make here, a user can unmake. + * ⚠ `schemaLocked` still gates the button inside ColumnMenu, so a pre-set link + * never shows it; a pre-set ROLLUP does, by the same owner ruling that made it + * editable, and the confirm says it will come back. + */ + isUserTable && + !menuField.custom && + menuField.key !== lockedKey && + (menuField.type === "link" || menuField.type === "rollup" || + menuField.type === "formula") + ? () => { + deleteDefinitionField(menuField.key); + setColumnMenu(null); + } + : undefined + } + onDuplicate={ + // Creatable strata only — a base Odoo field offers no Duplicate (contract). + menuField.custom ? () => duplicateField(menuField) : undefined + } + onPermissions={ + // Creatable strata only; ColumnMenu further gates on the viewer (creator/admin). + menuField.custom + ? (edit) => saveField({ ...menuField, permissions: { edit } }) + : undefined + } + onFormat={(format) => saveField({ ...menuField, format })} + /** + * ⭐ WAVE-29 T33 (owner item 17) — the column summary, through whichever door owns this + * column's definition. Same two-strata split as `onRename` above: a `ut_*` definition + * field PATCHes the definition (`_clean_field` stores `agg`), everything else rides the + * ordinary overlay upsert. ⚠ `undefined` CLEARS it, and the clear has to travel — a + * PATCH that merely omits `agg` keeps the stored one, which is the stickiness the money + * columns depend on. + */ + onAggregate={(agg) => + isUserTable && !menuField.custom + ? setFieldConfig(menuField.key, { agg: agg ?? "" }) + : saveField({ ...menuField, agg }) + } + onSort={(dir) => + updateConfig({ ...config, sorts: [{ colId: menuField.key, dir }] }) + } + onClearSort={() => + updateConfig({ + ...config, + sorts: config.sorts.filter((s) => s.colId !== menuField.key), + }) + } + onFilterBy={() => + // Item 6c — a MEASURE-carrying column seeds the equivalent MEASURE condition + // (same measure key + window, value empty); every other filterable column seeds + // an ordinary column condition. Same signal, one consumer (the Toolbar). + setFilterSeed((prev) => ({ + n: (prev?.n ?? 0) + 1, + ...(menuField.measure + ? { measure: { key: menuField.measure.key, window: menuField.measure.window } } + : { key: menuField.key }), + })) + } + pinnedTo={menuPinnedTo} + onPinTo={ + // Item 11c — pinning the identity column is the same as unpinning; skip it. + menuVisIndex > 0 && !menuPinnedTo + ? () => pinFieldTo(menuField.key) + : undefined + } + onUnpin={frozenN > 1 ? unpinFields : undefined} + scopeChoice={scopeChoice} + onClearFilter={() => + updateConfig({ + ...config, + filters: dropFieldFromTree(config.filters, menuField.key, measureCols), + }) + } + onGroupByField={() => updateConfig({ ...config, groupBy: menuField.key })} + onClearGroup={() => updateConfig({ ...config, groupBy: null })} + userOptions={payload?.userOptions ?? []} + measures={measures} + /> + )} + + {/* Wave-6 item 4 — the header "+": the SAME ColumnMenu, opened straight into its + create form in insert-at-end mode. Anchored on the button; the nominal field is the + locked identity column (only its position matters — the create form never shows it). + The action/pane handlers below are unreachable in this mode (create-only; Cancel + closes), supplied inert to satisfy the surface. */} + {plusMenu && fieldByKey.get(lockedKey) && ( + setPlusMenu(null)} + onNote={() => undefined} + onHide={() => undefined} + onCreate={createField} + onChangeField={() => undefined} + onCreateAndSwap={() => undefined} + onSort={() => undefined} + onClearSort={() => undefined} + onFilterBy={() => undefined} + onClearFilter={() => undefined} + onGroupByField={() => undefined} + onClearGroup={() => undefined} + scopeChoice={scopeChoice} + userOptions={payload?.userOptions ?? []} + measures={measures} + // ⛔⛔ 2026-08-09 — THE SECOND HALF OF THE SAME DEFECT. The sibling instance above has + // passed these since the relational types shipped; this one never did, so a rollup + // created through "+" had no databases to resolve its link against and no source offer + // to switch modes with — the Column picker held its placeholder and nothing else. + // ⚠ TWO THINGS WERE BOTH WRONG (the fetch was gated on `columnMenu` and these props + // were absent), so fixing either alone would still have shipped an empty picker and + // read as "the fix did not work" — [[defects-that-mask-each-other]]. + // ⚠ 2026-08-10 — THE SELF-EXCLUSION IS GONE FROM BOTH SITES (D-113). The line this + // replaces said the two call sites "must not disagree about that", and it was right for + // a reason it did not anticipate: the exclusion was removed from the sibling first and + // this copy kept it, so for one edit the "+" menu and the column menu offered DIFFERENT + // databases — caught by grepping the prop rather than by the build, which is happy + // either way. See the sibling's note for why self-links are offered now. + linkTargets={linkTargets} + rollupSourceOffer={rollupSourceOffer} + /> + )} + + {/* ⭐ WAVE 21 item 11 (R10 / contract C5) — "Select records from a list". + ⚠ `inView` is `visibleRows`, NOT `displayRows`. `displayRows` is a capped slice + (`DISPLAY_PAGE`) of what the view matched; matching over it would report every + record past the cap as "not found" — a lie that grows with the table. The whole + pool goes in beside it so a value the FILTER hides is reported as filtered, not + as missing. */} + {/* ⭐ WAVE-29 T25 — the import dialog, mounted as a conditional sibling exactly like + `SelectFromFile` above it. `scope` is the `ut_*` table key the import door is per. */} + {importOpen && ( + { + signal(TOAST_EVENT, + `${count.toLocaleString()} record${count === 1 ? "" : "s"} imported.`); + signal(ROWS_STALE_EVENT); + }} + onClose={() => setImportOpen(false)} + /> + )} + + {selectFromFile && ( + (vr.kind === "data" ? [vr.record] : []))} + wholeTable={computedRows} + viewName={activeView?.name ?? "this view"} + onSelect={(pids) => selectPids(pids, "replace")} + onClose={() => setSelectFromFile(false)} + /> + )} + + {/* The selection bar's cohort picker: existing cohorts, or a new one by name. Mirrors the + view menu's Add-to-list surface so the two ways of building a cohort read identically. */} + {selAddOpen && selAddRef.current && selectedPids.size > 0 && ( + setSelAddOpen(false)} + dataKind="selection-add-to-list" + > +
Add to locked view
+
+ Adds the {selectedPids.size.toLocaleString()} checked customer + {selectedPids.size === 1 ? "" : "s"}. The cohort stays fixed as the data changes. +
+ {lists.map((l) => ( + + ))} +
+ + setSelListName(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Enter" && selListName.trim()) + addSelectionToList("", selListName.trim()); + }} + /> +
+ +
+
+
+ )} + + {/* ⭐ Owner item 9 — the remove picker. Every row states how many of the CHECKED rows it + would take out ("3 of 4"), because the selection and the cohort are two different sets + and the difference is the whole reason this needs a picker rather than a button. */} + {selRemoveOpen && selRemoveRef.current && removableLists.length > 0 && ( + setSelRemoveOpen(false)} + dataKind="selection-remove-from-list" + > +
Remove from locked view
+
+ Only the locked views holding one of the {selectedPids.size.toLocaleString()} checked + customer{selectedPids.size === 1 ? "" : "s"} are listed. The records themselves are + not touched. +
+ {removableLists.map((l) => ( + + ))} +
+ )} + + {/* Cohort mode's "+ Add customers" picker (item 2c): search the POOL, tick many, confirm + once. Candidates exclude current members; picks ACCUMULATE across searches, so the + confirm button carries the running total. The 50-row render cap is stated beside the + list — nothing silently truncated. */} + {cohortMode && addCustOpen && addCustRef.current && activeCohortId && ( + setAddCustOpen(false)} + dataKind="cohort-add-customers" + > +
Add customers
+
+ Search your customer pool and tick who to add to{" "} + {activeCohort?.name ?? "this locked view"}. +
+ setAddCustQuery(e.target.value)} + /> +
+ {addCandidates.slice(0, 50).map((c) => ( + + ))} + {addCandidates.length === 0 && ( +
+ {addCustQuery.trim() + ? "No customer outside this locked view matches your search." + : "Every customer in your pool is already in this locked view."} +
+ )} +
+ {addCandidates.length > 50 && ( +
+ Showing the first 50 of {addCandidates.length.toLocaleString()} matches — keep + typing to narrow. +
+ )} +
+ + +
+
+ )} + + {/* ⭐ Wave-23 C7 — the document viewer. `onSave` is OMITTED, not disabled, when the reader + may not edit: an absent handler is what makes the raw tab a `
` instead of a
+          textarea, so there is no editor to be refused by. */}
+      {jsonField && jsonAt && (
+         patchAndRecord(jsonAt.pid, { [jsonField.key]: next }, "a document")
+              : undefined
+          }
+          onClose={closeJson}
+        />
+      )}
+      {linkField?.type === "link" && linkField.link?.table.startsWith("ut_") && linkAt && (
+         patchAndRecord(
+            linkAt.pid,
+            { [linkField.key]: ids.join(",") },
+            "linked records"
+          )}
+          onClose={closeLink}
+        />
+      )}
+      {pickerField && picker && (
+         {
+            const fwd = e.key === "ArrowDown" || e.key === "ArrowRight";
+            const back = e.key === "ArrowUp" || e.key === "ArrowLeft";
+            if (!fwd && !back) return;
+            const items = Array.from(
+              e.currentTarget.querySelectorAll(
+                ".cg-pick-row, .cg-star-btn"
+              )
+            );
+            if (!items.length) return;
+            e.preventDefault();
+            const at = items.indexOf(document.activeElement as HTMLButtonElement);
+            const next = fwd
+              ? (at + 1) % items.length
+              : (at - 1 + items.length) % items.length;
+            items[next]?.focus();
+          }}
+        >
+          {/* Wave-5 item 11 — a RATING cell's picker is the star row itself. */}
+          {pickerField.type === "rating" && (
+            
+ {Array.from({ length: ratingMax(pickerField) }, (_, i) => i + 1).map( + (n) => ( + + ) + )} +
+ )} + {pickerField.type !== "rating" && pickerChoices.length === 0 && ( +
+ {pickerField.type === "user" + ? "No assignable people were supplied for this workspace." + : "This field has no choices yet. Add them from the column menu."} +
+ )} +
+ {pickerChoices.map((choice) => { + const tint = optionTint(pickerField, choice); + const on = pickerMulti + ? pickerParts.includes(choice) + : pickerValue === choice; + return ( + + ); + })} +
+ {pickerMulti && ( +
+ +
+ )} + {/* Clearing must be possible, and it must write "" rather than delete the key — + the overlay patch contract is a value per field, and a missing key would leave + the previous value standing on the next render. */} + {pickerValue !== "" && ( + + )} +
+ )} + + {!embedded && detailRecord && detailPid !== null && ( + go(-1)} + onNext={() => go(1)} + onClose={() => setDetailPid(null)} + // Item 5 / C-LAYOUT — see `recordLayout` above. Both optional on RECORD's side, so + // this compiles whether or not their reorder UI has landed yet. + recordLayout={recordLayout} + onRecordLayout={onRecordLayout} + // Item 13 / C-EMBED — the host names WHICH question Insights asks. Without this the + // tab deliberately does not render (RECORD's no-safe-default rule: a wrong scope + // succeeds with plausible wrong numbers, so absence must mean absent, not guessed). + scope={scope} + docs={payload?.docs?.[String(detailPid)] ?? []} + docPayload={payload?.docPayload} + // C5 — documents ride the ordinary event log. The host answers a fetch + // on the NEXT render via `payload.docPayload`; Documents.tsx matches it + // to its own pending request before touching it. + onDocAdd={ + payload?.docs + ? (file) => + emitHostEvent({ + id: eventId("docadd"), + type: "doc_add", + pid: detailPid, + docId: `doc_${eventId("d").slice(2, 14).replace(/[^a-z0-9]/gi, "")}`, + ...file, + }) + : undefined + } + onDocFetch={ + payload?.docs + ? (docId) => + emitHostEvent({ + id: eventId("docget"), + type: "doc_fetch", + pid: detailPid, + docId, + }) + : undefined + } + onDocDelete={ + payload?.docs + ? (docId) => + emitHostEvent({ + id: eventId("docdel"), + type: "doc_delete", + pid: detailPid, + docId, + }) + : undefined + } + onNotesChange={(key, value) => + setOverlayEdits((current) => ({ + ...current, + [detailPid]: { ...current[detailPid], [key]: value }, + })) + } + onNotesCommit={(key, value) => patchAndRecord(detailPid, { [key]: value })} + viewer={viewer} + // Wave-14 close-out stitch (HOST): TSREC's two light-switch props, unwired when + // the GRID session ended — item 9's picker vocabulary + item 11's photos. Same + // values the grid's own surfaces already read (the wave-13 sixth-prop precedent). + userOptions={payload?.userOptions ?? []} + userAvatars={userAvatars} + /> + )} +
+ ); +} + +export default memo(CustomerGrid); diff --git a/web/src/customer-grid/FormInterface.css b/web/src/customer-grid/FormInterface.css new file mode 100644 index 0000000000000000000000000000000000000000..d716625a38e3ffdcc0c5bba7dd22119ede362b0c --- /dev/null +++ b/web/src/customer-grid/FormInterface.css @@ -0,0 +1,269 @@ +/* --------------------------------------------------------------------------- + customer-grid / FormInterface.css + ⭐ WAVE-29 R7 — the FORM builder's own stylesheet. + + Its own file rather than `index.css` for two reasons, and only one of them is this wave's + fence: a mode's surface that ships with its component can be read, moved or retired in one + place (`RecordDetail.css` and `ImportDialog.css` set the precedent), and `index.css` is + already the app's largest single file. + + Every colour, radius and type size is a `--lp-*` token. Nothing here invents a hue: the token + parity gate compares `index.css` against `theme.ts`, and a raw hex would be invisible to it. + --------------------------------------------------------------------------- */ + +.cg-form-build { + display: flex; + flex-wrap: wrap; + gap: 24px; + align-items: flex-start; + overflow: auto; + padding: 20px 24px 32px; + height: 100%; + background: var(--lp-surface); + color: var(--lp-ink); + font-size: var(--lp-fs-xs); +} + +.cg-form-pane { + flex: 1 1 320px; + min-width: 300px; + max-width: 560px; +} + +.cg-form-h { + margin: 0 0 12px; + font-size: var(--lp-fs-lg); + font-weight: 600; +} + +.cg-form-sub { + margin: 22px 0 8px; + font-size: var(--lp-fs-2xs); + font-weight: 600; + letter-spacing: 0.02em; + color: var(--lp-muted); +} + +/* The first `.cg-form-sub` in a pane sits under no heading of its own. */ +.cg-form-pane > .cg-form-sub:first-child { + margin-top: 0; +} + +.cg-form-note { + margin: 6px 0; + color: var(--lp-muted); + font-size: var(--lp-fs-2xs); + line-height: 1.5; +} + +/* An honest warning, not an alarm: it is a state the reader can fix, so it is tinted rather + than shouted. Red is reserved for a refusal that has already cost something. */ +.cg-form-warn { + margin: 8px 0; + padding: 8px 10px; + border: 1px solid var(--lp-red); + border-radius: var(--lp-r-sm); + background: var(--lp-red-tint); + color: var(--lp-red-deep); + font-size: var(--lp-fs-2xs); + line-height: 1.5; +} + +.cg-form-empty { + display: grid; + place-items: center; + height: 100%; + padding: 40px; + color: var(--lp-muted); + font-size: var(--lp-fs-xs); + text-align: center; +} + +/* --- the spec fields ------------------------------------------------------------------- */ + +.cg-form-row { + display: block; + margin: 0 0 12px; +} + +.cg-form-row > span { + display: block; + margin-bottom: 4px; + color: var(--lp-muted); + font-size: var(--lp-fs-3xs); + font-weight: 600; +} + +.cg-form-row input[type="text"], +.cg-form-row textarea, +.cg-form-addmail input { + width: 100%; + padding: 7px 9px; + border: 1px solid var(--lp-line); + border-radius: var(--lp-r-sm); + background: var(--lp-surface); + color: var(--lp-ink); + font-family: inherit; + font-size: var(--lp-fs-xs); + resize: vertical; +} + +.cg-form-row input:disabled, +.cg-form-row textarea:disabled { + background: var(--lp-wash); + color: var(--lp-muted); +} + +/* --- the question list ----------------------------------------------------------------- */ + +.cg-form-fields { + margin: 0; + padding: 0; + list-style: none; + border: 1px solid var(--lp-line); + border-radius: var(--lp-r-md); + overflow: hidden; +} + +.cg-form-field { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + padding: 7px 10px; + border-bottom: 1px solid var(--lp-line); +} + +.cg-form-field:last-child { + border-bottom: 0; +} + +.cg-form-field.on { + background: var(--lp-blue-tint); +} + +.cg-form-pick { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; + cursor: pointer; +} + +.cg-form-label { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.cg-form-type { + color: var(--lp-muted); + font-size: var(--lp-fs-3xs); +} + +.cg-form-req { + display: flex; + align-items: center; + gap: 5px; + flex: none; + color: var(--lp-muted); + font-size: var(--lp-fs-3xs); + cursor: pointer; +} + +/* --- who can submit -------------------------------------------------------------------- */ + +.cg-form-radio { + display: flex; + align-items: flex-start; + gap: 9px; + margin-bottom: 10px; + cursor: pointer; +} + +.cg-form-radio strong { + display: block; + font-weight: 600; +} + +.cg-form-radio em { + display: block; + margin-top: 2px; + color: var(--lp-muted); + font-size: var(--lp-fs-3xs); + font-style: normal; + line-height: 1.5; +} + +.cg-form-emails ul { + margin: 0 0 8px; + padding: 0; + list-style: none; +} + +.cg-form-emails li { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + padding: 5px 0; + border-bottom: 1px solid var(--lp-line); +} + +.cg-form-emails li:last-child { + border-bottom: 0; +} + +.cg-form-addmail { + display: flex; + gap: 8px; +} + +/* --- the share link -------------------------------------------------------------------- */ + +.cg-form-link { + padding: 10px 12px; + border: 1px solid var(--lp-line); + border-radius: var(--lp-r-md); + background: var(--lp-surface-2); +} + +.cg-form-link code { + display: block; + margin-bottom: 8px; + overflow-wrap: anywhere; + color: var(--lp-blue-deep); + font-size: var(--lp-fs-2xs); +} + +.cg-form-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.cg-form-build button { + padding: 5px 11px; + border: 1px solid var(--lp-line); + border-radius: var(--lp-r-pill); + background: var(--lp-surface); + color: var(--lp-ink); + font-family: inherit; + font-size: var(--lp-fs-2xs); + cursor: pointer; +} + +.cg-form-build button:hover:not(:disabled) { + border-color: var(--lp-blue-solid); + color: var(--lp-blue-deep); +} + +.cg-form-build button:disabled { + color: var(--lp-muted); + cursor: default; +} + +.cg-form-emails button { + padding: 2px 8px; + font-size: var(--lp-fs-3xs); +} diff --git a/web/src/customer-grid/FormInterface.tsx b/web/src/customer-grid/FormInterface.tsx new file mode 100644 index 0000000000000000000000000000000000000000..4c6456c6235becebf9e02ee57bd367fec116c438 --- /dev/null +++ b/web/src/customer-grid/FormInterface.tsx @@ -0,0 +1,504 @@ +// --------------------------------------------------------------------------- +// customer-grid / FormInterface.tsx +// ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3 / C4) — THE FORM INTERFACE. +// +// The owner's ask, verbatim: "a user fills in information; the form has a shareable link that +// accepts submissions either from authorized people (by email) or from the public. Google Forms +// is the reference." +// +// This file is the BUILDER — the in-app surface a `form`-kind view renders. The page a stranger +// fills in is `FormPublic.tsx` and it already shipped in wave 23, along with the whole public +// door (`api/routes_forms.py`). What had never existed was any way to CREATE the thing they +// serve: `form` was not a legal display mode and `_clean_display` dropped `display.form`, so the +// key the public door reads could not be written by anything. That is D-90, and it is why this +// component is mostly plumbing rather than a new idea — the door was built first and the handle +// never fitted. +// +// THE TWO HALVES, and they are stored in two different places on purpose: +// * the SPEC (questions, title, who may submit) lives on the view, at `config.display.form`, +// written through the host's ordinary view-save path like every other mode's configuration; +// * the TOKEN lives in a server-owned index a browser cannot write, because a share token that +// rides the wire is a token a browser can CHOOSE, and the public resolver answers with the +// first tenant that matches. See `routes_forms.py`'s note for the whole argument. +// --------------------------------------------------------------------------- + +import { useCallback, useEffect, useMemo, useState } from "react"; +import { API_V1, CREDENTIALS } from "../apiContract"; +import type { Field, FieldType, FormSpec } from "./types"; +import { TYPE_LABELS } from "./iconShapes"; +import "./FormInterface.css"; + +/** + * The field types a form may COLLECT. **Mirrors `platform/aios_grid.py::FORM_FIELD_TYPES` and + * `verify_forms.py` compares the two files name-for-name** — a type offered here and refused + * there is a question that silently vanishes from the published form, which is the failure a + * one-sided list produces every time. + * + * An ALLOW-LIST, not a list of exclusions: a type missing here is a question the builder cannot + * ask yet, while a type wrongly present is a public door writing values its column cannot mean. + * The two mistakes do not cost the same, so the list fails closed. + */ +export const FORM_FIELD_TYPES: ReadonlySet = new Set([ + "text", "select", "multiselect", "int", "currency", "pct", "date", + "checkbox", "phone", "email", "url", "rating", +]); + +/** + * The stored bag at `view.config.display.form` is `FormSpec`, and it is DECLARED IN `./types` + * (imported above), not here. ⛔ Declaring it in this file made a type-only circular import — + * `types.ts` needs it for `DisplaySpec.form` while this file needs `Field` from `types.ts` — which + * vite erased and `tsc --ignoreConfig` could not, killing `verify_optimism` and + * `verify_live_workspace` with TS6142. Re-exported here so this file still names the contract it + * edits; the single declaration stays in the type module. ⛔ No `token` — see the header. + */ +export type { FormSpec } from "./types"; + +export interface FormInterfaceProps { + /** The database this form collects into (`topicForScope(scope)`), e.g. `ut_leads`. */ + topic: string; + /** The view holding the spec. `null` while no saved view is active — a form has nowhere to live. */ + viewId: string | null; + /** Every column of the table; this component decides which can be questions. */ + fields: Field[]; + /** + * The stored spec, or `null` when this view has no form yet. + * ⚠ REQUIRED and NULLABLE, never optional: the host passes `displaySpec?.form ?? null`, and the + * `?? null` is the point — an optional prop degrades to "the feature does not exist", which is + * indistinguishable from never having been built. + */ + spec: FormSpec | null; + /** Writes the spec back into `config.display.form`. `null` clears it. */ + onSpec: (next: FormSpec | null) => void; + /** May this viewer change the form's configuration (the view-config grant)? */ + canEdit: boolean; + /** Why not, in one sentence, when `canEdit` is false. `null` when it is true. */ + readOnlyReason: string | null; + /** Does this database accept new records at all? A locked one can never be a form's target. */ + canCollect: boolean; +} + +interface LinkState { + token: string | null; + url: string | null; + stored: { fields: number; access: string; title: string } | null; +} + +const EMPTY_LINK: LinkState = { token: null, url: null, stored: null }; + +/** The share URL the page shows. The server returns an origin-qualified one when it knows the + * public base; otherwise it returns a path, and only the browser knows where it is. */ +function absolute(url: string): string { + return url.startsWith("http") ? url : `${window.location.origin}${url}`; +} + +export function FormInterface({ + topic, + viewId, + fields, + spec, + onSpec, + canEdit, + readOnlyReason, + canCollect, +}: FormInterfaceProps) { + const [link, setLink] = useState(EMPTY_LINK); + const [busy, setBusy] = useState(false); + const [problem, setProblem] = useState(""); + const [copied, setCopied] = useState(false); + const [emailDraft, setEmailDraft] = useState(""); + + /** + * The columns that can be questions. + * + * ⚠ The server enforces this too, and both walls are needed rather than one: a column can + * BECOME computed (a formula, a rollup, a metric binding) long after it was added to a live + * form, and this component is not running when that happens. Here it is a courtesy that keeps + * the builder honest; `routes_forms._public_form` is the wall. + */ + const askable = useMemo( + () => + fields.filter( + (f) => + FORM_FIELD_TYPES.has(f.type) && + f.source !== "odoo" && + !f.automation && + !f.rollup && + !f.metric + ), + [fields] + ); + const byKey = useMemo(() => new Map(fields.map((f) => [f.key, f])), [fields]); + + const chosen = spec?.fields ?? []; + const required = useMemo(() => new Set(spec?.required ?? []), [spec]); + const emails = spec?.emails ?? []; + const restricted = spec?.access === "emails"; + + /** + * ⭐ THE DEAD-STATE DETECTOR, and it earns its four lines. + * + * The spec reaches this panel through the host's `cleanDisplay`, which builds its output key by + * key — so a host that does not carry `display.form` hands back `null` forever. The panel would + * then look exactly like a form nobody has built yet: every keystroke accepted, nothing stored, + * no error anywhere. That is [W-07]'s failure verbatim — "the feature does not exist" and "the + * feature was never wired" are indistinguishable from the outside, and four consecutive waves + * lost a hand-off to it. + * + * One write plus one render answers it: if this panel has SENT a spec and the next render still + * has none, the carry is missing. It cannot false-positive on a slow save — `onSpec` is + * synchronous host state, not a request. + */ + const [wrote, setWrote] = useState(false); + const carryBroken = wrote && spec === null; + + const patch = useCallback( + (next: Partial) => { + if (!canEdit) return; + setWrote(true); + onSpec({ ...(spec ?? {}), ...next }); + }, + [canEdit, onSpec, spec] + ); + + // ---- the share link, read from the server (never minted on read) ------------------------- + const url = `${API_V1}/form-link?topic=${encodeURIComponent(topic)}&view=${encodeURIComponent( + viewId ?? "" + )}`; + + useEffect(() => { + if (!viewId) { + setLink(EMPTY_LINK); + return; + } + let live = true; + fetch(url, { credentials: CREDENTIALS }) + .then((r) => (r.ok ? r.json() : EMPTY_LINK)) + .then((body: LinkState) => { + if (live) setLink(body ?? EMPTY_LINK); + }) + .catch(() => { + if (live) setLink(EMPTY_LINK); + }); + return () => { + live = false; + }; + }, [url, viewId]); + + const call = useCallback( + async (method: "POST" | "DELETE", body?: unknown) => { + setBusy(true); + setProblem(""); + try { + const response = await fetch(method === "POST" ? `${API_V1}/form-link` : url, { + method, + credentials: CREDENTIALS, + ...(body + ? { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) } + : {}), + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + setProblem( + (payload as { error?: { message?: string } } | null)?.error?.message ?? + `The server answered ${response.status}.` + ); + return; + } + setLink( + method === "DELETE" + ? { ...EMPTY_LINK, stored: link.stored } + : { ...(payload as LinkState), stored: link.stored } + ); + } finally { + setBusy(false); + } + }, + [link.stored, url] + ); + + const copy = useCallback(() => { + if (!link.url) return; + void navigator.clipboard?.writeText(absolute(link.url)); + setCopied(true); + window.setTimeout(() => setCopied(false), 1600); + }, [link.url]); + + // ---- the states that are not a form ------------------------------------------------------- + if (!viewId) { + return ( +
+ A form belongs to a saved view. Save this view first, then build the form on it. +
+ ); + } + if (!canCollect) { + // ⚠ NOT `readOnlyReason`, and the two must not be merged: this is a fact about the DATABASE + // (a locked one accepts no new records, so a form could only ever collect refusals), while + // `readOnlyReason` is about the VIEWER's grant. + return ( +
+ This database does not accept new records, so a form has nowhere to put an answer. +
+ ); + } + + const diverged = spec === null && (link.stored?.fields ?? 0) > 0; + + return ( +
+
+

Form

+ {!canEdit && readOnlyReason &&

{readOnlyReason}

} + {/* ⚠ The divergence banner. The `spec` prop comes from the host's `cleanDisplay`, and if + that ever stops carrying `display.form` the panel would render an empty form over a + stored one — and the next save would erase real questions. Saying so beats losing + them quietly. */} + {diverged && ( +

+ The server holds {link.stored?.fields} question + {link.stored?.fields === 1 ? "" : "s"} for this view that this page is not carrying. + Reload before editing — saving now would replace them. +

+ )} + {/* ⚠ The banner above needs a STORED spec to compare against, so it can only speak once a + form has been saved successfully at least once. This one covers the state before + that: nothing has ever been stored, so there is nothing to diverge from. */} + {carryBroken && ( +

+ Nothing is being saved: this view gave back no form after the last change. Reload the + page — if it repeats, this build cannot store forms and an administrator should be + told, because everything typed here is being discarded. +

+ )} + + +