diff --git a/RELEASES.json b/RELEASES.json
index 79739322f0ad7638c9a9d6104bb0b07800124e13..89464650f708851a27e44b6348ae86f194a494f7 100644
--- a/RELEASES.json
+++ b/RELEASES.json
@@ -1,5 +1,5 @@
{
- "current": "7269240",
+ "current": "bac40c2",
"releases": [
{
"version": "v24",
diff --git a/VERSION b/VERSION
index 0cef34bb6a7571bbcbb4e0f8ed0f94c5b3835538..690f16a6a301ce64adf22bceb4860225f7d09088 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-7269240
+bac40c2
diff --git a/api/automation_engine.py b/api/automation_engine.py
index 65ce5ae5f937ac71cdc0924897504053e8b309cc..80dccc87f59b1736cd62e97248c173f25e61339b 100644
--- a/api/automation_engine.py
+++ b/api/automation_engine.py
@@ -1,10483 +1,11194 @@
-"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME.
-
-THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant
-store and a RUN that lives in this process. The definition is durable, small and rarely written;
-the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file
-is built to avoid, and it has two halves:
-
- * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the
- store outlives the process that wrote it, so a container restart mid-run leaves an automation
- that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives
- in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written
- exactly ONCE per run, at the end.
- * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s
- floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape
- by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes
- its whole result first and commits it in a single `update` callback.
-
-WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not
-deployed, so its extraction core is copied into this file (the wave doc's instruction), ported
-httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with
-it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point:
-the skill version takes a URL from a developer on a CLI; this takes one from a request body.
-"""
-from __future__ import annotations
-
-import datetime as _dt
-import hashlib
-import hmac
-import ipaddress
-import json
-import os
-import re
-import socket
-import threading
-import time
-from urllib.parse import urljoin, urlparse
-
-import requests
-
-# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability
-# router had exactly one consumer in this file — the views top-up inside the Bright Data rung —
-# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the
-# connector that routes, which is the shape the split was for: this file no longer knows that
-# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore
-# a signal, not a convenience — it means something in the automation runtime started making a
-# vendor decision, and that belongs one layer down.
-
-try: # the parser the extraction half needs
- from bs4 import BeautifulSoup
-except Exception: # pragma: no cover — deploy lag; see mailbox ②
- BeautifulSoup = None
-
-# ---------------------------------------------------------------------------------------------
-# THE BUCKET (C4-AUTO)
-# ---------------------------------------------------------------------------------------------
-
-#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern.
-STORE_KEY = "automations"
-#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through
-#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for
-#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the
-#: session-D mailbox as a finding against A's file rather than fixed from here.
-UT_STORE_KEY = "user_tables"
-UT_PREFIX = "ut_"
-
-MAX_AUTOMATIONS = 40
-MAX_RUNS = 20 # trimmed history per automation
-MAX_NAME = 80
-MAX_UT_ROWS = 5000 # mirrors core.user_tables.MAX_ROWS
-MAX_UT_TABLES = 40
-
-#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste.
-#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the
-#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull
-#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's
-#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour
-#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still
-#: report success. A time series that stops after five days without saying so is worse than one
-#: that was never built.
-#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE
-#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it:
-#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day
-#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts
-#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had
-#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the
-#: silent version would have flatlined a chart inside a fortnight with a green dot over it.
-#:
-#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises
-#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables,
-#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3
-#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a
-#: smaller number.
-#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.)
-MAX_UT_IG_ROWS = 200_000
-IG_TABLE_PREFIX = "ut_ig_"
-
-#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was
-#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table —
-#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY
-#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would
-#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket
-#: serialisation cost, purely by an accident of naming. Naming the append tables is the version
-#: that stays true when the next `ut_ig_*` table is not one.
-IG_SNAPSHOTS_TABLE = "ut_ig_snapshots"
-IG_POSTS_TABLE = "ut_ig_posts"
-IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots"
-IG_COMMENTS_TABLE = "ut_ig_comments"
-AUTOMATION_RECORD_MODE = "automation"
-APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE})
-
-#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT.
-#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger
-#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram
-#: column, an Instagram search — and they survive on the automations that already use them
-#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG).
-#:
-#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth:
-#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind
-#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was
-#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole
-#: description: three machine nodes it does not have, and a one-line summary about searching
-#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`.
-KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram")
-#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores
-#: when the body names none, which after R6 is every create the client makes.
-DEFAULT_KIND = "plain"
-#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate —
-#: the ruling retires the door, not the two automations behind it (see `create`).
-#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger.
-RETIRED_KINDS = ("scrape_db", "field_instagram")
-#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were
-#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name
-#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence
-#: per kind, in one place, because `create` and `clean_definition` both say it.
-RETIRED_KIND_REPLACEMENT = {
- "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead",
- "scrape_db": "Add a scrape step to any automation instead",
-}
-#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the
-#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`.
-#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any
-#: flow can take, so refusing the kind costs a person nothing but a different click.
-#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared
-#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a
-#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch
-#: door open on purpose, and there is a gate check that says so in those words. A kind may only be
-#: walled off at a door once something else answers the same need.
-#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.)
-KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database",
- "field_instagram": "Instagram profile column",
- "discover_instagram": "Find Instagram profiles"}
-EXTRACTS = ("table", "jsonld")
-STATES = ("idle", "running", "ok", "error", "partial")
-#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid).
-#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a
-#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle).
-TIERS = ("anonymous", "brightdata")
-#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap
-#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown
-#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation
-#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps
-#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile.
-TIER_ALIASES = {"hiker": "brightdata"}
-
-
-def clean_tier(raw):
- """A stored/posted tier → a tier this engine runs, or '' when it is neither."""
- t = str(raw or "").strip().lower()
- t = TIER_ALIASES.get(t, t)
- return t if t in TIERS else ""
-
-
-def row_cap(table_key):
- """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`.
-
- An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything
- else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one.
- """
- return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS
-
-#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI
-#: shows and the vocabulary the parser accepts cannot drift.
-CRON_PRESETS = [
- {"cron": "*/15 * * * *", "label": "Every 15 minutes"},
- {"cron": "0 * * * *", "label": "Hourly"},
- {"cron": "0 6 * * *", "label": "Daily at 06:00"},
- {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"},
- {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"},
-]
-
-UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)"
-
-
-def _now():
- return _dt.datetime.now()
-
-
-def _stamp(dt=None):
- return (dt or _now()).strftime("%Y-%m-%d %H:%M")
-
-
-def _iso(dt=None):
- """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18).
-
- ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables
- and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A
- naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC
- minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is
- not expressible at all. With the offset the same string is an instant, and relative rendering
- becomes possible without migrating a single stored row.
-
- ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in
- this module is against a naive `_now()`, and mixing aware and naive datetimes raises
- `TypeError` — so the offset rides on the WIRE and never enters the arithmetic.
- """
- dt = dt or _now()
- if dt.tzinfo is None:
- dt = dt.astimezone() # a naive stamp from this process IS local time
- return dt.isoformat(timespec="seconds")
-
-
-#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns.
-#:
-#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one
-#: function down: `first_found` cells exist as post-D-18 offset stamps
-#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s
-#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A
-#: converter that understood only the newest shape would blank the oldest rows — and a migration
-#: that empties cells is indistinguishable from one that moved them.
-#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats ""
-#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and
-#: shows up as itself instead of disappearing.
-def _day(s):
- """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there."""
- raw = str(s or "").strip()
- if not raw:
- return ""
- head = raw.replace("T", " ").split(" ")[0]
- try:
- _dt.date.fromisoformat(head)
- except ValueError:
- dt = _parse_iso(raw)
- return dt.strftime("%Y-%m-%d") if dt else ""
- return head
-
-
-#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`.
-#:
-#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct`
-#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every
-#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse
-#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1
-#: dialect and a 0–100 dialect, and they meet here).
-#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null
-#: — and `0.0` would claim we measured zero engagement.
-def _pct100(v):
- """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert."""
- if v is None or (isinstance(v, str) and not v.strip()):
- return ""
- try:
- return _s(round(float(v) * 100.0, 4))
- except (TypeError, ValueError):
- return ""
-
-
-def _parse_iso(s):
- """A stamp → a NAIVE LOCAL datetime, with or without an offset.
-
- Both shapes exist in the store simultaneously and always will: every run committed before
- D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new
- shape would silently return None for them — and `is_due` reads None as "never ran", which
- would re-fire every schedule once. Reading both is what makes the change additive.
- """
- raw = str(s or "").strip().replace("Z", "+00:00")
- dt = None
- try:
- dt = _dt.datetime.fromisoformat(raw)
- except Exception: # noqa: BLE001
- try:
- dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S")
- except Exception: # noqa: BLE001
- return None
- return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt
-
-
-# ---------------------------------------------------------------------------------------------
-# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher
-# ---------------------------------------------------------------------------------------------
-# The skill version guards the URL a developer typed. This one guards a URL that arrived in a
-# request body, which changes the threat model in two ways the original does not cover:
-#
-# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)`
-# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…`
-# sails straight past a guard that only ever saw the first URL. The negative control ("a
-# localhost URL is refused") would still pass while the rail was wide open. `fetch` below
-# therefore takes the hops MANUALLY and re-guards every one.
-# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the
-# host and checks EVERY answer, not just the literal.
-#
-# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the
-# record between the guard and the socket is not stopped by it. Closing that needs a pinned
-# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather
-# than implied away — the rail refuses the realistic cases and says what it does not cover.
-
-class Refused(ValueError):
- """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error."""
-
-
-def _ip_public(ip):
- return not (ip.is_private or ip.is_loopback or ip.is_link_local
- or ip.is_reserved or ip.is_unspecified or ip.is_multicast)
-
-
-def guard(url, allowed=()):
- """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers
- checked too, plus the optional `--allowed` domain rail scrape.py carries."""
- p = urlparse(url or "")
- if p.scheme not in ("http", "https"):
- raise Refused(f"only http(s) allowed, not {p.scheme!r}")
- host = (p.hostname or "").strip()
- if not host:
- raise Refused("no host in the URL")
- low = host.lower()
- if (low in ("localhost", "localhost.localdomain")
- or low.endswith(".local") or low.endswith(".internal")
- or low.endswith(".localhost")):
- raise Refused(f"refused non-public host {host!r} (public web only)")
- try: # a bare-IP host is decided on the literal
- ip = ipaddress.ip_address(low)
- except ValueError:
- ip = None
- if ip is not None and not _ip_public(ip):
- raise Refused(f"refused non-public IP {host} (public web only)")
- if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed):
- raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}")
- if ip is None:
- try:
- infos = socket.getaddrinfo(host, None)
- except Exception as e:
- raise Refused(f"could not resolve {host!r}: {type(e).__name__}")
- for info in infos:
- addr = info[4][0]
- try:
- resolved = ipaddress.ip_address(addr.split("%")[0])
- except ValueError:
- continue
- if not _ip_public(resolved):
- raise Refused(
- f"refused {host!r}: it resolves to the non-public address {resolved}")
- return True
-
-
-def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None):
- """GET `url`, taking redirects BY HAND so every hop passes `guard`.
-
- Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal,
- not a silent truncation — an endless redirect is indistinguishable from an attempt to walk
- the fetcher somewhere it was told not to go.
- """
- current = url
- hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"}
- hdrs.update(headers or {})
- for _hop in range(max_hops + 1):
- guard(current, allowed)
- r = requests.get(current, timeout=timeout, headers=hdrs,
- allow_redirects=False, stream=True)
- if r.status_code in (301, 302, 303, 307, 308):
- loc = r.headers.get("Location")
- r.close()
- if not loc:
- raise Refused(f"redirect {r.status_code} with no Location header")
- current = urljoin(current, loc)
- continue
- body = r.raw.read(max_kb * 1024, decode_content=True) or b""
- status, final = r.status_code, str(r.url)
- r.close()
- return status, final, body
- raise Refused(f"more than {max_hops} redirects — refusing to follow further")
-
-
-def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None):
- """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`.
-
- ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by
- hand because a redirected GET is ordinary. A redirected POST is not: re-sending a
- credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail
- exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever
- re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request.
- (The vendor calls below are the only POSTs this module makes, and they carry the API key.)
- """
- guard(url)
- hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"}
- hdrs.update(headers or {})
- r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False,
- data=json.dumps(body if body is not None else {}), stream=True)
- try:
- if r.status_code in (301, 302, 303, 307, 308):
- raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, "
- f"never re-sent to a location the server picked")
- return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"")
- finally:
- r.close()
-
-
-# ---------------------------------------------------------------------------------------------
-# EXTRACTION — vendored from scrape.py, unchanged in behaviour
-# ---------------------------------------------------------------------------------------------
-
-def _soup(body):
- if BeautifulSoup is None:
- raise RuntimeError(
- "beautifulsoup4 is not installed in this environment — the automation engine's "
- "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to "
- "aios-web/requirements.txt).")
- try:
- return BeautifulSoup(body, "lxml")
- except Exception:
- return BeautifulSoup(body, "html.parser")
-
-
-def tables(soup):
- """Every HTML table as rows. Dict rows when the first row looks like a header."""
- out = []
- for t in soup.find_all("table"):
- rows = []
- for tr in t.find_all("tr"):
- cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])]
- if cells:
- rows.append(cells)
- if not rows:
- continue
- head, body = rows[0], rows[1:]
- if body and len(head) == len(body[0]) and all(head):
- out.append([dict(zip(head, r)) for r in body if len(r) == len(head)])
- else:
- out.append(rows)
- return out
-
-
-def jsonld(soup):
- out = []
- for s in soup.find_all("script", attrs={"type": "application/ld+json"}):
- try:
- out.append(json.loads(s.string or s.get_text()))
- except Exception:
- pass
- return out
-
-
-def meta(soup):
- m = {}
- if soup.title and soup.title.string:
- m["title"] = soup.title.string.strip()
- for tag in soup.find_all("meta"):
- k = tag.get("name") or tag.get("property")
- v = tag.get("content")
- if k and v and (k in ("description", "keywords", "author")
- or k.startswith(("og:", "twitter:"))):
- m[k] = v.strip()
- return m
-
-
-def preview(url, extract="table", table_index=0, allowed=()):
- """The field-map preview the editor calls BEFORE anything is created: what columns does this
- page actually offer, and what do the first rows look like? Never writes."""
- status, final, body = fetch(url, allowed=allowed)
- if not (200 <= status < 300):
- return {"ok": False, "status": status, "url": final,
- "note": f"the page answered {status} — it may be bot-gated or gone",
- "columns": [], "sample": [], "rowCount": 0}
- soup = _soup(body)
- rows = []
- if extract == "jsonld":
- blocks = jsonld(soup)
- flat = []
- for b in blocks:
- if isinstance(b, list):
- flat.extend([x for x in b if isinstance(x, dict)])
- elif isinstance(b, dict):
- items = b.get("itemListElement")
- flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list)
- else [b])
- rows = [{k: _scalar(v) for k, v in d.items()} for d in flat]
- else:
- found = tables(soup)
- idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0
- picked = found[idx] if found else []
- rows = [r for r in picked if isinstance(r, dict)]
- cols, seen = [], set()
- for r in rows[:50]:
- for k in r:
- if k not in seen:
- seen.add(k)
- cols.append(k)
- return {"ok": True, "status": status, "url": final, "columns": cols,
- "sample": rows[:8], "rowCount": len(rows),
- "tableCount": len(tables(soup)) if extract != "jsonld" else 0,
- "title": (meta(soup) or {}).get("title", "")}
-
-
-def _scalar(v):
- if isinstance(v, (str, int, float)) and not isinstance(v, bool):
- return str(v)
- if isinstance(v, bool):
- return "1" if v else ""
- if isinstance(v, dict):
- return str(v.get("name") or v.get("@id") or "")
- if isinstance(v, list):
- return ", ".join(_scalar(x) for x in v[:8])
- return ""
-
-
-# ---------------------------------------------------------------------------------------------
-# CRON — a 5-field parser and a "what was the last fire time" walk
-# ---------------------------------------------------------------------------------------------
-# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward
-# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute
-# between the last run and now?"* — and answering it that way is what makes a missed tick
-# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and
-# not never.
-
-_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)]
-
-
-def _parse_field(spec, lo, hi):
- out = set()
- for part in str(spec).split(","):
- part = part.strip()
- if not part:
- raise ValueError("empty cron field part")
- step = 1
- if "/" in part:
- part, _, s = part.partition("/")
- step = int(s)
- if step < 1:
- raise ValueError("cron step must be >= 1")
- if part in ("*", "?"):
- a, b = lo, hi
- elif "-" in part.lstrip("-"):
- a_s, _, b_s = part.partition("-")
- a, b = int(a_s), int(b_s)
- else:
- a = b = int(part)
- if a < lo or b > hi or a > b:
- raise ValueError(f"cron value {part!r} out of range {lo}..{hi}")
- out.update(range(a, b + 1, step))
- return out
-
-
-def parse_cron(expr):
- """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`.
-
- Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at
- WRITE time, because a cron string nobody can evaluate is an automation that silently never
- runs and reports no error anywhere.
- """
- parts = str(expr or "").split()
- if len(parts) != 5:
- raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday")
- sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)]
- dom_r = parts[2].strip() not in ("*", "?")
- dow_r = parts[4].strip() not in ("*", "?")
- return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()),
- dom_r, dow_r)
-
-
-def _day_matches(day, doms, months, dows, dom_r, dow_r):
- if day.month not in months:
- return False
- # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the
- # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes
- # a schedule that looks right fire almost never.
- dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0
- if dom_r and dow_r:
- return day.day in doms or dow in dows
- if dom_r:
- return day.day in doms
- if dow_r:
- return dow in dows
- return True
-
-
-def prev_fire(expr, now=None, lookback_days=400):
- """The most recent scheduled minute at or before `now`, or None inside the lookback.
-
- Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first,
- and only a matching day needs its hours/minutes searched.
- """
- minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr)
- now = (now or _now()).replace(second=0, microsecond=0)
- for back in range(lookback_days + 1):
- day = (now - _dt.timedelta(days=back)).date()
- if not _day_matches(day, doms, months, dows, dom_r, dow_r):
- continue
- same_day = back == 0
- for h in sorted(hours, reverse=True):
- if same_day and h > now.hour:
- continue
- for m in sorted(minutes, reverse=True):
- if same_day and h == now.hour and m > now.minute:
- continue
- return _dt.datetime(day.year, day.month, day.day, h, m)
- return None
-
-
-def is_due(defn, now=None):
- """Should the scheduler run this automation right now?
-
- Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the
- moment the schedule was enabled, else creation) and at or before now. Anchoring on
- `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at
- 14:00" from running immediately and looking like a bug.
- """
- if not isinstance(defn, dict):
- return False
- sched = defn.get("schedule") or {}
- if not sched.get("enabled"):
- return False
- try:
- fire = prev_fire(sched.get("cron"), now=now)
- except ValueError:
- return False
- if fire is None:
- return False
- since = (_parse_iso((defn.get("status") or {}).get("lastRunAt"))
- or _parse_iso(sched.get("enabledAt"))
- or _parse_iso(defn.get("created")))
- return since is None or fire > since
-
-
-def next_fire(expr, now=None, lookahead_days=400):
- """The next scheduled minute strictly after `now` — display only (the rail is `is_due`)."""
- try:
- minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr)
- except ValueError:
- return None
- now = (now or _now()).replace(second=0, microsecond=0)
- for ahead in range(lookahead_days + 1):
- day = (now + _dt.timedelta(days=ahead)).date()
- if not _day_matches(day, doms, months, dows, dom_r, dow_r):
- continue
- same_day = ahead == 0
- for h in sorted(hours):
- if same_day and h < now.hour:
- continue
- for m in sorted(minutes):
- if same_day and h == now.hour and m <= now.minute:
- continue
- return _dt.datetime(day.year, day.month, day.day, h, m)
- return None
-
-
-# ---------------------------------------------------------------------------------------------
-# THE UPSERT — pure, so the arithmetic is testable without a store or a network
-# ---------------------------------------------------------------------------------------------
-
-def upsert_rows(existing, incoming, key_field, cap=None):
- """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`.
-
- THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped
- appearing on the source page has not necessarily stopped existing — the page changed its
- filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream
- hiccup into silent data loss, and the row may be carrying user-typed overlay values in
- columns the scrape never touches. So the run reports `orphans: N` and leaves them alone.
-
- Only MAPPED keys are written: a re-run never clobbers a column a user added by hand.
-
- ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on
- entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to
- call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS`
- that same loop is hundreds of millions of dict copies and the automation simply never
- finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.)
-
- ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different
- facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the
- DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" —
- a property of the SYSTEM, and never benign. One number for both meant a table hitting its
- ceiling was indistinguishable from a page with a few blank cells, which is how a time series
- stops silently. The runners turn any `capped` into a `partial` run that NAMES the table.
- """
- cap = MAX_UT_ROWS if cap is None else int(cap)
- rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()}
- counts = {"inserted": 0, "updated": 0, "unchanged": 0,
- "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0}
- by_key, dupe_ids = {}, set()
- for rid, row in rows.items():
- kv = str(row.get(key_field, "") or "").strip()
- if not kv:
- continue
- if kv in by_key:
- dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left
- continue
- by_key[kv] = rid
- next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1
- seen_keys, incoming_dupes = set(), 0
- for src in incoming or []:
- kv = str((src or {}).get(key_field, "") or "").strip()
- if not kv:
- counts["skipped"] += 1 # no key -> cannot be upserted; never guessed
- continue
- if kv in seen_keys:
- incoming_dupes += 1 # the SOURCE listed it twice; first wins
- continue
- seen_keys.add(kv)
- rid = by_key.get(kv)
- if rid is None:
- if len(rows) >= cap:
- counts["capped"] += 1 # LOUD: the runner turns this into a partial run
- continue
- rid = str(next_id)
- next_id += 1
- rows[rid] = dict(src)
- by_key[kv] = rid
- counts["inserted"] += 1
- continue
- before = rows[rid]
- changed = {k: v for k, v in src.items() if str(before.get(k, "")) != str(v)}
- if changed:
- before.update(src)
- counts["updated"] += 1
- else:
- counts["unchanged"] += 1
- counts["duplicates"] = incoming_dupes + len(dupe_ids)
- counts["orphans"] = sum(
- 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids)
- return rows, counts
-
-
-def dedupe_canonical_rows(existing, key_field, newest_by=""):
- """Collapse duplicate logical rows while preserving the lowest stable row id.
-
- Canonical entity tables use this before every upsert. Snapshot tables deliberately do not:
- repeated shortcodes there are new timestamped observations, not duplicate posts. Values are
- taken newest-first and then filled from older rows, so a sparse fresh projection does not
- erase a field an earlier row knew.
- """
- rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()}
- groups = {}
- for rid, row in rows.items():
- identity = str(row.get(key_field) or "").strip()
- if identity:
- groups.setdefault(identity, []).append((rid, row))
- removed = 0
- for members in groups.values():
- if len(members) < 2:
- continue
- keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r))
- ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \
- if newest_by else members
- merged = {}
- for _rid, row in ordered:
- for key, value in row.items():
- if key not in merged or str(merged.get(key) or "").strip() == "":
- merged[key] = value
- rows[keep] = merged
- for rid, _row in members:
- if rid != keep:
- rows.pop(rid, None)
- removed += 1
- return rows, removed
-
-
-# ---------------------------------------------------------------------------------------------
-# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key)
-# ---------------------------------------------------------------------------------------------
-
-def _ut_slug(label):
- s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_")
- return (s or "table")[:40]
-
-
-def ut_all(rt):
- try:
- return dict(rt.get(UT_STORE_KEY) or {})
- except Exception:
- return {}
-
-
-def disable_for_table(rt, table_key, note="target database deleted"):
- """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted.
-
- The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still
- shows what existed and why it stopped — silently deleting a user's automation because its
- target died would read as data loss. Returns the ids it touched."""
- key = str(table_key or "")
- touched = []
-
- def _up(cur):
- for aid, d in (cur or {}).items():
- if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key:
- sch = d.get("schedule")
- if not isinstance(sch, dict):
- sch = d["schedule"] = {}
- sch["enabled"] = False
- d["statusNote"] = note
- touched.append(str(aid))
- return cur
-
- if key:
- rt.update(STORE_KEY, _up, flush="sync")
- return touched
-
-
-def ut_get(rt, key):
- return ut_all(rt).get(str(key))
-
-
-def retire_automation_stage_fields(rt):
- """Delete obsolete Board-only fields and their hidden cells, never user fields.
-
- The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField``
- metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old
- generated timestamp/cycle cells are cleared too, including rows whose field definition was
- removed by an interrupted earlier migration. Re-running this migration is a no-op.
- """
- tables = ut_all(rt)
- stage_keys = set()
- for table in tables.values():
- if not isinstance(table, dict):
- continue
- for field in table.get("fields") or []:
- auto = field.get("automation") if isinstance(field, dict) else None
- if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")):
- stage_keys.add(str(field.get("key") or ""))
- for row in (table.get("rows") or {}).values():
- for key in (row or {}):
- text = str(key)
- if text.startswith("stage_auto_"):
- stage_keys.add(text.removesuffix("_at").removesuffix("_cycles"))
- stage_keys.discard("")
- if not stage_keys:
- return {"tables": 0, "fields": 0, "cells": 0}
-
- changed = {"tables": set(), "fields": 0, "cells": 0}
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- for table_key, table in cur.items():
- if not isinstance(table, dict):
- continue
- kept, removed = [], set()
- for field in table.get("fields") or []:
- auto = field.get("automation") if isinstance(field, dict) else None
- key = str(field.get("key") or "") if isinstance(field, dict) else ""
- if key in stage_keys and isinstance(auto, dict) and \
- (auto.get("stageField") or auto.get("cyclesField")):
- removed.add(key)
- changed["fields"] += 1
- continue
- kept.append(field)
- if removed:
- table["fields"] = kept
- changed["tables"].add(str(table_key))
- for row in (table.get("rows") or {}).values():
- if not isinstance(row, dict):
- continue
- for stage_key in stage_keys:
- for key in (stage_key, stage_key + "_at", stage_key + "_cycles"):
- if key in row:
- row.pop(key, None)
- changed["cells"] += 1
- changed["tables"].add(str(table_key))
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- return {"tables": len(changed["tables"]), "fields": changed["fields"],
- "cells": changed["cells"]}
-
-
-def bind_unbound_fields(rt):
- """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId`
- are bound to the definition that already writes them — matched on the (targetTable,
- fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no
- definition references is DISABLED with the reason on it rather than deleted or guessed:
- the column was already dead (nothing runs a column no flow names), and now it says so.
- Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate."""
- by_binding = {}
- for aid, d in all_definitions(rt).items():
- cfg = d.get("config") or {}
- if cfg.get("fieldKey") and cfg.get("targetTable"):
- by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid)
- plan = {}
- for tk, t in ut_all(rt).items():
- for f in (t.get("fields") or []):
- a = f.get("automation")
- # An already-DISABLED bag is a decision this migration made on a previous pass —
- # re-planning it every call would turn the once-per-process sweep into a write
- # per read.
- if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \
- and not a.get("disabled"):
- plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key"))))
- if not plan:
- return 0, 0
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- for (tk, fk), aid in plan.items():
- for f in ((cur.get(tk) or {}).get("fields") or []):
- if f.get("key") == fk and isinstance(f.get("automation"), dict):
- if aid:
- f["automation"]["flowId"] = aid
- else:
- f["automation"]["disabled"] = True
- f["automation"]["statusNote"] = (
- "not bound to any flow — create an automation for this column "
- "or delete it")
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- bound = sum(1 for v in plan.values() if v)
- return bound, len(plan) - bound
-
-
-def ut_key_for(label, key=None):
- """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which
- must not create anything) resolves the same key by construction rather than by copying the
- derivation and drifting from it."""
- k = str(key or (UT_PREFIX + _ut_slug(label)))
- return k if k.startswith(UT_PREFIX) else UT_PREFIX + k
-
-
-#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same
-#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging
-#: `core` into the API's boot path — and the two are held in step by a gate check.
-MACHINE_OWNERS = ("automation", "scheduler")
-
-
-def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
- record_mode="", lock_fields=False):
- """Create the table if it is missing; return its key. Idempotent — a re-run of an automation
- that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given)
- and an existing table with that key is adopted, not duplicated.
-
- Fields are MERGED, never replaced: a user who added a column to an automation's table keeps
- it, and a new source column joins on the next run.
-
- `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped
- `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their
- provenance. Fields already on the table keep whatever tag they have (first flow wins;
- shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not
- ownership).
-
- ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever
- or WHATEVER ran the automation, so the same table belonged to a person if its first run was
- manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open`
- admits only the creator or an admin, so **whether you could open your own Instagram
- snapshots depended on a race you never saw**. The owner is now the automation's creator, and
- a table already stamped with a machine name is ADOPTED the next time a run knows a human
- one. Adoption is a repair, not a widening: the automation's creator is the person who asked
- for the table in the first place.
- """
- key = ut_key_for(label, key)
- # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point.
- #
- # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it
- # would leave every table already in production on the old `text` schema forever while new
- # ones got the honest types — the split schema R3's note warned about, arriving through the
- # very function the note was written on. Migrating here means a table is brought forward
- # immediately BEFORE anything appends to it, so no caller has to remember anything and no
- # tenant is left behind by a script nobody ran.
- # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is
- # already current, which after the first run is every call.
- if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS):
- try:
- migrate_ig_tables(rt, log=lambda *_a: None, only=key)
- except Exception: # noqa: BLE001
- # A migration that cannot run must not stop the automation from writing its rows.
- # The old schema still reads; a refused write loses the pull we just paid for.
- pass
- wanted = []
- for raw_field in (fields or []):
- field = dict(raw_field)
- if flow_tag or lock_fields:
- automation = dict(field.get("automation") or {})
- if flow_tag:
- automation.setdefault("flowId", str(flow_tag))
- if lock_fields:
- automation["preset"] = True
- field["automation"] = automation
- wanted.append(field)
- created = _iso()
- human = username if username and username not in MACHINE_OWNERS else ""
-
- # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store
- # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo
- # budget, an idempotent helper that always writes is the same defect as a per-row insert.
- have = ut_get(rt, key)
- have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])}
- missing = [f for f in wanted if f.get("key") not in have_fields]
- missing_locks = [f for f in wanted
- if lock_fields and f.get("key") in have_fields
- and (not isinstance(have_fields[f.get("key")].get("automation"), dict)
- or have_fields[f.get("key")]["automation"].get("preset") is not True)]
- if (have is not None and not missing and not missing_locks
- and not (record_mode and have.get("recordMode") != record_mode)
- and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)):
- return key
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- t = cur.get(key)
- if t is None:
- if len(cur) >= MAX_UT_TABLES:
- return cur
- cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation",
- "createdBy": username, "created": created,
- "fields": wanted, "rows": {}}
- if record_mode:
- cur[key]["recordMode"] = record_mode
- return cur
- have = {f.get("key") for f in (t.get("fields") or [])}
- for f in wanted:
- if f.get("key") not in have:
- t.setdefault("fields", []).append(f)
- have.add(f.get("key"))
- elif lock_fields:
- stored = next((g for g in (t.get("fields") or [])
- if g.get("key") == f.get("key")), None)
- if stored is not None:
- automation = dict(stored.get("automation") or {})
- if flow_tag:
- automation.setdefault("flowId", str(flow_tag))
- automation["preset"] = True
- stored["automation"] = automation
- # ADOPTION: a machine name is not an owner. It never overwrites a human one.
- if human and (t.get("createdBy") or "") in MACHINE_OWNERS:
- t["createdBy"] = human
- # An automation's table SAYS an automation owns it — the nav badge reads from this.
- t["source"] = t.get("source") or "Automation"
- if record_mode:
- t["recordMode"] = record_mode
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- return key
-
-
-def ut_write_rows(rt, key, rows):
- """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header)."""
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- t = cur.get(key)
- if t is not None:
- t["rows"] = rows
- return cur
- rt.update(UT_STORE_KEY, _up, flush="sync")
-
-
-def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None):
- """Write one automation-owned cell (+ optional whole extra tables) in ONE update."""
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- t = cur.get(key)
- if t is not None:
- t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value
- for k, rws in (extra_rows or {}).items():
- tt = cur.get(k)
- if tt is not None:
- tt["rows"] = rws
- return cur
- rt.update(UT_STORE_KEY, _up, flush="sync")
-
-
-IG_FIELD_DESCRIPTIONS = {
- "alt_text": "Accessibility text attached to the Instagram post.",
- # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would
- # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED
- # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off
- # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same
- # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a
- # description that named only the first would have quietly excluded the second.
- "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus "
- "rather than measured at the time shown.",
- "avg_comments_12": "Average comments across the 12 most recent captured posts.",
- "avg_engagement": "Average engagement rate reported for the profile.",
- "avg_likes_12": "Average likes across the 12 most recent captured posts.",
- "avg_plays_12": "Average plays across the 12 most recent captured posts.",
- "avg_views_12": "Average views across the 12 most recent captured posts.",
- "views": "View count for the post, as Instagram displays it.",
- "video_duration": "Length of the video in seconds.",
- "comments_disabled": "Checked when the creator has turned comments off for this post.",
- "plays": "Times the video started playing, including replays.",
- "bio": "Biography shown on the Instagram profile.",
- "bio_hashtags": "Hashtags listed in the profile biography.",
- "business_category": "Instagram's business category for the account.",
- "caption": "Caption published with the Instagram post.",
- "category": "Instagram's category for the profile.",
- "comment_key": "Unique ID for the captured comment record.",
- "commented_at": "Date the comment was posted.",
- "comments": "Comment count reported for the post.",
- "comments_captured": "Number of distinct comment records linked to this profile.",
- "comments_link": "Comment records linked to this record.",
- "country_code": "Country code reported for the profile.",
- "created_by": "User whose automation first added this lead.",
- "enriched_at": "Date the profile was last enriched.",
- # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number
- # beside it is the only thing that says how much of one.
- "location_guess": "Most common city tagged across this profile's captured posts - a guess, "
- "not a stated location.",
- "location_confidence": "Share of this profile's geotagged posts that agree on the guessed "
- "city.",
- "external_url": "Website linked from the Instagram biography.",
- "external_url_title": "Title Instagram shows for the biography link.",
- "fbid": "Facebook ID associated with the Instagram account.",
- "first_found": "Date an automation first found this profile.",
- "followers": "Latest reported follower count.",
- "following": "Latest reported number of accounts followed.",
- "found_count": "Number of times discovery automations found this profile.",
- "full_name": "Display name shown on the Instagram profile.",
- "handle": "Instagram username without the @ symbol.",
- "has_channel": "Checked when the profile has an Instagram channel.",
- "hashtags": "Hashtags extracted from the post caption.",
- "highlights_count": "Total story highlight collections reported for the profile.",
- "ig_id": "Instagram's internal ID for the account.",
- "influencer_key": "Normalized handle linking this row to its profile.",
- "is_business": "Checked when Instagram marks the account as a business.",
- "is_joined_recently": "Checked when Instagram marks the account as recently joined.",
- "is_private": "Checked when the Instagram profile is private.",
- "is_professional": "Checked when Instagram marks the account as professional.",
- "last_found": "Date an automation most recently found this profile.",
- "likes": "Like count reported for this record.",
- "measured_at": "Date engagement metrics on this post were last read.",
- "measurements_captured": "Number of measurement rows stored for this post.",
- "paid_partnership": "Checked when Instagram marks the post as a paid partnership.",
- "partner": "Brand named in the post's paid partnership metadata.",
- "partner_id": "Partner ID reported for the Instagram profile.",
- "platform": "Social network for this profile.",
- "plays": "Video plays, including repeat plays, reported for the post.",
- "post_link": "Post record linked to this measurement or comment.",
- "post_measurements_captured": "Number of distinct post measurements linked to this profile.",
- "post_snapshot_key": "Unique ID for this post measurement.",
- "post_snapshots_link": "Engagement measurement rows linked to this record.",
- "posted_at": "Date the Instagram post was published.",
- "posts_captured": "Number of distinct post records captured for this profile.",
- "posts_count": "Total posts reported for the Instagram profile.",
- "posts_link": "Post records linked to this profile.",
- "profile_name": "Profile name returned by the data provider.",
- "profile_reads": "Number of profile snapshots stored for this profile.",
- "profile_snapshots_link": "Profile snapshot rows linked to this profile.",
- "profile_url": "Direct URL to the Instagram profile.",
- "pronouns": "Pronouns shown on the Instagram profile.",
- "pulled_at": "Date this snapshot was collected.",
- "related_accounts": "Accounts Instagram suggests alongside this profile.",
- "replies": "Reply count reported for the comment.",
- "shortcode": "Instagram shortcode that uniquely identifies the post.",
- "source_payload": "Full source response for this record.",
- "snapshot_key": "Unique ID for this profile snapshot.",
- "source": "Method or provider used to read this profile.",
- "tagged_location": "Location tagged on the post; it is not the creator's residence.",
- "type": "Instagram media type: image, video, or carousel.",
- "url": "Direct URL to the Instagram post.",
- "verified": "Checked when Instagram marks the profile as verified.",
-}
-
-
-def field_def(text_key, label, ftype="text", **extra):
- """One machine-spawned column definition.
-
- ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`,
- `profile`), and it stays ONE constructor rather than growing a second for "special" fields.
- ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s
- W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole
- preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]).
-
- ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a
- feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so
- `core.user_tables._clean_field` — the single validator every field created through the ordinary
- door passes — has never judged a single field this engine spawned. The difference was exactly
- one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f`
- for every automation column in the product.
- ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an
- absent key was already not that, so the bypass was fail-closed and therefore silent. It is
- stated now, and `verify_automation` asserts the equality field-by-field, so the next key
- `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]).
- """
- description = " ".join(str(extra.pop(
- "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split())
- out = {"key": text_key, "label": label, "type": ftype, "source": "overlay",
- "editRole": "admins"}
- if description:
- out["description"] = description
- out.update(extra)
- return out
-
-
-# ---------------------------------------------------------------------------------------------
-# DEFINITIONS — validation and the bucket's read/write half
-# ---------------------------------------------------------------------------------------------
-
-def _s(v, n=200):
- return str(v if v is not None else "")[:n]
-
-
-#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S.
-#:
-#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12
-#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config
-#: disagree with what the run can possibly do.
-#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect
-#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve
-#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is
-#: worse than one that refuses it, because the refusal is the only place the ceiling can be
-#: taught. So this REFUSES, and the sentence names the cap and who set it.
-#:
-#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the
-#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the
-#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask
-#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30`
-#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a
-#: property of the call we happened to make.
-#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of
-#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE
-#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to
-#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it
-#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12.
-MAX_POSTS_PER_PULL = 30
-DEFAULT_POSTS_PER_PULL = 10
-
-#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately
-#: NOT `MAX_POSTS_PER_PULL`.
-#:
-#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap
-#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so
-#: raising the capture cap to 30 silently made every one of them average thirty posts under a
-#: label that says twelve. Nobody would have looked at those columns again to check.
-#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different
-#: decisions, and only the first one was made. To widen the average, change this constant AND the
-#: four keys and labels together — a column whose meaning changes underneath its own name is
-#: worse than one that is merely narrow.
-AVG_WINDOW_POSTS = 12
-
-
-#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a
-#: filter cannot ask for a category that can never match a stored row.
-POST_TYPES = ("video", "image", "carousel")
-
-
-def clean_post_groups(raw, max_posts):
- """`(groups, error)` for `config.postGroups` — "last N reels, last M images".
-
- ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape:
- `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`.
-
- ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today
- carries no such key, and inventing a filter for it would silently start DROPPING posts those
- automations have always captured. Empty list and missing are the same answer.
-
- ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the
- difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them
- would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken
- one, so the only way to see one is a person typing it, and a filter that silently ignores the
- type you asked for is how you end up paying for reels and storing carousels.
- """
- if raw in (None, "", [], {}):
- return [], None
- if not isinstance(raw, list):
- return None, "the post groups have to be a list of {type, limit} entries"
- out, seen = [], set()
- for item in raw:
- if not isinstance(item, dict):
- return None, "each post group has to be a {type, limit} entry"
- t = str(item.get("type") or "").strip().lower()
- if t not in POST_TYPES:
- return None, (f"'{t or 'blank'}' is not a post type — use one of "
- f"{', '.join(POST_TYPES)}")
- if t in seen:
- return None, f"the post groups name '{t}' twice — one limit per type"
- seen.add(t)
- try:
- lim = int(item.get("limit"))
- except (TypeError, ValueError):
- return None, f"how many {t} posts to keep has to be a whole number"
- if lim < 1:
- return None, f"a {t} group has to keep at least one post"
- # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures
- # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is
- # refused HERE, where the number they typed is still on the screen in front of them.
- if lim > max_posts:
- return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} "
- f"group cannot keep {lim} — raise the post count first")
- out.append({"type": t, "limit": lim})
- return out, None
-
-
-def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True):
- """`(maxPosts, error)` — refuses out-of-range rather than clamping into range.
-
- ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this
- takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old
- clamp's default, which this cap now forbids. If an inherited value were refused the same way
- a typed one is, every one of those automations would 400 on its next Save **forever**, for a
- number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator
- that refuses stored data does not protect the user from it, it locks them out of their own
- automation.
- So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely
- INHERITED gets silently brought inside the cap it was already effectively subject to at the
- vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]).
- """
- if raw is None or (isinstance(raw, str) and not raw.strip()):
- return default, None
- try:
- n = int(raw)
- except (TypeError, ValueError):
- if not submitted:
- return default, None
- return None, "the post limit must be a whole number"
- if not submitted:
- return max(1, min(n, MAX_POSTS_PER_PULL)), None
- if n < 1:
- return None, "keep at least one post per profile, or turn the post capture off"
- if n > MAX_POSTS_PER_PULL:
- return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts — that is the "
- f"vendor's cap, not ours, so {n} would store the same "
- f"{MAX_POSTS_PER_PULL} and read as more")
- return n, None
-
-
-def clean_config(kind, raw, previous=None):
- """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence.
-
- ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail
- rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not
- edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save"
- from a panel that never knew about them would silently turn off the dry run, or turn ON a
- per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep;
- present ⇒ take it, including `false`.
- """
- raw = raw if isinstance(raw, dict) else {}
- prev = previous if isinstance(previous, dict) else {}
-
- def flag(name):
- return bool(raw[name]) if name in raw else bool(prev.get(name))
- if kind == "plain":
- # A plain automation has no machine step, so it only needs the database its records walk.
- #
- # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on
- # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else-
- # trigger — so requiring a second copy of that fact here would be a mandatory field with
- # exactly one legal answer, and a Save that refuses until you retype what you just picked.
- # There is no `dryRun`: dry-run means "run the machine step but do not write", and there
- # is no machine step to run.
- target = _s(raw.get("targetTable"), 60).strip()
- if target and not target.startswith(UT_PREFIX):
- target = UT_PREFIX + target
- return {"targetTable": target,
- "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None
- if kind == "scrape_db":
- url = _s(raw.get("url"), 2000).strip()
- if not url:
- return None, "a source URL is required"
- try:
- guard(url)
- except Refused as e:
- return None, str(e)
- extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table"
- fmap = {}
- for k, v in list((raw.get("fieldMap") or {}).items())[:60]:
- tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_")
- if tk:
- fmap[_s(k, 120)] = tk[:60]
- if not fmap:
- return None, "map at least one source column to a field"
- key_field = _s(raw.get("keyField"), 60).strip()
- if key_field not in fmap.values():
- return None, "the key field must be one of the mapped fields"
- target = _s(raw.get("targetTable"), 60).strip()
- if target and not target.startswith(UT_PREFIX):
- target = UT_PREFIX + target
- return {"url": url, "extract": extract,
- "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)),
- "fieldMap": fmap, "keyField": key_field,
- "targetTable": target,
- "dryRun": flag("dryRun"),
- "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None
- if kind == "field_instagram":
- table = _s(raw.get("targetTable"), 60).strip()
- if not table.startswith(UT_PREFIX):
- return None, "an Instagram automation runs against a blank database (ut_*)"
- fkey = _s(raw.get("fieldKey"), 80).strip()
- if not fkey:
- return None, "pick the automation column this run writes into"
- # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2).
- # They are read from nothing and written to nothing: a stored definition carrying either
- # still SAVES — it simply loses them on the next write — and neither is ever a reason to
- # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown
- # key would 400 every automation a tenant stored before this wave, forever, on a screen
- # that gives them no way to remove it. Dropping a retired key is a migration; refusing it
- # is an outage.
- # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary
- # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired
- # USER concept — renaming them would be a second, unrelated change wearing this one's
- # justification.
- # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel
- # that does not edit the post count must not reset it to the default on Save. And an
- # INHERITED value is clamped rather than refused; see `clean_max_posts`.
- sent = "maxPosts" in raw
- max_posts, perr = clean_max_posts(
- raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent)
- if perr:
- return None, perr
- return {"targetTable": table, "fieldKey": fkey,
- "urlField": _s(raw.get("urlField"), 80).strip(),
- # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
- # for: the Profiles row carries post IDENTITY for free but NO engagement
- # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
- "postMetrics": flag("postMetrics"),
- # The full Comments dataset can bill many rows per post. It is an explicit,
- # independent opt-in and never follows the per-post switch automatically.
- "commentMetrics": flag("commentMetrics"),
- "dryRun": flag("dryRun"),
- "maxPosts": max_posts}, None
- if kind == "discover_instagram":
- limit = int(raw.get("recordsLimit") or 0)
- if limit < 1:
- return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the "
- "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)")
- if limit > BD_MAX_RECORDS:
- return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles"
- # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what
- # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading
- # the operator afterwards is how the OR hole survived: the guard was handed the branches
- # and never told they were a union.
- join = "or" if str(raw.get("operator") or "").lower() == "or" else "and"
- preds, perr = clean_predicates(raw.get("predicates"), join)
- if perr:
- return None, perr
- target = _s(raw.get("targetTable"), 60).strip() or DISCOVER_TABLE
- if not target.startswith(UT_PREFIX):
- target = UT_PREFIX + target
- seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed"))
- if serr:
- return None, serr
- return {"recordsLimit": limit, "predicates": preds,
- "operator": join,
- "targetTable": target,
- "targetLabel": _s(raw.get("targetLabel"), 60).strip() or DISCOVER_LABEL,
- # C6/R5: WHERE the conditions above came from, when they were derived. Stored so
- # the surface can say "these were filled in from the 'Florists' view, over 12
- # records" instead of presenting them as if somebody typed them.
- "seed": seed,
- "dryRun": flag("dryRun")}, None
- return None, f"unknown automation kind {kind!r}"
-
-
-def clean_seed(raw):
- """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter
- somebody typed by hand has no seed, and that is the ordinary case.
-
- ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that
- the derived conditions are "written into `config.predicates` as ordinary conditions — it is a
- filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`,
- like every other search. Keeping it means the surface can say where those rows came from, and
- a user editing them freely is exactly what is supposed to happen.
- """
- if raw in (None, "", {}):
- return {}, None
- if not isinstance(raw, dict):
- return None, "the seed must be an object"
- source = _s(raw.get("source"), 12).strip().lower()
- if source not in SEED_SOURCES:
- return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES)
- table = _s(raw.get("table"), 60).strip()
- if table and not table.startswith(UT_PREFIX):
- return None, f"a seed reads a blank database (ut_*) — {table!r} is not one"
- basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {}
- derived, _err = clean_predicates(raw.get("derived") or [], "and")
- return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(),
- # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what
- # was suggested; the conditions that matter are already in `predicates` and were
- # validated there. Refusing a Save because a stored provenance note aged badly would
- # block the user from editing the very filter it describes.
- "derived": derived or [],
- "basis": {"rows": max(0, int(basis.get("rows") or 0)),
- "fields": [f for f in (basis.get("fields") or [])
- if isinstance(f, dict)][:SEED_MAX_PREDICATES],
- "related": basis.get("related")
- if isinstance(basis.get("related"), dict) else {},
- "note": _s(basis.get("note"), 200)}}, None
-
-
-def clean_schedule(raw, previous=None):
- """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due`
- measures from — without it, enabling a daily job fires it immediately)."""
- raw = raw if isinstance(raw, dict) else {}
- cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *"
- parse_cron(cron) # raises -> the route answers 400
- enabled = bool(raw.get("enabled"))
- prev = previous or {}
- out = {"cron": cron, "enabled": enabled}
- if enabled:
- out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso()
- return out
-
-
-# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ─────────────
-# The owner's words: "we can make this automation a loop, so the ending should always be defined,
-# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically,
-# or the user have to click a button to reset, or after a certain amount of time it can
-# automatically reset to first cycle."
-#
-# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition
-# that predates this field must not start moving records on its own the day the code ships. A
-# loop is a thing somebody turns on.
-def clean_flow(raw, previous=None):
- """Validate the builder's ordered action list. `(flow, error)`."""
- raw = raw if isinstance(raw, dict) else {}
- prev = previous if isinstance(previous, dict) else {}
- actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions"))
- if err:
- return None, err
- return {"actions": actions}, None
-
-
-#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the
-#: definition carries none.
-#:
-#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at
-#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the
-#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly
-#: created automation. Two numbers for one fact, in the panel, before anybody had typed
-#: anything. **Read off the screenshot; every assertion in the battery was green.**
-#:
-#: This is the same species as the default-versus-guard split fixed the same day: a value
-#: decided in one file and a value decided in another, describing the same thing. Aligning the
-#: constants closes the only window in which they can disagree — the seed — because every later
-#: state comes from the stored config.
-DISCOVER_SEED_RECORDS = 25
-
-
-def _ensure_ig_action(flow_raw, table):
- """Create record is PERMANENT step 1 on `ig_profile_match` — it cannot be deleted or moved.
-
- ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once,
- on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a
- comment warning that re-seeding on every clean would be "a control that will not take no for
- an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be
- deleted, because the nature of that automation is that it needs to first create a record in a
- database somewhere from the list of profiles to fetch."*
-
- Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER
- produces rows has nowhere to put them until something writes them, so an Instagram search
- with no Create record is not a customised automation — it is a search whose results are
- discarded. That is not a preference to respect.
-
- ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and
- the values it maps are theirs. An existing `create_record` further down is MOVED to the front
- rather than duplicated — re-seeding a second one would quietly double every run's writes.
- """
- flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {}
- actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)]
- if actions and actions[0].get("kind") == "create_record":
- return _pin_unique(_ensure_enrich_step(
- flow_raw if isinstance(flow_raw, dict) else flow))
- at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1)
- if at > 0:
- actions.insert(0, actions.pop(at))
- else:
- # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner
- # and no card: the wave-23 header records four kinds that shipped with illegal seeds and
- # did exactly that. `create_record` needs a ut_-prefixed table and at least one value.
- actions.insert(0, {
- "id": "act_1", "kind": "create_record", "enabled": True, "when": None,
- # `config.label` follows the `review` action's precedent in `_clean_action_config` —
- # an action naming itself, inside the untyped config bag, so no shared TS type changes.
- "config": {"table": str(table or DISCOVER_TABLE),
- "label": "Save the profile",
- # C5: the picture and the real write finally agree — see `_pin_unique`.
- "uniqueOn": "handle",
- "values": {"handle": "{{handle}}"}},
- })
- flow["actions"] = actions
- return _pin_unique(_ensure_enrich_step(flow))
-
-
-def _ensure_enrich_step(flow_raw):
- """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON AN INSTAGRAM SEARCH.
-
- Owner: *"make it default that this enrichment action is Step 2 always, and under Config of
- Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for
- the same reason: a search that finds profiles and never reads them has done half a job. The
- difference is the control — step 1 is permanent because a flow without it discards its
- results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and
- disabling would be two ways to say one thing, and only one of them survives a re-save.
-
- ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains
- this step by upgrading does not quietly start spending; switching the Source to the paid
- provider is an explicit choice a person makes in front of the sentence that names the cost.
- ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing —
- a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow
- re-buying the same profile nightly. Off-by-default would make the expensive behaviour the
- accident.
-
- ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an
- If/then branch has one; seeding a second at the top level would enrich twice and bill twice,
- which is the duplication `apply_actions` already paid for once with the pinned step 1.
- """
- flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {}
- actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)]
- if any(a.get("kind") == "enrich_instagram" for a in walk_actions(actions)):
- return flow_raw if isinstance(flow_raw, dict) else flow
- seeded = list(actions)
- # Index 1 — after the pinned Create record, because there is nothing to enrich until the
- # profiles have been written as records. `insert` past the end is a plain append, so a flow
- # with only step 1 lands this at the end, which IS step 2.
- seeded.insert(1, {
- "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
- "config": {"postMetrics": False, "commentMetrics": False,
- "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
- "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
- "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
- "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS},
- })
- return {**flow, "actions": seeded}
-
-
-#: The key the discovery writer really upserts on. `run_discover_instagram` keys candidates on
-#: `_ck(handle, created_by)`; `handle` is the half a person can see and the half this action's
-#: values carry.
-IG_PINNED_UNIQUE = "handle"
-
-
-def _pin_unique(flow):
- """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save.
-
- ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` —
- the builder posts the whole action list on every save — so a client that omitted the key would
- silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's
- own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming
- "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this
- module refuses everywhere else.
-
- ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason
- this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action
- does not write, so the stamp is applied ONLY when `handle` is among the action's own values.
- A user who remaps the card to write different columns keeps their edit and still saves; the
- alternative — stamping unconditionally — is a product-seeded default that its own validator
- would 400, which is precisely the post-W24 hotfix this rule exists because of.
-
- ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes
- `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED
- definition's own dict, so writing into it would edit the live store object in memory, before
- (and regardless of) any commit. Every touched level is copied instead.
- """
- if not isinstance(flow, dict):
- return flow
- acts = flow.get("actions")
- if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict):
- return flow
- first = acts[0]
- if first.get("kind") != "create_record":
- return flow
- cfg = first.get("config")
- if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}):
- return flow
- if str(cfg.get("uniqueOn") or "").strip():
- return flow
- return {**flow,
- "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]}
-
-
-#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so
-#: the seeder and the un-seeder cannot disagree about what "seeded" means.
-_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich"}
-
-
-def _is_untouched_ig_seed(action, table):
- """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15)
-
- ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and
- that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build
- the same dicts one screen above, so a step gaining a config key next wave gains it here too.
- A remembered key list would quietly start calling every seeded action "edited".
-
- ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The
- stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the
- kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW
- dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person",
- and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only
- version where the two sides are the same kind of object.
-
- ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the
- TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about
- a step they still want, and an off seed is still a seed.
- """
- if not isinstance(action, dict):
- return False
- kind = str(action.get("kind") or "")
- if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""):
- return False
- seeded, _seed_err = clean_actions(
- (_ensure_ig_action({"actions": []}, table).get("actions") or []))
- fresh = {a.get("kind"): a for a in (seeded or [])}
- seed = fresh.get(kind)
- if not seed:
- return False
- cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {})
- if kind == "enrich_instagram":
- cfg.pop("enabled", None)
- seed_cfg.pop("enabled", None)
- return cfg == seed_cfg and (action.get("when") or None) is None
-
-
-def _drop_ig_seeds(flow_raw, table):
- """Strip the trigger's own seeded steps, keeping any the person has since made theirs.
-
- ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT
- THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it
- does not delete the seeded actions … a create_record the person has since re-pointed at their
- own table with their own values is THEIR action now"*. That reasoning is still correct and is
- exactly what this function preserves — but it was applied to EVERY seeded action, including
- the ones nobody had ever opened, and the result is what the owner reported: switch an
- Instagram search to Manual and you are left holding a "Save the profile" step writing
- `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not
- somebody's work being protected; it is the machine's own leftover, pointed at a table the
- automation has nothing to do with any more.
-
- ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as
- `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a
- branch that destroys something, so the conservative half is the load-bearing half.
- """
- flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {}
- actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)]
- kept = [a for a in actions if not _is_untouched_ig_seed(a, table)]
- if len(kept) == len(actions):
- return flow_raw
- return {**flow, "actions": kept}
-
-
-def ig_action_pinned(defn, index):
- """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned`
- flag is a second copy of the rule, and the copy is what a hand-written PATCH omits."""
- return (defn or {}).get("kind") == "discover_instagram" and index == 0
-
-
-def clean_definition(raw, previous=None, username=""):
- """Whole-definition validation. Returns `(defn, error)`."""
- raw = raw if isinstance(raw, dict) else {}
- prev = previous or {}
- # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
- # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
- # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
- # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
- # invalid in BOTH its trigger and its config now answers with the trigger's sentence.
- trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw
- else prev.get("trigger"), prev.get("trigger"))
- if terr:
- return None, terr
- # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
- # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a
- # dead end the moment the wizard was deleted and the client's create body became {name}.
- kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND
- drop_ig_seeds = False
- if (trigger or {}).get("key") == "ig_profile_match":
- # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather
- # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and
- # whose kind says something else is not a preference to respect, it is two halves of one
- # answer disagreeing, and the trigger is the half the person actually picked.
- kind = "discover_instagram"
- elif (kind == "discover_instagram"
- and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match"
- and (trigger or {}).get("key") != "ig_profile_match"):
- # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG.
- #
- # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram
- # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch
- # the trigger to Manual and the automation stayed `discover_instagram`, which means
- #
- # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow
- # the person had just made MANUAL fired a PAID Bright Data corpus search;
- # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed
- # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the
- # instagram trigger, the 'always first' create record just stuck there"*. The server
- # would have accepted the delete; the CLIENT hid the control, because both halves ask
- # the kind and the kind was lying.
- #
- # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in
- # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as
- # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY
- # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`.
- # That turned five `section_bd` checks red and crashed the suite on a `StopIteration`
- # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest
- # question: did this automation stop being an Instagram search?
- #
- # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram
- # trigger — a `discover_instagram` created with an explicit kind and some other trigger is
- # somebody's deliberate state, not a mistake to correct. And only FROM
- # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose
- # triggers are their own business, and clobbering them would retire them by accident.
- #
- # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this
- # replaces argued the other way, so here is why it was half right. It read: *"it does not
- # delete the seeded actions … a create_record the person has since re-pointed at their own
- # table with their own values is THEIR action now, and quietly destroying it is the silent
- # data loss this module refuses everywhere else."* Every word of that is still true and is
- # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the
- # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to
- # Manual still carrying a "Save the profile" step writing `{{handle}}` into
- # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces
- # handles, unpinned but still there, still runnable, and still pointed at a table that has
- # nothing to do with this automation.
- # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line.
- kind = DEFAULT_KIND
- drop_ig_seeds = True
- if kind not in KINDS:
- return None, f"unknown automation kind {kind!r}"
- # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION.
- # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain`
- # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement.
- # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way
- # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two
- # live automations save through this function every time their owner edits them. The gate's
- # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved
- # from `create` into `clean_definition` (the tempting simplification), this helper would go
- # red across a dozen sections, which is the alarm that change deserves."* It did, and the
- # alarm worked.
- # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every
- # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody
- # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an
- # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored
- # ones are rewritten under their owner — and that is a ruling, not a refactor.
- name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split())
- if not name:
- return None, "name the automation"
- cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config")
- if kind == "discover_instagram" and prev.get("kind") != "discover_instagram":
- # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very
- # first Save after picking this trigger 400s — against the stored-inert-with-
- # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note).
- # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard
- # exists because an unbounded query is the one shape the vendor refuses outright, and
- # coercing a blank into a number would be exactly the silent widening it protects against.
- # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until
- # a person presses Run now or arms the schedule.
- cfg_raw = dict(cfg_raw or {})
- if not _ig_int(cfg_raw.get("recordsLimit")):
- cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS
- config, err = clean_config(kind, cfg_raw, prev.get("config"))
- if err:
- return None, err
- if trigger:
- # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own
- # configuration IS the config, and `clean_trigger` could not see it.
- trigger["configured"] = _trigger_configured(trigger, config)
- try:
- schedule = clean_schedule(
- raw.get("schedule") if isinstance(raw.get("schedule"), dict)
- else prev.get("schedule"), prev.get("schedule"))
- except ValueError as e:
- return None, str(e)
- # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.)
- flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow")
- # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only
- # call was what made the action deletable; running it on every save is what makes step 1
- # permanent, and it is deliberate rather than a widened condition nobody noticed.
- if (trigger or {}).get("key") == "ig_profile_match":
- flow_raw = _ensure_ig_action(flow_raw, config.get("targetTable") or DISCOVER_TABLE)
- elif drop_ig_seeds:
- # ITEM 15: the trigger just STOPPED being the Instagram one. Un-seed what the trigger
- # planted and nobody has since made theirs — measured against the table the seeds were
- # planted for, which is the PREVIOUS config's, not the one being saved.
- flow_raw = _drop_ig_seeds(
- flow_raw, (prev.get("config") or {}).get("targetTable") or DISCOVER_TABLE)
- flow, ferr = clean_flow(flow_raw, prev.get("flow"))
- if ferr:
- return None, ferr
- if (trigger or {}).get("key") == "ig_profile_match":
- # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and
- # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since
- # wave 24: the pinned step 1 says where profiles are written, and the config says where
- # the automation's records live — and for a discovery flow those are the same database by
- # construction. When they disagreed, every surface picked a different winner (the canvas
- # Write node read the config, the actual write read the action), which is a bug you can
- # only see by comparing two panels.
- #
- # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the
- # action's own configuration, so the table named there is the one they were looking at.
- #
- # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established:
- # a stored definition carrying two different values is reconciled on its next clean,
- # silently and exactly once, because nothing writes the losing value back. A refusal here
- # would 400 every Save of a definition that was legal yesterday.
- # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the
- # TIE-BREAK rather than the whole rule. The owner's report: change the database on the
- # TRIGGER, save, and the picker springs back to the old one with no message. The cause is
- # the branch below read unconditionally — the pinned action still named yesterday's table,
- # so it overwrote a change the person had just made in front of it, silently, every time.
- #
- # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled
- # "when the two disagree, whose value is real?" — the action's, because R2b puts the
- # preset list on the action's own panel, so that is the one they were looking at. That
- # answers a definition at REST. A save is different: it carries an INTENT, and the honest
- # question is which side this particular save changed. So:
- # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to
- # follow it (leaving the action behind would just re-revert on the next save);
- # · only the action moved, or neither did and they were already inconsistent -> R2's
- # migration, unchanged, reconciling a stored definition exactly once.
- # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is
- # also the only reading that cannot lose a value: the action's table is the one with
- # the field mapping attached to it.
- first = (flow.get("actions") or [{}])[0]
- pinned = str((first.get("config") or {}).get("table") or "") \
- if first.get("kind") == "create_record" else ""
- prev_target = str((prev.get("config") or {}).get("targetTable") or "")
- prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0]
- ).get("config") or {}).get("table") or "")
- target_moved = bool(prev_target) and config.get("targetTable") != prev_target
- action_moved = bool(prev_pinned) and pinned != prev_pinned
- if pinned and target_moved and not action_moved:
- # The person changed the trigger's database. Carry it INTO the pinned step so the two
- # agree, instead of throwing their edit away to keep a stale copy consistent.
- cfg1 = dict(first.get("config") or {})
- cfg1["table"] = str(config.get("targetTable") or "")
- flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]]
- elif pinned and pinned != config.get("targetTable"):
- config["targetTable"] = pinned
- # The stored label described a DIFFERENT database, so keeping it would caption the new
- # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure`
- # never relabels a table that already exists — so an adopted hand-made database keeps
- # the name its owner gave it.
- config["targetLabel"] = ""
- return {
- "id": _s(prev.get("id") or raw.get("id"), 40),
- "name": name, "kind": kind, "config": config, "schedule": schedule,
- "trigger": trigger,
- # The Builder's ordered actions. An absent flow is simply an automation with no actions.
- "flow": flow,
- "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""},
- "runs": list(prev.get("runs") or [])[:MAX_RUNS],
- # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20
- # minutes to build, so a run persists its id here and the NEXT run collects it. It is
- # carried through `patch` untouched because a user pressing Save must not silently orphan
- # a set the vendor is already building (and already counting against the funds gate).
- "state": dict(prev.get("state") or {}),
- "created": prev.get("created") or _iso(),
- "createdBy": prev.get("createdBy") or username,
- }, None
-
-
-def set_state(rt, auto_id, patch_state):
- """Merge into ONE automation's engine state. A key set to None is removed.
-
- Its own tiny store write rather than a field on `_commit_run`, because the handoff must
- survive a run that ends in `error` — a snapshot the vendor is building does not stop existing
- because the run that started it failed afterwards.
- """
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- d = cur.get(str(auto_id))
- if d is None:
- return cur
- st = dict(d.get("state") or {})
- for k, v in (patch_state or {}).items():
- st.pop(k, None) if v is None else st.__setitem__(k, v)
- d["state"] = st
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
-
-
-def _without_retired_board(definition):
- """Return a definition with retired Board-only state removed, without mutating the store.
-
- This protects scheduled/manual execution before the next UI list request persists the
- migration. It removes only the former Board configuration, actions, and audit trail; normal
- actions and all user data remain intact.
- """
- if not isinstance(definition, dict):
- return definition, False
- out, changed = dict(definition), False
- cfg = out.get("config")
- if isinstance(cfg, dict) and "lanes" in cfg:
- cfg = dict(cfg)
- cfg.pop("lanes", None)
- out["config"] = cfg
- changed = True
-
- def _actions(actions):
- nonlocal changed
- clean = []
- for action in actions if isinstance(actions, list) else []:
- if not isinstance(action, dict):
- clean.append(action)
- continue
- if action.get("kind") == "review":
- changed = True
- continue
- item = dict(action)
- if item.get("kind") == "group" and isinstance(item.get("config"), dict):
- group_cfg = dict(item["config"])
- branches = group_cfg.get("branches")
- if isinstance(branches, list):
- kept = []
- for branch in branches:
- if not isinstance(branch, dict):
- changed = True
- continue
- next_actions = _actions(branch.get("actions"))
- if not next_actions:
- changed = True
- continue
- kept.append({**branch, "actions": next_actions})
- if not kept:
- changed = True
- continue
- if kept != branches:
- changed = True
- group_cfg["branches"] = kept
- item["config"] = group_cfg
- elif isinstance(group_cfg.get("actions"), list):
- nested = _actions(group_cfg["actions"])
- if not nested:
- changed = True
- continue
- if nested != group_cfg["actions"]:
- group_cfg["actions"] = nested
- item["config"] = group_cfg
- clean.append(item)
- return clean
-
- flow = out.get("flow")
- if isinstance(flow, dict):
- next_flow = dict(flow)
- actions = _actions(flow.get("actions"))
- if actions != flow.get("actions"):
- next_flow["actions"] = actions
- if "ending" in next_flow:
- next_flow.pop("ending", None)
- changed = True
- out["flow"] = next_flow
- if "reviews" in out:
- out.pop("reviews", None)
- changed = True
- return out, changed
-
-
-def retire_automation_board_state(rt):
- """Persist the idempotent Board retirement, then remove its generated table fields."""
- fields = retire_automation_stage_fields(rt)
- try:
- stored = dict(rt.get(STORE_KEY) or {})
- except Exception:
- return {"definitions": 0, **fields}
- plan = {str(aid): cleaned for aid, raw in stored.items()
- for cleaned, changed in [_without_retired_board(raw)] if changed}
- if not plan:
- return {"definitions": 0, **fields}
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- for aid, cleaned in plan.items():
- if aid in cur:
- cur[aid] = cleaned
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
- return {"definitions": len(plan), **fields}
-
-
-def all_definitions(rt):
- try:
- raw = dict(rt.get(STORE_KEY) or {})
- except Exception:
- return {}
- return {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()}
-
-
-def _new_id(existing):
- n = 1
- while f"auto_{n}" in existing:
- n += 1
- return f"auto_{n}"
-
-
-#: C2's three answers to "which database does this automation work on?" — the FIRST question the
-#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is).
-TARGET_MODES = ("existing", "new", "automated")
-
-
-def resolve_target(rt, raw, username=""):
- """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`.
-
- - `existing` — bind a database that is already there.
- - `new` — mint a blank one now, so the automation has somewhere to write before its first
- run instead of conjuring a table the person never agreed to.
- - `automated` — leave it to the runner: a scraping automation MINTS its target on first run
- (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database.
- Nothing is created here, deliberately — an empty table created up front for a scrape that
- never runs is litter nobody can explain.
- """
- if not isinstance(raw, dict) or not raw:
- return {}, None # no target block = the pre-wizard shape
- mode = _s(raw.get("mode"), 20).strip() or "existing"
- if mode not in TARGET_MODES:
- return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES)
- if mode == "automated":
- label = " ".join(_s(raw.get("label"), 60).split())
- return ({"targetLabel": label} if label else {}), None
- if mode == "existing":
- key = _s(raw.get("table"), 60).strip()
- if not key:
- return None, "choose the database this automation works on"
- t = ut_get(rt, key)
- if t is None:
- return None, f"{key!r} is not a database in this workspace"
- return {"targetTable": key, "targetLabel": t.get("label") or key}, None
- label = " ".join(_s(raw.get("label"), 60).split())
- if not label:
- return None, "name the new database"
- import core.user_tables as _ut_new
- key = _ut_new.create(label, username or "automation", st=rt)
- if not key:
- return None, ("the database could not be created — this workspace may be at its table "
- "limit")
- return {"targetTable": key, "targetLabel": label}, None
-
-
-def _unmint(rt, key):
- """Delete a database this call minted moments ago, because the call is refusing (D-50).
-
- ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user
- needs to read; a rollback that raised would replace that sentence with a 500 and lose the
- reason the create failed in the first place. The worst case of a swallowed failure here is
- the orphan we had before — strictly no worse, and the refusal still reaches the person.
- """
- if not key:
- return
- try:
- import core.user_tables as _ut_del
- _ut_del.delete(key, st=rt)
- except Exception: # noqa: BLE001
- pass
-
-
-def create(rt, raw, username=""):
- existing = all_definitions(rt)
- if len(existing) >= MAX_AUTOMATIONS:
- return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit"
- raw = dict(raw or {})
- # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation
- # happens after it, so a correct refusal ("a source URL is required") used to leave an
- # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close:
- # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form
- # twice silently accumulated empty databases in their nav.
- #
- # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It
- # does not test for `mode == "new"`; it asks "did the table this call's target resolved to
- # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and
- # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target
- # rather than "any table that appeared", so a concurrent create in another request is never
- # collateral.
- before = set(ut_all(rt) or {})
- patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username)
- if terr:
- return None, terr
- minted = str((patch_cfg or {}).get("targetTable") or "")
- minted = minted if minted and minted not in before else ""
- if patch_cfg:
- raw["config"] = {**(raw.get("config") or {}), **patch_cfg}
- # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY
- # has, instead of letting the pure validator mint a second empty one. See
- # `discover_default_table`; a target the caller named is never rewritten.
- raw = _apply_discover_default(rt, raw)
- defn, err = clean_definition(raw, None, username)
- if err:
- _unmint(rt, minted)
- return None, err
- # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations
- # that already use them and NO NEW ONE CAN BE CREATED."
- #
- # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live
- # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing
- # the kind there would make both of them unsaveable — the ruling retires the door, not the
- # automations behind it. Enforced on the server rather than left to the deleted wizard: a
- # wall that holds only because the client stopped asking is a convention, not a wall.
- if defn["kind"] in RETIRED_KINDS:
- _unmint(rt, minted)
- return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no "
- f"longer created — the ones already using it keep working. "
- f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}")
- defn["id"] = _new_id(existing)
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- cur[defn["id"]] = defn
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
- _presets_after_write(rt, defn, username) # R10: the columns exist before a run
- if defn.get("trigger"):
- _seed_event_state(rt, defn)
- return defn, None
-
-
-def patch(rt, auto_id, raw, username=""):
- existing = all_definitions(rt)
- prev = existing.get(str(auto_id))
- if prev is None:
- return None, "no such automation"
- merged = dict(prev)
- # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a
- # key missing here is accepted by the route, validated by `clean_definition`, and then
- # DISCARDED — the client shows a saved flow that the store never received, and nothing goes
- # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
- # ordering that never has a window where the bug exists.
- for k in ("name", "config", "schedule", "kind", "trigger", "flow"):
- if k in (raw or {}):
- merged[k] = raw[k]
- # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
- # Instagram trigger on an existing automation flips the kind, and the very next Save runs a
- # config that has never carried a target through the pure validator, which mints the empty
- # `ut_ig_candidates`. `create` alone would have left the commonest path untouched.
- merged = _apply_discover_default(rt, merged)
- defn, err = clean_definition(merged, prev, username)
- if err:
- return None, err
- defn["id"] = str(auto_id)
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- cur[defn["id"]] = defn
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
- _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS
- # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching
- # DISARMED — enabling never fires for records already in the state.
- if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}):
- _seed_event_state(rt, defn)
- return defn, None
-
-
-def _presets_after_write(rt, defn, username):
- """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run.
-
- Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset
- columns immediately, "so R2b's list is real and the database is visible before anything is
- spent". That last clause is the point — a person can look at the database, see the sixteen
- columns, and decide whether to arm the schedule, instead of paying a vendor to find out what
- they are agreeing to.
-
- ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs
- nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first
- run of a saved automation finds the table already correct and spends no commit.
-
- ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the
- search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the
- preset set, and spawning a subset here would mean the columns changed between Save and the
- first run — two different answers to "what does this database look like", which is the exact
- disagreement R10 exists to remove.
-
- ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure`
- returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an
- automation because the workspace is full would be a worse answer than saving one whose table
- arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the
- honest failure still has exactly one home.
- """
- cfg = defn.get("config") or {}
- target = str(_flow_table(defn) or cfg.get("targetTable") or "")
- if not target:
- return
- try:
- if (defn.get("trigger") or {}).get("key") == "ig_profile_match":
- ut_ensure(rt, cfg.get("targetLabel") or DISCOVER_LABEL, CANDIDATE_FIELDS, username,
- key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True)
- return
- enrich_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []),
- "enrich_instagram")
- table = ut_get(rt, target) or {}
- named = next((str((a.get("config") or {}).get("profileField") or "").strip()
- for a in enrich_actions
- if str((a.get("config") or {}).get("profileField") or "").strip()), "")
- bound = next((str(f.get("key") or "") for f in table.get("fields") or []
- if isinstance(f.get("profile"), dict)), "") or named
- if enrich_actions and bound:
- ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username,
- key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True)
- except Exception as e: # noqa: BLE001
- # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so
- # raising here would answer 500 for an automation that IS stored — the caller would retry
- # and create a second one. The columns then arrive on the first run, which is precisely
- # the behaviour this function is an improvement on rather than a replacement for.
- print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}")
-
-
-def remove(rt, auto_id):
- aid = str(auto_id)
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- cur.pop(aid, None)
- return cur
- rt.update(STORE_KEY, _up, flush="sync")
- # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned
- # silently: the column keeps its values and its config, and says why it stopped. Scanned
- # first so a delete with no tagged fields costs no store commit.
- hit = any(str((f.get("automation") or {}).get("flowId") or "") == aid
- for t in ut_all(rt).values() for f in (t.get("fields") or []))
- if not hit:
- return
-
- def _disable(cur):
- cur = cur if isinstance(cur, dict) else {}
- for t in cur.values():
- for f in (t.get("fields") or []):
- a = f.get("automation")
- if isinstance(a, dict) and str(a.get("flowId") or "") == aid:
- a["disabled"] = True
- a["statusNote"] = f"its automation was deleted {_stamp()}"
- return cur
- rt.update(UT_STORE_KEY, _disable, flush="sync")
-
-
-# ---------------------------------------------------------------------------------------------
-# RUN STATE — process memory only (see the module header for why this may never be persisted)
-# ---------------------------------------------------------------------------------------------
-
-_RUN_LOCK = threading.RLock()
-_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'}
-
-
-def running(tenant, auto_id):
- with _RUN_LOCK:
- return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None
-
-
-def _claim(tenant, auto_id):
- with _RUN_LOCK:
- if (tenant, str(auto_id)) in _RUNNING:
- return False
- _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"}
- return True
-
-
-def _step(tenant, auto_id, text):
- with _RUN_LOCK:
- cur = _RUNNING.get((tenant, str(auto_id)))
- if cur is not None:
- cur["step"] = text
-
-
-def _no_step(_text):
- """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner
- directly). A runner must never REQUIRE the slot — the slot is process state, and the runner
- is the part a test is allowed to call on its own."""
-
-
-def _release(tenant, auto_id):
- with _RUN_LOCK:
- _RUNNING.pop((tenant, str(auto_id)), None)
-
-
-def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None):
- """THE ONE store write a run performs on the `automations` bucket.
-
- `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here
- because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to
- invent a node status when this map is absent (every run stored before W19-C has no `steps`,
- and painting those nodes green from the run's overall state would be a fabrication).
-
- ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the
- comprehension below drops everything else — so the per-record sentence a vendor gave us had
- literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the
- product could say. MEASURED on nurilab: the same record blocked on three separate runs and
- the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by
- calling the vendors by hand. The note is the most valuable thing a run produces — a dead
- handle, a vendor hiccup and an exhausted key are three different actions — and it was the one
- thing thrown away.
- ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it
- did not ask for, and W25/R4 already retired writing status STRINGS into people's grids.
- """
- entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400),
- "counts": {k: int(v) for k, v in (counts or {}).items()
- if isinstance(v, (int, float))},
- "steps": {str(k): str(v) for k, v in (steps or {}).items()},
- # Bounded on both axes: a 100-profile run must not put 100 sentences into a store
- # entry that is kept 20 deep per automation.
- "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25],
- "affected": list(affected or [])[:200]}
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- d = cur.get(str(auto_id))
- if d is None:
- return cur
- d["status"] = {"state": state, "lastRunAt": entry["ts"],
- "lastSummary": entry["summary"]}
- d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS]
- # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with
- # the reason on it — a dead credential must not burn quota (or a paid vendor's records)
- # 96 times a day while a green toggle sits over a buried error log. Errors only:
- # `partial` is the honest-progress state and pausing on it would punish honesty.
- runs = d.get("runs") or []
- if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all(
- not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]):
- sch = d.get("schedule")
- if isinstance(sch, dict) and sch.get("enabled"):
- sch["enabled"] = False
- trg = d.get("trigger")
- if isinstance(trg, dict):
- trg["paused"] = True
- d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive "
- f"failures — fix the cause, then re-enable: "
- f"{entry['summary'][:120]}")
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
- _notify_outcome(rt, auto_id, state, entry)
- return entry
-
-
-def _notify_outcome(rt, auto_id, state, entry):
- """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7).
-
- Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is
- written, so it is the only place this can go without a second definition of "what happened".
-
- ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED.
- 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the
- MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a
- background thread with no session, so an omitted `st` files nurilab's automation failures
- in Royal Imports' inbox — D-16's class, silently, forever.
- 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler
- has no session at all, so the alternative is not "the wrong person" but "nobody", and
- `notify` returns None on a blank owner — the notification would simply never exist, on
- exactly the runs nobody was watching.
- ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only
- view-alert CREATION; `notify()` writes straight into the notifications bucket and
- `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell.
- ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that
- SUCCEEDED into one that raised — the run entry is already committed one line above, and the
- bell is a courtesy on top of it.
- """
- try:
- defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {}
- owner = str(defn.get("createdBy") or "").strip()
- if not owner:
- return
- from core import alerts as _alerts
- _alerts.notify(
- owner=owner,
- label=_s(defn.get("name") or "Automation", 80),
- topic="automation",
- key=str(auto_id),
- detail=f"{state}: {entry.get('summary') or ''}"[:300],
- st=rt)
- except Exception: # noqa: BLE001
- pass
-
-
-# ---------------------------------------------------------------------------------------------
-# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN
-# ---------------------------------------------------------------------------------------------
-# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and
-# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down)
-# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper:
-#
-# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie,
-# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the
-# supplier takes the scraping risk; it is never an Instagram credential and this module must
-# never be given one.
-#
-# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind).
-#
-# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress
-# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been
-# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and
-# the cell says which. A pull that quietly wrote zero posts and reported success would be worse
-# than no automation at all: the table would look maintained and be empty.
-#
-# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`,
-# so a run history answers "what still works anonymously?" from data rather than from memory.
-
-PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5)
-
-#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's
-#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number
-#: is a NOW value, so the series starts the day capture starts and a field we do not capture
-#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows
-#: has exactly the same cost as delaying capture — for that column — and is justified by exactly
-#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to
-#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices.
-#: Every returned provider field is retained in Source data. Promoted columns make commonly used
-#: values filterable; the raw source document prevents a new or uncommon field from being lost.
-#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose
-#: almost none of them, which is what the `source` column is for.
-SNAPSHOT_FIELDS = [
- field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"),
- field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"),
- field_def("following", "Following", "int"),
- field_def("posts_count", "Post count", "int"),
- field_def("full_name", "Name"), field_def("bio", "Bio"),
- field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"),
- field_def("approx", "Counts are approximate", "checkbox"),
- # The link in bio — the paid rung's field, and commercially the most useful single string an
- # influencer row can carry (it is the shop/affiliate destination).
- field_def("external_url", "Link in bio", "url"),
- # --- the rest of the vendor's Profiles schema.
- field_def("ig_id", "Instagram id"),
- field_def("profile_url", "Profile", "url"),
- field_def("avg_engagement", "Avg engagement", "pct"),
- field_def("category", "Category"),
- field_def("business_category", "Business category"),
- field_def("is_business", "Business account", "checkbox"),
- field_def("is_professional", "Professional account", "checkbox"),
- field_def("is_private", "Private", "checkbox"),
- field_def("highlights_count", "Story highlight count", "int"),
- field_def("bio_hashtags", "Bio hashtags"),
- field_def("pronouns", "Pronouns"),
- # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
- # row maps the WHOLE schema by design, so these belong here the moment the map reads them;
- # leaving them out would be the "captured but unrecorded" half of the same loss the note at
- # the top of this list is about.
- field_def("profile_name", "Profile name"),
- field_def("is_joined_recently", "Joined recently", "checkbox"),
- field_def("has_channel", "Has channel", "checkbox"),
- field_def("partner_id", "Partner id"),
- field_def("external_url_title", "Link title"),
- field_def("fbid", "Facebook id"),
- field_def("related_accounts", "Related accounts"),
- field_def("country_code", "Country"),
- field_def("source_payload", "Source data", "json"),
-]
-POST_FIELDS = [
- field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
- field_def("posted_at", "Posted at", "date"),
- field_def("type", "Type", "select", options=["image", "video", "carousel"]),
- field_def("caption", "Caption"), field_def("url", "URL", "url"),
- # A tagged place is useful content context and can be filtered as ordinary text. It is never
- # presented as the creator's location: a creator can tag a holiday, venue or brand location.
- field_def("tagged_location", "Tagged location"),
- # The per-field map below gives operational fields first; this locked JSON document retains
- # every other value Bright Data supplied, so a vendor schema addition is preserved
- # instead of being silently discarded while the canonical field model catches up.
- field_def("source_payload", "Source data", "json"),
- # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous
- # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
- field_def("paid_partnership", "Paid partnership", "checkbox"),
- field_def("partner", "Partner brand"),
- field_def("hashtags", "Hashtags"),
- field_def("alt_text", "Alt text"),
- # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT
- # ONE HOP.
- #
- # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's
- # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a
- # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated
- # transitively). So the post row carries its own LATEST value and the rollup reads it
- # directly.
- #
- # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile
- # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in
- # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that
- # was just appended — they are a projection of that store, never a second copy of it, and
- # deleting them would cost a convenience rather than a measurement.
- # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is
- # off by default (it buys one vendor record per post), so on most pulls this stays empty, and
- # empty means "not read", which is what makes an honest average possible at all. A 0 here
- # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or
- # COMMENT count is a real measurement and must survive.
- #
- # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08,
- # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought).
- # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct
- # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in
- # the very same rows. Mapping it here is what put one number on many unrelated posts. We
- # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers;
- # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays
- # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained
- # whole, so if the vendor ever populates it per-reel the history is re-derivable.
- # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30
- # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the
- # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named
- # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10
- # (folded into a single "Views"), not because we are reading the wrong key.
- # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED.
- # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the
- # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` —
- # measured against a browser-read ground truth to the digit. Same column name, same type, same
- # rollup; the engine underneath is swappable and the preset database never moved. That is the
- # owner's schema ruling working exactly as intended.
- # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung.
- field_def("views", "Views", "int"),
- field_def("plays", "Plays", "int"),
- field_def("likes", "Likes", "int"),
- field_def("comments", "Comments", "int"),
- # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data
- # further"*). Two fields the primary source does not return AT ALL, already paid for inside
- # the same engagement response — so capturing them costs nothing extra.
- # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on
- # every write door, so a normaliser that emits `video_duration` without this line writes
- # nothing and reports success — the wave-28 defect that swallowed nine preset cells.
- field_def("video_duration", "Video length (s)", "int"),
- field_def("comments_disabled", "Comments off", "checkbox"),
- field_def("measured_at", "Engagement read at", "date"),
- # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED":
- # a post reaches its own comment rows the same way a profile reaches its posts. Derived from
- # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance.
- # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state
- # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when
- # `postMetrics` is off.
- field_def("comments_link", "Comment rows", "link",
- description="Comment records linked to this post.",
- link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}),
- field_def("post_snapshots_link", "Measurement rows", "link",
- description="Engagement measurements linked to this post.",
- link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}),
- field_def("measurements_captured", "Measurements captured", "rollup",
- rollup={"link": "post_snapshots_link", "fn": "countall"}),
-]
-
-#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE.
-#:
-#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to
-#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default
-#: is the capture.
-#:
-#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED.
-#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE:
-#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of
-#: `instagram-capture.md`). They are the single most expensive thing this product can buy.
-#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has
-#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment
-#: thread ingests identifiable people who never entered anybody's influencer list.
-#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code
-#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments`
-#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on
-#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is
-#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md`
-#: is written against.
-#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as
-#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs
-#: nothing; the spend and the third-party ingest are a click somebody makes knowingly.
-#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because
-#: they are part of a record already paid for. The separate full Comments dataset remains OFF
-#: unless `commentMetrics` is explicitly enabled on the enrichment action.
-COMMENT_FIELDS = [
- field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
- field_def("influencer_key", "Influencer"),
- field_def("commented_at", "Commented at", "date"),
- field_def("likes", "Likes", "int"),
- field_def("replies", "Replies", "int"),
- field_def("source_payload", "Source data", "json"),
- field_def("post_link", "Post", "link",
- description="Post record linked to this comment.",
- link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
- "single": True}),
- # Author/text and every provider-specific value stay intact in Source data. The promoted
- # columns intentionally keep the grid concise while Likes and Replies make comment engagement
- # directly filterable and rollup-ready.
-]
-POST_SNAPSHOT_FIELDS = [
- field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
- field_def("influencer_key", "Influencer"),
- field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"),
- field_def("comments", "Comments", "int"),
- # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits
- # before any rollup argument is made.
- #
- # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with
- # only the first can describe a measurement but not its AGE, so the single most useful thing
- # this series can compute — views at N days old, i.e. velocity — is not expressible over it at
- # all. `type` is the same shape one column over: "reels only" is the default lens on this data
- # and without it every question has to go back through `ut_ig_posts` to ask what kind of post
- # this was.
- #
- # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row
- # from the same vendor record), so this costs one dict key each and no extra call.
- #
- # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24
- # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the
- # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied
- # from it deliberately rather than shared, because these are two different tables' columns that
- # happen to agree today, and `verify_automation` asserts the agreement.
- # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO
- # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`,
- # and `ut_ig_posts` already performs the first of them. See the verdict in
- # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for
- # moving the window here.
- field_def("posted_at", "Posted at", "date"),
- field_def("type", "Type", "select", options=["image", "video", "carousel"]),
- # Plays move independently of likes on video, so it is its own series rather than a thing to
- # derive. Paid rung only; blank means not read.
- # The series regains Views alongside the latest-value projection on the Post row — one store
- # for one series (R3) is untouched; this IS that store.
- field_def("views", "Views", "int"),
- field_def("plays", "Plays", "int"),
- field_def("source_payload", "Source data", "json"),
- field_def("post_link", "Post", "link",
- description="Post record linked to this measurement.",
- link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
- "single": True}),
-]
-
-
-# ---------------------------------------------------------------------------------------------
-# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`.
-# ---------------------------------------------------------------------------------------------
-# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE.
-# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation
-# runtime needs that, and the runtime is what this file is for.
-#
-# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation
-# runtime still reaches for after the split — twenty, down from the seventy-one that used to be
-# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor
-# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and
-# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that
-# true without those files having to learn where the wire went.
-# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked
-# that would be green whether or not anything moved. `verify_automation`'s `section_split`
-# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING
-# system instead of trusting this import line ([[gate-answers-the-wrong-question]]).
-#
-# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back
-# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily,
-# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail
-# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a
-# module-level import, the names it wants already exist and the cycle still resolves.
-from connectors_ig import ( # noqa: E402
- BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT,
- BD_RECORD_PRICE_SPEC,
- _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_profile, _bd_rows,
- _bd_tagged_location,
- _first, _ig_int,
- DEFERRED_MARK,
- apify_profile,
- bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready,
- bd_snapshot_progress, depth_refusal,
- ig_handle, public_source, pull_profile, select_post_groups, top_up_views,
-)
-
-
-# ---------------------------------------------------------------------------------------------
-# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23)
-# ---------------------------------------------------------------------------------------------
-# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's
-# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns
-# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND
-# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower
-# Georgia/Florida floral-design company — Fisch Floral's actual market.
-#
-# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE:
-#
-# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/`
-# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is
-# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the
-# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other.
-# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104,
-# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH,
-# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still
-# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a
-# snapshot that is not ready when it looks.
-# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery
-# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time`
-# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is
-# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the
-# next run collects it. A `partial` that says "still building, the next run collects it" is
-# the honest report of exactly what happened.
-# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and
-# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into
-# a wider profile set would silently multiply the enrichment bill AND pollute the
-# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich.
-# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important
-# test died on it.
-#
-# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE
-# the API returns a number, `price: 0` means "not priced" rather than "free", and
-# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to
-# report. Every estimate below is derived from the PRICING PAGE and says so.
-
-#: The table a discovery run writes when the tenant has no Instagram profile database yet — the
-#: FIRST one, never a second (`discover_default_table` elects an existing one before this is
-#: reached). Upsert by handle, so re-finding a profile is free.
-#:
-#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG
-#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said
-#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`,
-#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept
-#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it
-#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on
-#: any tenant who renamed the table, which is exactly the mismatch the owner hit.
-#:
-#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three
-#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the
-#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key
-#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant
-#: would keep working; there simply is not one.
-#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration
-#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family.
-DISCOVER_TABLE = "ut_ig_profile"
-#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels,
-#: so changing this can rename nothing that already exists. One constant because it was three
-#: copies of the same string literal, and a default spelled three times is one edit from drifting.
-DISCOVER_LABEL = "IG profile"
-#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at
-#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed
-#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real
-#: headroom. ⚠ If the row size grows, this number is the thing to re-derive.
-BD_MAX_RECORDS = 500
-#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above).
-BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120)
-BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15)
-
-#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error
-#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact.
-BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes",
- "array_includes", "not_array_includes", "is_null", "is_not_null")
-#: Operators that take NO value (everything else requires one).
-BD_NULLARY_OPS = ("is_null", "is_not_null")
-
-#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED**
-#: by the API and is therefore absent rather than offered-and-broken).
-#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY
-#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier
-#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the
-#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by
-#: accident; adding them back is a decision, not a typo.
-BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography",
- "category_name", "business_category_name", "is_business_account",
- "is_professional_account", "is_verified", "account", "full_name",
- "external_url", "bio_hashtags", "post_hashtags", "profile_url",
- "related_accounts", "profile_name", "id", "fbid", "highlights_count")
-#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste:
-#: these are the fields seen carrying values on real CORPUS rows. `category_name` and
-#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter
-#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist".
-BD_FILTER_LEAD = ("followers", "biography", "avg_engagement")
-
-#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count.
-#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still
-#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not
-#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a
-#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the
-#: scanner still has to walk, so they refine a search and cannot BE one.
-BD_NARROWING_FIELDS = frozenset({
- "biography", "account", "full_name", "profile_name", "category_name",
- "business_category_name", "external_url", "bio_hashtags", "post_hashtags",
- "related_accounts", "id", "fbid", "profile_url"})
-#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the
-#: whole corpus, which is breadth wearing a condition's clothes.
-BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"})
-#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed
-#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60%
-#: populated: below it a filter misses more corpus than it matches, and the surface should
-#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%,
-#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never
-#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked.
-#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding
-#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true.
-BD_POPULATED_FIELDS = frozenset({
- "account", "followers", "following", "posts_count", "avg_engagement", "biography",
- "external_url", "is_business_account", "is_professional_account", "is_verified",
- "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"})
-BD_MIN_NARROWING = 1
-
-# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ──────────────────
-# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY
-# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what
-# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner:
-# *"look at each damn schema and only provide toggles operator that make sense"*.
-#
-# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx`
-# defended raw names on the grounds that a refusal names the field exactly as the row does — and
-# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI
-# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio".
-BD_FIELD_LABELS = {
- "followers": "Followers", "following": "Following", "posts_count": "Post count",
- "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category",
- "business_category_name": "Business category", "is_business_account": "Business account",
- "is_professional_account": "Professional account", "is_verified": "Verified",
- "account": "Handle", "full_name": "Name", "profile_name": "Profile name",
- "external_url": "Link in bio", "profile_url": "Profile link",
- "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts",
- "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID",
- "highlights_count": "Story highlight count",
-}
-#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident;
-#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson).
-BD_FIELD_HINTS = {
- "account": "The @username",
- "avg_engagement": "Likes and comments as a share of followers",
- "id": "Instagram's internal number for the account — for matching a list you already have",
- "fbid": "The linked Facebook id — for matching a list you already have",
- "profile_name": "The name shown above the bio, when it differs from the account name",
- "related_accounts": "Accounts Instagram suggests alongside this one",
-}
-#: The KIND decides the comparisons offered and the control drawn for the value.
-BD_FIELD_KINDS = {
- **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement",
- "highlights_count")},
- **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")},
- **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")},
- **{n: "choice" for n in ("category_name", "business_category_name")},
- **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url",
- "profile_url", "id", "fbid")},
-}
-#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of
-#: each list is what `default_operator` must agree with.
-BD_OPS_BY_KIND = {
- # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second
- # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one
- # idea — and the second one is the one written in vendor grammar.
- "text": ("includes", "not_includes", "=", "is_not_null", "is_null"),
- "choice": ("=", "!=", "is_not_null", "is_null"),
- "number": (">=", "<=", "=", ">", "<"),
- # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean
- # is `=` with the other value, written the confusing way.
- "boolean": ("=",),
- "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"),
-}
-#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire
-#: (it is what the API accepts); nobody has to read it.
-BD_OP_LABELS = {
- "includes": "contains", "not_includes": "does not contain",
- "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of",
- ">=": "at least", "<=": "at most", ">": "more than", "<": "less than",
- "array_includes": "has any of", "not_array_includes": "does not have",
- "is_null": "is empty", "is_not_null": "is not empty",
-}
-#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into.
-BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}]
-
-#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many
-#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start`
-#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing
-#: the whole search to "match any" (which would also drag Followers into the union — the hole the
-#: OR guard closed the same day).
-#:
-#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not
-#: contain floral OR does not contain flower" matches nearly every account alive — a negative over
-#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a
-#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs
-#: them enough to write the AND branch.
-BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"})
-MAX_PREDICATE_VALUES = 12
-
-
-def field_label(name):
- """The human name for a searchable field — used by the SURFACE and by every REFUSAL."""
- return BD_FIELD_LABELS.get(name) or str(name or "that field")
-
-
-def field_kind(name):
- return BD_FIELD_KINDS.get(name, "text")
-
-
-def ops_for(name):
- """The comparisons this field may be asked. A tuple, in offer order."""
- return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"])
-
-#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing
-#: law, because the two drifted apart and the drift cost a user their first automation.
-#:
-#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no
-#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its
-#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS`
-#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door
-#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in —
-#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one
-#: sentence of drift, and nothing red anywhere at any point.
-#:
-#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field
-#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks
-#: one itself — a client-side default is a second copy of this table, free to disagree with the
-#: guard that judges it, which is exactly what happened.
-BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"})
-BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"})
-#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a
-#: worse question than an equality, and both narrow.
-BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"})
-
-
-def default_operator(name):
- """The operator a NEW condition on `name` starts with.
-
- Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from
- saveable rather than two (change the comparison, then type a value) — with the second step
- being one the surface never told anyone to take.
- """
- kind = field_kind(name)
- if kind == "tags":
- return "array_includes"
- if kind in ("boolean", "choice"):
- # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through
- # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so
- # the default would have been an operator the same module refuses to offer — the exact
- # default-versus-guard split this function was written to close, reintroduced one commit
- # later by adding a kind. Asserted per field in `verify_automation.py`.
- return "="
- if kind == "number":
- # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person
- # reaching for Followers means, and the guard's sentence is the honest answer when it is
- # the ONLY condition present.
- return ">="
- return "=" if name in BD_EXACT_FIELDS else "includes"
-
-
-def predicate_narrows(p):
- """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else."""
- return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS
- and str((p or {}).get("operator") or "") in BD_NARROWING_OPS)
-
-
-def narrowing_refusal(preds, operator="and"):
- """C4's server law: the sentence when a predicate set does not narrow, else ''. One function
- so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law)
- refuse in the same words.
-
- ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY
- predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a
- UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR
- followers >= 10000` saved clean and asked the vendor for every account over ten thousand
- followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop.
- A money wall that a dropdown three rows above it can walk through is not a wall.
-
- So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows
- either way — `all([])` is True, which would have made "no conditions at all" the widest legal
- query of the lot.
- """
- preds = list(preds or [])
- if str(operator or "").lower() == "or":
- if preds and all(predicate_narrows(p) for p in preds):
- return ""
- return ("with Match set to any, every condition has to describe the account itself "
- "(Bio, Handle, Name, a hashtag). Matching any means the results are added "
- "together, so one follower or yes/no condition widens the whole search. Narrow "
- "every condition, or set Match to all")
- if any(predicate_narrows(p) for p in preds):
- return ""
- return ("add a condition that describes the account itself — Bio contains, Handle is, a "
- "hashtag. Follower counts and yes/no conditions alone match too much of Instagram "
- "to search")
-
-
-def filter_meta():
- """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]`
- over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent,
- R3).
-
- `defaultOperator` rides here rather than being a client constant for the reason written over
- `default_operator`: the client's own default was `is_not_null` for every field, which the
- narrowing guard refuses on every field.
- """
- return [{"name": n,
- "label": field_label(n),
- "hint": BD_FIELD_HINTS.get(n, ""),
- "kind": field_kind(n),
- "populated": n in BD_POPULATED_FIELDS,
- "narrowing": n in BD_NARROWING_FIELDS,
- "defaultOperator": default_operator(n),
- # The comparisons THIS field may be asked, already in plain English and already in
- # offer order. A client that filtered a global list by field kind would be a second
- # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale.
- "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o),
- "nullary": o in BD_NULLARY_OPS,
- # `in`/`is any of` and the tag operators take a LIST — the control
- # has to know that, and deriving it from the token in the client is
- # the same second-copy mistake one level down.
- "multi": o in BD_MULTI_VALUE_OPS}
- for o in ops_for(n)],
- "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []}
- for n in BD_FILTER_FIELDS]
-
-#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET.
-#:
-#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what
-#: "unified across tenant" means and what makes the pooled master series joinable at all — two
-#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query.
-#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET
-#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat.
-#:
-#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in
-#: `ut_ig_snapshots`, which the metric fields already read. ONE STORE FOR ONE SERIES: no per-record
-#: `json` history column, no second copy of a number that already has a home.
-#:
-#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION,
-#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields
-#: available to us from Bright Data to be pre-set Fields for us and populated."*
-#:
-#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule
-#: and the next reader will otherwise re-derive it and prune these columns back. It said: a
-#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on
-#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles,
-#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is
-#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were
-#: captured into the snapshots and were NOT preset columns.
-#:
-#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be
-#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the
-#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus;
-#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does
-#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and
-#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a
-#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make.
-#: Source data retains every field the provider returns. The only separate decision is whether to
-#: request the paid full Comments dataset (`commentMetrics`, default OFF).
-#:
-#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is
-#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`).
-#:
-#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person
-#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and
-#: written down here rather than inferred from a runner's module name, because the day a second
-#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the
-#: profile simply appears twice.
-#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it.
-#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals
-#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as
-#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names
-#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things,
-#: which is exactly why both are written down instead of inferred.
-PROFILE_SOURCE_IG = "instagram"
-
-PLATFORM_INSTAGRAM = "Instagram"
-PLATFORM_TIKTOK = "TikTok" # D-9's runner will stamp this; nothing writes it yet
-PLATFORM_FACEBOOK = "Facebook"
-PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK)
-
-#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT.
-#:
-#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as
-#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring
-#: `followers` as `int` gives NEW tables a schema every table already in production does not have
-#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument
-#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason
-#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format?
-#: change this."*
-#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without
-#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it
-#: would do it silently, because a merged-by-key field list produces no error when two tables
-#: disagree about what a column IS.
-#:
-#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor
-#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this
-#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction
-#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the
-#: mapping and in the migration both. See `_pct100`.
-PRESET_PROFILE_FIELDS = [
- # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now.
- # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different
- # people as often as not, so the dedup key is (platform, handle) and never handle alone
- # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung
- # answered* (`field_def("source", "Read via")`), and one word meaning two things across two
- # tables in one product is the drift this whole list exists to prevent.
- field_def("platform", "Platform"),
- # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING.
- # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be
- # REPLACED or made as the Handle. in any case the user can add their own record right and edit
- # those records."* Two declarations, one field, and they are the same sentence read twice:
- #
- # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT
- # this the primary is an accident of creation order. It is a DECLARATION, not a
- # reorder: the stored field order is left alone (saved view orders keep working)
- # and the client pins the column to position 0 wherever it sits. That is why this
- # fixes tables that already exist without moving a single stored field.
- # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to
- # (`profile_field_key` step 2), which is what turns "I added a profile to my
- # database, how do I enrich it?" from an unanswerable question into a step. It
- # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle
- # all normalise to the bare handle) and routes a typed value to the SHARED
- # definition row rather than one user's overlay (C3-A1) — which is precisely what
- # makes the owner's second clause true: a record you add BY HAND is a record the
- # engine can read and enrich.
- #
- # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag
- # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on
- # anything but `text`.
- field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}),
- field_def("profile_url", "Profile", "url"),
- field_def("full_name", "Name"), field_def("followers", "Followers", "int"),
- field_def("following", "Following", "int"),
- field_def("avg_engagement", "Avg engagement", "pct"),
- field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"),
- field_def("verified", "Verified", "checkbox"), field_def("category", "Category"),
- # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields
- # MEASURED ≥60% populated (see the note above).
- field_def("posts_count", "Post count", "int"),
- field_def("highlights_count", "Story highlight count", "int"),
- field_def("is_business", "Business account", "checkbox"),
- field_def("is_professional", "Professional account", "checkbox"),
- # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would
- # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it.
- field_def("ig_id", "Instagram id"),
- # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction
- # (see the block comment above for what that reverses). Types are HONEST from birth, which
- # costs nothing here: no table has ever carried these columns, so there is no stored `text`
- # definition for the migration to convert — R3's conversion rule applies to the columns that
- # already exist, and these are new everywhere.
- field_def("business_category", "Business category"),
- field_def("is_private", "Private account", "checkbox"),
- field_def("bio_hashtags", "Bio hashtags"),
- field_def("pronouns", "Pronouns"),
- field_def("profile_name", "Profile name"),
- field_def("is_joined_recently", "Joined recently", "checkbox"),
- field_def("has_channel", "Has channel", "checkbox"),
- field_def("partner_id", "Partner id"),
- field_def("external_url_title", "Link title"),
- # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity.
- field_def("fbid", "Facebook id"),
- field_def("related_accounts", "Related accounts"),
- field_def("country_code", "Country"),
- field_def("source_payload", "Source data", "json"),
- # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT.
- #
- # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is
- # linked to the profile automatically… and this rollup needs to have formula that we can use
- # to calculate things like average Views over last N posts."* Both halves are these five
- # columns, and they are PRESETS rather than something a person assembles, because "linked
- # automatically" is the instruction — a relation you have to wire up by hand is the feature
- # not existing.
- #
- # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged
- # column and then the pinned one, both of which are `handle` on every preset table — so this
- # links correctly on a database where the handle column was renamed, and on one where the
- # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject
- # [[gate-answers-the-wrong-question]] warns about, one layer down.
- # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total.
- # Two count-like columns with the same label would be unreadable. Caught by the label-collision
- # gate rather than on screen, which is what that gate is for.
- field_def("posts_link", "Post rows", "link",
- description="Post records linked to this profile.",
- link={"table": IG_POSTS_TABLE, "on": "influencer_key"}),
- field_def("profile_snapshots_link", "Profile history", "link",
- description="Profile snapshots linked to this profile.",
- link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}),
- field_def("post_snapshots_link", "Post measurement rows", "link",
- description="Post engagement measurements linked to this profile.",
- link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}),
- field_def("comments_link", "Comment rows", "link",
- description="Comment records linked to this profile.",
- link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}),
- # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup
- # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further
- # out in `ut_ig_post_snapshots`.
- # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be
- # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30
- # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant
- # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a
- # `limit`, and this is why — "the last 12" has to name what makes one post later than another.
- # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours
- # because its input was an account-grain constant; with the input now a real per-reel
- # measurement the average means what its label says again.
- field_def("avg_views_12", "Avg views · last 12 posts", "rollup",
- rollup={"link": "posts_link", "field": "views", "fn": "average",
- "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
- "distinctBy": "shortcode"}),
- field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
- rollup={"link": "posts_link", "field": "plays", "fn": "average",
- "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
- "distinctBy": "shortcode"}),
- field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup",
- rollup={"link": "posts_link", "field": "likes", "fn": "average",
- "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
- "distinctBy": "shortcode"}),
- field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup",
- rollup={"link": "posts_link", "field": "comments", "fn": "average",
- "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
- "distinctBy": "shortcode"}),
- # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO
- # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after
- # a paid enrich. This counts the post rows actually captured — a different number and a true
- # one, which is why it gets its own column and its own label rather than quietly filling
- # `posts_count` with something that is not what that field means.
- field_def("posts_captured", "Posts captured", "rollup",
- rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}),
- field_def("profile_reads", "Profile reads", "rollup",
- rollup={"link": "profile_snapshots_link", "fn": "countall",
- "distinctBy": "snapshot_key"}),
- field_def("post_measurements_captured", "Post measurements captured", "rollup",
- rollup={"link": "post_snapshots_link", "fn": "countall",
- "distinctBy": "post_snapshot_key"}),
- field_def("comments_captured", "Comments captured", "rollup",
- rollup={"link": "comments_link", "fn": "countall",
- "distinctBy": "comment_key"}),
- # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING.
- #
- # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we
- # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one
- # thing a buyer most wants and the one thing the profile row cannot answer. The competitor
- # teardown found the same gap solved by GUESSING from bio words, two buckets deep
- # ([[janney-ai-teardown]]).
- #
- # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place
- # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the
- # post row, plus the vendor's fuller `location`/`location_details` inside the paid
- # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a
- # creator's own posts is a far better residency signal than a word in a bio, and it costs a
- # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1.
- #
- # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator
- # posting from twelve cities gets a low confidence rather than a confident wrong answer, and
- # `location_confidence` is the number a person filters on before trusting the guess. A single
- # geotagged post produces NO guess at all — n=1 dressed as a pattern is what
- # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain.
- field_def("location_guess", "Location (guess)"),
- field_def("location_confidence", "Location confidence", "pct"),
- # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never
- # enriched" and a stale one means "enriched in March", and no cell on the row can tell them
- # apart. It is the single field that turns the other fifteen from numbers into measurements.
- field_def("enriched_at", "Enriched at", "date"),
- # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE.
- # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the
- # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the
- # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying
- # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two,
- # rewritten each run, so a person reading the profile row can see the recent posts without a
- # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2.
-]
-#: The keys the preset set owns — derived, so a field added above cannot be forgotten here.
-PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS)
-
-#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile
-#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what
-#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the
-#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now
-#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to
-#: enforce a rule nobody declares any more.
-#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" —
-#: never as "field number zero".
-PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "")
-PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "")
-
-#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the
-#: alternative is two lists that describe the same columns and drift one label at a time, which is
-#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found
-#: them, who for, and a human's decision) rather than about the profile, so they are discovery's
-#: and not part of the cross-tenant set.
-CANDIDATE_FIELDS = [
- *PRESET_PROFILE_FIELDS,
- # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell.
- # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner
- # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a
- # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its
- # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what
- # changed is that "when did we first see this account" is a DAY, and a day is all anybody
- # asks it for. `migrate_ig_field_types()` converts the stored strings.
- field_def("found_count", "Times found", "int"),
- field_def("first_found", "First found", "date"),
- field_def("last_found", "Last found", "date"),
- # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it
- # describes, which is the first place anybody looks up what this column means. It read: *"the
- # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two
- # people discovering the same profile each get their own row, their own found_count and their
- # own review card."* Wave 26 · R4/R5 retired every clause of that.
- #
- # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two
- # people in one workspace who discover the same profile share ONE row, one `found_count` and
- # one history — because they are looking at one company's leads, not two private lists.
- # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it
- # first. It is no part of the dedup key, and a run must never branch on it.
- # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged,
- # and the one clause of the old note that was still true.
- field_def("created_by", "Found by"),
- # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have
- # a Field called stage to track the progress of the Automation per Record. We don't need
- # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage
- # column: every candidate carried BOTH a `stage_` select saying where it was AND a
- # boolean saying whether it was kept — two progress fields that could disagree, with no rule
- # about which one won. The stage field is the one progress column. Nothing replaces this.
-]
-
-
-def _profile_backlink_field(table_key, table_label, profile_key):
- """A deterministic reciprocal link from one canonical IG table to one profile database.
-
- The key includes the target table identity, so ten separate Profile databases can all point
- into the same canonical Posts/Comments/history tables without one relation overwriting the
- next. The join is derived in both directions and therefore needs no cross-table fan-out write.
- """
- digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10]
- return field_def(
- f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link",
- description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.",
- link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"},
- )
-
-
-def _profile_schema_for(bound_key):
- """Preset fields for an existing Profile table without inventing a second identity column."""
- out = []
- for field in PRESET_PROFILE_FIELDS:
- item = dict(field)
- if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
- item.pop("profile", None)
- out.append(item)
- return out
-
-
-def _locked_ig_field(field):
- """One canonical Instagram field with the immutable preset declaration attached."""
- item = dict(field)
- automation = dict(item.get("automation") or {})
- automation["preset"] = True
- item["automation"] = automation
- return item
-
-
-def _profile_binding(table):
- """The declared Profile identity field, falling back only to the canonical handle."""
- fields = list((table or {}).get("fields") or [])
- flagged = next((str(f.get("key") or "") for f in fields
- if isinstance(f.get("profile"), dict)), "")
- if flagged:
- return flagged
- return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else ""
-
-
-def _ig_schema_contract(rt, profile_tables=None):
- """The complete per-tenant Instagram graph contract, including every Profile backlink.
-
- The fixed child tables are shared. Profile tables are discovered structurally, so an older
- user-named target and the built-in candidate database receive the same preset columns, Links,
- Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted.
- """
- tables = ut_all(rt)
- profile_keys = sorted(set(profile_tables if profile_tables is not None
- else _ig_profile_tables(rt)))
- backlinks, wanted, drops = [], {}, {
- IG_SNAPSHOTS_TABLE: {"post_hashtags"},
- # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today
- # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS
- # and fed by the `ig_post_views` capability. Leaving it here would have made the migration
- # delete, on every authenticated read, the column the same release just added — the
- # drop set and the field list are two halves of ONE contract and must move together.
- }
- for table_key in profile_keys:
- table = tables.get(table_key) or {}
- bound = _profile_binding(table)
- if not bound:
- continue
- label = str(table.get("label") or table_key)
- backlinks.append(_profile_backlink_field(table_key, label, bound))
- schema = _profile_schema_for(bound)
- # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram
- # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT
- # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION.
- #
- # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every
- # contract key it owns — `rollup` included, by its own note — so a column converted beside
- # the contract is re-typed back to `int` on the next reconcile, silently, hours later.
- # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the
- # schema instead of two that disagree.
- #
- # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a
- # column only where this tenant's own rows show the fold already equals the stored value —
- # measured, because a named list would have blanked 221 live cells on nurilab alone
- # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts
- # fewer columns and loses nothing; as its history fills in, later passes convert more.
- derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {}))
- wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f
- for f in schema]
- # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
- # to disagree, so it is retired by key just like the old nested Posts JSON.
- drops[table_key] = {"posts", "post_hashtags", "tracked"}
- wanted.update({
- IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
- IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
- IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks],
- IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks],
- })
- return wanted, drops
-
-
-# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use
-# the scalar strings their renderers expect; engagement is the exception because Bright Data's
-# 0-1 fraction becomes this product's 0-100 percentage.
-_IG_RETYPE_CONVERTERS = {
- (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100,
-}
-
-
-def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
- """Repair machine-owned IG field declarations in one coalesced store update.
-
- `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not
- sufficient for a canonical relation: a stale `link.table`, rollup function, or type would
- survive forever. This pass overwrites the contract keys for the fields we own, preserves
- unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
- """
- drop_by_table = drop_by_table or {}
- wanted_by_table = {
- key: [_locked_ig_field(f) for f in fields]
- for key, fields in wanted_by_table.items()
- }
- changed = False
-
- def _up(cur):
- nonlocal changed
- cur = cur if isinstance(cur, dict) else {}
- for table_key, wanted_fields in wanted_by_table.items():
- table = cur.get(table_key)
- if table is None:
- continue
- wanted = {str(f.get("key")): dict(f) for f in wanted_fields}
- drops = set(drop_by_table.get(table_key) or ())
- fields, seen = [], set()
- drops = _guarded_drops(table, drops)
- for stored in table.get("fields") or []:
- key = str(stored.get("key") or "")
- if key in drops:
- changed = True
- continue
- desired = wanted.get(key)
- if desired is None:
- fields.append(stored)
- continue
- # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A
- # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten
- # from the shipped contract, `rollup` included, so without this the newly
- # unlocked "edit a preset rollup" would save, render, recompute and then revert
- # at the next enrichment run — an edit that looks like it worked and undoes
- # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp.
- if _ut().user_edited(stored):
- fields.append(stored)
- seen.add(key)
- continue
- repaired = dict(stored)
- for dkey, value in desired.items():
- if dkey != "automation":
- repaired[dkey] = value
- automation = dict(stored.get("automation") or {})
- automation.update(desired.get("automation") or {})
- repaired["automation"] = automation
- # These bags define behaviour, not decoration. If the desired field does not use
- # one, a stale bag from an earlier type must not remain attached to it.
- for bag in ("link", "rollup", "profile", "pinned"):
- if bag not in desired:
- repaired.pop(bag, None)
- if desired.get("type") not in ("select", "multiselect"):
- repaired.pop("options", None)
- converter = _IG_RETYPE_CONVERTERS.get((table_key, key))
- if (converter and stored.get("type") in ("text", "", None)
- and repaired.get("type") != stored.get("type")):
- for row in (table.get("rows") or {}).values():
- if str(row.get(key) or "").strip():
- converted = converter(row.get(key))
- if converted:
- row[key] = converted
- changed = True
- if repaired != stored:
- changed = True
- fields.append(repaired)
- seen.add(key)
- for key, desired in wanted.items():
- if key not in seen and not any(str(f.get("key") or "") == key for f in fields):
- fields.append(desired)
- changed = True
- if drops:
- for row in (table.get("rows") or {}).values():
- for key in drops:
- if key in row:
- row.pop(key, None)
- changed = True
- table["fields"] = fields
- return cur
-
- # Avoid a commit when the declarations are already exact.
- snapshot = ut_all(rt)
- needs = False
- for table_key, wanted_fields in wanted_by_table.items():
- if table_key not in snapshot:
- continue
- table = snapshot.get(table_key) or {}
- by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
- # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip
- # the commit when nothing would change; a spared `tracked` column left in the drop set
- # answers "yes, work to do" on every single call, forever, and the schema pass would
- # rewrite the table on every authenticated read without ever changing a byte.
- drops = _guarded_drops(table, drop_by_table.get(table_key) or ())
- if drops & set(by_key):
- needs = True
- break
- for desired in wanted_fields:
- stored = by_key.get(str(desired.get("key") or ""))
- mismatch = stored is None
- if stored is not None:
- for key, value in desired.items():
- if key == "automation":
- if any((stored.get("automation") or {}).get(k) != v
- for k, v in value.items()):
- mismatch = True
- break
- elif stored.get(key) != value:
- mismatch = True
- break
- if not mismatch:
- stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired)
- mismatch = any(k in stored for k in stale_bags)
- if (not mismatch and desired.get("type") not in ("select", "multiselect")
- and "options" in stored):
- mismatch = True
- if mismatch:
- needs = True
- break
- if needs:
- break
- if needs:
- rt.update(UT_STORE_KEY, _up, flush="sync")
- return changed
-
-
-def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""):
- """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys.
-
- Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new
- Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect
- each Profile database to the same Posts, Comments, profile-history, and post-history stores.
- """
- profile = ut_get(rt, profile_table) if profile_table else None
- bound = str(profile_field or "").strip()
- if profile is not None and not bound:
- bound = next((str(f.get("key") or "") for f in profile.get("fields") or []
- if isinstance(f.get("profile"), dict)), "")
- profile_label = str((profile or {}).get("label") or profile_table)
- backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \
- if profile_table and profile is not None and bound else []
-
- graph = {
- IG_SNAPSHOTS_TABLE: ("IG snapshots", [*SNAPSHOT_FIELDS, *backlink]),
- IG_POSTS_TABLE: ("IG posts", [*POST_FIELDS, *backlink]),
- IG_POST_SNAPSHOTS_TABLE: ("IG post snapshots", [*POST_SNAPSHOT_FIELDS, *backlink]),
- IG_COMMENTS_TABLE: ("IG comments", [*COMMENT_FIELDS, *backlink]),
- }
- keys = {}
- for key, (label, fields) in graph.items():
- keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
- record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
-
- if profile_table and profile is not None and bound:
- profile_fields = _profile_schema_for(bound)
- ut_ensure(rt, profile_label or profile_table, profile_fields, username,
- key=profile_table, flow_tag=flow_tag, lock_fields=True)
- # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older
- # Profile database and IG candidates on the same universal schema while all of them point to
- # the same four children. Existing flow provenance is merged, never replaced.
- wanted, drops = _ig_schema_contract(rt)
- _reconcile_ig_graph_fields(rt, wanted, drops)
- return keys
-
-
-# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ───────────────
-#
-# Two changes land on data that already exists, and neither is optional once the field defs move:
-#
-# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a
-# `date` column and a "20872" string in an `int` column are the split-schema the old
-# "types are text on purpose" note correctly feared. Re-typing without converting is worse
-# than not re-typing.
-# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as
-# `(handle, created_by)` DUPLICATES must be merged rather than left to collide.
-#
-# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a
-# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would
-# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a
-# column is converted only while its STORED definition still says `text`; the moment it says `pct`
-# the work is done and re-running is a no-op. That makes the migration safe to call on every boot,
-# which is what it is for.
-#
-# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice.
-# `observed_categories` named its two source tables and went silently empty the day wave 25 let a
-# user point an automation somewhere else. Naming tables here would leave exactly those user-named
-# databases on the old schema, which is the same bug wearing a migration's clothes.
-
-#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset
-#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told.
-MIGRATE_MIN_PRESET_FIELDS = 3
-#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/'').
-_MIGRATE_CONVERT = {
- "first_found": _day, "last_found": _day, "enriched_at": _day,
- "avg_engagement": _pct100,
-}
-
-
-def _ig_profile_tables(rt):
- """Every `ut_*` table in this tenant that looks like an Instagram profile table."""
- out = []
- for key, tbl in sorted((ut_all(rt) or {}).items()):
- keys = {f.get("key") for f in (tbl or {}).get("fields") or []}
- if "handle" in keys and len(keys & set(PRESET_PROFILE_KEYS)) >= MIGRATE_MIN_PRESET_FIELDS:
- out.append(key)
- return out
-
-
-def discover_default_table(rt):
- """Which database should a discovery automation write to when nobody has said? (2026-08-10)
-
- Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there.
- We only need ONE IG profile so it's not confusing."*
-
- ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_
- leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the
- label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own
- note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is
- a constant rather than a question, a tenant that already had an Instagram profile database got
- a SECOND, empty one the first time a discovery automation was saved without a target.
- MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside
- `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them
- pure confusion.
-
- So the default becomes a question asked of the tenant:
-
- exactly one profile database (excluding the fallback) -> that one
- several, exactly one of which holds rows -> the one in use
- several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE)
-
- ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and
- picking one silently would write a paid discovery run into a database the user did not name —
- worse than the empty table this exists to prevent. "" means "no opinion", and the caller's
- existing fallback stands, which is exactly today's behaviour for that case.
- ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset
- schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the
- empty table would keep re-electing it and nothing would ever change.
- """
- try:
- tables = ut_all(rt) or {}
- found = [k for k in _ig_profile_tables(rt) if k != DISCOVER_TABLE]
- except Exception: # noqa: BLE001
- return ""
- if len(found) == 1:
- return found[0]
- if len(found) > 1:
- used = [k for k in found if (tables.get(k) or {}).get("rows")]
- if len(used) == 1:
- return used[0]
- return ""
-
-
-def _apply_discover_default(rt, raw):
- """Fill in a discovery automation's target BEFORE the pure validator invents one.
-
- ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that
- turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure
- validator with a two-argument contract every gate and route depends on). Resolving only at the
- RUN sites would be cosmetic: the literal is written into the STORED config at save time, so
- `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted.
- `create`/`patch` are the one pair that both know the tenant and sit above that validator.
- ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save
- stored — is the user's choice and is never rewritten.
- """
- cfg = raw.get("config")
- if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip():
- return raw
- default = discover_default_table(rt)
- if not default:
- return raw
- return {**raw, "config": {**cfg, "targetTable": default}}
-
-
-def _vestigial_name_field(fields, rows):
- """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half.
-
- `user_tables.create()` mints EXACTLY ONE column on a hand-made database —
- `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the
- automation's columns after it, never reordering. So on every Instagram database somebody
- created before pointing an automation at it, column 1 is a `Name` nothing will ever write.
-
- ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative
- because this is the one branch in the migration that DESTROYS something. A column carrying any
- declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with
- a value in any row is somebody's data. Either way it survives as an ordinary column and merely
- stops being the locked primary, which the `pinned` half achieves on its own.
-
- ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted
- `custom__` (`buildOverlayField`) and every machine one comes from a `field_def`
- list, so nothing but `create()`'s default can hold this key. The label and type are checked
- anyway — a renamed or retyped column is a column someone touched on purpose.
- """
- f = next((f for f in fields if f.get("key") == "name"), None)
- if f is None or f.get("label") != "Name" or f.get("type") != "text":
- return None
- if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")):
- return None
- if any(str(r.get("name") or "").strip() for r in rows.values()):
- return None
- return f
-
-
-def _merge_candidates(rows):
- """C3's merge rule, applied to rows now colliding on `(platform, handle)`.
-
- Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell
- takes the first non-blank, preferring the most recently enriched row.
-
- ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and
- board cards; minting a new one would orphan all of them, so a merge picks a survivor rather
- than creating one.
- """
- groups, keepers = {}, {}
- for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))):
- r = rows[rid]
- h = str(r.get("handle") or "").strip()
- if not h:
- # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a
- # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a
- # manual handle in my 'Beauty influencer leads' database, after an automation run, it
- # gets deleted?"*
- #
- # A row you add by hand is born blank and stays blank until you type into it, so the
- # window is not narrow — it is every row, from creation until the cell is committed.
- # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S
- # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was
- # dropped even after it looked filled in on screen.
- #
- # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is
- # not a candidate — it is somebody's row. It cannot collide with anything (there is
- # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A
- # merge pass that deletes what it cannot classify is not a merge, it is a filter.
- keepers[rid] = r
- continue
- groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h),
- []).append((rid, r))
- out, merged = dict(keepers), 0
- for _key, members in groups.items():
- if len(members) == 1:
- out[members[0][0]] = members[0][1]
- continue
- merged += len(members) - 1
- # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement.
- ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True)
- keep_id = sorted(rid for rid, _r in members)[0]
- acc = {}
- for _rid, r in ordered:
- for k, v in r.items():
- if k not in acc and str(v or "").strip():
- acc[k] = v
- firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")]
- lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")]
- if firsts:
- acc["first_found"] = min(firsts)
- if lasts:
- acc["last_found"] = max(lasts)
- total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members)
- if total:
- acc["found_count"] = str(total)
- out[keep_id] = acc
- return out, merged
-
-
-#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER
-#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites
-#: *"we already have a Field called stage… we don't need another checkbox for this"* — and
-#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still
-#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing
-#: writes, nothing reads, and anybody can still click.
-#:
-#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the
-#: record of who approved what before the wave, or leave them and let the next hand-made `tracked`
-#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving
-#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT
-#: as a column nobody declared is the worse half.
-TRACKED_KEY = "tracked"
-
-
-def _retired_tracked_field(fields):
- """The retired `tracked` column when it is still the MACHINE's own, else None.
-
- ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that
- DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a
- column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED,
- or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as
- an ordinary column and merely stops being fed by anything, which was already true.
-
- ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are
- dropped unguarded: both are machine-derived cells with no history of anyone editing them.
- `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test.
- """
- f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None)
- if f is None or f.get("type") not in ("checkbox", "", None):
- return None
- if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")):
- return None
- return f
-
-
-def _guarded_drops(table, drops):
- """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81).
-
- ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check.
- `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks
- `_retired_tracked_field` whether the column is still the machine's own, and the schema
- reconciliation, which drops by KEY with no question asked. Guarding only the first was
- decorative — the schema pass runs immediately afterwards on the same table and deleted the
- retyped column the guard had just spared. Two paths to one deletion with one of them checked
- is the shape where a control looks present and enforces nothing
- ([[defects-that-mask-each-other]]).
- """
- drops = set(drops or ())
- if TRACKED_KEY in drops:
- stored = [f for f in (table or {}).get("fields") or []
- if str(f.get("key") or "") == TRACKED_KEY]
- if stored and _retired_tracked_field(stored) is None:
- drops.discard(TRACKED_KEY)
- return drops
-
-
-def _orphan_tracked_rows(fields, rows):
- """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject.
-
- ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE
- BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose
- declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every
- surface that renders from `fields` — and a migration that only looks at `fields` finds nothing
- to do and `continue`s past the table. Storage, export and any future re-declaration of the key
- all still see them.
- """
- if any(str(f.get("key") or "") == TRACKED_KEY for f in fields):
- return [] # the column above owns its own cells
- return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})]
-
-
-def backfill_corpus_snapshots(rt, log=print):
- """The BACKFILL half: every profile row holding a measurement with no series behind it gets
- its corpus observation. Returns how many were written.
-
- ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows
- that arrived before the forward fix existed. A second row-shape here would be two ideas of
- what a corpus observation is, and they would disagree on the next promoted column.
-
- ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is
- deliberate. A profile that HAS been enriched already owns real observations; synthesising a
- corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and
- replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind
- it, which is the only case where a corpus observation can only improve matters.
-
- ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on
- every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has
- a series, so it stops matching and the scan costs a dict comparison.
- """
- snaps = ut_get(rt, IG_SNAPSHOTS_TABLE)
- if snaps is None:
- return 0 # no series store — see `append_ig_snapshots` on why we never mint one
- known = {str((r or {}).get("influencer_key") or "").strip().lower()
- for r in (snaps.get("rows") or {}).values()}
- known.discard("")
- #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived
- #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a
- #: named list goes silently empty the day the schema moves).
- measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS
- if fd["key"] not in _SNAPSHOT_OWN_KEYS
- and fd.get("type") in ("int", "pct"))
- incoming = []
- for table_key in _ig_profile_tables(rt):
- for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values():
- handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower()
- if not handle or handle in known:
- continue
- if not any(str((row or {}).get(k) or "").strip() for k in measured):
- continue # nothing was ever read about this row — there is no observation to record
- built = corpus_snapshot_row(row)
- if built:
- incoming.append(built)
- known.add(handle) # two profile tables holding one handle write ONE observation
- return append_ig_snapshots(rt, incoming, log=log)
-
-
-def backfill_post_snapshot_grain(rt, log=print):
- """Copy `posted_at`/`type` off the POST row onto every measurement missing them.
-
- ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's
- is not, so a pass that has already run matches nothing and a post that genuinely has no date
- is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration
- around it.
-
- ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN
- THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own
- record of the moment is the one to keep — a fact table that lets a dimension rewrite its
- history is not a fact table.
- """
- psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE)
- posts = ut_get(rt, IG_POSTS_TABLE)
- if psnaps is None or posts is None:
- return 0
- have = {f.get("key") for f in (psnaps.get("fields") or [])}
- carried = [k for k in ("posted_at", "type") if k in have]
- if not carried:
- return 0 # the columns are not declared yet — `ut_ensure` adds them first
- by_shortcode = {}
- for row in (posts.get("rows") or {}).values():
- code = str((row or {}).get("shortcode") or "").strip()
- if code:
- by_shortcode[code] = row or {}
- changes = {}
- for rid, row in (psnaps.get("rows") or {}).items():
- post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip())
- if not post:
- continue
- for key in carried:
- want = str(post.get(key) or "").strip()
- if want and not str((row or {}).get(key) or "").strip():
- changes.setdefault(str(rid), {})[key] = _s(want, 200)
- if not changes:
- return 0
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- table = cur.get(IG_POST_SNAPSHOTS_TABLE)
- if table is not None:
- for rid, values in changes.items():
- table.setdefault("rows", {}).setdefault(rid, {}).update(values)
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind")
- return len(changes)
-
-
-#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work
-#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS.
-#:
-#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same
-#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset
-#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already
-#: relational (the 12 links and rollups), or have no observation behind them at all
-#: (`location_guess`/`location_confidence`, which WE infer rather than read).
-#:
-#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST.
-#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells —
-#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those
-#: values were never recorded as observations (the hole this same wave closed going forward, in its
-#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three
-#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a
-#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author
-#: assumed.
-#:
-#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that
-#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`.
-#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not
-#: safe today is not refused forever, it is simply not converted yet.
-IG_DERIVABLE_KEYS = tuple(
- fd["key"] for fd in PRESET_PROFILE_FIELDS
- if fd.get("type") not in ("link", "rollup")
- and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found",
- "found_count", "created_by")
- and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS})
-
-
-def _derived_profile_bag(key):
- """The rollup that replaces the mapper's write of ONE profile column.
-
- ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's
- value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an
- expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the
- scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest
- observation that ACTUALLY READ this field", which is what the materialised cell meant.
- MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the
- observation itself, which is why the guard below is a proof and not a hope.
- """
- return {"link": "profile_snapshots_link", "field": key, "fn": "latest",
- "sortBy": "pulled_at", "sortDir": "desc",
- "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"}
-
-
-def _derived_profile_columns(table, snapshots):
- """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table.
-
- = the ones already converted, PLUS the ones whose derived value equals the stored value on
- every row. Returns keys.
-
- ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with
- a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns —
- `rollup` included, by design — so a converted column that this function stopped naming would be
- re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written
- by the mapper. The edit would look applied and undo itself later, which is exactly
- [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore
- included unconditionally rather than re-proved: its cells ARE the derived values, so a proof
- would be comparing the fold against itself.
-
- ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody
- measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere
- else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest
- authority is the data in front of the migration.
- ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an
- oversight: there is nothing to lose, and leaving it materialised would mean the schema differs
- between two tenants for no reason a reader could discover.
- """
- fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []}
- rows = (table or {}).get("rows") or {}
- by_subject = {}
- for snap in ((snapshots or {}).get("rows") or {}).values():
- key = str((snap or {}).get("influencer_key") or "").strip().lower()
- if key:
- by_subject.setdefault(key, []).append(snap or {})
- # The engine's own ordering, once per subject — newest first, unrankable rows last.
- ordered = {}
- for subject, observations in by_subject.items():
- keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations]
- rankable = [(o, k) for o, k in keyed if k is not None]
- rankable.sort(key=lambda ok: ok[1], reverse=True)
- ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None]
- # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph
- # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose
- # link resolves to nothing — a blank cell wearing a configured column's clothes.
- if "profile_snapshots_link" not in fields:
- return []
- out = []
- for key in IG_DERIVABLE_KEYS:
- field = fields.get(key)
- if not field:
- continue
- if field.get("type") == "rollup":
- out.append(key) # already converted — see the note above on why
- continue
- declared = str(field.get("type") or "text")
- safe = True
- for row in rows.values():
- stored = str((row or {}).get(key) or "").strip()
- window = [o for o in ordered.get(
- str((row or {}).get("handle") or "").strip().lower(), [])
- if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here
- derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared)
- or "").strip()
- if stored != derived:
- safe = False
- break
- if safe:
- out.append(key)
- return out
-
-
-def _as_derived_field(field):
- """One preset scalar declaration → the same column declared as a rollup over the series."""
- key = str(field.get("key") or "")
- return {**{k: v for k, v in field.items() if k not in ("agg",)},
- "type": "rollup", "rollup": _derived_profile_bag(key)}
-
-
-def migrate_ig_tables(rt, log=print, only=""):
- """Bring every Instagram Profile and canonical child table onto one current contract.
-
- Returns counted changes rather than a reassuring line. Profile cells are converted before
- their type declarations change; then the graph reconciliation adds any missing preset
- Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes
- only retired machine fields.
-
- `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides
- the WRITE PATH, so a table is brought forward immediately before anything appends to it and no
- caller has to remember to run anything.
- """
- stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0,
- # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from
- # `stamped` because they answer different questions: that one is "how many ROWS got a
- # platform", these are "how many TABLES changed shape".
- "pinned": 0, "flagged": 0, "droppedName": 0,
- "droppedRecentPosts": 0, "droppedPostHashtags": 0,
- "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0}
- want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
- # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red
- # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
- # first run the table does not exist yet: with the detection bypassed it fell through to the
- # writer and stamped a stub table with empty fields, clobbering the creation that was the whole
- # point of the call (the owner stamp and the flow tag went with it). A table that does not
- # exist, or that is not an Instagram profile table, has nothing to migrate.
- detected = set(_ig_profile_tables(rt))
- targets = ([only] if only in detected else []) if only else sorted(detected)
- for table_key in targets:
- tbl = ut_get(rt, table_key) or {}
- fields = [dict(f) for f in tbl.get("fields") or []]
- rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()}
- # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched,
- # so the cell conversions below can never run twice on the same value.
- stale = [f for f in fields
- if f.get("key") in want and f.get("type") != want[f["key"]]
- and f.get("type") in ("text", "", None)]
- stale_keys = {f["key"] for f in stale}
- has_platform = any(f.get("key") == "platform" for f in fields)
- needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()]
- # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING.
- # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the
- # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in
- # production needs them stamped here, and that is the whole reason this branch exists.
- #
- # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule,
- # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which
- # is exactly why these are safe to run on every write where a value conversion would not be.
- pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \
- if PRESET_PINNED_KEY else None
- flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \
- if PRESET_FLAG_KEY else None
- # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write
- # doors, and a migration is not exempt from it. If somebody has already flagged another
- # column on this table, stamping the handle too would leave the table in a state its own
- # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would
- # silently bind to whichever came first in the field list. Leave it alone and pin only.
- other_profile = next((f for f in fields if f is not flag_f
- and isinstance(f.get("profile"), dict)), None)
- want_pin = pin_f is not None and pin_f.get("pinned") is not True
- want_flag = (flag_f is not None and other_profile is None
- and not isinstance(flag_f.get("profile"), dict))
- vestigial = _vestigial_name_field(fields, rows)
- retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
- if _retired_tracked_field(fields) is not None:
- retired.add(TRACKED_KEY)
- orphan_tracked = _orphan_tracked_rows(fields, rows)
- if (not stale_keys and has_platform and not needs_stamp
- and not want_pin and not want_flag and vestigial is None and not retired
- and not orphan_tracked):
- continue
- stats["tables"] += 1
- if want_pin:
- pin_f["pinned"] = True
- stats["pinned"] += 1
- if want_flag:
- flag_f["profile"] = {"source": PROFILE_SOURCE_IG}
- stats["flagged"] += 1
- if vestigial is not None:
- # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer
- # exists is invisible everywhere except the next export, where it reappears as a column
- # nobody declared. `_vestigial_name_field` has already proven every one of them blank.
- fields = [f for f in fields if f is not vestigial]
- for r in rows.values():
- r.pop("name", None)
- stats["droppedName"] += 1
- if retired:
- fields = [f for f in fields if f.get("key") not in retired]
- for r in rows.values():
- for key in retired:
- r.pop(key, None)
- stats["droppedRecentPosts"] += int("posts" in retired)
- stats["droppedPostHashtags"] += int("post_hashtags" in retired)
- stats["droppedTracked"] += int(TRACKED_KEY in retired)
- for rid in orphan_tracked:
- # D-74: a tick whose column was already gone. Counted apart from the column drop
- # because they answer different questions — "how many tables still declared it" and
- # "how many rows were still carrying it after the declaration went".
- rows[rid].pop(TRACKED_KEY, None)
- stats["droppedTrackedCells"] += 1
-
- for key in stale_keys:
- conv = _MIGRATE_CONVERT.get(key)
- if not conv:
- continue # int/checkbox already store the right text shape
- for r in rows.values():
- if str(r.get(key) or "").strip():
- new = conv(r[key])
- # "" from a converter means UNREADABLE, and the cell keeps its original text.
- # A migration that empties a cell is indistinguishable from one that moved it.
- if new:
- r[key], stats["converted"] = new, stats["converted"] + 1
- for f in stale:
- f["type"] = want[f["key"]]
- stats["retyped"] += 1
- for rid in needs_stamp:
- rows[rid]["platform"] = PLATFORM_INSTAGRAM
- stats["stamped"] += 1
- if not has_platform:
- fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields
- rows, merged = _merge_candidates(rows)
- stats["merged"] += merged
-
- def _up(cur, _f=fields, _r=rows):
- cur = cur or {}
- t = dict(cur.get(table_key) or {})
- t["fields"], t["rows"] = _f, _r
- cur[table_key] = t
- return cur
-
- rt.update(UT_STORE_KEY, _up)
- # The profile loop owns value conversions and deduplication. The schema pass owns the
- # declarations across ALL related databases and is safe only after those conversions, because
- # the stored old type is the idempotency key for unit-changing values such as engagement.
- wanted, drops = _ig_schema_contract(rt)
- stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
- # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT
- # IDEMPOTENT, which is the one property everything else here is built on.
- #
- # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the
- # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs
- # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our
- # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired
- # profile cell against an UNREPAIRED observation, finds them different, and declines to convert
- # a column that is in fact perfectly derivable. The next call then converts it — and
- # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that
- # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1.
- #
- # ⚠ The module already knew this shape one level up — the profile loop's own note says the
- # schema pass "is safe only after those conversions, because the stored old type is the
- # idempotency key". Same argument, one table further down.
- # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same
- # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is
- # what moves the data, and it has already run.
- wanted2, drops2 = _ig_schema_contract(rt)
- stats["schema"] = int(bool(stats["schema"]
- | int(_reconcile_ig_graph_fields(rt, wanted2, drops2))))
- # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose:
- # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running
- # it first would find nothing to fill and report a truthful zero about the wrong moment.
- # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which
- # is what makes them safe on a function that rides every `ut_ensure`.
- stats["series"] = backfill_corpus_snapshots(rt, log=log)
- stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log)
- if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]:
- log(f"[ig-migrate] {stats}")
- return stats
-
-
-# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ──────────────────
-# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their
-# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never
-# hidden (R5).
-#
-# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN:
-# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the
-# corpus barely populates produces a filter that returns nothing and looks exactly like
-# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about).
-# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The
-# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`,
-# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them
-# narrows under its own `default_operator`, asserted in the gate rather than assumed.
-#
-# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`,
-# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and
-# nothing else — a search that returns what you gave it, at full price. So the ranker reads only
-# the three that describe a KIND of account: shared words in the bio, a shared domain in the link,
-# and shared words in the name.
-SEED_SOURCES = ("view", "cohort")
-SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N]
-SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter
-SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared
-SEED_MIN_ROWS = 2 # one row has nothing to share WITH
-#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two
-#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every
-#: bio word in a two-row seed qualified as "shared", and the surface would have offered a
-#: characteristic derived from a single record as the thing those records have in common. A
-#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is
-#: the fabrication this module refuses everywhere else.
-SEED_MIN_ROWS_SHARING = 2
-#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor
-#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down.
-SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"),
- ("full_name", "full_name"))
-#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT —
-#: an aggressive list would silently drop a real signal, and the coverage bar already removes most
-#: noise. Anything ≤2 characters is dropped by length instead of by enumeration.
-SEED_STOPWORDS = frozenset({
- "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from",
- "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more",
- "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"})
-
-
-def _seed_tokens(text):
- """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars."""
- words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower())
- return {w for w in words if w not in SEED_STOPWORDS}
-
-
-def _seed_domain(url):
- """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a
- domain and that IS the shared characteristic; the path is the thing that differs."""
- raw = str(url or "").strip()
- if not raw:
- return ""
- host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower()
- return host[4:] if host.startswith("www.") else host
-
-
-def seed_predicates(rows):
- """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network.
-
- `basis` is what makes the offer honest rather than magic: how many rows were read, which
- characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads
- as weak instead of being presented with the same confidence as a strong one.
-
- ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED
- (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24
- default-versus-guard defect rebuilt — and it would be worse here, because the user did not
- type it: they would be told their own product's suggestion is invalid.
-
- ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage
- bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks
- plausible, returns zero rows, and reads as "no such accounts exist".
- """
- rows = [r for r in (rows or []) if isinstance(r, dict)]
- total = len(rows)
- basis = {"rows": total, "fields": [],
- # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature.
- # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1
- # preset set, so on our own seed rows it is usually simply absent — which the surface
- # must SAY, or a lane that adds nothing reads as a lane that failed.
- "related": {"tried": total,
- "found": sum(1 for r in rows
- if str(r.get("related_accounts") or "").strip())},
- "note": ""}
- if total < SEED_MIN_ROWS:
- basis["note"] = ("a seed needs at least two records — one record has nothing to share "
- "with anything")
- return [], basis
- ranked = []
- for row_key, vendor_field in SEED_FROM_ROW:
- counts = {}
- for r in rows:
- vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url"
- else _seed_tokens(r.get(row_key)))
- for v in vals:
- if v:
- counts[v] = counts.get(v, 0) + 1
- for value, n in counts.items():
- cov = n / total
- if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING:
- ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3),
- "rows": n})
- # Strongest first; ties broken by the LONGER value, which is the more specific one.
- ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"]))
- derived, seen_fields = [], set()
- for f in ranked:
- if len(derived) >= SEED_MAX_PREDICATES:
- break
- # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the
- # accounts carrying both words, which is a much smaller search than the seed implies.
- if f["name"] in seen_fields:
- continue
- p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]}
- if not predicate_narrows(p):
- continue # cannot be offered; it would be refused at the save door
- seen_fields.add(f["name"])
- derived.append(p)
- basis["fields"].append({"name": f["name"], "label": field_label(f["name"]),
- "value": f["value"], "coverage": f["coverage"]})
- if derived and narrowing_refusal(derived, "and"):
- # Belt AND braces: the per-predicate check above should make this unreachable, and it is
- # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest
- # empty rather than offering something the save door will reject.
- basis["note"] = ("the shared characteristics found are not specific enough to search on "
- "— add a condition of your own")
- return [], basis
- if not derived:
- basis["note"] = basis["note"] or (
- "these records share no bio words, link domain or name in common — nothing could be "
- "derived, so the conditions are yours to write")
- return derived, basis
-
-
-def seed_rows(rt, table_key, view_id=""):
- """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by
- `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]])."""
- t = ut_get(rt, str(table_key or ""))
- if t is None:
- return [], f"{table_key!r} is not a database in this workspace"
- rows = list((t.get("rows") or {}).values())
- if str(view_id or "").strip():
- # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a
- # different row set from the view it names would be describing a view nobody has.
- tree, _fields, problem = view_filter(rt, table_key, view_id)
- if problem:
- return [], problem
- rows = [r for r in rows if lane_match(tree, r)]
- return rows[:SEED_MAX_ROWS], ""
-
-
-def automation_tables(defn):
- """Every `ut_*` table THIS automation writes into — its config target plus every
- `create_record` action's table, deduped, order preserved.
-
- ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition.
- Split out rather than parameterised because the two questions are different: that one asks
- "where could a Category value be, anywhere in this tenant" and is deliberately generous; this
- one asks "which rows has THIS automation already found" and must not reach into a table it
- does not write, or a nightly search would start excluding another automation's finds.
- """
- keys, seen = [], set()
- cfg = (defn or {}).get("config") or {}
- for raw in [cfg.get("targetTable"),
- *(((a or {}).get("config") or {}).get("table")
- for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]:
- k = str(raw or "").strip()
- if k.startswith(UT_PREFIX) and k not in seen:
- seen.add(k)
- keys.append(k)
- return keys
-
-
-def already_found_handles(rt, defn, cap=None):
- """The handles this automation has already written, for the vendor-side exclusion (item 7).
-
- ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a
- profile has always been harmless — and never free: the vendor bills for every record it
- returns, so a nightly search over stable keywords pays again, in full, for the same accounts
- every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty
- scout re-found its whole result set on every run. This is the list that stops it, sent as one
- `not_in` the vendor evaluates BEFORE billing.
-
- ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would
- hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords
- are two questions, and one must not answer with "somebody already looked at that".
- """
- cap = BD_EXCLUDE_MAX if cap is None else int(cap)
- out, seen = [], set()
- for table_key in automation_tables(defn):
- for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values():
- h = str((row or {}).get("handle") or "").strip().lstrip("@").lower()
- if h and h not in seen:
- seen.add(h)
- out.append(h)
- if len(out) >= cap:
- return out
- return out
-
-
-def observed_category_tables(rt):
- """Every table a Category value could have been written into, DERIVED from the automations
- this tenant actually has — the default discovery table and the snapshot table are the FLOOR,
- not the list.
-
- ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared:
- `config.targetTable` (the discovery runner's own write) and any `create_record` action's
- `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker
- writes). Both are read, deduped, order preserved so the defaults come first.
- """
- keys, seen = [], set()
-
- def _add(k):
- k = str(k or "").strip()
- if k and k.startswith("ut_") and k not in seen:
- seen.add(k)
- keys.append(k)
-
- _add(DISCOVER_TABLE)
- _add("ut_ig_snapshots")
- try:
- for defn in (all_definitions(rt) or {}).values():
- cfg = (defn or {}).get("config") or {}
- _add(cfg.get("targetTable"))
- for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")):
- _add(((act or {}).get("config") or {}).get("table"))
- except Exception: # noqa: BLE001
- # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults —
- # the same posture as the master lane below: degrade to less vocabulary, never to an error.
- pass
- return keys
-
-
-def observed_categories(rt, limit=60):
- """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts
- free text.
-
- D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen
- (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that
- still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**:
- transcribe a published taxonomy and hope it lines up."* This is the first branch.
-
- ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the
- corpus does not use returns zero rows and looks exactly like an honest "no such accounts
- exist" — so a picker built from Instagram's own category list would mislead precisely when it
- looked most authoritative. Every option here has been observed on a real row, and each carries
- its COUNT so a value seen once reads differently from one seen forty times.
-
- ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a
- control that refused anything else would be a second, quieter version of the same lie.
- """
- seen = {}
-
- def _eat(rows, key):
- for r in rows or []:
- v = " ".join(str((r or {}).get(key) or "").split())[:60]
- if v:
- seen[v] = seen.get(v, 0) + 1
-
- # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped
- # green. The two constants below were the whole list, which was correct until **wave 25 R2
- # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a
- # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely
- # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows
- # while the tenant's two real target tables held 20 each, so this function honestly reported
- # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that
- # names its sources goes quietly empty the moment the product lets a user choose one.
- for key in observed_category_tables(rt):
- _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category")
- try:
- # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three
- # candidates still gets a vocabulary drawn from every profile the platform has captured.
- import ig_master
- if ig_master.configured():
- # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which
- # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers
- # None, so the master lane would have degraded to nothing INSIDE the except below and
- # this would have shipped as a silent no-op with the gate green. Verified against
- # `ig_master.series_for`, which reads the same bucket the same way.
- _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category")
- except Exception: # noqa: BLE001
- # A master that cannot be read costs the tenant its own values and nothing else.
- pass
- return [{"value": v, "count": n}
- for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]]
-
-
-def preset_plan(rt, table_key):
- """C1 / R2b: the preset set diffed against ONE database's CURRENT fields.
-
- `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}`
-
- ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns
- an automation will USE and which it will CREATE — and the only thing that can answer it is
- whatever holds both lists. A client that diffed the preset set against a table's fields would
- be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that
- actually runs; it would be wrong precisely when the two lists differ, which is the only case
- anybody is asking about.
-
- ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save
- moment, and the honest answer is "all of them will be created". `present` is a fact about the
- table, so it reads False for every field rather than the call refusing.
-
- ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG
- (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the
- relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY.
- That is CORRECT for this payload — it answers "which columns will be used vs created", where
- a name and a type are the whole question — and it is safe today because nothing RENDERS a
- column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes
- `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from
- `PRESET_PROFILE_FIELDS`, which carries the bag.
- ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would
- render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or
- widen this projection deliberately; do not assume a bag is here because the field has one.
- """
- have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields")
- or [])}
- fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text",
- "present": f["key"] in have}
- for f in PRESET_PROFILE_FIELDS]
- return {"table": str(table_key or ""),
- "exists": ut_get(rt, str(table_key or "")) is not None,
- "fields": fields,
- "willUse": [f for f in fields if f["present"]],
- "willCreate": [f for f in fields if not f["present"]]}
-
-
-def clean_predicates(raw, operator="and"):
- """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`.
-
- Refuses rather than coerces: a predicate naming a field the API rejects comes back as a
- 400 with the field named, because the alternative — dropping it — turns "find me verified
- accounts in Georgia" into "find me any account" and bills for the difference.
- """
- out = []
- for p in (raw or [])[:12]:
- if not isinstance(p, dict):
- continue
- name = _s(p.get("name") or p.get("field"), 60).strip()
- op = _s(p.get("operator") or p.get("op"), 20).strip()
- if name not in BD_FILTER_FIELDS:
- return None, ("that field cannot be searched — the searchable ones are: "
- + ", ".join(field_label(f) for f in BD_FILTER_FIELDS))
- # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly
- # because every operator was legal on every field; a yes/no column offering "at least"
- # is a question with no meaning that the vendor is nevertheless asked.
- allowed = ops_for(name)
- if op not in allowed:
- return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r} — "
- "it takes: "
- + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed))
- entry = {"name": name, "operator": op}
- if op not in BD_NULLARY_OPS:
- raw_v = p.get("value")
- # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and
- # it is stored as a list rather than as N sibling predicates so the row a person sees
- # and the row that is stored are the same thing.
- vals = raw_v if isinstance(raw_v, list) else [raw_v]
- vals = [v.strip() if isinstance(v, str) else v for v in vals]
- vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))]
- if not vals:
- return None, f"give {field_label(name)} something to compare against"
- if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS:
- return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value — "
- f"{field_label(name)} can only be given a list with "
- + ", ".join(BD_OP_LABELS[o] for o in ops_for(name)
- if o in BD_MULTI_VALUE_OPS))
- if len(vals) > MAX_PREDICATE_VALUES:
- return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values "
- "in one condition")
- if field_kind(name) == "boolean":
- ok = {o["value"] for o in BD_BOOLEAN_OPTIONS}
- bad = [v for v in vals if str(v).lower() not in ok]
- if bad:
- return None, f"{field_label(name)} is answered Yes or No"
- # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers
- # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and
- # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field
- # was a free-text box until this wave, so the Yes/No dropdown that makes it easy
- # to ask is also the thing that would have made every boolean condition fail.
- # The dropdown's option VALUES stay "true"/"false" (a carries strings);
- # the conversion belongs here, at the edge that talks to the vendor.
- vals = [str(v).lower() == "true" for v in vals]
- if field_kind(name) == "number":
- try:
- vals = [float(v) if "." in str(v) else int(v) for v in vals]
- except (TypeError, ValueError):
- return None, f"{field_label(name)} takes a number"
- # A single value stays a SCALAR on the wire. The vendor has only ever been sent
- # scalars for these operators; the list form is expanded at send time and the
- # one-value case must not quietly start exercising an untested shape.
- entry["value"] = vals[0] if len(vals) == 1 else vals
- out.append(entry)
- if not out:
- # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES.
- # This used to refuse, which was right while a wizard collected the filters before the
- # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger
- # CREATES the automation, and the filters are typed afterwards — so a save-time refusal
- # here meant the trigger could never be picked at all. It is the A3
- # stored-inert-with-`configured:false` pattern this module already runs on.
- #
- # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient:
- # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a
- # search, deliberately, because "a stored config can predate the law, and the vendor
- # bills for breadth whether the filter was saved yesterday or last month". The RUN is the
- # wall. Every MALFORMED predicate above is still refused here, and the narrowing law
- # below still refuses a non-empty filter that narrows nothing — only EMPTY changed.
- return [], None
- # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the
- # measured hang this guards against. Same sentence at create/patch and at run.
- guard_err = narrowing_refusal(out, operator) or depth_refusal(out, operator)
- if guard_err:
- return None, guard_err
- return out, None
-
-
-def discover_estimate(records_limit):
- """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the
- section header. Returned as structured data so the UI cannot accidentally drop the caveat."""
- n = max(0, int(records_limit or 0))
- return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4),
- "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC",
- "note": "estimated from the published rate — an exact price is only known after a run"
- "a price before a run, and this account's token cannot read a balance"}
-
-
-def _candidate_row(row, stamp):
- """One corpus row → a `ut_ig_candidates` row. None when it has no handle.
-
- ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already
- stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted
- `tracked` either; R6 deleted that column in wave 26.)
- """
- if not isinstance(row, dict):
- return None
- handle = str(_first(row, "account", "username", default="") or "").strip()
- if not handle:
- return None
- return {
- # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads
- # Instagram, so it is a constant here rather than something derived from the row; the
- # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart.
- "platform": PLATFORM_INSTAGRAM,
- "handle": handle,
- "profile_url": str(_first(row, "profile_url", "url",
- default=f"https://www.instagram.com/{handle}/")),
- "full_name": str(_first(row, "full_name", "profile_name", default="") or ""),
- "followers": _s(_ig_int(_first(row, "followers"))),
- "following": _s(_ig_int(_first(row, "following"))),
- # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is
- # the path where it carries values (measured on all five of the 2026-08-04 result set).
- # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`.
- "avg_engagement": _pct100(_first(row, "avg_engagement", default="")),
- "bio": _s(_first(row, "biography", "bio", default=""), 500),
- "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300),
- "verified": "1" if _first(row, "is_verified", default=False) else "",
- "category": _s(_first(row, "category_name", "business_category_name", default="")),
- # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp
- # still exists on the snapshot series, which is where a time axis belongs.
- "last_found": _day(stamp),
- }
-
-
-# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ──────────────────────────────────────────
-#
-# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a
-# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a
-# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via
-# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers,
-# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at
-# all, so discovery was the one rung in this module that read numbers and recorded no observation.
-#
-# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold
-# real corpus numbers that are already on screen and already filterable. The gap is that the
-# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law
-# exists to forbid one level up.
-#
-# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make
-# it honest already existed: `source` says **Discovery** (this was read off the vendor's
-# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not
-# read this as an exact count taken at `pulled_at`" — which is the whole difference between
-# recording what we know and inventing what we do not.
-#
-# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it
-# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as
-# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over
-# the series therefore prefers the exact read over the corpus read whenever both happened on one
-# day — the ordering you would have to hand-write, falling out of the grain.
-
-#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell.
-IG_VIA_DISCOVERY = "brightdata:discovery"
-
-#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement
-#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather
-#: than typed out — a hand-list is what silently stops carrying the next promoted column.
-_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx")
-
-
-def corpus_snapshot_row(row, day=""):
- """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable.
-
- `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's
- only date is `first_found`.
-
- ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is
- `@`, so re-running this over the same rows on the same day rewrites the same key
- and `upsert_rows` merges it — the migration is safe to call on every write, which is the only
- way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys
- idempotency on the stored TYPE).
-
- ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty
- ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed.
- Here the merge is the hazard instead: a second discovery on the same day returning a thinner
- corpus record would otherwise ERASE what the first one learned. An absent key renders exactly
- as an empty cell, so nothing is lost by leaving it out.
- """
- handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower()
- if not handle:
- return None
- when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "")
- if not when:
- return None
- out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when,
- "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"}
- for fd in SNAPSHOT_FIELDS:
- key = fd["key"]
- if key in _SNAPSHOT_OWN_KEYS:
- continue
- value = (row or {}).get(key)
- if str(value or "").strip():
- out[key] = str(value) if key == "source_payload" else _s(value, 400)
- return out
-
-
-def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print):
- """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written.
-
- ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only
- honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls
- `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the
- creator from there is unbounded recursion. A tenant with no series store has nothing to
- back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this
- function then has somewhere to write.
- """
- rows_in = [r for r in (incoming or []) if r]
- if not rows_in:
- return 0
- existing = ut_get(rt, snap_key)
- if existing is None:
- return 0
- merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key",
- cap=row_cap(snap_key))
- if counts["capped"]:
- # D-11's law: a full append table means the SERIES has stopped growing, which is the one
- # failure a chart cannot show you.
- log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by "
- f"{snap_key}'s row cap")
- written = counts["inserted"] + counts["updated"]
- if not written:
- return 0
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- if cur.get(snap_key) is not None:
- cur[snap_key]["rows"] = merged
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- return written
-
-
-# ---------------------------------------------------------------------------------------------
-# RUNNERS
-# ---------------------------------------------------------------------------------------------
-
-def cap_note(capped, missing=()):
- """The sentence(s) a cap breach adds to a run summary.
-
- `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit.
- `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not
- be created at all.
-
- ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the
- ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that
- does not say WHERE is only marginally better than a silent one.
-
- ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused
- SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the
- runners' `if tgt is not None` write guard then skipped that bucket, and the run reported
- success over a database that does not exist. Exactly the silent-stop failure the row cap
- already had, one level up — and harder to spot, because the table is not there to look at.
- """
- out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} "
- f"from this run were NOT written" for k, n in (capped or []) if n]
- out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its "
- f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it"
- for k in (missing or [])]
- return "; ".join(out)
-
-
-def ut_missing(rt, *keys):
- """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11)."""
- have = ut_all(rt)
- return [k for k in keys if k and k not in have]
-
-
-def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None):
- """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key."""
- cfg = defn.get("config") or {}
- fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField")
- dry = bool(cfg.get("dryRun"))
- # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from
- # the rollup — inferring is exactly how a green dot ended up over an empty table (see the
- # rollup comment in `run_field_instagram`).
- steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"}
- step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}")
- status, final, body = fetch(cfg["url"])
- if not (200 <= status < 300):
- steps["fetch"] = "error"
- return ("error", f"{cfg['url']} answered {status} — nothing was written",
- {"status": status}, [], steps)
- steps["fetch"] = "ok"
- soup = _soup(body)
- if cfg.get("extract") == "jsonld":
- blocks = jsonld(soup)
- flat = []
- for b in blocks:
- if isinstance(b, list):
- flat.extend([x for x in b if isinstance(x, dict)])
- elif isinstance(b, dict):
- items = b.get("itemListElement")
- flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list)
- else [b])
- raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat]
- else:
- found = tables(soup)
- idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0
- raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)]
- if not raw_rows:
- steps["extract"] = "error"
- return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [],
- steps)
- steps["extract"] = "ok"
- step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}")
-
- incoming = []
- for r in raw_rows:
- mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()}
- if any(v for v in mapped.values()):
- incoming.append(mapped)
-
- fields = [field_def(tgt, src if len(src) <= 60 else src[:60])
- for src, tgt in fmap.items()]
- # the key column leads, so the table opens on the identity it is upserted by
- fields.sort(key=lambda f: 0 if f["key"] == key_field else 1)
- label = cfg.get("targetLabel") or defn.get("name") or "Scraped table"
- if dry:
- # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`,
- # which would create the table: a dry run that leaves a new empty database behind is not
- # a dry run.
- table_key = ut_key_for(label, cfg.get("targetTable") or None)
- else:
- table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None)
- t = ut_get(rt, table_key) or {}
- rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field,
- cap=row_cap(table_key))
- touched = [rid for rid, row in rows.items()
- if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}]
- if not dry:
- ut_write_rows(rt, table_key, rows)
- counts["source_rows"] = len(incoming)
- affected = touched[:200]
- summary = (f"{counts['inserted']} new, {counts['updated']} updated, "
- f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page "
- f"(kept)")
- capped = [(table_key, counts.get("capped", 0))]
- missing = [] if dry else ut_missing(rt, table_key)
- counts["missing_tables"] = len(missing)
- note = cap_note(capped, missing)
- if note:
- summary += f" — {note}"
- steps["write"] = "partial"
- if dry:
- summary = f"Test run — nothing saved. Would have written: {summary}"
- steps["write"] = "skipped"
- elif not note:
- steps["write"] = "ok"
- state = "partial" if (counts.get("capped") or counts.get("skipped") or missing
- or dry) else "ok"
- log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}")
- return (state, summary, counts, affected, steps)
-
-
-def capture_rows(res, pulled):
- """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`.
-
- ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT
- RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path —
- do not fork them." A second copy of this mapping is the shape that goes wrong invisibly,
- because the two copies would each be *plausible* and would disagree only on the rows a
- particular rung happened to return.
-
- ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE:
- * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every
- time and a blank is unambiguous — it means THIS PULL did not read it.
- * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are
- readable on some rungs and not others (the Bright Data PROFILE row carries post identity
- with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and
- `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run
- ERASE what an expensive one learned.
- * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is
- the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement
- rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k
- ceiling and drawing a chart of nothing. A row in a time series should mean "this was true
- then"; a row meaning "nobody looked" belongs nowhere.
- """
- prof = (res or {}).get("profile") or {}
- snap_row = {
- "snapshot_key": f"{prof.get('username')}@{pulled}",
- "influencer_key": prof.get("username"), "pulled_at": pulled,
- # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called
- # "Read via". `res["via"]` keeps the real key for the call graph and the server log.
- "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")),
- "bio": _s(prof.get("bio"), 500),
- "external_url": _s(prof.get("external_url"), 300),
- }
- for fd in SNAPSHOT_FIELDS:
- k = fd["key"]
- if k not in snap_row:
- value = prof.get(k)
- # Profile history declares the same percentage dialect as the current Profile row.
- # Converting only the latest projection left the append series at 0-1 while its field
- # now said pct (0-100), so the two views of one measurement disagreed by 100x.
- if k == "source_payload":
- snap_row[k] = str(value or "")
- else:
- snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
- idents, metrics_rows, comment_rows = [], [], []
- for p in (res or {}).get("posts") or []:
- ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
- "url": _s(p.get("url"), 300)}
- for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
- ("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
- ("alt_text", 400), ("tagged_location", 400)):
- if p.get(k):
- ident[k] = _s(p.get(k), n)
- if p.get("source_payload"):
- ident["source_payload"] = str(p["source_payload"])
- metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
- # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
- #
- # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
- # views over the last 12 posts" would have to walk profile → posts → each post's most
- # recent snapshot, and a two-hop rollup is a much larger feature with a much worse
- # invalidation story. The post row now carries its own latest, exactly as the profile row
- # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3).
- #
- # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the
- # authoritative engagement series and still gets its appended row. These cells are a
- # projection of the row being appended in the same breath, never a second source.
- # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent
- # key leaves the previous pull's value standing — which is the correct behaviour for a
- # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing
- # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the
- # same failure the snapshot-append rule above is written against.
- measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""}
- if measured:
- ident.update({k: _s(v, 40) for k, v in measured.items()})
- # The stamp is what makes the numbers readable: a blank `plays` beside a
- # `measured_at` of last week means "we looked and the vendor had nothing", and with
- # no stamp it means that AND "we never looked", indistinguishably.
- ident["measured_at"] = _day(pulled)
- idents.append(ident)
- if any(v is not None for v in metrics.values()):
- metrics_rows.append({
- "post_snapshot_key": f"{p['shortcode']}@{pulled}",
- "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
- "pulled_at": pulled,
- # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the
- # SAME vendor record the identity row above is built from, so they cannot disagree
- # with it on this pull.
- # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split`
- # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull
- # that learns a post's engagement often does not learn its date in the same
- # breath. `backfill_post_snapshot_grain` fills those from the post row, which by
- # then may have learned it from a different rung.
- "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200),
- "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
- "views": _s(metrics["views"]), "plays": _s(metrics["plays"]),
- "source_payload": str(p.get("source_payload") or "")})
- for comment in p.get("embedded_comments") or []:
- # The embedded shape often names only the commenter. Its parent Post is the
- # authoritative owner, so bind every preview to this profile and post explicitly.
- comment_rows.append({**comment, "shortcode": p["shortcode"],
- "influencer_key": prof.get("username") or ""})
- comment_rows.extend((res or {}).get("comments") or [])
- # The same comment can be returned as both `latest_comments` and `top_comments`, or by the
- # optional full endpoint after an embedded preview. The canonical key is the one row truth.
- unique_comments = {}
- for row in comment_rows:
- key = str((row or {}).get("comment_key") or "")
- if key:
- unique_comments[key] = {**(unique_comments.get(key) or {}), **row}
- return snap_row, idents, metrics_rows, list(unique_comments.values())
-
-
-#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched.
-#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of
-#: `row[...] = prof.get(...)` that a future field addition can silently miss.
-#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did
-#: not read it", and `enriched_at` is what distinguishes that from "never enriched").
-PRESET_FROM_PROFILE = {
- "username": "handle", "profile_url": "profile_url", "full_name": "full_name",
- "followers": "followers", "following": "following", "avg_engagement": "avg_engagement",
- "bio": "bio", "external_url": "external_url", "verified": "verified",
- "category": "category", "posts_count": "posts_count",
- "highlights_count": "highlights_count", "is_business": "is_business",
- "is_professional": "is_professional", "ig_id": "ig_id",
- # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column
- # that can only ever be blank, so "make the preset fields populated" is THIS half of the
- # owner's instruction and the field list is only the other half.
- # ⚠ The two lists are held in step by a derived gate rather than by care: every
- # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared
- # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`).
- "business_category": "business_category", "is_private": "is_private",
- "bio_hashtags": "bio_hashtags",
- "pronouns": "pronouns", "profile_name": "profile_name",
- "is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
- "partner_id": "partner_id", "external_url_title": "external_url_title",
- "fbid": "fbid", "related_accounts": "related_accounts",
- "country_code": "country_code", "source_payload": "source_payload",
-}
-
-#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a
-#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate
-#: could only be written as "these three are fine" — a hard-coded exception list, which is the
-#: shape [[gate-answers-the-wrong-question]] warns about.
-#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an
-#: enrichment run — which is the whole point of them: they stay true when the LINKED table
-#: changes, and a pull that touched no profile still updates a profile's post count.
-PRESET_WRITTEN_ELSEWHERE = (
- "platform", "handle", "enriched_at",
- "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
- "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12",
- "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
-)
-
-
-#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a
-#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING`
-#: refuses on the discovery side for exactly the same reason.
-LOCATION_MIN_POSTS = 2
-
-
-def _post_place(post):
- """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload
- second.
-
- ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the
- vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the
- `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details`
- shape. The complete vendor row is retained in `source_payload` on the same record, already
- paid for, so a post whose normalised field came back blank can still be read here.
- ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region,
- country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to
- "where is this person"; "Indonesia" mostly is not.
- """
- if not isinstance(post, dict):
- return ""
- raw = str(post.get("tagged_location") or "").strip()
- if not raw:
- try:
- payload = json.loads(post.get("source_payload") or "{}")
- except Exception: # noqa: BLE001
- payload = {}
- raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else ""
- return raw.split(",")[0].strip()
-
-
-def location_guess(posts):
- """`(city, confidence_pct)` from the places a creator's own posts were tagged in.
-
- `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not
- the share of ALL posts, because a creator who geotags three of twelve is telling us about
- three, and dividing by twelve would report a real signal as weak. How thin the evidence is
- stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two.
- """
- places = [p for p in (_post_place(x) for x in posts or []) if p]
- if len(places) < LOCATION_MIN_POSTS:
- return "", None
- counts = {}
- for p in places:
- counts[p] = counts.get(p, 0) + 1
- # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be
- # resolved alphabetically, which is an arbitrary answer wearing a confident number.
- best = max(places, key=lambda p: (counts[p], -places.index(p)))
- return best, round(100.0 * counts[best] / len(places), 1)
-
-
-def preset_cells(res, pulled):
- """C4/R3: one pull → the LATEST-value cells written onto the enriched record.
-
- ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the
- other fifteen readable: without it a blank `followers` means both "never enriched" and
- "enriched in March and the vendor had nothing", and no cell on the row can tell them apart.
- """
- prof = (res or {}).get("profile") or {}
- cells = {}
- for src, dest in PRESET_FROM_PROFILE.items():
- v = prof.get(src)
- if v is not None and v != "":
- # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The
- # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a
- # straight-through copy here would write the same wrong number the discovery path
- # used to write. Both writers convert; neither is the exception.
- cells[dest] = (_pct100(v) if dest == "avg_engagement" else
- (str(v) if dest == "source_payload" else _s(v, 500)))
- # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an
- # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
- # arrived with no platform at all — and a blank half of the dedup key is how one account
- # becomes two rows.
- cells["platform"] = PLATFORM_INSTAGRAM
- # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when
- # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose
- # twelve posts happened to carry no geotag would make the column worse the more it ran.
- place, confidence = location_guess((res or {}).get("posts"))
- if place:
- cells["location_guess"] = _s(place, 120)
- cells["location_confidence"] = str(confidence)
- # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a
- # question in days; the full stamp keeps its precision on the snapshot series.
- cells["enriched_at"] = _day(pulled)
- return cells
-
-
-def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step,
- rows=None):
- """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public
- profile through the paid capability chain, write a status string into the automation column,
- and append a timestamped row to each of the three IG tables.
-
- ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per
- CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two
- "which rung answered" counters this function used to keep have nothing left to distinguish."""
- cfg = defn.get("config") or {}
- table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
- post_metrics = bool(cfg.get("postMetrics"))
- comment_metrics = bool(cfg.get("commentMetrics"))
- dry = bool(cfg.get("dryRun"))
- # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a
- # status written under a node id `graph()` no longer emits is a dot nothing renders, which is
- # indistinguishable from a step that never ran.
- steps = {"trigger": "ok", "source": "idle",
- "capture_posts": "idle" if post_metrics else "skipped",
- "capture_comments": "idle" if comment_metrics else "skipped",
- "write": "idle"}
- t = ut_get(rt, table_key)
- if not t:
- steps["source"] = "error"
- return ("error", f"{table_key} is not a database in this workspace", {}, [], steps)
- url_field = cfg.get("urlField") or _auto_url_field(t, fkey)
- if not url_field:
- steps["source"] = "error"
- return ("error", "the automation column has no URL field bound to it", {}, [], steps)
- steps["source"] = "ok"
- rows = dict(t.get("rows") or {})
- # the relational tables the pull lands in (R7): every row timestamped for time-range filters
- if dry:
- # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
- snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
- IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE)
- else:
- graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
- profile_table=table_key)
- snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
- graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
- missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key)
- snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
- posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
- psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {})
- comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {})
-
- counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0,
- "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0,
- "missing_tables": len(missing)}
- cells, affected, notes = {}, [], []
- pending_metrics = []
- # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows`
- # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows.
- # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
- # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
- in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
- targets = [(rid, str(r.get(url_field, "") or "").strip())
- for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
- for i, (rid, url) in enumerate(targets):
- if i:
- time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7)
- # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile
- # column automation blocks for minutes by design. A counting step is the difference
- # between "working through them" and "stuck".
- step(f"Capturing profile {i + 1} of {len(targets)}")
- counts["profiles"] += 1
- res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
- post_metrics=post_metrics, comment_metrics=comment_metrics,
- pending_metrics=(pending_metrics if not dry else None))
- pulled = _iso()
- via = res.get("via") or ""
- read_ok = res["state"] in ("ok", "partial")
- # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID
- # vendor actually answered, which is the run's spend report — the thing the owner reads to
- # reconcile a bill. What died is the free rung it used to be contrasted with, so the test
- # is now simply "did a vendor answer" rather than "did the vendor we were told to try".
- if read_ok and via in ("brightdata", "apify"):
- counts["paid"] += 1
- if read_ok:
- counts["ok" if res["state"] == "ok" else "partial"] += 1
- prof = res["profile"]
- snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled)
- in_snaps.append(snap_row)
- in_posts.extend(ident_rows)
- in_psnaps.extend(metric_rows)
- in_comments.extend(captured_comments)
- counts["metrics"] += len(metric_rows)
- counts["comment_rows"] += len(captured_comments)
- counts["posts"] += len(res.get("posts") or [])
- detail = f"{prof.get('followers') or '?'} followers"
- if prof.get("approx"):
- detail += " (approx)"
- if res.get("posts"):
- detail += f", {len(res['posts'])} posts"
- elif res["state"] == "partial":
- detail += ", posts not readable on the rung that answered"
- cells[rid] = f"{res['state']} · {_stamp()} · {detail}"
- elif res["state"] == "blocked":
- counts["blocked"] += 1
- cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}"
- notes.append(res.get("note") or "blocked")
- else:
- counts["error"] += 1
- cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}"
- notes.append(res.get("note") or "error")
- affected.append(rid)
-
- # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows.
- snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key))
- posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
- posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
- c_post["duplicates"] += collapsed_posts
- psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
- comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key")
- comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key))
- c_comments["duplicates"] += collapsed_comments
- counts["new_posts"] = c_post["inserted"]
- capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
- (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])]
- counts["capped"] = sum(n for _k, n in capped)
-
- if not dry:
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- tt = cur.get(table_key)
- if tt is not None:
- for rid, val in cells.items():
- tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
- for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
- (comment_key, comments)):
- tgt = cur.get(k)
- if tgt is not None:
- tgt["rows"] = rws
- _refresh_relations_inplace(cur, log=log)
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables
- queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics)
- if queued:
- counts["metric_batches_pending"] = queued
-
- # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated:
- # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment),
- # `error` a LOUD partial — a pooled history with silent holes is worse than none.
- master_note = ""
- if not dry and (in_snaps or in_posts or in_psnaps):
- import ig_master
- m_status, m_note = ig_master.append_run(getattr(rt, "key", ""),
- in_snaps, in_posts, in_psnaps)
- if m_status == "error":
- master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \
- f"is complete and the next run re-appends"
- counts["master_failed"] = 1
- elif m_status == "ok":
- counts["master"] = len(in_snaps) + len(in_psnaps)
-
- # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while
- # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail
- # showed a green dot over a table with no posts in it. A summary that disagrees with the
- # cells it summarises is the failure this whole module's honest-status rule exists to
- # prevent, so `partial` now propagates. Measured 2026-08-04.
- read = counts["ok"] + counts["partial"]
- if not read:
- state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok"
- elif counts["blocked"] or counts["error"] or counts["partial"]:
- state = "partial"
- else:
- state = "ok"
- if counts.get("metric_batches_pending"):
- state = "partial"
- # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`)
- if not targets:
- cap_state = "idle"
- elif not read:
- cap_state = "error" if counts["error"] else "blocked"
- else:
- cap_state = "partial" if (counts["blocked"] or counts["error"]
- or counts["partial"]) else "ok"
- # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to
- # read a profile now, and the node that names the profile set is the honest owner of "did
- # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch
- # that no longer exists in behaviour OR on screen.
- steps["source"] = cap_state if targets else steps["source"]
- # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually
- # appended. "It was switched on" is not the same fact as "it answered", and painting the
- # second from the first is the fabrication `node_status` refuses to make.
- # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A
- # run that found no posts to enrich did not have a rung refuse it — nothing was requested —
- # so that reads `skipped`, the same word an off switch earns.
- steps["capture_posts"] = ("ok" if counts["metrics"]
- else "partial" if counts.get("metric_batches_pending")
- else "blocked" if (post_metrics and counts["posts"])
- else "skipped")
- steps["capture_comments"] = ("ok" if counts.get("comments")
- else "blocked" if (comment_metrics and counts["posts"])
- else "skipped")
-
- summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts "
- f"({counts['new_posts']} new)")
- if counts.get("metric_batches_pending"):
- summary += (f", {counts['metric_batches_pending']} post-engagement batch"
- f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building "
- "(collected automatically)")
- if counts["paid"]:
- summary += f", {counts['paid']} with exact counts"
- if counts["metrics"]:
- summary += f", {counts['metrics']} post engagement snapshots"
- elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"):
- # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there
- # was no engagement", which is a claim about Instagram rather than about our run.
- summary += ", no post engagement was readable"
- if counts["comment_rows"]:
- summary += f", {counts['comment_rows']} comment rows captured"
- elif comment_metrics and counts["posts"]:
- summary += ", no comment engagement was readable"
- if counts["partial"]:
- summary += f", {counts['partial']} profile-only (posts not readable)"
- if counts["blocked"]:
- summary += f", {counts['blocked']} blocked"
- note = cap_note(capped, missing)
- if note:
- summary += f" — {note}"
- state = "partial" if state != "error" else state
- steps["write"] = "partial"
- elif not dry:
- steps["write"] = "ok"
- if master_note:
- summary += f" — {master_note}"
- state = "partial" if state != "error" else state
- steps["write"] = "partial"
- if dry:
- summary = f"Test run — nothing saved. Would have written: {summary}"
- steps["write"] = "skipped"
- state = "partial" if state != "error" else state
- if notes:
- summary += f" — {notes[0][:120]}"
- if not dry:
- # C7: the fresh master rows are exactly what the table's metric cells summarise.
- try:
- compute_metric_cells(rt, table_key)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}")
- return (state, summary, counts, affected, steps)
-
-
-def _auto_url_field(table, fkey):
- """The URL field an automation column is bound to: its own `automation.urlField`, else the
- first url-typed column on the table. Never guessed silently — the caller reports which."""
- for f in table.get("fields") or []:
- if f.get("key") == fkey:
- bound = ((f.get("automation") or {}).get("urlField") or "").strip()
- if bound:
- return bound
- for f in table.get("fields") or []:
- if f.get("type") == "url":
- return f.get("key")
- return ""
-
-
-def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_step,
- rows=None):
- """Automation #3 (owner ruling R7 / D-23): FIND handles nobody here has typed in.
-
- Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus
- query and hands its snapshot id to the next run, or COLLECTS one a previous run started.
- Every outcome is a sentence about what actually happened; none of them is a green zero.
- """
- cfg = defn.get("config") or {}
- dry = bool(cfg.get("dryRun"))
- limit = int(cfg.get("recordsLimit") or 5)
- preds = list(cfg.get("predicates") or [])
- auto_id = str(defn.get("id") or "")
- steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"}
- counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0,
- "missing_tables": 0}
-
- if not bd_ready():
- # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is
- # the vendor's — so this is where the run stops, saying which env var is missing.
- steps["find"] = "blocked"
- return ("partial", "Profile search is not set up yet — nothing was searched and nothing "
- "was charged", counts, [], steps)
-
- pending = str((defn.get("state") or {}).get("pendingSnapshot") or "")
- if not pending:
- # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and
- # the vendor bills for breadth whether the filter was saved yesterday or last month.
- # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config
- # stored before this check existed must not reach the vendor either.
- guard_err = (narrowing_refusal(preds, cfg.get("operator"))
- or depth_refusal(preds, cfg.get("operator")))
- if guard_err:
- steps["find"] = "blocked"
- return ("error", f"the search was not started — {guard_err}", counts, [], steps)
- if dry:
- est = discover_estimate(limit)
- steps["find"] = steps["write"] = "skipped"
- return ("partial",
- f"Test run — nothing saved. Would look for up to {limit} "
- f"profile{'' if limit == 1 else 's'} matching "
- f"{_predicate_sentence(preds, cfg.get('operator'))}, for about "
- f"${est['usd']}", counts, [], steps)
- step("Starting the search")
- # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never
- # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the
- # vendor refuses. Reported either way, because "why did this run find fewer" and "why did
- # this run cost the same as last night" are both questions this line answers.
- excl = {}
- sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit,
- exclude_handles=already_found_handles(rt, defn),
- applied=excl)
- if excl.get("excluded"):
- step(f"excluded {excl['excluded']} already-found "
- f"profile{'' if excl['excluded'] == 1 else 's'} at the provider")
- elif excl.get("dropped"):
- step(f"searching without the already-found list — {excl['dropped']}")
- if note:
- steps["find"] = "blocked"
- return ("error", f"the search was not started — {note}", counts, [], steps)
- pending = sid
- # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes
- # through `rt.update`, which can RAISE on an unavailable store — and that exception
- # becomes an error run, so a `log()` placed after it never executes. The snapshot would
- # then exist, be billed, and have its id in neither the store nor the logs. Logging first
- # means the worst case is still recoverable by a human reading the Space output.
- log(f"[aios-auto] discover {auto_id}: started {sid}")
- # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop
- # existing because this process dies thirty seconds later, and a lost id is a set we
- # paid attention to and can never collect.
- set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()})
- steps["find"] = "ok"
-
- waited, status, size, note = 0.0, "", 0, ""
- while True:
- # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to
- # BD_FILTER_WAIT (120 s) at the vendor, and D measured that this wait IS the whole of
- # "Run once now is laggy / looks stuck". A counter here is what makes a legitimate
- # two-minute wait distinguishable from a hung thread; without it the rendered step reads
- # the same word for both, which is a progress indicator that cannot indicate progress.
- step(f"Searching — {int(waited)}s of "
- f"{int(BD_FILTER_WAIT)}s")
- status, size, note = bd_filter_status(pending)
- if note:
- steps["collect"] = "error"
- set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
- return ("error", note, counts, [], steps)
- if status != "building" or waited >= BD_FILTER_WAIT:
- break
- time.sleep(BD_FILTER_POLL)
- waited += BD_FILTER_POLL
- step("Reading the profiles the search found")
-
- if status == "building":
- # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20
- # minutes; holding a worker thread that long on a free-tier container to poll would be
- # the wrong shape. Say plainly that it is still running and who collects it.
- steps["collect"] = "partial"
- # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made
- # this read as "stuck": *"The next run picks up the results"* is a promise about a run
- # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick
- # finish it, so the promise is now kept by something rather than by the reader.
- mins = 0
- try:
- since = (defn.get("state") or {}).get("pendingSince")
- if since:
- mins = max(0, int((_dt.datetime.now(_dt.timezone.utc)
- - _dt.datetime.fromisoformat(str(since))).total_seconds()
- // 60))
- except Exception: # noqa: BLE001
- mins = 0
- been = f" — {mins} min so far" if mins else ""
- # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by
- # naming the corpus size: *"the wait is the SEARCH over 620 million profiles, not the 10
- # rows"*. It was TRUE, and it was answering a question the person had not asked. Somebody
- # watching a run that has taken eighteen minutes wants to know whether it is broken and
- # when it ends; how big the index is, is our fact about our supplier. Keeping it also
- # meant this line was the one place a vendor's catalogue size was quoted to a customer, so
- # it would have gone stale the day the vendor grew — a number nobody would think to
- # re-measure because nothing depends on it.
- # ⚠ THE SHAPE OF THE ANSWER SURVIVES and is the part that mattered: the wait does not
- # shrink when you ask for fewer records, and a person who does not know that reads
- # "20 minutes for 10 rows" as a fault.
- return ("partial",
- f"Still searching at the provider{been}. A corpus search takes about 20 minutes "
- f"however few records you asked for. Nothing is lost, you are not charged twice, "
- f"and the results are collected automatically as soon as they are ready",
- counts, [], steps)
- if status == "empty":
- # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword
- # that is too specific, so it says what to do about it instead of reporting a fault.
- steps["collect"] = "ok"
- steps["write"] = "skipped"
- set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
- return ("partial", "No profiles matched. Try a shorter or more common keyword — "
- "'floral' finds more accounts than 'floral design studio'",
- counts, [], steps)
- if status != "ready":
- steps["collect"] = "error"
- set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
- return ("error", f"the search ended as {status or 'unreadable'} and returned nothing",
- counts, [], steps)
-
- rows, dnote = bd_filter_rows(pending)
- if dnote:
- # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it.
- steps["collect"] = "partial"
- return ("partial", f"the search finished with {size} match"
- f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps)
- set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
- steps["collect"] = "ok"
-
- stamp = _iso()
- incoming = [c for c in (_candidate_row(r, stamp) for r in rows) if c]
- counts["found"] = len(incoming)
- # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and
- # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to
- # "the vendor delivered 3". It also catches the shape where an error document comes back
- # down the rows path and parses as one unusable row instead of an error.
- counts["dropped"] = max(0, len(rows) - len(incoming))
- label = cfg.get("targetLabel") or DISCOVER_LABEL
- table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry
- else ut_ensure(rt, label, CANDIDATE_FIELDS, username,
- key=cfg.get("targetTable") or DISCOVER_TABLE,
- flow_tag=str(defn.get("id") or ""), lock_fields=True))
- existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
- # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`.
- #
- # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning
- # that built it is worth stating before it is discarded rather than after: per-user rows gave
- # each person their own `found_count` and their own review card, so two colleagues scouting the
- # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The
- # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no
- # longer part of the identity.
- #
- # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN
- # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on
- # TikTok are routinely different people, so keying on the handle ALONE would have silently
- # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with
- # no error, discovered months later by someone wondering why a creator's followers halved.
- # Hence `platform`, and hence it being part of the key rather than a label beside it.
- def _ck(platform, handle):
- return f"{str(platform or PLATFORM_INSTAGRAM).strip()}\n{str(handle or '').strip()}"
-
- # ⚠ A PRE-WAVE-26 ROW HAS NO `platform` CELL, and `_ck` defaults it to Instagram rather than to
- # blank on purpose: every row that exists today was written by the Instagram runner, so that
- # default is a FACT about the stored data, not an assumption. Blank-keyed rows would fail to
- # match their own re-find and duplicate the whole table on the next run.
- for r in existing.values():
- if not str(r.get("platform") or "").strip():
- r["platform"] = PLATFORM_INSTAGRAM
- existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))}
- for rid, r in existing.items()}
- # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding
- # a profile is the signal that it keeps matching, so the counter grows instead of
- # resetting. `first_found` is written only when the pair is new, so it never moves.
- seen = {r["_ckey"]: r for r in existing2.values()}
- for c in incoming:
- # "Found by" — the first finder is recorded and later finders do not overwrite them, which
- # is what the column means now that it is no longer part of the identity.
- c["created_by"] = username
- c["_ckey"] = _ck(c.get("platform"), c["handle"])
- prev_row = seen.get(c["_ckey"])
- if prev_row and str(prev_row.get("created_by") or "").strip():
- c["created_by"] = prev_row["created_by"]
- if prev_row:
- counts["seen_again"] += 1
- c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1)
- else:
- counts["new"] += 1
- c["found_count"] = "1"
- c["first_found"] = stamp
- # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF
- # "is this row new to the TABLE".
- #
- # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's
- # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true,
- # and then drew nothing, because this stamp lived inside the `else` above. `skey` is
- # `stage_`; the branch it sat in asks whether some OTHER automation had
- # already created the row. So the first automation to reach a handle claimed it, and every
- # later automation sharing that database silently had no cards — with a summary saying it
- # had found them. Nothing errored, and both boards are green surfaces.
- #
- # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an
- # unconditional write: a re-find must never pull a card somebody has already moved back
- # into Review. So the question is "has THIS automation placed this record yet?" — not "is
- # this record new?" A blank stage cell for this automation means unplaced, which is
- # exactly the state a new card is in.
- merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key))
- merged = {rid: {k: v for k, v in r.items() if k != "_ckey"}
- for rid, r in merged.items()}
- counts["capped"] = mc["capped"]
- missing = [] if dry else ut_missing(rt, table_key)
- counts["missing_tables"] = len(missing)
- if not dry and not missing:
- ut_write_rows(rt, table_key, merged)
- steps["write"] = "ok"
- # ⭐⭐ THE FORWARD HALF (2026-08-10). Every candidate this run wrote now also leaves an
- # OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the
- # next discovery run re-opens exactly the gap the backfill just closed, which is how a
- # data defect becomes a recurring one.
- #
- # ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change
- # worth owning: a tenant that has never enriched now gets the four canonical child
- # databases the first time discovery runs (four of `MAX_UT_TABLES = 40`). The alternative
- # writes a series into a store the profile table has no LINK to — the numbers would be
- # recorded and unreachable, which is half a feature wearing a whole one's clothes. A
- # tenant running discovery is an Instagram tenant; this moves the spawn earlier, it does
- # not add one.
- # ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not
- # to every row the table happens to hold.
- try:
- graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key)
- counts["series"] = append_ig_snapshots(
- rt, [corpus_snapshot_row(c, day=stamp) for c in incoming],
- graph[IG_SNAPSHOTS_TABLE], log=log)
- except Exception as e: # noqa: BLE001
- # The candidates already landed. A series write that fails must not throw away the
- # rows we just paid the vendor for — the same posture the preset top-up takes.
- log(f"[aios-auto] discover {auto_id}: corpus series write failed: "
- f"{type(e).__name__}: {e}")
- elif dry:
- steps["write"] = "skipped"
-
- affected = [rid for rid, r in merged.items()
- if str(r.get("handle")) in {c["handle"] for c in incoming}][:200]
- est = discover_estimate(counts["found"])
- summary = (f"{counts['found']} profile{'' if counts['found'] == 1 else 's'} found — "
- f"{counts['new']} new, {counts['seen_again']} seen before. About "
- f"${est['usd']}")
- if counts["dropped"]:
- summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no "
- f"username and could not be saved")
- cnote = cap_note([(table_key, counts["capped"])], missing)
- if cnote:
- summary += f" — {cnote}"
- steps["write"] = "partial"
- if dry:
- summary = f"Test run — nothing saved. Would have written: {summary}"
- state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok"
- if not dry and not missing:
- try:
- compute_metric_cells(rt, table_key) # C7 — see the field runner
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}")
- log(f"[aios-auto] discover {auto_id} -> {table_key}: {summary}")
- return (state, summary, counts, affected, steps)
-
-
-def _predicate_sentence(preds, operator="and"):
- """A predicate list as something a person can read back. Used in summaries and the canvas.
-
- Human words on BOTH halves — it used to render `biography includes floral`, the vendor's
- column name beside the vendor's operator token, in a sentence shown to a customer.
- """
- joiner = " or " if str(operator or "").lower() == "or" else " and "
- parts = []
- for p in preds or []:
- v = p.get("value")
- shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v
- parts.append(f"{field_label(p.get('name'))} "
- f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}"
- + ("" if shown is None else f" {shown}"))
- return joiner.join(parts) or "no conditions"
-
-
-#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather
-#: than a second, smaller number nobody could explain: a plain automation's records ARE its
-#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a
-#: flow could be pointed at and which would not finish.
-MAX_FLOW_RECORDS = MAX_UT_ROWS
-
-
-def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None):
- """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole
- automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special
- case and `apply_actions` still runs at the ONE call site that already exists.
-
- Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed
- `error: unknown automation kind 'plain'` — on the kind every new automation now has.
-
- ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here.
- `apply_actions` returns immediately when there is no table, so a flow whose only action is
- `create_record` into some other database does NOTHING — and a green dot over nothing is the
- exact defect this module names in three other places. It reports the honest state and says
- which fact is missing.
- """
- table = _flow_table(defn)
- if not table:
- return ("partial",
- "no database is bound yet — a plain automation walks the records of its target "
- "database, and this one names none (pick one on the trigger, or in Properties)",
- {}, [], {})
- all_rows = (ut_get(rt, table) or {}).get("rows") or {}
- # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug
- # on the kind every new automation now has. `plain` has no machine step, so it has nothing of
- # its own to call "the records this run touched" — and walking the WHOLE table would mean
- # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the
- # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them
- # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what
- # those genuinely mean.
- ids = ([r for r in (rows or []) if str(r) in all_rows] if rows
- else sorted(all_rows, key=_rid_num))
-
- # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its
- # saved View and quota are not merely how the action decides whether to spend; they ARE the
- # set of records the automation was asked to process. Walking every row first meant a run
- # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows.
- # Besides being misleading, that shape made the runtime scale with unrelated historical rows.
- #
- # Keep this deliberately narrow: any flow with another action must still hand that action the
- # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record
- # semantics to preserve, so it can begin at its selected records safely.
- flow_actions = list((defn.get("flow") or {}).get("actions") or [])
- enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)]
- enrich_scope_note = ""
- if rows is None and len(enabled_direct) == 1 \
- and enabled_direct[0].get("kind") == "enrich_instagram":
- enrich_cfg = enabled_direct[0].get("config") or {}
- profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField"))
- if profile_key:
- # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES.
- # `enrich_selection` is called TWICE per run — here, to decide which records the
- # runner walks and announces, and again inside `_walk` to decide which the action
- # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so
- # the run announced *"(1 of the 30 asked for)"* in its summary while its own note
- # from the second call said *"0 of the 30"* — one run, two answers, both printed.
- # ⚠ `gone` is read from the definition here rather than passed in, because this
- # function has the definition and `_walk` does the same read; a third source of the
- # same fact is how the two would drift again ([[one-evaluator-per-question]]).
- selected, enrich_scope_note = enrich_selection(
- rt, table, enrich_cfg, profile_key,
- gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}))
- if enrich_scope_note.startswith("the enrich step names a view it cannot read"):
- return ("partial", f"nothing walked in {table} — {enrich_scope_note}", {}, [], {})
- ids = selected
- step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}")
- counts = {"records": len(ids)}
- if len(ids) > MAX_FLOW_RECORDS:
- # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names
- # both numbers so "it only processed some of them" is readable rather than deducible.
- total = len(ids)
- ids = ids[:MAX_FLOW_RECORDS]
- return ("partial",
- f"{total} records in {table} — this run walked the first "
- f"{MAX_FLOW_RECORDS}", counts, ids, {})
- n = len(ids)
- # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK.
- # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)?
- # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours:
- # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product
- # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key.
- # · `(all)` meant "every record, because a manual run has no trigger records to narrow to",
- # which is unguessable from the word. It reads as a filter nobody chose.
- # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole
- # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that
- # this sentence never claimed to report. What was wrong was that nothing said so.
- label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table
- scope = ("the records that fired the trigger" if rows
- else ("the records selected for Instagram enrichment"
- if len(enabled_direct) == 1
- and enabled_direct[0].get("kind") == "enrich_instagram"
- else "every record — this run was started by hand, so nothing narrowed it"))
- if enrich_scope_note:
- scope += f" ({enrich_scope_note})"
- # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400
- # characters by `_commit_run`, and this note is the longest thing a run says — it names
- # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING
- # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy
- # of this sentence is never produced. Without this line the explanation exists only in a
- # string that is about to be truncated, on exactly the runs that look like the automation
- # has stopped working.
- counts[RUN_NOTES_KEY] = [enrich_scope_note]
- return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}",
- counts, ids, {})
-
-
-RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db,
- "field_instagram": run_field_instagram,
- "discover_instagram": run_discover_instagram}
-
-
-# ---------------------------------------------------------------------------------------------
-# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b)
-# ---------------------------------------------------------------------------------------------
-# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI →
-# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness
-# boolean on the wire, four client strings and two gate sections. That is what a hard-coded
-# vendor costs, and TikTok (D-9) would have paid it again from scratch.
-#
-# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the
-# three verbs it can do — and the surfaces read the declaration instead of naming a vendor:
-#
-# probe "is this configured?" -> the honest readiness bit the UI shows
-# capture enrich a handle we know -> `pull_profile`-shaped
-# discover find handles we do not -> the corpus query
-#
-# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in
-# later; it is the honest statement that this source has no discovery route, and a surface that
-# offers one anyway is offering a button that must refuse.
-SOURCES = {
- "instagram": {
- # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing
- # copy, and which provider serves a capability is a routing decision the product owns —
- # `providers.py` still holds the real keys, chains and costs.
- "key": "instagram", "label": "Instagram", "vendor": "Scraper",
- "probe": bd_ready,
- "capture": pull_profile,
- "discover": bd_filter_start,
- "captureKind": "field_instagram",
- "discoverKind": "discover_instagram",
- "tiers": TIERS,
- },
- # D-9 (TikTok) lands here as one more entry: a new field map and a new `capture`, with the
- # canvas, the append tables, the caps and the honest-status contract already in place.
-}
-
-
-def source_status():
- """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source.
-
- ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface
- needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the
- credential to a browser would put a billable secret in every user's devtools.
- """
- # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which
- # supplier this rung actually uses — that history is worth keeping in the code — but a
- # customer has no use for it, and a server-supplied string rendered by a generic component
- # is precisely the half a client-file scan cannot see.
- return [{"key": s["key"], "label": s["label"],
- "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))}
- for s in SOURCES.values()]
-
-
-# ---------------------------------------------------------------------------------------------
-# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client
-# ---------------------------------------------------------------------------------------------
-# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are
-# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible —
-# a canvas that still shows a step the engine stopped running looks completely fine. So the node
-# list is derived from the DEFINITION by the same module that executes it, rides on the payload
-# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`.
-# The client owns layout (pixels, pan/zoom, hit targets) and nothing else.
-#
-# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is
-# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine
-# walks it linearly. Nothing here should be read as a general branching runtime.
-
-#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be
-#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw
-#: away the only information the dot is there to carry.
-NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped")
-
-#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a
-#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is
-#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones
-#: here are all REAL — each changes what the next run does:
-#: schedule the trigger fires on its cron, or only by hand
-#: postMetrics likes/comments per post are bought, or the engagement series does not grow —
-#: it costs a vendor record PER POST rather than per profile (measured: the
-#: profile row carries post identity and no engagement)
-#: commentMetrics the Comments dataset is bought, which can bill many rows per post
-#: write DRY RUN — read everything, compute the counts, write nothing anywhere
-#:
-#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is
-#: the point of the ruling: the money switches used to be "which rung do we try" (a question about
-#: our plumbing), and they are now "what do you want captured" (a question about the user's data).
-#: Profile is always captured and has no switch — an automation that fetches nothing is not an
-#: automation with a step turned off, it is a broken one with a lie on it.
-#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and
-#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the
-#: three toggles" names the CAPTURE ladder it is reshaping.
-NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics",
- "capture_comments": "commentMetrics", "write": "write"}
-
-
-def _cron_label(cron):
- return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "")
-
-
-def node_status(steps, nid):
- """A node's dot — from the last run's MEASURED step outcome, or `idle`.
-
- ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely
- so a negative control can prove that. Every run stored before W19-C has no `steps` map at all,
- and painting those nodes green because the run said `ok` would manufacture a measurement — the
- same defect as the green dot over the empty posts table recorded in `run_field_instagram`.
- No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI.
- """
- v = str((steps or {}).get(nid) or "")
- return v if v in NODE_STATES else "idle"
-
-
-def graph(defn):
- """One automation → `{nodes, edges}`, left to right. Pure over the definition."""
- defn = defn if isinstance(defn, dict) else {}
- cfg = defn.get("config") or {}
- sched = defn.get("schedule") or {}
- runs = defn.get("runs") or []
- last = runs[0] if runs and isinstance(runs[0], dict) else {}
- steps = last.get("steps") if isinstance(last.get("steps"), dict) else {}
-
- def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True):
- return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail,
- "col": col, "row": row, "panel": panel or nid, "enabled": bool(on),
- "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)}
-
- on = bool(sched.get("enabled"))
- trg = defn.get("trigger") or {}
- if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"):
- # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail
- # read the same node, so an event-triggered flow must not read "Schedule". The membership
- # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old
- # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so
- # the else-branch was reachable) — green, compiling, and wrong on screen.
- t_on = bool(trg.get("enabled", True)) and not trg.get("paused")
- sub = TRIGGER_LABELS.get(trg["key"], trg["key"])
- watched = ", ".join(trg.get("fields") or []) or "any field"
- # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the
- # whole of it) and `record_updated` no longer names a condition (its watched fields are).
- # These two lines are the migration's visible half: a node still describing a `.field`
- # the validator has stopped storing would be the surface and the store disagreeing.
- det = {"event_field": (f"{trg.get('table', '')}"
- + (f" — {_lane_sentence(trg.get('when'))}"
- if trg.get("when") else "")),
- "record_updated": f"{trg.get('table', '')} — {watched}",
- "record_created": str(trg.get("table") or ""),
- "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}",
- "form_submitted": str(trg.get("table") or ""),
- "webhook": "POST the hook URL to fire it",
- "ig_profile_match": (_predicate_sentence(cfg.get("predicates"),
- cfg.get("operator"))[:80]
- if cfg.get("predicates")
- else "No filters yet — nothing to search for"),
- "email": str(trg.get("query") or "")}.get(trg["key"], "")
- if not trg.get("configured", True):
- det = "Finish setting this trigger up before it can fire"
- if trg.get("paused"):
- det = str(defn.get("statusNote") or "paused")
- nodes = [node("trigger", "trigger", sub,
- "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)]
- else:
- # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off
- # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with
- # the user's own pick — the same class of defect as the Database picker that could not be
- # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the
- # disabled-schedule gate asserts, and it was never the wrong half.
- # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires
- # TODAY, not that it may never be scheduled — the cron has to stay one click away, and
- # this is the only node that offers it.
- nodes = [node("trigger", "trigger", "Schedule" if on else "Manual",
- _cron_label(sched.get("cron")) if on else "Manual only", 0,
- panel="schedule", on=on,
- detail="" if on else "It still runs when you press Run now")]
- edges = []
-
- if defn.get("kind") == "field_instagram":
- # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED.
- # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and
- # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5
- # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the
- # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners
- # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas.
- # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an
- # implementation detail: the profile is always captured (R6), Posts and Comments are the
- # opt-ins, and both are OFF until asked for.
- metrics_on = bool(cfg.get("postMetrics"))
- comments_on = bool(cfg.get("commentMetrics"))
- max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL
- nodes += [
- node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1,
- panel="source",
- detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField")
- else "URL column: the one the field is bound to"),
- # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY
- # and no engagement (measured), so likes/comments are a SECOND vendor call per post.
- # A switch that multiplies a run's cost by the post count deserves to be visible on
- # the canvas rather than buried in a config panel.
- node("capture_posts", "capture", "Post data",
- "Likes and comments per post" if metrics_on else "Off",
- 2, panel="capture", on=metrics_on,
- detail=(f"Up to {max_posts} posts per profile" if metrics_on else
- "ut_ig_post_snapshots only grows while this is on")),
- node("capture_comments", "capture", "Comment data",
- "Comments on those posts" if comments_on else "Off",
- 3, panel="capture", on=comments_on,
- detail=("The full comments dataset — many rows per post" if comments_on else
- "Comments already embedded in a paid post row are kept either way")),
- node("write", "write", "Write",
- cfg.get("targetTable") or "No database", 4, panel="write",
- on=not cfg.get("dryRun"),
- detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"),
- ]
- edges = [{"from": "trigger", "to": "source", "label": ""},
- {"from": "source", "to": "capture_posts", "label": ""},
- {"from": "capture_posts", "to": "capture_comments", "label": ""},
- {"from": "capture_comments", "to": "write", "label": ""}]
- elif defn.get("kind") == "discover_instagram":
- limit = int(cfg.get("recordsLimit") or 0)
- est = discover_estimate(limit)
- pending = str((defn.get("state") or {}).get("pendingSnapshot") or "")
- nodes += [
- node("find", "source", "Find profiles",
- _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1,
- panel="find",
- detail=(f"Up to {limit} profiles · about ${est['usd']}"
- if bd_ready()
- else "Profile search is not set up yet")),
- node("collect", "capture", "Collect results",
- "Searching…" if pending else "Takes about 20 minutes", 2,
- panel="find",
- detail=""),
- node("write", "write", "Save results",
- cfg.get("targetTable") or DISCOVER_TABLE, 3, panel="write",
- on=not cfg.get("dryRun"),
- detail=""),
- ]
- edges = [{"from": "trigger", "to": "find", "label": ""},
- {"from": "find", "to": "collect", "label": ""},
- {"from": "collect", "to": "write", "label": ""}]
- elif defn.get("kind") == "scrape_db":
- url = str(cfg.get("url") or "")
- host = (urlparse(url).hostname or url or "No URL") if url else "No URL"
- which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld"
- else f"HTML table #{int(cfg.get('tableIndex') or 0)}")
- mapped = len(cfg.get("fieldMap") or {})
- nodes += [
- node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]),
- node("extract", "capture", "Extract", which, 2, panel="columns",
- detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: "
- f"{cfg.get('keyField') or '—'}"),
- node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3,
- panel="write", on=not cfg.get("dryRun"),
- detail=cfg.get("targetTable") or "a new database"),
- ]
- edges = [{"from": "trigger", "to": "fetch", "label": ""},
- {"from": "fetch", "to": "extract", "label": ""},
- {"from": "extract", "to": "write", "label": ""}]
- # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point.
- # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught
- # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML
- # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A
- # surface that draws nothing is obviously incomplete; one that draws a fetch step that does
- # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at
- # once because all three read this.
- #
- # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE
- # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node,
- # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on
- # that promise, so it is stated rather than left to be inferred from the control flow.
- return {"nodes": nodes, "edges": edges}
-
-
-def toggle_node(rt, auto_id, node_id):
- """Flip ONE node's switch. Returns `(defn, error)`.
-
- The node → config mapping lives on the SERVER for the reason the whole graph does: a client
- that knew which field each node writes would be a second copy of that knowledge, free to
- drift. The canvas just says "this node was clicked".
- """
- prev = all_definitions(rt).get(str(auto_id))
- if prev is None:
- return None, "no such automation"
- target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None)
- if target is None:
- return None, f"there is no {node_id!r} step in this automation"
- if not target.get("toggle"):
- return None, (f"the {target['title']} step cannot be turned off — it is what this "
- f"automation is")
- cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {})
- which = target["toggle"]
- trg_key = str((prev.get("trigger") or {}).get("key") or "")
- if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS:
- # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping
- # the cron under a node labelled "When a field changes" would be a switch that lies.
- # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that
- # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so
- # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so
- # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half.
- trg = dict(prev.get("trigger") or {})
- trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused"))
- trg["paused"] = False # a deliberate flip clears an auto-pause: human wins
- return patch(rt, auto_id, {"trigger": trg})
- if which == "schedule":
- sched["enabled"] = not sched.get("enabled")
- elif which == "postMetrics":
- cfg["postMetrics"] = not cfg.get("postMetrics")
- elif which == "commentMetrics":
- cfg["commentMetrics"] = not cfg.get("commentMetrics")
- elif which == "write":
- cfg["dryRun"] = not cfg.get("dryRun")
- return patch(rt, auto_id, {"config": cfg, "schedule": sched})
-
-
-# ---------------------------------------------------------------------------------------------
-# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed
-# ---------------------------------------------------------------------------------------------
-# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were
-# records of the target database, and each record's position lived in a single-select STAGE FIELD
-# the engine created on the customer's own database. Wave 26 deleted the board's built-in
-# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`,
-# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review
-# branch of the action walk, and the endings that sent a terminal card back round.
-#
-# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote
-# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles`
-# counter per automation — to render a view of state the RUN LOG already holds. The customer paid
-# for that in columns they did not ask for, in a store commit on every scheduled run, and in a
-# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we
-# had added. `migrate_ig_tables` now drops those columns wherever an automation created them.
-#
-# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE:
-# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the
-# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was
-# one of four callers. The name is the last thing the board left behind here.
-# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board`
-# — the MIGRATION. It must outlive the thing it retires, or a definition written before the
-# retirement walks into a runtime that no longer understands it.
-# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are
-# parked with no caller and say so at their own definitions.
-
-LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes",
- "is_empty", "is_not_empty")
-LANE_NULLARY_OPS = ("is_empty", "is_not_empty")
-
-# ─────────────────────────────────────────────────────────────────────────────────────────────
-# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}`
-#
-# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why
-# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches
-# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc
-# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the
-# same function applied one level up, so the cheaper honest version is to evaluate the leaf where
-# it lies and never touch the bytes.)
-#
-# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An
-# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and
-# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep.
-# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an
-# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that
-# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine
-# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it.
-MAX_COND_DEPTH = 3
-MAX_COND_CHILDREN = 12
-COND_GROUP_KEYS = ("all", "any")
-
-
-def clean_cond(raw, depth=0, where=""):
- """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no
- condition", which is legal everywhere a condition is optional (the catch-all lane, an
- unconditioned trigger, an action group that always runs)."""
- if raw in (None, "", {}):
- return None, None
- at = f" on {where}" if where else ""
- if not isinstance(raw, dict):
- return None, f"the condition{at} must be an object"
- conj = [k for k in COND_GROUP_KEYS if k in raw]
- if len(conj) > 1:
- return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the "
- f"other")
- if conj:
- key = conj[0]
- if depth + 1 > MAX_COND_DEPTH:
- return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — "
- f"the group{at} is deeper")
- kids_raw = raw.get(key)
- if not isinstance(kids_raw, list) or not kids_raw:
- return None, f"the '{key}' group{at} needs at least one condition inside it"
- if len(kids_raw) > MAX_COND_CHILDREN:
- return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — "
- f"the '{key}' group{at} has {len(kids_raw)}")
- kids = []
- for child in kids_raw:
- c, err = clean_cond(child, depth + 1, where)
- if err:
- return None, err
- if c is None:
- return None, (f"an empty condition sits inside the '{key}' group{at} — "
- f"finish it or remove it")
- kids.append(c)
- return {key: kids}, None
- field = _s(raw.get("field"), 80).strip()
- op = _s(raw.get("op") or raw.get("operator"), 20).strip()
- if not field:
- # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could
- # act on. It states a fact about the stored tree; it never said what to DO, and the usual
- # cause is not a typo but an empty field picker (the flow had no walking record to offer
- # columns from). The fix belongs in the message.
- return None, (f"the condition{at} names no field — pick a column, or remove the "
- f"condition")
- if op not in LANE_OPS:
- return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS)
- cond = {"field": field, "op": op}
- if op not in LANE_NULLARY_OPS:
- v = raw.get("value")
- if v is None or (isinstance(v, str) and not v.strip()):
- return None, f"give a value to compare {field!r} against{at}"
- cond["value"] = v.strip() if isinstance(v, str) else v
- return cond, None
-
-
-def cond_fields(cond):
- """Every field name a tree reads — the set a caller must have on the row image before the
- answer means anything."""
- if not isinstance(cond, dict):
- return set()
- for key in COND_GROUP_KEYS:
- if key in cond:
- out = set()
- for child in cond.get(key) or []:
- out |= cond_fields(child)
- return out
- f = cond.get("field")
- return {f} if f else set()
-#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would
-#: collide with the stage the label is a choice FOR, and the board could no longer tell a
-#: routed card from a gated one.
-# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the
-# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a
-# legacy stamped cell would read as the same lane while behaving as a different one.
-def _lane_num(v):
- try:
- return float(str(v).replace(",", ""))
- except (TypeError, ValueError):
- return None
-
-
-def lane_match(cond, row):
- """Does `row` satisfy this condition TREE? None matches everything (the catch-all).
-
- Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22
- caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping
- this name rather than growing a second entry point that could disagree with it.
-
- ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse
- as a number answers False — the record simply does not enter the lane — never "treat blank
- as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank
- cell is an unknown, and an unknown cannot be less than 10,000.
- """
- if cond is None:
- return True
- if isinstance(cond, dict):
- if "all" in cond:
- return all(lane_match(c, row) for c in cond.get("all") or [])
- if "any" in cond:
- return any(lane_match(c, row) for c in cond.get("any") or [])
- raw = (row or {}).get(cond.get("field"))
- op = cond.get("op")
- if op == "is_empty":
- return raw in (None, "")
- if op == "is_not_empty":
- return raw not in (None, "")
- want = cond.get("value")
- if op in (">", ">=", "<", "<="):
- a, b = _lane_num(raw), _lane_num(want)
- if a is None or b is None:
- return False
- return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op]
- have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "")
- if op == "includes":
- return want_s.lower() in have_s.lower()
- if op == "not_includes":
- return want_s.lower() not in have_s.lower()
- a, b = _lane_num(raw), _lane_num(want)
- same = (a is not None and b is not None and a == b) or have_s == want_s
- return same if op == "=" else (not same) if op == "!=" else False
-
-
-# ─────────────────────────────────────────────────────────────────────────────────────────────
-# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things.
-#
-# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run.
-# A `group` holds nested actions behind a condition — Airtable's "conditional action group",
-# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's
-# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this
-# builder is a column, not a canvas).
-#
-# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a
-# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack /
-# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty
-# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready
-# kind with a sentence, so the faded state is a wall and not a styling choice.
-MAX_ACTIONS = 20 # per automation, counting nested ones
-MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow
-MAX_BRANCHES = 6 # per If / then
-MAX_ACTION_VALUES = 20 # cells one update/create action may write
-FIND_LIMIT_MAX = 100
-#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder`
-#: and never by a literal list of group names: a client-side ordering is a second copy of this
-#: table, and the way it fails is that a group added here renders last, or not at all, with
-#: nothing red anywhere.
-ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4}
-ACTION_CATALOG = [
- # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ────────
- # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row
- # is a WALL, not a styling choice — and each `detail` says the true reason rather than a
- # placeholder, because "coming soon" on five rows is how a menu stops being believed.
- #
- # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24
- # scheduled this build for wave 25 and wave 25's six owner items do not include it, so
- # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its
- # own promises has to be re-read by whoever misses the date, and a stale date is worse than
- # none because it reads as a commitment somebody already broke. The replacement names the
- # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule.
- {"kind": "web_goto", "label": "Open a page", "group": "Web action", "ready": False,
- "detail": "Needs the browser job — a real browser running inside an ephemeral worker"},
- {"kind": "web_fill", "label": "Fill a field", "group": "Web action", "ready": False,
- "detail": "Needs the recorder that captures a selector instead of a coordinate"},
- {"kind": "web_click", "label": "Click something", "group": "Web action", "ready": False,
- "detail": "Needs the recorder that captures a selector instead of a coordinate"},
- {"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": False,
- "detail": "Needs the browser job — the scrape action reads static pages today"},
- {"kind": "web_repair", "label": "Repair a broken step", "group": "Web action",
- "ready": False,
- "detail": "Self-healing selectors arrive with the browser engine (Scrapling)"},
- # ── 2. DATABASE ──────────────────────────────────────────────────────────────────────────
- {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True,
- "detail": "Write values onto the record walking the flow"},
- {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True,
- "detail": "Add a row to another database"},
- {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True,
- "detail": "Look rows up by condition; the run log opens them"},
- # ── 3. CONNECTED ─────────────────────────────────────────────────────────────────────────
- # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND.
- # `field_instagram` was an automation you created to fill one column; this is a step any flow
- # can take, which is the shape it should always have had — enriching a profile is something
- # you do TO a record, not a species of automation.
- # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu,
- # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`.
- {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected",
- "ready": True, "connector": "scraper",
- "detail": "Fill this record's Instagram columns from its profile, and add a point to its "
- "history"},
- {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False,
- "detail": "Needs a send scope on the Gmail connection"},
- {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
- "detail": "Needs the Slack connector"},
- # ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
- # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
- # and R8 made it a FORK with lettered branches, which is a thing people already have a name
- # for. The KIND is untouched — renaming it would orphan every stored action for a caption.
- {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True,
- "detail": "Send the record down one of several branches, by condition"},
- {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic",
- "ready": False, "detail": "Run the same actions on every item in a list"},
- {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False,
- "detail": "Not built — a sandbox is its own decision"},
- {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic",
- "ready": False, "detail": "Needs an AI action implementation"},
-]
-
-
-#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can
-#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an
-#: "Integrations" header, one row per connector, a chevron into that connector's own submenu).
-#:
-#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are
-#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the
-#: key and render the label, never join it against the directory.
-#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key.
-#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one
-#: connector wearing two names on two screens somebody sees within a second of each other, and a
-#: second literal is how that happens. This tuple says only WHICH connectors an action may name;
-#: what they are CALLED has exactly one source.
-ACTION_CONNECTORS = ("scraper",)
-
-
-def _connector_meta(key):
- """`{key, label}` for a connector key, from the one place either picker declares it."""
- key = str(key or "")
- if key not in ACTION_CONNECTORS:
- return None
- return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None)
-
-
-def action_catalog():
- """The catalog as the wire carries it — a copy, because a caller that mutated the module
- constant would change every later reader's answer.
-
- ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from
- `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two
- different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST
- (not first) — a new group appearing above "Web action" because its order defaulted to 0 is
- the failure that would look deliberate.
-
- ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client
- nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two
- rows in one nest cannot disagree about what that nest is called. A row naming an unknown
- connector loses the stamp instead of inventing a nest with a raw slug for a title.
- """
- last = max(ACTION_GROUP_ORDER.values()) + 1
- out = []
- for a in ACTION_CATALOG:
- row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)}
- conn = _connector_meta(a.get("connector"))
- if conn:
- row["connector"] = dict(conn)
- else:
- row.pop("connector", None)
- out.append(row)
- return out
-
-
-def _action_label(act):
- for row in ACTION_CATALOG:
- if row["kind"] == act.get("kind"):
- return row["label"]
- return str(act.get("kind") or "Action")
-
-
-def clean_actions(raw, depth=0, _seen=None, _count=None):
- """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces.
-
- Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder
- survives a Save. A missing or colliding one is minted `act_`; minting on every clean would
- move the selection under the person editing it.
- """
- if raw in (None, ""):
- return [], None
- if not isinstance(raw, list):
- return None, "actions must be a list"
- seen = _seen if _seen is not None else set()
- count = _count if _count is not None else [0]
- out = []
- for entry in raw:
- if not isinstance(entry, dict):
- return None, "each action must be an object with a kind"
- kind = _s(entry.get("kind"), 30).strip()
- row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None)
- if row is None:
- return None, (f"{kind or 'that action'!r} is not one of: "
- + ", ".join(a["kind"] for a in ACTION_CATALOG))
- if not row["ready"]:
- return None, (f"{row['label']!r} is on the menu but not built yet — "
- f"{row['detail'][0].lower()}{row['detail'][1:]}")
- count[0] += 1
- if count[0] > MAX_ACTIONS:
- return None, f"an automation runs at most {MAX_ACTIONS} actions"
- aid = _s(entry.get("id"), 40).strip()
- if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen:
- n = 1
- while f"act_{n}" in seen:
- n += 1
- aid = f"act_{n}"
- seen.add(aid)
- # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER.
- #
- # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't
- # think you can even add Conditions at all. That's not how the Create Record works."*
- # Airtable agrees, and so does the shape: every other action operates ON the record the
- # flow is walking, so "run this only when …" is a question about something
- # that exists. Create record MAKES one. There is nothing to test yet, which is why the
- # picker on this panel was empty and why every condition saved against it named no field —
- # the owner's *"the condition on action act_1 names no field"*. The condition belongs on
- # the trigger, or on a conditional group wrapping the action.
- #
- # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored
- # automation that already carries one — forever, with no way to edit it out, because the
- # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a
- # locked door.
- # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26
- # removes the condition editor from this action kind entirely (C4), so there is no control
- # whose value could appear to be ignored. If a `create_record` condition editor ever comes
- # back, this needs a disclosure channel — `clean_actions` has none today.
- if kind == "create_record":
- when = None
- else:
- when, cerr = clean_cond(entry.get("when"), where=f"action {aid}")
- if cerr:
- return None, cerr
- cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {}
- cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count)
- if cerr:
- return None, cerr
- out.append({"id": aid, "kind": kind,
- "enabled": bool(entry.get("enabled", True)),
- "when": when, "config": cfg})
- return out, None
-
-
-def _clean_action_config(kind, cfg, depth, seen, count):
- """One action's `config`, per kind. Returns `(config, error)`."""
- if kind == "group":
- # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`,
- # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an
- # if with no else, so expressing "otherwise" meant a second group carrying the negation
- # by hand — two conditions a later edit could put out of step with each other.
- if depth + 1 > MAX_GROUP_DEPTH:
- return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — "
- f"past that a flow is a flowchart and belongs on the board")
- raw_branches = cfg.get("branches")
- if not isinstance(raw_branches, list) or not raw_branches:
- # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and
- # the live automations carry it — refusing it would 400 every Save of a flow that
- # was legal yesterday. Read-side, so nothing has to be rewritten in the store.
- raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"),
- "actions": cfg.get("actions")}]
- if len(raw_branches) > MAX_BRANCHES:
- return None, (f"an If / then has at most {MAX_BRANCHES} branches — past that the "
- f"record is being routed, which is what the board's lanes are for")
- out_branches, bseen = [], set()
- for i, br in enumerate(raw_branches):
- if not isinstance(br, dict):
- return None, "each branch of an If / then is an object"
- cond, cerr = clean_cond(br.get("cond"), where="the branch")
- if cerr:
- return None, cerr
- if cond is None and i != len(raw_branches) - 1:
- # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches
- # and a null condition matches everything, so a catch-all above another branch
- # would make every branch below it dead code — silently, and only for the
- # records that reached it. Refused with the reason rather than reordered: this
- # module does not quietly rewrite what somebody built.
- return None, ("only the LAST branch of an If / then can be the Otherwise leg — "
- "a catch-all above another branch makes the ones below it "
- "unreachable")
- kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count)
- if kerr:
- return None, kerr
- if not kids:
- return None, "a branch with no actions inside it does nothing"
- bid = _s(br.get("id"), 40).strip()
- if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen:
- bid = f"b{i + 1}"
- bseen.add(bid)
- label = " ".join(_s(br.get("label"), 40).split()) or (
- "Otherwise" if cond is None else _branch_letter(i))
- out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids})
- return {"branches": out_branches}, None
- if kind in ("update_record", "create_record"):
- values = cfg.get("values")
- if not isinstance(values, dict) or not values:
- return None, f"the {kind.replace('_', ' ')} action writes no values"
- if len(values) > MAX_ACTION_VALUES:
- return None, f"one action writes at most {MAX_ACTION_VALUES} cells"
- clean_vals = {}
- for k, v in values.items():
- key = _s(k, 80).strip()
- if not key:
- return None, "a value is written to a field with no name"
- if v is not None and not isinstance(v, (str, int, float, bool)):
- return None, (f"the value for {key!r} must be text or a number — "
- f"an automation writes cells, not objects")
- clean_vals[key] = _s(v, 500) if isinstance(v, str) else v
- out = {"values": clean_vals}
- # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is
- # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic
- # "Create record". Follows the `review` arm below, which has carried `config.label` since
- # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config`
- # bag so no shared client type has to change for it.
- lbl = " ".join(_s(cfg.get("label"), 60).split())
- if lbl:
- out["label"] = lbl
- if kind == "create_record":
- table = _s(cfg.get("table"), 60).strip()
- if not table:
- return None, "the create record action names no database"
- if not table.startswith(UT_PREFIX):
- return None, (f"actions write to blank databases (ut_*) this wave — "
- f"{table!r} is not one")
- out["table"] = table
- # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS
- # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a
- # Create record duplicates a row per run — silently, and worse every day.
- #
- # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a
- # guess: a stored automation must not change meaning because this key arrived. The
- # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`.
- #
- # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the
- # action never sets means every incoming row carries a BLANK key — `upsert_rows`
- # counts those as `skipped` and writes nothing at all. That is a control that reads
- # as "no duplicates" and delivers "no rows", which is the worse failure by a distance.
- # Same shape as `scrape_db`'s "the key field must be one of the mapped fields"
- # (`clean_config`, above) — one precedent, not a second mechanism.
- unique = _s(cfg.get("uniqueOn"), 80).strip()
- if unique and unique not in clean_vals:
- return None, (f"this action does not write {unique!r}, so it cannot keep records "
- f"unique on it — it writes: " + ", ".join(sorted(clean_vals)))
- out["uniqueOn"] = unique
- return out, None
- if kind == "enrich_instagram":
- # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid
- # rung, its fallback, the per-post engagement buy and the dry run — because they are the
- # things that decide what a run COSTS and R4 moves the capability, not the controls.
- #
- # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than
- # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a
- # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a
- # sentence at run (`profile_field_key`), which is the honest half — refusing the save
- # would make the action unaddable on a database that has not been flagged yet.
- # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent
- # clamps on the same key is how the action and the legacy kind come to disagree about
- # what "10 posts" means.
- #
- # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference
- # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH,
- # so a key being there is a person having typed it. This function gets the whole action
- # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a
- # `maxPosts` here is usually just the client echoing back what was already stored — and
- # every action written before this wave stored the old default of 24. Refusing would
- # therefore 400 those automations forever on a number nobody chose, which is D-65.
- # The ceiling is still taught, in the place where a person can act on it: the panel's
- # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more).
- max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False)
- if perr:
- return None, perr
- # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a
- # working default. That is D-65's lesson applied ahead of time rather than after: an enrich
- # action stored before this wave carries none of these, and it must keep saving and keep
- # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT;
- # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching.
- # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door
- # sees a person's typed number and can refuse politely, while the selection sees whatever
- # is STORED — including configs written before the cap existed. Clamping only here would
- # let an old 5000 through at run time.
- try:
- e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT)
- except (TypeError, ValueError):
- return None, "how many records to enrich has to be a number"
- if e_limit < 1:
- return None, "an enrich step has to be allowed at least one record"
- try:
- e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
- except (TypeError, ValueError):
- return None, "the number of days has to be a number"
- groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts)
- if gerr:
- return None, gerr
- return {"profileField": _s(cfg.get("profileField"), 80).strip(),
- "fromView": _s(cfg.get("fromView"), 80).strip(),
- "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT,
- # Anything that is not "asc" is newest-first, so a typo cannot invert the order a
- # person is spending against — it lands on the documented default instead.
- "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc"
- else "desc",
- "limit": min(e_limit, MAX_ENRICH_PER_RUN),
- "skipRecent": bool(cfg.get("skipRecent")),
- "skipRecentDays": max(1, e_days),
- # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in
- # `clean_config` (R5 / C2): a stored action carrying them still saves and simply
- # loses them, because refusing a retired key 400s every definition written before
- # this wave (D-65).
- # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
- "postMetrics": bool(cfg.get("postMetrics")),
- # The separate Comments dataset is more granular than a post read and stays off
- # until the user intentionally enables it.
- "commentMetrics": bool(cfg.get("commentMetrics")),
- "dryRun": bool(cfg.get("dryRun")),
- # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so
- # every action stored before today keeps saving and keeps running unchanged.
- "postGroups": groups,
- "maxPosts": max_posts}, None
- if kind == "find_records":
- table = _s(cfg.get("table"), 60).strip()
- if not table:
- return None, "the find records action names no database"
- cond, cerr = clean_cond(cfg.get("cond"), where="the find")
- if cerr:
- return None, cerr
- try:
- limit = int(cfg.get("limit") or 25)
- except (TypeError, ValueError):
- return None, "the find records limit must be a whole number"
- if limit < 1 or limit > FIND_LIMIT_MAX:
- return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}"
- return {"table": table, "cond": cond, "limit": limit}, None
- return {}, None
-
-
-def _branch_letter(i):
- """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a
- label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is
- worse than an ugly name."""
- return chr(65 + i % 26) * (1 + i // 26)
-
-
-def group_branches(act):
- """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape.
-
- ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying
- for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored
- group might carry either shape, and the one that forgot would silently skip a live
- automation's actions. A definition cleaned since wave 24 always carries `branches`; one read
- straight out of the store (the runner reads definitions, not payloads) may not.
- """
- cfg = (act or {}).get("config") or {}
- brs = cfg.get("branches")
- if isinstance(brs, list) and brs:
- return [b for b in brs if isinstance(b, dict)]
- return [{"id": "b1", "label": "A", "cond": cfg.get("cond"),
- "actions": cfg.get("actions") or []}]
-
-
-def walk_actions(actions):
- """Every action in the tree, depth-first, groups included. One walker, so "how many actions
- does this flow have" and "which review stages exist" cannot answer differently.
-
- ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action
- COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version
- that still read `config.actions` would report a fork's contents as zero actions, hide every
- review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing.
- """
- for act in actions or []:
- yield act
- if act.get("kind") == "group":
- for br in group_branches(act):
- for kid in walk_actions(br.get("actions")):
- yield kid
-
-
-def interpolate(text, row):
- """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an
- action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would
- put template syntax in front of a customer, and a blank is the visible failure."""
- if not isinstance(text, str) or "{{" not in text:
- return text
- def _sub(m):
- v = (row or {}).get(m.group(1).strip())
- return "" if v is None else str(v)
- return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text)
-
-
-# ─────────────────────────────────────────────────────────────────────────────────────────────
-# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order.
-#
-# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits
-# them once at the end (the 256-commits/hr budget every writer in this module respects). A
-# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget
-# on one busy flow.
-# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the
-# runner stamps its stage and stops walking THAT record — later actions belong to the branch a
-# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause.
-# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the
-# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write
-# cannot fire an event trigger, its own or a sibling's.
-
-def _act_row_patch(patches, table, rid, values):
- patches.setdefault(table, {}).setdefault(str(rid), {}).update(values)
-
-
-def profile_field_key(table, named=""):
- """C3/C4: WHICH COLUMN on this database carries the Instagram handle. '' when none does.
-
- Resolution order, and the missing branch is the important one:
- 1. the column the action NAMES — the person picked it, so it is not a guess;
- 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`;
- 3. otherwise **NOTHING**, and the run says so.
-
- ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url`
- column" and is exactly right for `field_instagram`, whose whole configuration was a URL column
- — but here it would silently bind the enrich step to whatever URL column happens to be first,
- on a database where nobody has said which column is a profile. The failure would be a run that
- reports success having enriched from the wrong column, which is worse than one that refuses:
- a wrong number is harder to notice than a missing one, and it would be written into the
- permanent history as if it had been measured.
- """
- fields = (table or {}).get("fields") or []
- want = str(named or "").strip()
- if want:
- return want if any(str(f.get("key")) == want for f in fields) else ""
- for f in fields:
- p = f.get("profile")
- if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG:
- return str(f.get("key") or "")
- return ""
-
-
-def _flow_table(defn):
- """The database a flow's actions and ending operate on: the automation's own target, or the
- table its event trigger watches when the flow has no target of its own."""
- cfg = (defn or {}).get("config") or {}
- if defn.get("kind") == "discover_instagram":
- return cfg.get("targetTable") or DISCOVER_TABLE
- return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "")
-
-
-#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN.
-#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the
-#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]`
-#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own.
-MAX_ENRICH_PER_RUN = 100
-DEFAULT_ENRICH_LIMIT = 25
-DEFAULT_ENRICH_COOLDOWN_DAYS = 30
-#: Which column the selection orders by when nobody has said. `first_found` newest-first is the
-#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what
-#: discovery just found rather than re-walking the oldest leads in the book.
-DEFAULT_ENRICH_SORT = "first_found"
-
-#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL.
-#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence
-#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is
-#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored.
-#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with
-#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
-#: accident, and `_commit_run`'s numeric filter is the second net under it.
-RUN_NOTES_KEY = "_notes"
-#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES
-#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval.
-#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence
-#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so
-#: the verdict should expire. But nothing about US changes on day 31 — the only new information
-#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per
-#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a
-#: hundred stale handles pays a hundred times a month to be told the same thing.
-#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors:
-#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no
-#: wiring at all, because the corrected handle simply is not the one we recorded; and
-#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle
-#: (a person saying "try it again") also re-arms.
-#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered
-#: them: a tombstone the owner cannot see is just a disappearance.
-#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
-#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
-#: opposite mask ([[gate-answers-the-wrong-question]]).
-PENDING_PROFILE_MAX_HOURS = 24
-
-
-def _order_key(value):
- """Order one CELL. Numbers numerically, everything else as text.
-
- ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text —
- which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting
- paid for. Numbers are tried first and text is the fallback, never the other way round.
- ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where
- lexicographic order IS chronological order. If that format ever changes, this comment is the
- thing that was relied on.
- """
- s = str(value if value is not None else "").strip()
- try:
- return (0, float(s.replace(",", "")), "")
- except ValueError:
- return (1, 0.0, s.lower())
-
-
-def _days_since(day, today=None):
- """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read.
-
- ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a
- large number — the cooldown below reads an unreadable stamp as "never enriched", which spends
- money rather than skipping. That is the right way round: a skipped record is invisible, and a
- stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data.
- """
- import datetime as _dt
- s = str(day or "").strip()[:10]
- if not s:
- return None
- try:
- d = _dt.date.fromisoformat(s)
- except ValueError:
- return None
- return ((today or _dt.date.today()) - d).days
-
-
-def _hours_since(stamp):
- """Whole hours since a full ISO stamp, or None when it cannot be read.
-
- ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date`
- column and truncates to 10 characters. A pending snapshot's age is measured in hours and its
- stamp carries a time, so reusing that function would read every entry as "queued at
- midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid,
- outstanding snapshot out of the queue.
- """
- import datetime as _dt
- s = str(stamp or "").strip()
- if not s:
- return None
- try:
- t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
- except ValueError:
- return None
- now = _dt.datetime.now(_dt.timezone.utc)
- if t.tzinfo is None:
- t = t.replace(tzinfo=_dt.timezone.utc)
- return max(0, int((now - t).total_seconds() // 3600))
-
-
-def _gone_key(row, handle):
- """The identity a "this account does not exist" verdict is remembered under.
-
- `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on
- Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what
- every row on a preset profile table is and what `preset_cells` stamps.
- """
- plat = str((row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower()
- return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
-
-
-def clear_gone(rt, table_key, handle, row=None):
- """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle.
-
- Returns the number of automations whose memory was changed — 0 is the ordinary answer and is
- not an error, because most cell edits are not on a dead handle.
-
- ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER.
- The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to,
- because it is a fact about what a run learned and paid for. But the thing that re-arms it is a
- ROW event, and the row writer must not need to know the shape of an automation's state to
- trigger it. So the seam is one call with the three things the writer already has, and every
- walk of `all_definitions` stays on this side of the fence.
- ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which
- handle is this", so a verdict can never be recorded under a key this cannot find
- ([[one-evaluator-per-question]]).
-
- ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first
- door — correcting a typo — works with no wiring at all, because the key IS the handle and a
- different handle is simply not the one we recorded. This second door only opens when the row
- write path calls it, which lives in `routes_tables.patch_row` (another session's fence).
- Until that line lands, re-typing the SAME dead handle stays skipped
- ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later).
- """
- key = _gone_key(row or {}, handle)
- if not str(handle or "").strip():
- return 0
- changed = 0
- for auto_id, defn in (all_definitions(rt) or {}).items():
- if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key):
- continue
- known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {})
- if key not in known:
- continue
- known.pop(key, None)
- # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what
- # every other state writer here does and what keeps a definition from growing a graveyard
- # of empty maps.
- set_state(rt, str(auto_id), {"enrichNotFound": known or None})
- changed += 1
- return changed
-
-
-def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch,
- step=_no_step, log=print):
- """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved.
-
- ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a
- LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an
- hour"); the only profile caller passed a single-element list from inside the per-record walk.
- MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25
- vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand.
-
- ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from
- `prefetch`, and the per-record rung then behaves exactly as it does today — including its
- corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a
- batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure
- returns 0 instead of raising.
-
- ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now
- serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they
- share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the
- handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch`
- behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time.
- """
- want, dests, handles = [], {}, []
- for rid in (chosen or []):
- row = rows.get(str(rid)) or {}
- h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower()
- if not h or h in prefetch:
- continue
- want.append((h, str(rid)))
- if h not in dests:
- handles.append(h)
- dests.setdefault(h, []).append(str(rid))
- if not handles:
- return 0
- step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source")
- deferrals = []
- try:
- nodes, note = bd_profiles_batch(handles, deferred=deferrals)
- except Exception as exc: # noqa: BLE001 — never fatal, see docstring
- log(f"[aios-auto] profile batch failed, falling back to per-record reads: "
- f"{type(exc).__name__}: {exc}")
- return 0
- prefetch.update(nodes)
- if note:
- log(f"[aios-auto] profile batch note: {_s(note, 300)}")
- for d in deferrals:
- sid = str(d.get("snapshotId") or "")
- if not sid:
- continue
- for u in (d.get("urls") or []):
- h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower()
- if not h or h in nodes:
- continue
- prefetch[h] = {DEFERRED_MARK: sid}
- for rid in dests.get(h, []):
- enrich["pendingProfiles"].append(
- {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""),
- "urls": [str(u)], "kind": "profile", "influencer": h,
- "table": table, "rowId": rid, "requestedAt": _iso()})
- return len(nodes)
-
-
-def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
- """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order.
-
- Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did
- and is surfaced in the run summary; `""` means there is nothing worth saying.
-
- ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's
- ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is
- enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every
- skipped record until 30 have actually been enriched or the list runs out. The naive reading —
- take the top 30, then filter — would have billed for 10 and reported success, and the number in
- the box would have meant something different every run depending on how much of the top of the
- list happened to be fresh. A quota is predictable; a filtered window is not.
-
- The order of operations, each step narrowing the one above:
- 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view`
- and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an
- empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich
- the entire database", at vendor prices.
- 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions
- (a blank is unknown, not smallest).
- 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on.
- 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`.
-
- ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is
- nothing to enrich and it must not consume a slot somebody paid for.
- """
- t = ut_get(rt, str(table_key or ""))
- if t is None:
- return [], f"{table_key!r} is not a database in this workspace"
- rows = dict(t.get("rows") or {})
- notes = []
-
- view_id = str(cfg.get("fromView") or "").strip()
- if view_id:
- tree, fields, problem = view_filter(rt, table_key, view_id)
- if problem:
- # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet
- # fallback costs real money.
- return [], f"the enrich step names a view it cannot read — {problem}"
- # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS
- # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE.
- #
- # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}],
- # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/
- # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition —
- # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on
- # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`,
- # got `""`, and answered **False for every row**: MEASURED on the owner's own automation,
- # a view matching exactly one record selected NOTHING, reported NO error, and the run
- # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual
- # record... how come it says 51 records walked?"*
- #
- # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch
- # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has
- # always been correct. `_seed_event_state`'s own docstring states the law this line broke:
- # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it
- # is supposed to arm."* There were three. Now there are two callers of one function.
- # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date
- # as a date and a number as a number, which is the half `lane_match` could not have had.
- import harness.filter_eval as filter_eval
- rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)}
-
- sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT
- newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc"
- # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN".
- #
- # Owner: *"treat a missing first found as now, and manual entry should go first, instead of
- # treated as last. I want to see my inayma manual entry works with automation enrichment."*
- # And they are right about the semantics, not just the preference: on these tables the ONLY
- # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it
- # on every row it writes. So a blank is not missing data — it is a row that was first found
- # today, by the person sitting in front of it, and the one they most want enriched.
- #
- # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position
- # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one
- # it never reached.
- #
- # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about
- # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and
- # treating it as the maximum would spend the budget on the rows we know least about. So a
- # non-date sort keeps the old rule: unknown sorts last, in both directions.
- #
- # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real
- # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at
- # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One
- # rule, both directions, no branch that can disagree with itself.
- ftype = str(((next((f for f in (t.get("fields") or [])
- if str(f.get("key")) == sort_field), None)) or {}).get("type") or "")
- if ftype == "date":
- ordered = sorted(rows.items(),
- key=lambda x: (0, str(x[1].get(sort_field) or "").strip())
- if str(x[1].get(sort_field) or "").strip() else (1, ""),
- reverse=newest_first)
- else:
- have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()]
- blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()]
- have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first)
- blank.sort(key=lambda x: _rid_num(x[0]))
- ordered = have + blank
-
- try:
- quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT)
- except (TypeError, ValueError):
- quota = DEFAULT_ENRICH_LIMIT
- quota = max(1, min(quota, MAX_ENRICH_PER_RUN))
- if cfg.get("limit") and quota != int(cfg.get("limit") or 0):
- notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run")
-
- cooling = bool(cfg.get("skipRecent"))
- try:
- days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
- except (TypeError, ValueError):
- days = DEFAULT_ENRICH_COOLDOWN_DAYS
- days = max(1, days)
-
- picked, cooled, blank_handle = [], 0, 0
- # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST.
- #
- # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT.
- # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is
- # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it
- # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at
- # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix
- # removes that loop; only a memory of the verdict does.
- #
- # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the
- # note goes into the run every single time, not only on the run that discovered it, because a
- # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf".
- skipped_gone = []
- for rid, r in ordered:
- if len(picked) >= quota:
- break
- raw_handle = str(r.get(profile_key) or "").strip()
- if not raw_handle:
- blank_handle += 1
- continue
- # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is
- # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the
- # owner reads to know when we last paid to be told this), it is simply not a clock.
- if isinstance((gone or {}).get(_gone_key(r, raw_handle)), dict):
- skipped_gone.append(raw_handle)
- continue
- if cooling:
- since = _days_since(r.get("enriched_at"), today=today)
- if since is not None and since < days:
- cooled += 1
- continue
- picked.append(str(rid))
-
- # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the
- # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and
- # `run_plain` follow. Silence here would make a shrinking selection invisible.
- if cooled:
- notes.append(f"{cooled} skipped as enriched in the last {days} days")
- if skipped_gone:
- shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
- # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can
- # do, because nothing else will ever retry them — under R9 this note is the only path
- # back from a tombstone, so a vaguer version would strand the row permanently.
- notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
- f"({shown}{', …' if len(skipped_gone) > 5 else ''}) — delete the row or "
- f"correct the handle; they are not retried automatically")
- if blank_handle:
- notes.append(f"{blank_handle} skipped with no handle")
- if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
- notes.append(f"{len(picked)} of the {quota} asked for — the list ran out")
- return picked, "; ".join(notes)
-
-
-def _has_action(actions, kind):
- """Does this flow contain `kind` ANYWHERE, forks included?
-
- ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under
- `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an
- enrich step somebody put inside an If — and the caller would then skip a schema top-up the
- run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24
- recorded when four hand-rolled walks all forgot to descend.
- """
- for a in (actions or []):
- if not isinstance(a, dict):
- continue
- if a.get("kind") == kind:
- return True
- for br in ((a.get("config") or {}).get("branches") or []):
- if _has_action((br or {}).get("actions") or [], kind):
- return True
- return False
-
-
-def _actions_of_kind(actions, kind):
- """Every action of `kind`, including actions nested inside If branches."""
- out = []
- for action in actions or []:
- if not isinstance(action, dict):
- continue
- if action.get("kind") == kind:
- out.append(action)
- for branch in ((action.get("config") or {}).get("branches") or []):
- out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind))
- return out
-
-
-def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print,
- step=_no_step):
- """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts.
-
- ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery
- automation's review stage comes from `stages_for`, not from actions,
- so an early return on an empty action list would have silently disabled the owner's loop
- feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in
- review; the tests all happened to pass a non-empty action list, which is why it was green.
- """
- defn, _retired = _without_retired_board(defn)
- flow = (defn or {}).get("flow") or {}
- actions = flow.get("actions") or []
- # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE.
- #
- # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the
- # "Save results" node) and THEN walked the flow over the rows it had just written — where
- # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on
- # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner
- # ruling, same day) would have made the duplication permanent and unavoidable with it.
- #
- # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what
- # it finds in a database" is exactly what the engine does. What it must not do is do it
- # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the
- # server still refuses to let it be removed.
- if ig_action_pinned(defn, 0) and actions:
- actions = actions[1:]
- counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0}
- # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected
- # across the WHOLE walk and written once at the end, for the reason the module header states
- # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
- # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an
- # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
- enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
- # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor)
- # and handles a vendor has said do not exist. Both are collected across the whole
- # walk and written once, for the same reason every other list here is.
- "pendingProfiles": [], "gone": {},
- "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False}
- # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them
- # per record and re-reading the definition per row would be a store read per record.
- known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})
- # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
- # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
- # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked
- # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps
- # in one flow are two independent budgets.
- enrich_plan = {}
- # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK.
- # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a
- # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was
- # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever
- # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are
- # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth
- # buying twice.
- enrich_prefetch = {}
- if not table_key:
- return counts
- table = str(table_key)
- # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN.
- #
- # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled
- # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a
- # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door.
- # So every preset the target table did not already happen to have was silently dropped, and
- # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET
- # FILLED"* — this is the half that was swallowing them.
- #
- # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`,
- # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose
- # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the
- # feature worked on discovery tables purely because discovery creates them for its own reasons.
- #
- # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present:
- # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady
- # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no
- # enrich step never touches the schema of the database it walks.
- # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset
- # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an
- # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a
- # step at it, and (b) add a SECOND profile column to a table that already flagged a different
- # one, which `user_tables` forbids at both write doors. A table with no profile column is an
- # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its
- # own honest refusal, not something to paper over by inventing the binding.
- _tbl_now = ut_get(rt, table) or {}
- _enrich_actions = _actions_of_kind(actions, "enrich_instagram")
- _bound = next((f for f in (_tbl_now.get("fields") or [])
- if isinstance(f.get("profile"), dict)), None)
- if _bound is None:
- _named = next((str((a.get("config") or {}).get("profileField") or "").strip()
- for a in _enrich_actions
- if str((a.get("config") or {}).get("profileField") or "").strip()), "")
- _bound = next((f for f in (_tbl_now.get("fields") or [])
- if str(f.get("key") or "") == _named), None)
- if _bound and _enrich_actions:
- # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists —
- # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a
- # table whose profile column is `ig_handle` would otherwise gain a rival `handle`.
- topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")}
- if f.get("key") != _bound.get("key") else dict(f))
- for f in PRESET_PROFILE_FIELDS]
- try:
- if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions):
- ensure_ig_graph(rt, username, str(defn.get("id") or ""),
- profile_table=table, profile_field=str(_bound.get("key") or ""))
- else:
- # Dry run keeps its original no-create contract; only the already-established
- # Profile table is topped up, and no canonical related database is spawned.
- ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table,
- flow_tag=str(defn.get("id") or ""), lock_fields=True)
- except Exception as exc: # noqa: BLE001
- # A schema top-up that cannot run must not stop the enrichment: the cells that DO
- # have columns still land, which is strictly better than the pull being thrown away.
- log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}")
- rows = dict((ut_get(rt, table) or {}).get("rows") or {})
- if not rows:
- return counts
- # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The
- # walk used to keep a `reached` map and stamp each record with the furthest action it made —
- # a board column's worth of bookkeeping written into the tenant's own table on every run.
- # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a
- # cell the customer has to look at and a store commit we pay for on every scheduled run.
- # The RUN LOG is the operational record of what happened now, and it is the only one.
- patches, creates = {}, {}
-
- def _walk(acts, row, rid, depth=0):
- """Walk the actions in order for ONE record.
-
- ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean
- "this record was SUSPENDED by a review gate" — the one branch that could stop a walk
- early. With review retired nothing suspends anything, so every record walks its whole
- flow; the signature stays so a future gate-style action has an obvious place to say so.
- """
- for act in acts:
- if not act.get("enabled", True):
- continue
- if not lane_match(act.get("when"), row):
- continue
- kind = act.get("kind")
- cfg = act.get("config") or {}
- counts["actionsRun"] += 1
- if kind == "group":
- # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs.
- # Declaration order is priority order, the same rule `route_record` applies to
- # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch
- # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions`
- # has already guaranteed a null condition can only be LAST.
- for br in group_branches(act):
- if lane_match(br.get("cond"), row):
- if _walk(br.get("actions") or [], row, rid, depth + 1):
- return True
- break
- elif kind == "update_record":
- vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()}
- row.update(vals) # later actions see the write, as they must
- _act_row_patch(patches, table, rid, vals)
- counts["updated"] += 1
- elif kind == "create_record":
- target = str(cfg.get("table") or "")
- vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()}
- # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions
- # may legitimately write to ONE database on different keys — collapsing them onto
- # the table would silently apply one action's uniqueness rule to the other's rows.
- creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals)
- counts["created"] += 1
- elif kind == "enrich_instagram":
- # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions
- # `run_field_instagram` calls, not a second implementation of either.
- pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"))
- if not pkey:
- # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a
- # property of the flow, so a 100-record run would otherwise put the same
- # sentence in the log a hundred times and bury everything else.
- if "unbound" not in enrich["notes"]:
- enrich["notes"].append("unbound")
- log("[aios-auto] enrich_instagram: no profile column on "
- f"{table} — name one on the action, or mark a text column as an "
- "Instagram profile")
- continue
- # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET?
- # Resolved once (see `enrich_plan`) and then a membership test. A record outside
- # the selection is NOT an error and NOT a skip worth logging per row — it is simply
- # not this run's work, and the selection's own note already accounts for it.
- aid = str(act.get("id") or "")
- if aid not in enrich_plan:
- chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone)
- enrich_plan[aid] = set(chosen)
- if sel_note:
- enrich["notes"].append(sel_note)
- # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else:
- # this is the only place the full chosen set and the row bodies are both in
- # hand. Anything it resolves the per-record rung below reads from memory.
- _prime_enrich_batch(chosen, rows, pkey, table, enrich,
- enrich_prefetch, step, log)
- if str(rid) not in enrich_plan[aid]:
- continue
- handle_raw = str(row.get(pkey, "") or "").strip()
- if not handle_raw:
- continue # nothing to enrich on this record, not an error
- # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED.
- # A handle already in `enrich_prefetch` was bought by the batch above and is read
- # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the
- # batching win straight back (25 records = ~62 s of pure waiting).
- if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \
- not in enrich_prefetch:
- time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does
- enrich["profiles"] += 1
- # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with
- # the row it belongs to: the collector writes preset cells back onto THAT record,
- # and a run-wide list would have no way to say which handle each snapshot was for.
- pend_prof = []
- res = pull_profile(handle_raw,
- max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
- log=log,
- post_metrics=bool(cfg.get("postMetrics")),
- comment_metrics=bool(cfg.get("commentMetrics")),
- pending_metrics=(enrich["pending"]
- if not cfg.get("dryRun") else None),
- pending_profile=(pend_prof
- if not cfg.get("dryRun") else None),
- prefetch=enrich_prefetch)
- # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels /
- # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is
- # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped.
- # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this
- # costs nothing and changes nothing until somebody configures it.
- # ⚠ IT FILTERS THE WINDOW WE ALREADY BOUGHT, it does not widen it. The profile
- # read is capped at 12 posts (MEASURED: asked 40, got 12), so "12 reels AND 12
- # images" needs `bd_profile_posts` — which is built and proven (30 rows in 90 s)
- # but deliberately NOT wired here: the views top-up runs INSIDE `pull_profile`
- # against the posts it captured, so swapping a wider list in afterwards would
- # hand back posts with no view counts. Widening belongs ahead of that step, in
- # the paid path, and that is its own change. See DEBT.
- if cfg.get("postGroups"):
- res = {**res, "posts": select_post_groups(res.get("posts") or [],
- cfg.get("postGroups"))}
- for _p in pend_prof:
- _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
- enrich["pendingProfiles"].extend(pend_prof)
- pulled = _iso()
- if res["state"] in ("ok", "partial"):
- enrich["ok"] += 1
- snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
- enrich["snaps"].append(snap_row)
- enrich["posts"].extend(idents)
- enrich["psnaps"].extend(metric_rows)
- enrich["comments"].extend(comment_rows)
- # R3: LATEST onto the record itself. `row.update` too, so a LATER action in
- # this same flow sees the enriched values — the rule `update_record` already
- # follows, and without it "enrich, then route by follower count" would judge
- # the record on the values it had before the pull.
- cells = preset_cells(res, pulled)
- if cfg.get("dryRun"):
- enrich["dry"] = True
- else:
- row.update(cells)
- _act_row_patch(patches, table, rid, cells)
- else:
- enrich["blocked"] += 1
- # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH.
- # It used to be `_s(note, 90)` with no handle attached: three runs blocked
- # the same record and the store could not say which record, let alone why.
- # 90 characters also truncated the one measured sentence mid-clause. This is
- # the most valuable thing the run produces and it now reaches the run entry.
- enrich["notes"].append(
- f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}")
- # A vendor STATING that the account does not exist is remembered, so the next
- # run stops paying to be told the same thing.
- if res.get("gone"):
- enrich["gone"][_gone_key(row, handle_raw)] = {
- "at": _iso(), "handle": handle_raw,
- "note": _s(res.get("note"), 300)}
- elif kind == "find_records":
- found = find_records(rt, cfg.get("table"), cfg.get("cond"),
- int(cfg.get("limit") or 25))
- counts["found"] += len(found)
- # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had
- # already stopped being reachable one wave earlier — which is the part worth reading.
- # `_without_retired_board` strips every `review` action out of a definition on the
- # READ path, so `all_definitions` and this function have not seen one since the board
- # was retired. What was left behind was a branch referencing `skey`, `stamp` and
- # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not
- # dead code that merely wasted space: it was a `NameError` held back by a migration
- # rather than by a guard, and any change that let one stored `review` action through
- # would have crashed the whole action walk for every record in that flow.
- # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps
- # review "as an AI decision without lanes", and the cheap-first provider ladder behind
- # it is real, working, measured work. They are PARKED, deliberately and in writing,
- # not orphaned; see their own notes.
- return False
-
- if actions:
- for rid in list(row_ids or [])[:FLOOD_LIMIT]:
- row = dict(rows.get(str(rid)) or {})
- if not row:
- continue
- _walk(actions, row, str(rid))
- counts.update(_enrich_flush(rt, defn, username, enrich, log))
- # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was
- # incremented once per create the flow decided to make; what landed is what the store says,
- # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches
- # every row it made last time — which is the whole point of the feature).
- counts.update(_commit_action_writes(rt, table, patches, creates, username, log))
- return counts
-
-
-def migrate_field_instagram(rt, defn):
- """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the
- `enrich_instagram` action. Returns `(definition, changed)`.
-
- R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live
- `field_instagram` automation to a plain flow carrying it; delete the kind and its label."
-
- ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration
- turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** —
- so a live automation whose `urlField` is blank has been working off that fallback for months.
- The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a
- blank `urlField` verbatim would produce an automation that USED to work and now refuses. The
- fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit
- binding — which is also the honest outcome: the column stops being implicit.
-
- ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The
- old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s
- column; the action writes the C1 PRESET CELLS instead. The status column is left in place and
- simply stops being written — deleting somebody's column as part of a migration would be data
- loss, and a stale cell beside a fresh `enriched_at` is readable for what it is.
-
- ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can
- run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN
- against the real stored definition" is meant to be satisfied.
- """
- if (defn or {}).get("kind") != "field_instagram":
- return defn, False
- cfg = dict(defn.get("config") or {})
- target = str(cfg.get("targetTable") or "")
- bound = str(cfg.get("urlField") or "").strip()
- if not bound and target:
- bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "")
- try:
- max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL)
- except (TypeError, ValueError):
- max_posts = 24
- act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
- "config": {"profileField": bound,
- "postMetrics": bool(cfg.get("postMetrics")),
- "commentMetrics": bool(cfg.get("commentMetrics")),
- "dryRun": bool(cfg.get("dryRun")),
- "maxPosts": max(1, min(max_posts, 200))}}
- flow = dict(defn.get("flow") or {})
- out = dict(defn)
- out["kind"] = DEFAULT_KIND
- # The enrich step goes FIRST — it is what the automation was, and every action the owner
- # added afterwards was written expecting the capture to have happened.
- out["flow"] = {"actions": [act] + list(flow.get("actions") or [])}
- out["config"] = {"targetTable": target,
- "targetLabel": str(cfg.get("targetLabel") or "")}
- return out, True
-
-
-def _enrich_flush(rt, defn, username, acc, log):
- """C4: the enrich action's canonical IG tables, written ONCE for the whole run.
-
- Returns the counts the run reports. Empty when no enrich action ran, so a flow without one
- pays nothing and its run history is unchanged (`run_now` merges only non-zero keys).
-
- ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three
- tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then
- the write-through to the platform master. Forking any of those would give the enrich action a
- history that the metric fields (which read `ut_ig_snapshots`) could not see.
- """
- if "unbound" in (acc.get("notes") or []):
- # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step
- # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so
- # this function used to return `{}`, `run_now` merged only non-zero keys, and the run
- # committed **`ok` — "N records walked"** having written nothing at all. The sentence
- # naming the fix went to `log()`, i.e. the server console, which no customer reads. That
- # is "green over nothing" on the one path where somebody is waiting for data.
- # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a
- # special case — and so the run entry itself records that this happened.
- return {"enrichUnbound": 1}
- # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail.
- # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate
- # was skipped as a known-dead handle — so the sentence explaining why the automation appears
- # to do nothing would have been dropped on precisely the runs that most need it, and the
- # owner would be back at "0 records walked, wtf". The selection note is produced before any
- # profile is read and must outlive that early return.
- if not acc.get("profiles"):
- notes = list(acc.get("notes") or [])
- return {RUN_NOTES_KEY: notes} if notes else {}
- out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
- if acc.get("notes"):
- out[RUN_NOTES_KEY] = list(acc["notes"])
- # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so
- # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel.
- if not acc.get("dry"):
- # The vendor's not-found verdicts, merged into engine state (never a tenant column and
- # never a status string — W25/R4 retired those). Merged rather than replaced: a run that
- # walked one record must not forget what earlier runs learned about the others.
- if acc.get("gone"):
- merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})
- merged.update(acc["gone"])
- set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged})
- pending_profiles = queue_pending_profile_snapshots(
- rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or [])
- if pending_profiles:
- out["enrichProfileBatchesPending"] = pending_profiles
- queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""),
- acc.get("pending") or [])
- if queued:
- out["enrichMetricBatchesPending"] = queued
- if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]):
- # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES.
- if acc.get("dry"):
- out["enrichDryRun"] = acc["profiles"]
- return out
- tag = str(defn.get("id") or "")
- profile_table = str(_flow_table(defn) or "")
- graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
- snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
- graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
- missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key)
- snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
- acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
- old_posts, collapsed_posts = dedupe_canonical_rows(
- dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
- posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
- c_post["duplicates"] += collapsed_posts
- psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
- acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
- old_comments, collapsed_comments = dedupe_canonical_rows(
- dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key")
- comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key",
- cap=row_cap(comment_key))
- c_comments["duplicates"] += collapsed_comments
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
- (comment_key, comments)):
- if cur.get(k) is not None:
- cur[k]["rows"] = rws
- _refresh_relations_inplace(cur, log=log)
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
- # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
- # is the failure a chart cannot show you.
- capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"]
- if capped:
- out["enrichCapped"] = capped
- log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap")
- if missing:
- out["enrichMissingTables"] = len(missing)
- log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}")
- out["enrichPoints"] = len(acc["snaps"])
- out["enrichPosts"] = c_post["inserted"]
- out["enrichComments"] = c_comments["inserted"]
- # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same
- # contract `run_field_instagram` follows, because a pooled history with silent holes is worse
- # than none.
- try:
- import ig_master
- m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"],
- acc["posts"], acc["psnaps"])
- if m_status == "error":
- out["enrichMasterFailed"] = 1
- log(f"[aios-auto] enrich_instagram: the platform master copy FAILED — {m_note}; "
- f"the tenant copy is complete and the next run re-appends")
- elif m_status == "ok":
- out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"])
- except Exception as e: # noqa: BLE001
- out["enrichMasterFailed"] = 1
- log(f"[aios-auto] enrich_instagram: master write-through raised "
- f"{type(e).__name__}: {e}")
- return out
-
-
-_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS))
-_PENDING_METRIC_KINDS = frozenset(("posts", "comments"))
-
-
-def _pending_metric_tasks(defn):
- """Read validated, deduplicated paid metric snapshots from automation continuation state.
-
- A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally
- stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or
- the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor
- endpoint, and the collector never starts a second scrape request.
- """
- raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or []
- out, seen = [], set()
- for item in raw if isinstance(raw, list) else []:
- if not isinstance(item, dict):
- continue
- sid = str(item.get("snapshotId") or "").strip()
- dataset = str(item.get("datasetId") or "").strip()
- kind = str(item.get("kind") or "").strip()
- handle = str(item.get("influencer") or "").strip().lstrip("@").lower()
- if (not sid.startswith("sd_") or dataset not in _PENDING_METRIC_DATASETS
- or kind not in _PENDING_METRIC_KINDS or not handle):
- continue
- key = (sid, dataset, kind, handle)
- if key in seen:
- continue
- seen.add(key)
- out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind,
- "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""),
- "lastChecked": str(item.get("lastChecked") or ""),
- "lastNote": _s(item.get("lastNote"), 160)})
- return out
-
-
-def queue_pending_metric_snapshots(rt, auto_id, pending):
- """Durably retain new engagement snapshots without creating another paid provider request."""
- aid = str(auto_id or "").strip()
- if not aid:
- return 0
- incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}})
- if not incoming:
- return 0
- added = [0]
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- definition = cur.get(aid)
- if not isinstance(definition, dict):
- return cur
- state = definition.setdefault("state", {})
- existing = _pending_metric_tasks(definition)
- known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"])
- for x in existing}
- for task in incoming:
- key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"])
- if key not in known:
- existing.append(task)
- known.add(key)
- added[0] += 1
- state["pendingMetricSnapshots"] = existing
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
- return added[0]
-
-
-def _match_profile_row(rows, handle):
- """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there.
-
- ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With
- `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and
- taking the first row would write ONE profile's followers onto all of them — a wrong number that
- looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the
- same field `_bd_profile` reads first.
-
- ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points
- at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that
- shape the task's own handle is still the best evidence. It applies only when the snapshot holds
- ONE row, so it can never mis-assign inside a batch.
- """
- want = str(handle or "").strip().lstrip("@").lower()
- usable = [n for n in (rows or []) if isinstance(n, dict)]
- for node in usable:
- got = str(_first(node, "account", "username", default="") or "").strip()
- if got.lstrip("@").lower() == want and want:
- return node
- if len(usable) == 1:
- return usable[0]
- return None
-
-
-def _pending_profile_tasks(defn):
- """Validated, deduplicated deferred PROFILE snapshots from automation state.
-
- ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather
- than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and
- `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to
- zero by its own validator, and even if it survived, the collector would hand it to
- `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile
- row. Reusing the name would have shipped a green no-op of exactly the class this change
- exists to remove.
-
- ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written
- back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be
- resolved from the handle alone: two databases may both hold `@x`.
- """
- raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or []
- out, seen = [], set()
- for item in raw if isinstance(raw, list) else []:
- if not isinstance(item, dict):
- continue
- sid = str(item.get("snapshotId") or "").strip()
- dataset = str(item.get("datasetId") or "").strip()
- handle = str(item.get("influencer") or "").strip().lstrip("@").lower()
- table = str(item.get("table") or "").strip()
- row_id = str(item.get("rowId") or "").strip()
- # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a
- # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor
- # endpoint at all.
- if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle
- or not table or not row_id):
- continue
- if sid in seen:
- continue
- seen.add(sid)
- out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile",
- "influencer": handle, "table": table, "rowId": row_id,
- "requestedAt": str(item.get("requestedAt") or ""),
- "lastChecked": str(item.get("lastChecked") or ""),
- "lastNote": _s(item.get("lastNote"), 200)})
- return out
-
-
-def queue_pending_profile_snapshots(rt, auto_id, pending):
- """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many
- were newly added."""
- aid = str(auto_id or "").strip()
- if not aid:
- return 0
- incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}})
- if not incoming:
- return 0
- added = [0]
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- definition = cur.get(aid)
- if not isinstance(definition, dict):
- return cur
- state = definition.setdefault("state", {})
- existing = _pending_profile_tasks(definition)
- known = {x["snapshotId"] for x in existing}
- for task in incoming:
- if task["snapshotId"] not in known:
- existing.append(task)
- known.add(task["snapshotId"])
- added[0] += 1
- state["pendingProfileSnapshots"] = existing
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
- return added[0]
-
-
-def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step):
- """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape.
-
- ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred`
- list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the
- vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown
- away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a
- 180 s budget, two abandoned `sd_…` snapshots in two runs.
-
- ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same
- forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero
- records ends the task with the vendor's reason attached — and, when the vendor blames the
- target rather than itself, records the not-found verdict so the handle stops being re-bought.
- """
- tasks = _pending_profile_tasks(defn)
- if not tasks:
- return ("ok", "No pending profile reads.", {}, [], {"source": "idle"})
- step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
- remaining, patches, affected, notes = [], {}, [], []
- acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
- "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0,
- "notes": [], "dry": False}
- ready = waiting = closed = 0
- # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME
- # `snapshotId`, and asking the vendor about it once per task would spend the round trips the
- # batching just saved. Memoised for this collection pass only — a snapshot's state must be
- # re-read on the NEXT run, which is a fresh call and a fresh dict.
- _progress_memo, _rows_memo = {}, {}
-
- def _progress_of(sid):
- if sid not in _progress_memo:
- _progress_memo[sid] = bd_snapshot_progress(sid)
- return _progress_memo[sid]
-
- def _store_profile(task, profile, via):
- """Write ONE collected profile — snapshot row + preset cells on the record that asked.
-
- ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the
- two paths cannot drift into writing different things — the same rule `top_up_views`
- follows for the inline/deferred split.
- """
- res = {"state": "ok", "profile": profile, "posts": [], "comments": [],
- "via": via, "note": ""}
- pulled = _iso()
- snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
- acc["snaps"].append(snap_row)
- acc["posts"].extend(idents)
- acc["psnaps"].extend(metric_rows)
- acc["comments"].extend(comment_rows)
- _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
- affected.append(task["rowId"])
-
- def _backup_profile(handle):
- """The second rung, asked ONLY once the first has finished and delivered nothing.
-
- ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS
- CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but
- a profile that DEFERRED never came back through `pull_profile` — it came back here, and
- this collector had no second rung at all. So the one path where the primary is most
- likely to have failed was the one path with no fallback.
-
- ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary
- answered is never re-bought from a second vendor — the mistake the capability split exists
- to prevent. A refusal returns None and the caller reports blocked exactly as before.
- """
- try:
- import providers as _p
- if not _p.PROVIDERS["apify"].can("ig_profile"):
- return None
- prof, note = apify_profile(str(handle))
- except Exception as exc: # noqa: BLE001 — a backup must not raise
- log(f"[aios-auto] backup profile rung failed for @{handle}: "
- f"{type(exc).__name__}: {exc}")
- return None
- if prof and prof.get("followers") is not None:
- return prof
- return None
-
- def _rows_of(sid):
- if sid not in _rows_memo:
- payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
- got = _bd_rows(payload) if not err else []
- if got and (_bd_deferral(payload) or
- (len(got) == 1 and str(got[0].get("status") or "") in
- ("running", "building", "collecting"))):
- got = []
- _rows_memo[sid] = got
- return _rows_memo[sid]
-
- for task in tasks:
- stale_h = _hours_since(task.get("requestedAt"))
- state, records, empty_note = _progress_of(task["snapshotId"])
- if state in ("done", "failed") and not records:
- acc["profiles"] += 1
- # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing,
- # so there is nothing left to wait for and no risk of buying the same record twice.
- backup = _backup_profile(task["influencer"])
- if backup is not None:
- ready += 1
- acc["ok"] += 1
- _store_profile(task, backup, "apify")
- note = (f"@{task['influencer']}: the primary source returned nothing, so a backup "
- f"source supplied the profile ({backup.get('followers')} followers)")
- notes.append(note)
- acc["notes"].append(note)
- continue
- closed += 1
- acc["blocked"] += 1
- note = f"@{task['influencer']}: {_s(empty_note, 240)}"
- notes.append(note)
- acc["notes"].append(note)
- # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
- # profile request that means the account could not be reached at all, so the verdict
- # is remembered and the selection stops re-buying it (R9: permanently, until a
- # human edits the handle cell — see `clear_gone`).
- # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about
- # the query, and turning it into "this account does not exist" would silently retire
- # live handles.
- if state == "failed":
- acc["gone"][_gone_key({}, task["influencer"])] = {
- "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)}
- continue
- rows = _rows_of(task["snapshotId"]) if state != "running" else []
- if not rows:
- # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of
- # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never
- # silently. An unbounded queue is the forever-loop this change removes, inverted.
- if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS:
- closed += 1
- note = (f"@{task['influencer']}: the source never finished the profile read "
- f"queued {int(stale_h)}h ago ({task['snapshotId']}) — it was dropped; "
- f"the next run will ask again")
- notes.append(note)
- acc["notes"].append(note)
- continue
- waiting += 1
- remaining.append({**task, "lastChecked": _iso(),
- "lastNote": _s("still building", 200)})
- continue
- # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot
- # holds several profiles and `rows[0]` would write one creator's numbers onto every record
- # in the chunk.
- node = _match_profile_row(rows, task["influencer"])
- if node is None:
- closed += 1
- acc["profiles"] += 1
- acc["blocked"] += 1
- note = (f"@{task['influencer']}: the source delivered {len(rows)} profile"
- f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle — "
- f"it was dropped from the batch and the next run will ask again")
- notes.append(note)
- acc["notes"].append(note)
- continue
- ready += 1
- acc["profiles"] += 1
- profile = _bd_profile(node, task["influencer"])
- if profile.get("followers") is None and profile.get("following") is None:
- acc["blocked"] += 1
- note = (f"@{task['influencer']}: the source delivered the profile but no "
- f"follower/following counts were readable in it")
- notes.append(note)
- acc["notes"].append(note)
- continue
- acc["ok"] += 1
- _store_profile(task, profile, "brightdata:deferred")
- notes.append(f"@{task['influencer']}: collected the profile the source had already been "
- f"paid for ({profile.get('followers')} followers)")
-
- set_state(rt, str(defn.get("id") or ""),
- {"pendingProfileSnapshots": remaining or None})
- counts = _enrich_flush(rt, defn, username, acc, log)
- counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below
- counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {},
- username, log))
- counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting,
- "profileBatchesEmpty": closed})
- if notes:
- counts[RUN_NOTES_KEY] = notes
- head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected"
- if ready else "no deferred profile read was ready")
- tail = "".join([f"; {closed} finished with nothing to collect" if closed else "",
- f"; {waiting} still building" if waiting else ""])
- state = "ok" if ready and not closed else "partial"
- return (state, head + tail, counts, affected,
- {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"})
-
-
-def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
- """Write a completed metric snapshot through the canonical Post/Comment graph once."""
- if not (idents or snapshots or comments):
- return {"posts": 0, "snapshots": 0, "comments": 0}
- profile_table = str(_flow_table(defn) or "")
- if not profile_table:
- raise Refused("the pending engagement snapshot has no Profile database to link to")
- graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
- profile_table=profile_table)
- post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE],
- graph[IG_COMMENTS_TABLE])
- old_posts, collapsed_posts = dedupe_canonical_rows(
- dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
- posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key))
- c_post["duplicates"] += collapsed_posts
- psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots,
- "post_snapshot_key", cap=row_cap(ps_key))
- old_comments, collapsed_comments = dedupe_canonical_rows(
- dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key")
- comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key",
- cap=row_cap(comment_key))
- c_comments["duplicates"] += collapsed_comments
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)):
- if cur.get(key) is not None:
- cur[key]["rows"] = rows
- _refresh_relations_inplace(cur, log=log)
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- try:
- import ig_master
- status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots)
- if status == "error":
- log(f"[aios-auto] deferred post metrics master copy FAILED: {note}")
- except Exception as exc: # noqa: BLE001
- log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}")
- return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"],
- "comments": c_comments["inserted"]}
-
-
-def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step):
- """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request."""
- tasks = _pending_metric_tasks(defn)
- if not tasks:
- return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"})
- step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
- remaining, idents, snapshots, comments = [], [], [], []
- ready, waiting, closed, run_notes = 0, 0, 0, []
- for task in tasks:
- # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
- # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued
- # as "still building" on every tick — forever, because nothing about it would ever
- # change. That is the same forever-loop the profile path was measured in, one dataset
- # over, and it was latent here the whole time.
- state, records, empty_note = bd_snapshot_progress(task["snapshotId"])
- if state in ("done", "failed") and not records:
- # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
- # keeping the entry would be a pending task that can never resolve.
- closed += 1
- run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}")
- continue
- payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
- rows = _bd_rows(payload) if not note else []
- building = (state == "running" or not rows or _bd_deferral(payload) or
- (len(rows) == 1 and str(rows[0].get("status") or "") in
- ("running", "building", "collecting")))
- if building:
- remaining.append({**task, "lastChecked": _iso(),
- "lastNote": _s(note or "still building", 160)})
- waiting += 1
- continue
- ready += 1
- pulled = _iso()
- if task["kind"] == "posts":
- mapped_posts = []
- for raw in rows:
- post = _bd_post_metrics(raw)
- if post:
- # The snapshot was created from this Profile's canonical post URLs. Keep
- # that explicit backlink even when a vendor response omits `user_posted`.
- post["influencer_key"] = task["influencer"]
- mapped_posts.append(post)
- if mapped_posts:
- # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the
- # owner's `theresalearns` run filled every column except Views.
- #
- # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a
- # Posts row physically cannot carry a view count — MEASURED on the stored
- # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them.
- # Views only ever comes from the Apify capability. That top-up lived INSIDE
- # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait
- # budget; when the batch deferred — which is routine, and what happened here — the
- # rows came back through THIS function and Apify was never asked.
- # ⇒ `top_up_views` is now one function with two callers rather than a copy, so
- # the inline and deferred paths cannot answer this differently again.
- # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which
- # is what freezes the values into the post + snapshot rows.
- v_note = top_up_views(mapped_posts, log=log)
- if v_note:
- run_notes.append(f"@{task['influencer']}: {v_note}")
- _unused, post_rows, metric_rows, embedded = capture_rows(
- {"state": "ok", "profile": {"username": task["influencer"]},
- "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled)
- idents.extend(post_rows)
- snapshots.extend(metric_rows)
- comments.extend(embedded)
- else:
- for raw in rows:
- comment = _bd_comment(raw, influencer_key=task["influencer"])
- if comment:
- comments.append(comment)
-
- set_state(rt, str(defn.get("id") or ""),
- {"pendingMetricSnapshots": remaining or None})
- written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log)
- counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting,
- "metricBatchesEmpty": closed,
- "postEngagementSnapshots": written["snapshots"],
- "commentsCollected": written["comments"]}
- # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved
- # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four
- # other call sites disagree about the shape of a run — and `_commit_run` already drops
- # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
- # than to a crash.
- if run_notes:
- counts[RUN_NOTES_KEY] = run_notes
- # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
- # a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
- tail = (f"; {closed} finished with nothing to collect" if closed else "")
- if waiting:
- return ("partial",
- f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
- f"{waiting} still building and will be collected automatically{tail}",
- counts, [], {"capture_posts": "partial", "write": "ok"})
- return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
- counts, [], {"capture_posts": "ok", "write": "ok"})
-
-
-def ai_decide(rt, defn, act, row, row_id="", log=print):
- """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or ""
- to leave the card for a person.
-
- ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed
- answer, or a label the review does not offer all return "" — and "" means the card sits at
- the review gate exactly as it would with no AI at all. The feature can be broken, absent or
- wrong and the worst outcome is a human doing the work.
-
- ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN
- ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board
- under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely
- worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter →
- anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is
- MISSING is only the door — an action kind that asks a question and takes an answer, without a
- stage column to write it into. **Do not delete this in a dead-code sweep without reading that
- sentence first**; equally, do not treat it as shipped — nothing reaches it today.
- """
- cfg = act.get("config") or {}
- options = list(cfg.get("next") or [])
- if not options:
- return ""
- try:
- import ai_review
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}")
- return ""
- fields = [f.get("key") for f in
- (ut_get(rt, ((defn.get("config") or {}).get("targetTable")
- or (defn.get("trigger") or {}).get("table") or "")) or {}
- ).get("fields") or []]
- choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options,
- row=row, fields=[f for f in fields if f],
- label=cfg.get("label") or "Review")
- if not choice:
- if meta.get("problem"):
- log(f"[aios-auto] ai review declined to answer: {meta['problem']}")
- return ""
- review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review",
- choice, meta.get("provider") or "ai", by="ai",
- note=meta.get("reason") or "", model=meta.get("model") or "")
- return choice
-
-
-def find_records(rt, table_key, cond, limit=25):
- """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts
- the count in the run log, where it drills — [[no-unverifiable-aggregates]])."""
- rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {}
- out = []
- for rid, row in rows.items():
- if lane_match(cond, row or {}):
- out.append(str(rid))
- if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)):
- break
- return out
-
-
-def _commit_action_writes(rt, table, patches, creates, username, log):
- """The ONE write. Row patches merge into the target's rows; creates land in their own tables —
- APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a).
-
- Returns the REALIZED create counts, and returning them is the point rather than a
- convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only
- this function knows how many of those became rows, how many matched one that was already
- there, and how many the cap refused. A run that reported the attempt as "created" would be
- the summary-disagrees-with-what-happened defect this module names in three other places.
-
- ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE
- implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand
- rolling a third inside this function is that debt returning with a new name — and it would
- quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED,
- NEVER DELETED, and `capped` is its own count rather than folded into `skipped`).
- """
- out = {"created": 0, "createUpdated": 0, "createUnchanged": 0,
- "createCapped": 0, "createSkipped": 0}
- if not patches and not creates:
- return out
- # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one
- # `ut_write_rows` per patched table, and that was free while the only patcher was a review
- # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the
- # step it reached, so the flow's own table is now patched on essentially every run — and a
- # flow that also creates into that same table would have committed it TWICE per run, against
- # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around.
- # Staged into `staged` and handed to the creates pass, which writes each table exactly once.
- staged = {}
- for tkey, rowpatch in (patches or {}).items():
- cur = dict((ut_get(rt, tkey) or {}).get("rows") or {})
- for rid, vals in rowpatch.items():
- cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals}
- staged[str(tkey)] = cur
- # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The
- # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once
- # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr
- # repo budget this whole module is shaped around.
- by_table = {}
- for (tkey, unique), new_rows in (creates or {}).items():
- by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows))
- for tkey, groups in by_table.items():
- t = ut_get(rt, tkey)
- if t is None:
- n = sum(len(r) for _u, r in groups)
- log(f"[aios-auto] create_record: {tkey} no longer exists — {n} skipped")
- out["createSkipped"] += n
- continue
- # The PATCHED rows when this table was also stamped this run, so the creates land on top
- # of the stamp rather than on a copy of the store that predates it.
- cur = staged.pop(tkey, None)
- cur = dict(t.get("rows") or {}) if cur is None else cur
- cap = row_cap(tkey)
- for unique, new_rows in groups:
- if unique:
- cur, c = upsert_rows(cur, new_rows, unique, cap=cap)
- out["created"] += c["inserted"]
- out["createUpdated"] += c["updated"]
- out["createUnchanged"] += c["unchanged"]
- out["createCapped"] += c["capped"]
- # ⚠ `skipped` here means "this row had no value for the unique key", which for a
- # create action is a mapped value that interpolated to nothing — worth surfacing,
- # because the symptom is otherwise a run that says it created less than it walked.
- out["createSkipped"] += c["skipped"]
- if c["capped"]:
- log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — "
- f"{c['capped']} row(s) not written")
- continue
- nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1
- for i, vals in enumerate(new_rows):
- if len(cur) >= cap:
- # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is
- # named, here and in the run's counts. This used to `break` with a log line
- # that said the cap was hit and never said how much was lost.
- out["createCapped"] += len(new_rows) - i
- log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — "
- f"{len(new_rows) - i} row(s) not written")
- break
- cur[str(nxt)] = dict(vals)
- nxt += 1
- out["created"] += 1
- ut_write_rows(rt, tkey, cur)
- # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and
- # once each, so "one store write per table" holds whether a table was patched, created into,
- # or both.
- for tkey, cur in staged.items():
- ut_write_rows(rt, tkey, cur)
- return out
-
-
-def _lane_sentence(cond, top=True):
- """One condition tree → the sentence a step's `detail` carries on the canvas.
-
- C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a
- label never claims a flat comparison the tree does not make. The SERVER composes it, for the
- same reason it composes every other `detail` — a client paraphrase of a structure the engine
- evaluates is a second implementation of the same sentence, free to drift from it.
-
- ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is
- live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a
- rename would be churn on a working function to fix a word.
- """
- if cond is None:
- return "Everything else" if top else ""
- if isinstance(cond, dict):
- for key, joiner in (("all", " and "), ("any", " or ")):
- if key in cond:
- parts = [_lane_sentence(c, False) for c in cond.get(key) or []]
- parts = [p for p in parts if p]
- if not parts:
- return ""
- inner = joiner.join(parts)
- return inner if top or len(parts) == 1 else f"({inner})"
- v = cond.get("value")
- return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}")
-
-
-def _rid_num(rid):
- return int(rid) if str(rid).isdigit() else 10 ** 9
-
-
-#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can
-#: grow a definition without limit is a serialisation cost wearing a compliance hat.
-MAX_REVIEWS = 100
-
-
-def review_audit(rt, auto_id, row_id, from_label, to_label, username,
- by="user", note="", model=""):
- """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to
- the definition (newest first, bounded).
-
- ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that
- made it and the one-line reason it gave. One audit log, not two — a reader asking "who
- decided this card" must not have to know there are two places to look, and the moment an
- AI decision is invisible beside a human one the log stops being an audit.
-
- ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone:
- `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote
- this shape off a stage field's `flowId` — a field the migration now drops. Kept because the
- SHAPE is the contract a future decision action would write, and re-deriving an audit format
- is how two of them end up existing.
- """
- entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id),
- "from": _s(from_label, 60), "to": _s(to_label, 60),
- "by": "ai" if by == "ai" else "user"}
- if note:
- entry["note"] = _s(note, 300)
- if model:
- entry["model"] = _s(model, 60)
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- d = cur.get(str(auto_id))
- if d is not None:
- d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS]
- return cur
-
- rt.update(STORE_KEY, _up, flush="sync")
-
-
-# ---------------------------------------------------------------------------------------------
-# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24)
-# ---------------------------------------------------------------------------------------------
-# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit —
-# the value reflects every pull the platform has, not just this tenant's). Computed by the
-# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files
-# no session owns this wave), human-write-refused at the grid door, and every number drills to
-# the exact snapshot rows behind it.
-#
-# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no
-# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero
-# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not
-# readable") — the `_bd_posts_count` law, one layer up.
-
-METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments")
-METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d")
-#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each
-#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer;
-#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned).
-PROFILE_MEASURES = ("followers", "avg_engagement")
-METRIC_AGGS = ("avg", "sum", "latest")
-
-
-def metric_value(series, measure, window, agg="", today=None):
- """One metric over one handle's master series → `(value_string_or_None, drill_rows)`.
-
- `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference
- day; post-count windows count back from the newest post. `drill_rows` are the exact rows
- the number came from, so the route can honour [[no-unverifiable-aggregates]] without
- recomputing differently."""
- series = series or {}
- today = today or _now()
- if measure in PROFILE_MEASURES:
- rows = [r for r in series.get("snapshots") or []
- if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""]
- if window in ("last_7d", "last_30d"):
- days = 7 if window == "last_7d" else 30
- floor = today - _dt.timedelta(days=days)
- rows = [r for r in rows
- if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor]
- if not rows:
- return None, []
- if window == "latest" or agg == "latest":
- picked = [rows[-1]]
- else:
- picked = rows
- vals = [_lane_num(r.get(measure)) for r in picked]
- vals = [v for v in vals if v is not None]
- if not vals:
- return None, []
- out = vals[-1] if (window == "latest" or agg == "latest") else \
- (sum(vals) if agg == "sum" else sum(vals) / len(vals))
- if measure == "avg_engagement":
- # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs
- # transform-pct scar) — scaled exactly once, here.
- return f"{out * 100:.2f}", picked
- return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked
- # --- post measures: select POSTS by window, then each post's LATEST measured snapshot.
- posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()]
- posts.sort(key=lambda p: str(p.get("posted_at")))
- if window == "latest":
- picked_posts = posts[-1:]
- elif window == "last_3_posts":
- picked_posts = posts[-3:]
- else:
- days = 7 if window == "last_7d" else 30
- floor = today - _dt.timedelta(days=days)
- picked_posts = [p for p in posts
- if (_parse_iso(str(p.get("posted_at")).replace(" ", "T"))
- or _dt.datetime.min) >= floor]
- vals, drill = [], []
- for p in picked_posts:
- snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or []
- for snap in reversed(snaps):
- raw = snap.get(measure)
- v = _lane_num(raw)
- if v is not None and str(raw).strip() != "":
- vals.append(v)
- drill.append(snap)
- break
- if not vals:
- return None, []
- out = sum(vals) if (agg or "sum") == "sum" else \
- (vals[-1] if agg == "latest" else sum(vals) / len(vals))
- return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill
-
-
-def metric_fields_of_table(t):
- return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)]
-
-
-def _table_handle(row, url_field):
- return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or ""))
- or "").strip().lower()
-
-
-def compute_metric_cells(rt, table_key, today=None):
- """Recompute every metric cell on ONE table from the master series. One coalesced write,
- only when something actually changed (the flush-ceiling law); zero reads when the table
- has no metric fields or the master is off. Returns the number of rows touched."""
- t = ut_get(rt, table_key)
- mfields = metric_fields_of_table(t)
- if not mfields:
- return 0
- import ig_master
- if not ig_master.configured():
- return 0
- url_field = next((f.get("key") for f in (t.get("fields") or [])
- if f.get("type") == "url"), "")
- rows = t.get("rows") or {}
- handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()}
- series = ig_master.series_for({h for h in handles.values() if h})
- changes = {}
- for rid, row in rows.items():
- s = series.get(handles.get(rid) or "")
- for f in mfields:
- bag = f["metric"]
- val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"),
- bag.get("agg") or "", today=today)
- if s else (None, []))
- want = "" if val is None else str(val)
- if str((row or {}).get(f["key"], "")) != want:
- changes.setdefault(str(rid), {})[f["key"]] = want
- if not changes:
- return 0
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- tt = cur.get(table_key)
- if tt is not None:
- for rid, vals in changes.items():
- tt.setdefault("rows", {}).setdefault(rid, {}).update(vals)
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- return len(changes)
-
-
-# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ────────────────
-#
-# Owner: *"adding relational database function with links and rollups… exactly like how Airtable
-# does it… and this rollup needs to have formula that we can use to calculate things like average
-# Views over last N posts."*
-#
-# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula
-# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not
-# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced
-# write only when something changed — and inherits its flush-ceiling discipline for free.
-#
-# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching
-# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is
-# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as
-# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED
-# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The
-# moment one writes a number nothing else can re-derive, it has become a third copy.
-
-def _ut():
- """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic.
-
- Pulling this module in at import time initialises the store layer earlier than
- `automation_engine` used to, and the last time that happened it moved a store-commit COUNT
- from three to four on an unrelated gate. The relational pass
- needs three constants and two predicates from the field layer; it does not need to change
- when this module is imported.
- """
- import core.user_tables as _m
- return _m
-
-
-#: The fns by family — how a value is folded, and what a blank means in each.
-_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"})
-_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"})
-#: The count/order family — folded by their own arms above the lanes.
-_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"})
-#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how
-#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED
-#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm
-#: claims fell past both lanes into the join below — so a "Std deviation" column rendered
-#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now
-#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question
-#: nothing can answer; a comma-joined list is a different question's answer wearing this label).
-_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
-#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED.
-#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name
-#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot
-#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the
-#: defect above, because the defect was that the two lists already disagreed.
-ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS
- | _ROLLUP_TEXT_FNS)
-_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
-#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing.
-_ROLLUP_JOIN = ", "
-#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a
-#: cell nobody can read is not an aggregate.
-ROLLUP_MAX_CHARS = 4000
-
-
-def _sort_key(value, ftype):
- """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell
- is blank or does not parse as its declared type.
-
- ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a
- string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides
- WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently
- averages the wrong twelve posts.
-
- ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION
- SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")`
- otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True`
- flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent
- real one out of the window. A sentinel inside the key cannot mean "last" in both directions,
- because the direction is applied to the sentinel too.
- ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as
- `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at`
- only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every
- key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed
- because its fixture posts all carried dates.
- ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it
- sorts, and "an unknown date is not a recent one" is finally what the code does.
- """
- raw = "" if value is None else str(value).strip()
- if raw == "":
- return None
- if ftype in ("int", "currency", "pct", "rating"):
- n = _lane_num(raw)
- return (n,) if n is not None else None
- if ftype == "date":
- d = _parse_iso(raw.replace(" ", "T"))
- return (d.timestamp(),) if d is not None else None
- return (raw.lower(),)
-
-
-def _sample_stdev(nums):
- """SAMPLE standard deviation (n-1) of `nums`, or None under two values.
-
- ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a
- function: `_rollup_fold` RENDERS this number into a "Std deviation" column and
- `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy
- would let a column and the filter beside it disagree about the same word on the same set —
- the exact drift `top_up_views` was extracted to prevent one module over
- ([[one-evaluator-per-question]]).
- ⛔ None means UNANSWERABLE and no caller may read it as 0.
- """
- if len(nums) < 2:
- return None
- mean = sum(nums) / len(nums)
- return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5
-
-
-def _rollup_ref_threshold(rows, field, sigmas):
- """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1).
-
- ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`,
- and NOT yet filtered by the conditions this threshold feeds. That order is the contract
- (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders
- produce a plausible number: computing it before `limit` answers "2 sigma of everything this
- account ever posted" under a column that says "of the last 10", and computing it after the
- filter makes the threshold depend on the rows it is choosing — a definition that chases
- itself.
-
- Returns None when the window cannot answer — fewer than two numeric values in `field`.
- ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question
- with a permissive answer; letting an unanswerable leaf pass everything would silently turn
- "the outliers" into "all of them", which is this module's worst failure mode wearing a filter.
- """
- nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None]
- sd = _sample_stdev(nums)
- if sd is None:
- return None
- return (sum(nums) / len(nums)) + float(sigmas) * sd
-
-
-def _rollup_fold(fn, values, ftype="text"):
- """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING.
-
- Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's
- oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a
- question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement
- that the creator gets no views.
- ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records"
- over an empty set is genuinely 0, not unknown.
-
- `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only
- `min`/`max` read it today; it defaults to `text` so every existing caller and every test that
- folds a bare list keeps its exact previous answer.
- """
- if fn == "countall":
- return str(len(values))
- if fn == "latest":
- # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve
- # a blank on the newest row rather than reaching backwards and presenting an older value
- # as current.
- return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS]
- if fn == "counta":
- return str(len([v for v in values if str(v or "").strip() != ""]))
- if fn == "count":
- # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them
- # distinct is the whole reason both exist.
- return str(len([v for v in values if _lane_num(v) is not None]))
- # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN.
- #
- # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered
- # BLANK forever while looking completely configured — the failure this module refuses
- # everywhere else, arriving through the one fold that had no type awareness. "Earliest order"
- # and "latest invoice due" are the two most ordinary date rollups there are, and
- # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates
- # and the fold did not.
- #
- # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a
- # string-compare fallback would have worked on well-formed data and silently mis-ordered a
- # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is
- # already at this call site (`linked_types`), and `_sort_key` is already the one function that
- # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather
- # than growing a second idea of what a date is.
- # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person
- # wants in the column is the stored date string exactly as the source row spells it.
- # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean
- # of two timestamps is a number this product has no column type for, and inventing one here
- # would be a value with no author.
- if fn in ("min", "max") and ftype == "date":
- keyed = [(k, str(v)) for k, v in
- ((_sort_key(v, "date"), v) for v in values) if k is not None]
- if not keyed:
- return ""
- return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS]
- if fn in _ROLLUP_NUM_FNS:
- nums = [n for n in (_lane_num(v) for v in values) if n is not None]
- if not nums:
- return ""
- if fn == "stdev":
- # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference
- # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to
- # be LINKED, which is a sample of an account's posting history and not its entirety.
- # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION
- # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently
- # switched divisors.
- # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the
- # honest reason is upstream of the arithmetic: one measurement has no spread to
- # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the
- # single most confident thing this column can say, asserted from a single row. Same
- # law as the blank `sum`, and it bites harder here.
- out = _sample_stdev(nums)
- if out is None:
- return ""
- else:
- out = (sum(nums) if fn == "sum" else min(nums) if fn == "min"
- else max(nums) if fn == "max" else sum(nums) / len(nums))
- return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"
- if fn in _ROLLUP_BOOL_FNS:
- # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero".
- flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values]
- if not flags:
- return ""
- hit = (all(flags) if fn == "and" else any(flags) if fn == "or"
- else sum(1 for f in flags if f) % 2 == 1)
- return "1" if hit else ""
- # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`.
- # An fn no arm above recognises returns "" rather than a comma-joined dump of every value,
- # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28.
- if fn not in _ROLLUP_TEXT_FNS:
- return ""
- vals = [str(v).strip() for v in values if str(v or "").strip() != ""]
- if fn == "arrayunique":
- seen, uniq = set(), []
- for v in vals:
- if v.lower() not in seen:
- seen.add(v.lower())
- uniq.append(v)
- vals = uniq
- if not vals:
- return ""
- text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals))
- return text[:ROLLUP_MAX_CHARS]
-
-
-def _link_from_key(fields, bag):
- """Which column on THIS table supplies the join value.
-
- Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
- fallback chain is what makes an Instagram database link up with no configuration at all —
- the "automatically" in the owner's instruction — and it is the same chain the grid already
- uses to decide a table's identity column, rather than a second opinion about it.
- """
- declared = str((bag or {}).get("from") or "").strip()
- if declared:
- return declared
- prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None)
- if prof:
- return str(prof.get("key") or "")
- pin = next((f for f in fields if f.get("pinned") is True), None)
- return str(pin.get("key") or "") if pin else ""
-
-
-def _linked_rows_by_join(linked, on_key):
- """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE.
-
- ⚠ Lower-cased because the join values this exists for are Instagram handles, which the
- profile flag already normalises to lower case on one side and which a hand-typed cell on the
- other side may not. A join that misses on case is a relation that silently reports zero.
- """
- idx = {}
- for rid, row in ((linked or {}).get("rows") or {}).items():
- k = str((row or {}).get(on_key) or "").strip().lower()
- if k:
- idx.setdefault(k, []).append((str(rid), row or {}))
- return idx
-
-
-def _rollup_condition_matches(row, condition, field_types, ref_value=None):
- """Evaluate one Airtable-style linked-record condition against a candidate row.
-
- `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the
- caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed
- here because this function sees one row and the statistic is a property of the whole window —
- a version that reached for the set from inside would be recomputing the same mean once per
- row, and would have to be handed the window anyway.
- ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the
- threshold helper for why the permissive reading is the dangerous one.
- """
- field = str((condition or {}).get("field") or "")
- op = str((condition or {}).get("op") or "")
- raw = (row or {}).get(field)
- text = str(raw or "").strip()
- if op == "is_empty":
- return text == ""
- if op == "is_not_empty":
- return text != ""
- if (condition or {}).get("ref") is not None:
- # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering
- # ops, and a row whose cell is blank or unparseable has no position relative to a computed
- # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key`
- # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]).
- left_num = _lane_num(text)
- if ref_value is None or left_num is None:
- return False
- return ((op == "gt" and left_num > ref_value)
- or (op == "gte" and left_num >= ref_value)
- or (op == "lt" and left_num < ref_value)
- or (op == "lte" and left_num <= ref_value))
- wanted = str((condition or {}).get("value") or "").strip()
- if op == "contains":
- return wanted.casefold() in text.casefold()
- if op == "not_contains":
- return wanted.casefold() not in text.casefold()
- if op in ("eq", "neq"):
- left_num, right_num = _lane_num(text), _lane_num(wanted)
- equal = (left_num == right_num if left_num is not None and right_num is not None
- else text.casefold() == wanted.casefold())
- return equal if op == "eq" else not equal
- ftype = (field_types or {}).get(field, "text")
- left = _sort_key(text, ftype)
- right = _sort_key(wanted, ftype)
- if left is None or right is None:
- return False
- return ((op == "gt" and left > right) or (op == "gte" and left >= right)
- or (op == "lt" and left < right) or (op == "lte" and left <= right))
-
-
-def compute_relation_cells(rt, table_key, tables=None):
- """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
-
- Zero store reads when the table declares neither kind — the same cheap-by-construction shape
- `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table.
- """
- store = tables if tables is not None else ut_all(rt)
- t = (store or {}).get(table_key)
- fields = list((t or {}).get("fields") or [])
- links = [f for f in fields if _ut().is_derived_link(f)]
- rollups = [f for f in fields if isinstance(f.get("rollup"), dict)]
- if not links and not rollups:
- return 0
- rows = (t or {}).get("rows") or {}
- by_key = {str(f.get("key")): f for f in fields}
-
- # --- resolve every link field ONCE per table, not once per row.
- # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
- resolved, linked_types = {}, {}
- for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
- lk_key = str((f or {}).get("key") or "")
- if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
- continue
- bag = f["link"]
- linked = (store or {}).get(str(bag.get("table") or "")) or {}
- linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text")
- for lf in (linked.get("fields") or [])}
- if bag.get("inverse"):
- # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary
- # link cell contains this row id. The source cell remains the one relationship truth.
- source_rows = linked.get("rows") or {}
- inverse_key = str(bag.get("inverse") or "")
- inverse_index = {}
- for source_id, source_row in source_rows.items():
- for target_id in [part.strip() for part in
- str((source_row or {}).get(inverse_key) or "").split(",")]:
- if target_id:
- inverse_index.setdefault(target_id, []).append(
- (str(source_id), source_row or {}))
- resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows}
- elif bag.get("on"):
- idx = _linked_rows_by_join(linked, str(bag["on"]))
- from_key = _link_from_key(fields, bag)
- resolved[lk_key] = {
- str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), [])
- for rid, row in rows.items()} if from_key else {}
- else:
- # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
- # are dropped rather than carried — a link to a deleted row is not a link.
- lrows = (linked.get("rows") or {})
- resolved[lk_key] = {
- str(rid): [(i, lrows[i]) for i in
- [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")]
- if i and i in lrows]
- for rid, row in rows.items()}
-
- changes = {}
- for rid, row in rows.items():
- rid = str(rid)
- row = row or {}
- for f in links:
- fk = str(f["key"])
- hits = (resolved.get(fk) or {}).get(rid) or []
- if f["link"].get("single"):
- hits = hits[:1]
- # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell
- # is a projection of `resolved`, which is uncapped and is what every rollup below
- # consumes — so a profile with 900 posts shows the first 500 ids and still averages
- # over all 900. Same argument the `posts` window already makes: shedding is safe
- # precisely because the authoritative store still holds everything.
- want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS])
- if str(row.get(fk, "")) != want:
- changes.setdefault(rid, {})[fk] = want
- for f in rollups:
- fk, bag = str(f["key"]), f["rollup"]
- lk_key = str(bag.get("link") or "")
- hits = list((resolved.get(lk_key) or {}).get(rid) or [])
- # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
- # and therefore to a blank cell — never to a stale number. A column that keeps
- # printing yesterday's answer after its input is gone is the worst of the options.
- # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE
- # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so
- # "last 10 posts where views > X" meant *the 10 most recent of the posts over X*
- # rather than *the ones over X among the last 10* — two different windows wearing one
- # sentence. Harmless while a threshold was a literal; incoherent the moment a
- # threshold is a statistic OF the window, because the set being described and the set
- # doing the describing would be different sets.
- # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s
- # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is
- # computed over that window, THEN the conditions filter it") and the validator half
- # was written against it.
- # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH
- # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational`
- # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall`
- # with no limit), so the blast radius is user-built rollups only.
- # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last
- # 12 posts, we also want to make it so its last N record, where the record's Status is
- # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose,
- # so they select among the rows the window already kept. `where` selects WHICH rows
- # the window is spent on.
- # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per
- # identity; run it first and a carousel could claim the slot its reel sibling needed,
- # so the window would come up short for a reason nothing on screen explains.
- # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this
- # list, because at this point there is no fixed set for a statistic to be about.
- where = list(bag.get("where") or [])
- if where:
- where_matches = lambda pair: [
- _rollup_condition_matches(pair[1], condition,
- linked_types.get(lk_key) or {}, ref_value=None)
- for condition in where]
- if str(bag.get("whereConj") or "and") == "or":
- hits = [pair for pair in hits if any(where_matches(pair))]
- else:
- hits = [pair for pair in hits if all(where_matches(pair))]
- sort_by = str(bag.get("sortBy") or "")
- if sort_by:
- ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
- # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not
- # rankable, and it must land at the END whichever direction is asked for — which
- # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel
- # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its
- # window on rows that HAVE the value before it falls back to ones that do not.
- keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits]
- rankable = [(p, k) for p, k in keyed if k is not None]
- rankable.sort(key=lambda pk: pk[1],
- reverse=str(bag.get("sortDir") or "desc") == "desc")
- hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None]
- distinct_by = str(bag.get("distinctBy") or "")
- if distinct_by:
- seen, unique = set(), []
- for pair in hits:
- identity = str(pair[1].get(distinct_by) or "").strip().lower()
- # A blank is not an identity. Keep it rather than collapsing every unknown
- # record into one synthetic duplicate.
- if identity and identity in seen:
- continue
- if identity:
- seen.add(identity)
- unique.append(pair)
- hits = unique
- limit = int(bag.get("limit") or 0)
- if limit:
- hits = hits[:limit]
- # --- the window is now FIXED, so a set-statistic threshold has a set to be about.
- conditions = list(bag.get("conditions") or [])
- if conditions:
- # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the
- # window; computing it inside `matches` would recompute the same mean for every
- # candidate and — worse — would invite computing it over a set that the filter is
- # already shrinking underneath it.
- # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond
- # the mean") is falsy, and the `or` spelling would silently rewrite it to the same
- # number by accident. It reads identically and is right for the wrong reason,
- # which is how it survives a review.
- refs = [_rollup_ref_threshold(
- [r for _i, r in hits], str(c.get("field") or ""),
- (c.get("ref") or {}).get("sigmas", 0.0))
- if isinstance(c, dict) and c.get("ref") is not None else None
- for c in conditions]
- matches = lambda pair: [
- _rollup_condition_matches(pair[1], condition,
- linked_types.get(lk_key) or {}, ref_value=ref)
- for condition, ref in zip(conditions, refs)]
- if str(bag.get("conditionConj") or "and") == "or":
- hits = [pair for pair in hits if any(matches(pair))]
- else:
- hits = [pair for pair in hits if all(matches(pair))]
- src = str(bag.get("field") or "")
- # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the
- # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at
- # all, so the default stands for it.
- want = _rollup_fold(str(bag.get("fn") or ""),
- [r.get(src) for _i, r in hits] if src else [1] * len(hits),
- (linked_types.get(lk_key) or {}).get(src, "text"))
- if str(row.get(fk, "")) != want:
- changes.setdefault(rid, {})[fk] = want
- if not changes:
- return 0
-
- # A run that just wrote linked rows passes its in-flight user_tables bucket here so the
- # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps
- # the public helper's old persist-on-change behaviour.
- if tables is not None:
- tt = tables.get(table_key)
- if tt is not None:
- for r, vals in changes.items():
- tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
- return len(changes)
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- tt = cur.get(table_key)
- if tt is not None:
- for r, vals in changes.items():
- tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- return len(changes)
-
-
-def _refresh_relations_inplace(tables, log=print):
- """Refresh every relation against one mutable user_tables bucket; perform no store write."""
- touched = 0
- for tk, table in list((tables or {}).items()):
- fields = (table or {}).get("fields") or []
- if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields):
- continue
- try:
- touched += compute_relation_cells(None, tk, tables=tables)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
- return touched
-
-
-def refresh_relations(rt, log=print):
- """The tick half of the relational pass — the twin of `refresh_metrics`.
-
- ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale
- when table B gains a row, and A has no way to know that happened. Cheap by construction — a
- table declaring neither kind costs one dict scan.
- """
- snapshot = {
- str(key): {**(table or {}),
- "rows": {str(rid): dict(row or {})
- for rid, row in ((table or {}).get("rows") or {}).items()}}
- for key, table in (ut_all(rt) or {}).items()
- }
- if not _refresh_relations_inplace(snapshot, log=log):
- return 0
- actual = [0]
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- actual[0] = _refresh_relations_inplace(cur, log=log)
- return cur
-
- # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug).
- #
- # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and
- # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget.
- # A derived-cell recompute has no business forcing a commit on someone typing.
- # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new
- # row lives only in the store cache for 2-20s, and this call — on the SAME key
- # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the
- # result. `POST /rows` answered 201 and the row was gone.
- #
- # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it
- # is what makes the other ~39 sync writers of this key safe. This line is not that fix and
- # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both
- # landed together on purpose: one is correctness, one is the hot path.
- rt.update(UT_STORE_KEY, _up, flush="async")
- return actual[0]
-
-
-def refresh_metrics(rt, today=None, log=print):
- """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window
- metric can never go staler than one tick while a scheduler exists. Cheap by construction —
- a table without metric fields costs a dict scan and nothing else."""
- touched = 0
- for tk, t in ut_all(rt).items():
- if metric_fields_of_table(t):
- try:
- touched += compute_metric_cells(rt, tk, today=today)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}")
- return touched
-
-
-def purge_subject(rt, handle):
- """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the
- tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half
- non-optional: a purge that missed the pooled copy would not be erasure). Returns
- `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is
- now ABSENT, which is the one aggregate whose drill is emptiness."""
- subject = str(handle or "").strip().lstrip("@").lower()
- if not subject:
- return {}
- counts = {}
- tables = ut_all(rt)
- post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {}
- codes = {str(r.get("shortcode") or "") for r in post_rows.values()
- if str((r or {}).get("influencer_key") or "").strip().lower() == subject}
-
- keeps = {
- "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key")
- or "").strip().lower() != subject,
- "ut_ig_posts": lambda r: str((r or {}).get("influencer_key")
- or "").strip().lower() != subject,
- "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes,
- DISCOVER_TABLE: lambda r: str((r or {}).get("handle")
- or "").strip().lower() != subject,
- }
-
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- for tk, keep in keeps.items():
- t = cur.get(tk)
- if t is None:
- continue
- rows = t.get("rows") or {}
- nxt = {rid: r for rid, r in rows.items() if keep(r)}
- counts[tk] = len(rows) - len(nxt)
- t["rows"] = nxt
- return cur
-
- rt.update(UT_STORE_KEY, _up, flush="sync")
- import ig_master
- for bucket, n in (ig_master.purge_handle(subject) or {}).items():
- counts[f"master:{bucket}"] = n
- return counts
-
-
-# ---------------------------------------------------------------------------------------------
-# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33)
-# ---------------------------------------------------------------------------------------------
-# Six ways an automation starts, exactly: manual | schedule | event_field | record_created |
-# webhook | email. The first two are what always existed (Run now; cron via the tick). The four
-# new ones are EVENTS, and A2 makes their discipline LAW rather than taste:
-#
-# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state,
-# not for being in it. Implemented as per-record ARMED state over successive evaluations
-# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms
-# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing
-# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the
-# disarmed set with everything currently matching, so already-matching records do not fire
-# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar).
-# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only
-# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers
-# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never
-# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by
-# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as
-# a statusNote) catches whatever construction did not foresee.
-# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds
-# instead of running — a partial run entry names the count and the deliberate way through
-# (Run now). C4's discovery guard is this rule's special case.
-# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record
-# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire);
-# event_field fires every transition.
-
-# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ───────────────────────────────
-# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that
-# renames the same events is a second vocabulary to learn for no gain.
-#
-# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions".
-# Renaming the key would have orphaned every stored trigger in production for a caption; the key
-# is the contract with the store, the label is the contract with the reader, and they are allowed
-# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the
-# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4
-# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath.
-#
-# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show
-# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a
-# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop
-# class wearing a friendlier face; here the two lists are separate on purpose and the refusal
-# names the state rather than pretending the key is unknown.
-# ── WAVE 24 · C-TRIG (owner ruling R6). ───────────────────────────────────────────────────────
-# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the
-# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway —
-# it is the event this flow starts from. Picking it sets the definition's kind to
-# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now.
-#
-# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the
-# schedule (or on Run now), so it stays out of both lists below and its node switch flips the
-# CRON — see `TRIGGER_SCHEDULE_KEYS`.
-TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created",
- "enters_view", "webhook", "email", "form_submitted", "ig_profile_match")
-#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` and `tiktok_profile_match` JOIN THE
-#: PLANNED LIST, and joining THIS tuple rather than `TRIGGER_KEYS` is the whole of their
-#: implementation. `clean_trigger` refuses everything here with a sentence, so the faded row is a
-#: wall; the picker shows it so the Scraper section is not a section of one and TikTok is visibly
-#: coming rather than silently absent.
-#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an
-#: action that drives a browser are different builds; this row must not be read as progress on
-#: either D-51 or D-9 (TikTok CAPTURE), both of which stay open with their own exit conditions.
-TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed", "tiktok_profile_match")
-TRIGGER_LABELS = {
- "manual": "Manual",
- "schedule": "At a scheduled time",
- "event_field": "When a record matches conditions",
- "record_updated": "When a record is updated",
- "record_created": "When a record is created",
- "enters_view": "When a record enters a view",
- "webhook": "When a webhook is received",
- "email": "When an email arrives",
- "form_submitted": "When a form is submitted",
- "ig_profile_match": "When an Instagram profile fits a criteria",
- "button_clicked": "When a button is clicked",
- "comment_added": "When a comment is added",
- "web_page_changed": "When a website page changes",
- "tiktok_profile_match": "When a TikTok profile fits a criteria",
-}
-
-# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ──
-# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a
-# distinction about where a trigger came FROM rather than about what a person is choosing. The
-# question the picker actually asks is: does this fire on TIME, on your own DATA, or because
-# something OUTSIDE said so. Three answers, and every trigger has exactly one.
-#
-# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map
-# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the
-# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS`
-# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation
-# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is
-# the surface. So:
-# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule
-# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because
-# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into
-# Database.
-# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`,
-# so the fallback is provably dead code in production and the classification is still
-# mandatory. The fallback catches the accident; the gate stops it shipping.
-TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1},
- "database": {"label": "Database", "order": 2},
- "connector": {"label": "Connector", "order": 3},
- "other": {"label": "Other", "order": 99}}
-#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to
-#: prevent — it is the soft landing, not a category anybody should be adding triggers to.
-TRIGGER_GROUP_FALLBACK = "other"
-TRIGGER_GROUP_OF = {
- "manual": "time", "schedule": "time",
- "event_field": "database", "record_updated": "database", "record_created": "database",
- "enters_view": "database", "form_submitted": "database",
- "button_clicked": "database", "comment_added": "database",
- "email": "connector", "webhook": "connector", "ig_profile_match": "connector",
- "web_page_changed": "connector", "tiktok_profile_match": "connector",
-}
-#: The SUB-group inside "Connector" — which connected thing this trigger comes through.
-#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two
-#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a
-#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key
-#: against `/connectors/directory` would match `scraper` and `tiktok` and miss `gmail` — so it
-#: must group by it and render `label`, never look it up. Said here because the miss would be
-#: silent and partial, which is the worst shape.
-TRIGGER_CONNECTOR = {
- "email": {"key": "gmail", "label": "Gmail"},
- "webhook": {"key": "webhooks", "label": "Webhooks"},
- # R9: the Scraper ships with ONE built trigger and one faded one.
- "ig_profile_match": {"key": "scraper", "label": "Scraper"},
- "web_page_changed": {"key": "scraper", "label": "Scraper"},
- "tiktok_profile_match": {"key": "tiktok", "label": "TikTok"},
-}
-#: Triggers that watch a database and therefore need one named before they can fire.
-TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view",
- "form_submitted")
-#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself.
-#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven,
-#: so flipping its node must flip the schedule.
-#:
-#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read
-#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`,
-#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For
-#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a
-#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns
-#: about eight lines into its own comment. Derived from one named set now, so a trigger added to
-#: `TRIGGER_KEYS` cannot silently join the wrong side of it.
-TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match")
-#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last.
-#:
-#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire
-#: reason this constant exists rather than the tuple above being sent. That set answers "which
-#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual
-#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw
-#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run
-#: now": a control contradicting its own description.
-#:
-#: D-55's history is why it is DERIVED rather than listed: the client carried
-#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact
-#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face).
-#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here.
-TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"}
-#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so
-#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples.
-TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view")
-MAX_WATCH_FIELDS = 12
-#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there,
-#: and so would a deployment that prefers immediacy over coalescing.
-EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15)
-FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses
-FIRE_WINDOW_SECONDS = 300
-FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on
-EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation
-EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest
-CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota
-
-EMAIL_FIELDS = [
- field_def("email_id", "Email id"), field_def("email_from", "From"),
- field_def("email_subject", "Subject"), field_def("email_date", "Date"),
- field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"),
-]
-
-
-def clean_trigger(raw, previous=None):
- """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and
- means what it always meant: manual + whatever `schedule` says.
-
- ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry
- with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT
- rather than refused — the picker writes `{key}` first and the table/field after, the
- wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the
- wire so the surface says "finish setting this up" instead of snapping back to Manual. It
- cannot fire while incomplete (the hooks match on the table it does not name), which is the
- fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a
- condition on a field the trigger does not watch) are still refused with the sentence —
- incomplete is a state, wrong is not.
- """
- if raw in (None, "", {}):
- return (dict(previous) if isinstance(previous, dict) and previous else None), None
- if not isinstance(raw, dict):
- return None, "the trigger must be an object"
- prev = previous if isinstance(previous, dict) else {}
- key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"),
- 30).strip()
- if key in TRIGGER_PLANNED:
- # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence
- # says WHY rather than "unknown trigger", because the picker legitimately showed it.
- return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — "
- f"it renders so you can see it is coming, and it cannot be saved")
- if key not in TRIGGER_KEYS:
- return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS))
- if key == "schedule":
- # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule`
- # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored
- # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it.
- return None, None
- if key == "manual":
- # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.**
- # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user
- # can just press Run to make the full flow work."*
- #
- # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys
- # together with one argument — *"storing a no-op trigger would be a second copy of that
- # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has
- # another home; **manual has none.** Nothing anywhere recorded "this automation is
- # manual", so storing it is not a duplicate: it is the only record there has ever been.
- #
- # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so
- # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the
- # "nobody has decided yet" empty state, and Configuration — including the Database picker
- # a plain automation cannot do without — never rendered. The owner reported it twice. The
- # previous note reasoned that a manual option *"would bounce straight back to this state
- # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion
- # was that it should be STORED.
- #
- # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off
- # (Run now always works, which is the whole of what it means) and a switch that governs
- # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the
- # cron stays reachable — picking Manual says how it fires today, never that it may not be
- # scheduled tomorrow.
- return {"key": "manual"}, None
- out = {"key": key,
- "enabled": bool(raw["enabled"]) if "enabled" in raw else
- bool(prev.get("enabled", True)),
- "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))}
- if key in TRIGGER_TABLE_KEYS:
- table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip()
- if table and not table.startswith(UT_PREFIX):
- return None, ("event triggers watch blank databases (ut_*) this wave — "
- f"{table!r} is not one")
- out["table"] = table
- if key == "event_field":
- # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record
- # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a
- # second, quieter way to express the same narrowing, and the owner asked for one.
- # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is
- # dropped on this definition's next clean — silently, and exactly once, because nothing
- # writes the key back. A refusal here would have 400'd the live automations that carry it.
- cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"),
- where="the trigger")
- if cerr:
- return None, cerr
- out["when"] = cond
- if key == "record_updated":
- # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty
- # is the WIDER reading and it is the default there too, so it stays the default here.
- watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields")
- if watch_raw in (None, ""):
- watch = []
- elif not isinstance(watch_raw, list):
- return None, "the watched-field list must be a list of field keys"
- else:
- watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()]
- if len(watch) > MAX_WATCH_FIELDS:
- return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} "
- f"fields — leave the list empty to watch every field")
- out["fields"] = watch
- # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A
- # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration:
- # "a record was updated" is an event, and asking it to also be a filter was the overlap
- # with `event_field` the owner asked to end. Stated loudly in the contract AND here so
- # nobody restores it as a bug fix.
- # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate`
- # naturally returns "no gate" for it — the write itself becomes the event — rather than
- # this needing a second removal anywhere.
- if key == "enters_view":
- out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"),
- 80).strip()
- if key == "form_submitted":
- # Blank = any form on that database. Naming one narrows to it, which is what a table
- # carrying an intake form AND a correction form needs.
- out["formToken"] = _s(raw.get("formToken") if "formToken" in raw
- else prev.get("formToken"), 64).strip()
- if key == "webhook":
- # The token is MINTED here, once, and survives every later patch — rotating it on
- # every Save would silently break the external caller the URL was given to.
- out["token"] = _s(prev.get("token"), 64) or _secrets_token()
- # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config.
- # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no
- # map behaves exactly as it did — it fires the flow and reads nothing — so the live
- # webhook automations are untouched. `webhook` deliberately stays OUT of
- # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and
- # every existing webhook trigger would go unconfigured and stop firing.
- table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip()
- if table and not table.startswith(UT_PREFIX):
- return None, ("a webhook writes into a blank database (ut_*) — "
- f"{table!r} is not one")
- out["table"] = table
- fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw
- else prev.get("fieldMap"))
- if ferr:
- return None, ferr
- out["fieldMap"] = fmap
- if key == "email":
- out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"),
- 200).strip() or "in:inbox is:unread"
- out["configured"] = _trigger_configured(out)
- return out, None
-
-
-#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody
-#: else's schema and is legitimately wider than an action's hand-written value list.
-MAX_BODY_FIELDS = 40
-MAX_BODY_DEPTH = 5
-
-
-def clean_body_map(raw):
- """D-41: `{"": ""}` for a webhook. Returns `(map, error)`.
-
- Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated
- rather than half-supported: `items.0.sku` would read as working for the first element and
- silently write nothing the day a payload arrives with the list empty, which is the shape of
- bug this module keeps paying for. A path that resolves to nothing writes nothing.
- """
- if raw in (None, ""):
- return {}, None
- if not isinstance(raw, dict):
- return None, "the webhook field map must be an object of {body path: field key}"
- if len(raw) > MAX_BODY_FIELDS:
- return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record"
- out = {}
- for path, field in raw.items():
- p = _s(path, 200).strip()
- if not p:
- return None, "a webhook mapping has an empty body path"
- if len(p.split(".")) > MAX_BODY_DEPTH:
- return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload — "
- f"map a shallower value")
- fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_")
- if not fk:
- return None, f"the value at {p!r} is not mapped to a field"
- out[p] = fk[:60]
- return out, None
-
-
-def body_value(body, path):
- """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is
- an object or a list answers None rather than a stringified `{...}` in a cell, because the
- Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable."""
- cur = body
- for part in str(path or "").split("."):
- if not isinstance(cur, dict):
- return None
- cur = cur.get(part)
- return cur if isinstance(cur, (str, int, float, bool)) else None
-
-
-def webhook_row(rt, defn, body):
- """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`.
-
- ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same
- `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable,
- from the only side the caller is on, from a payload whose paths did not resolve. That is the
- D-11 class (a table that quietly stops growing), and the caller here is a machine that will
- keep posting. The note rides the 200: the flow still fires, and the answer says why no row
- was written.
-
- ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop
- prevention law (A2(2)). A sibling automation watching this table does NOT fire on a
- webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow
- fires, because `hook_fire` fires it explicitly, which is the difference between "this
- trigger fired" and "a write happened".
- """
- trg = (defn or {}).get("trigger") or {}
- table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {})
- if not table or not fmap or not isinstance(body, dict):
- return "", 0, "" # no map configured — nothing to report
- t = ut_get(rt, table)
- if t is None:
- return "", 0, f"{table} no longer exists, so nothing was written"
- values = {}
- for path, fkey in fmap.items():
- v = body_value(body, path)
- if v is not None:
- values[fkey] = _s(v, 500) if isinstance(v, str) else v
- if not values:
- return "", 0, ("none of the mapped paths resolved to a value in this payload — "
- "check the paths against what you are sending")
- rows = dict((t.get("rows") or {}))
- if len(rows) >= row_cap(table):
- return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was "
- f"written (the flow still ran)")
- rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1)
- rows[rid] = values
- ut_write_rows(rt, table, rows)
- return rid, len(values), ""
-
-
-def _trigger_configured(trg, config=None):
- """Is this trigger complete enough to fire? One reader, because "configured" is asserted in
- three places (the wire, the graph node, the hooks) and three copies of a boolean is how a
- surface says "ready" about a trigger the engine skips.
-
- ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is
- the one trigger whose configuration lives in the DEFINITION's config (the discovery filters)
- rather than on the trigger. `clean_trigger` calls this without it and so answers
- conservatively (False); `clean_definition` calls it again with the validated config and
- refines. Conservative-then-refined is the fail-closed order — the reverse would flash
- "ready" on a trigger with nothing to search for.
- """
- key = str((trg or {}).get("key") or "")
- if key == "ig_profile_match":
- return bool((config or {}).get("predicates"))
- if key in TRIGGER_TABLE_KEYS and not trg.get("table"):
- return False
- if key == "event_field":
- # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything,
- # ever" — which is not a trigger, it is a description of the table.
- # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone
- # and carried no condition becomes `configured: false` on its next clean. It stops
- # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it
- # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert.
- return bool(trg.get("when"))
- if key == "enters_view":
- return bool(trg.get("viewId"))
- return True
-
-
-def _secrets_token():
- import secrets as _sec
- return _sec.token_urlsafe(24)
-
-
-# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the
-# process would keep punishing an automation for a storm that ended with the restart.
-_FIRES = {}
-_FIRES_LOCK = threading.Lock()
-
-
-def _breaker_trips(tenant, auto_id, now=None):
- now = now if now is not None else time.time()
- key = (tenant, str(auto_id))
- with _FIRES_LOCK:
- log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS]
- log.append(now)
- _FIRES[key] = log
- return len(log) > FIRE_LIMIT
-
-
-def _pause_trigger(rt, auto_id, note):
- def _up(cur):
- cur = cur if isinstance(cur, dict) else {}
- d = cur.get(str(auto_id))
- if d is not None:
- trg = d.get("trigger")
- if isinstance(trg, dict):
- trg["paused"] = True
- d["statusNote"] = _s(note, 200)
- return cur
- rt.update(STORE_KEY, _up, flush="sync")
-
-
-def trigger_fire(rt, tenant, auto_id, log=print, rows=None):
- """One trigger firing — breaker first, then the ordinary async run. False when it did not
- start (breaker, or already running — both are answers, not errors)."""
- if _breaker_trips(tenant, auto_id):
- note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in "
- f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's "
- f"subject in a loop")
- _pause_trigger(rt, auto_id, note)
- log(f"[aios-auto] breaker: {auto_id} {note}")
- return False
- return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows)
-
-
-# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE
-# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates).
-_SETTLE = {}
-_SETTLE_LOCK = threading.Lock()
-
-
-def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print):
- """Buffer ONE touched row for a coalesced evaluation, carrying the written cell.
-
- ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave
- dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the
- reason is worth stating because it is invisible from this file: an ordinary ut cell typed at
- the grid door lands in the editor's PER-USER OVERLAY stratum
- (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables`
- definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for
- the common case the value that just changed exists ONLY in this event, and a definition-row
- read sees the pre-write value — the trigger would evaluate stale and never fire.
- `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree
- read the record's other columns) with the written cells on top.
- """
- key = (tenant, str(auto_id))
- cell = {str(field): after} if field else {}
- if EVENT_SETTLE_SECONDS <= 0:
- with _SETTLE_LOCK:
- buf = _SETTLE.setdefault(key, {"rows": {}})
- buf["rows"].setdefault(str(row_id), {}).update(cell)
- _settle_eval(rt, tenant, auto_id, log=log)
- return
- with _SETTLE_LOCK:
- buf = _SETTLE.setdefault(key, {"rows": {}})
- buf["rows"].setdefault(str(row_id), {}).update(cell)
- timer = buf.get("timer")
- if timer is not None:
- timer.cancel() # the burst continues — push the window out
- timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval,
- args=(rt, tenant, auto_id), kwargs={"log": log})
- timer.daemon = True
- buf["timer"] = timer
- timer.start()
-
-
-def view_filter(rt, table_key, view_id):
- """`(tree, fields, problem)` for one saved view on a user table — the substrate the
- `enters_view` trigger tests membership against (C3-v2 / owner R2).
-
- Personal strata first (`find_view`), then the shared bucket, because a view somebody shared
- is exactly the kind an automation gets pointed at. A view that has been deleted answers a
- PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one
- would turn "when a record enters Overdue" into "on every write", which is the widening this
- module refuses everywhere else.
- """
- key = str(table_key or "")
- vid = str(view_id or "").strip()
- if not key or not vid:
- return None, [], "the trigger names no view"
- try:
- import core.table_store as table_store
- tops = table_store.make(f"{key}_table_workspace", st=rt)
- found = tops.find_view(vid)
- view = (found[1] if found else None) or tops.shared_view(vid)
- except Exception as e: # noqa: BLE001
- return None, [], f"the view could not be read ({type(e).__name__})"
- if not isinstance(view, dict):
- return None, [], f"view {vid!r} no longer exists on {key}"
- cfg = view.get("config") or {}
- tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"}
- return tree, list((ut_get(rt, key) or {}).get("fields") or []), ""
-
-
-def _row_gate(rt, defn, trg):
- """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string.
-
- None means the trigger has NO match gate — the write itself is the event (wave 22's "the
- field changed", and `record_updated` over any field). A problem means the gate cannot be
- built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that
- passes.
- """
- key = trg.get("key")
- if key == "enters_view":
- tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId"))
- if problem:
- return None, problem
- import harness.filter_eval as filter_eval
- return (lambda row: filter_eval.matches(tree, row, fields)), ""
- when = trg.get("when")
- if when:
- return (lambda row: lane_match(when, row)), ""
- return None, ""
-
-
-def _settle_eval(rt, tenant, auto_id, log=print):
- """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire."""
- with _SETTLE_LOCK:
- buf = _SETTLE.pop((tenant, str(auto_id)), None)
- written = dict((buf or {}).get("rows") or {})
- touched = list(written)
- if not touched:
- return
- d = all_definitions(rt).get(str(auto_id))
- trg = (d or {}).get("trigger") or {}
- if trg.get("key") not in ("event_field", "record_updated", "enters_view") \
- or trg.get("paused") or not trg.get("enabled", True) \
- or not trg.get("configured", True):
- return
- gate, problem = _row_gate(rt, d, trg)
- if problem:
- # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken
- # automation, not a quiet no-op — the note is what the surface shows instead of "On".
- if (d.get("statusNote") or "") != problem:
- _pause_trigger(rt, auto_id, problem)
- log(f"[aios-auto] trigger gate: {auto_id} {problem}")
- return
- fired = []
- if gate is not None:
- rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
- disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
- nxt = set(disarmed)
- for rid in touched:
- # The merge (see `_settle_buffer`): the shared definition row underneath so a C4
- # tree can read the record's other columns, the just-written cells on top because
- # for an ordinary ut column this event is the ONLY place that value exists yet.
- if gate({**(rows.get(rid) or rows.get(str(rid)) or {}),
- **(written.get(rid) or {})}):
- if rid not in disarmed:
- fired.append(rid) # false→true THIS evaluation: the edge
- nxt.add(rid)
- else:
- nxt.discard(rid) # left the state — re-armed (Airtable's rule)
- if nxt != disarmed:
- set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]})
- if not fired:
- return
- else:
- fired = list(touched) # "the field changed" — the burst is the edge
- if len(fired) > FLOOD_LIMIT:
- _commit_run(rt, auto_id, "partial",
- f"the trigger matched {len(fired)} records in one evaluation — more than "
- f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to "
- f"process them deliberately", {"held": len(fired)}, True)
- return
- trigger_fire(rt, tenant, auto_id, log=log, rows=fired)
-
-
-def _seed_event_state(rt, defn):
- """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so
- turning the trigger on fires nothing — the first fire needs a real false→true transition.
- Evaluated over the definition rows (the machine-written truth this trigger class watches).
-
- Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation
- reads. Two implementations of "does this row match" is how a seed disagrees with the edge it
- is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it.
- """
- trg = (defn or {}).get("trigger") or {}
- if trg.get("key") not in ("event_field", "record_updated", "enters_view") \
- or not trg.get("configured"):
- return
- gate, problem = _row_gate(rt, defn, trg)
- if gate is None or problem:
- return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
- rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
- matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
- set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
-
-
-def grid_hook(evt):
- """THE listener the human write doors emit into (registered onto
- `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may
- import both sides). Never raises into a write path; a broken trigger listener must not
- break typing into a cell."""
- try:
- st = evt.get("st")
- if st is None:
- return
- tenant = str(getattr(st, "key", "") or "royal-imports")
- table = str(evt.get("table") or "")
- kind = str(evt.get("type") or "")
- defs = all_definitions(st)
- field = str(evt.get("field") or "")
- rid = str(evt.get("rowId") or "")
- for aid, d in defs.items():
- trg = (d or {}).get("trigger") or {}
- key = str(trg.get("key") or "")
- if trg.get("paused") or not trg.get("enabled", True) \
- or not trg.get("configured", True) \
- or key not in TRIGGER_ROW_KEYS \
- or str(trg.get("table") or "") != table:
- continue
- if kind == "record_created" and key == "record_created":
- hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
- ridn = int(rid) if rid.isdigit() else None
- if ridn is None or ridn <= hw:
- continue # once per record EVER (A2(4)) — undo-proof
- set_state(st, aid, {"rcHighwater": ridn})
- trigger_fire(st, tenant, aid, rows=[rid])
- elif kind == "event_field" and key == "event_field":
- # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every
- # write on the watched table settles, and the CONDITION decides whether it fires
- # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would
- # now always take its first arm anyway; leaving a read of a key the validator no
- # longer writes is the drift seat this migration exists to close.
- _settle_buffer(st, tenant, aid, rid, field, evt.get("after"))
- elif kind == "event_field" and key == "record_updated" and (
- not (trg.get("fields") or []) or field in (trg.get("fields") or [])):
- _settle_buffer(st, tenant, aid, rid, field, evt.get("after"))
- elif key == "enters_view":
- # BOTH kinds feed it: a row can enter a view by being edited into its filter or
- # by being CREATED already inside it. Listening only to edits would silently miss
- # every new record — the half of the definition a reader assumes is covered.
- _settle_buffer(st, tenant, aid, rid, field, evt.get("after"))
- except Exception as e: # noqa: BLE001
- print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}")
-
-
-def form_fired(rt, table_key, row_id, values=None, form_token=""):
- """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7).
-
- One submitted form row → every `form_submitted` automation watching that database fires.
- Returns the list of automation ids that started, so the door can log what it set off (and so
- the gate can assert it, rather than asserting a side effect nobody can see).
-
- ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form,
- so it fires triggers exactly like typing into a cell does. The loop-prevention law is not
- weakened by that — the engine's own writers still never reach here (only `routes_forms` calls
- it), so an automation cannot create a form row and re-fire itself.
-
- `values` is accepted and unused today: the row is already written when this is called, and
- the flow reads it from the table. It stays in the signature because the caller HAS it and a
- later refire policy ("only when field X was submitted") needs it — a parameter added later
- would mean changing D's call site in a wave that does not own it.
- """
- started, table = [], str(table_key or "")
- if not table:
- return started
- tenant = str(getattr(rt, "key", "") or "royal-imports")
- for aid, d in all_definitions(rt).items():
- trg = (d or {}).get("trigger") or {}
- if trg.get("key") != "form_submitted" or trg.get("paused") \
- or not trg.get("enabled", True) or not trg.get("configured", True):
- continue
- if str(trg.get("table") or "") != table:
- continue
- want = str(trg.get("formToken") or "")
- if want and not hmac.compare_digest(want, str(form_token or "")):
- continue # this automation watches a DIFFERENT form on that table
- if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None):
- started.append(aid)
- return started
-
-
-def hook_fire(rt, tenant, auto_id, token, body=None):
- """The webhook trigger's decision, separated from FastAPI so the gate can drive it.
- Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind,
- 409 already running, 200 started.
-
- ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or
- sent something that is not JSON. It is written onto a record BEFORE the flow fires — the
- flow's actions walk the record the webhook just created, which is the whole point of mapping
- it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected.
- """
- import hmac as _hmac
- defn = all_definitions(rt).get(str(auto_id))
- if defn is None:
- return 404, {"error": "unknown_automation"}
- trg = defn.get("trigger") or {}
- want = str(trg.get("token") or "")
- if trg.get("key") != "webhook" or not want:
- return 403, {"error": "no_webhook", "message":
- "this automation has no webhook trigger"}
- if trg.get("paused") or not trg.get("enabled", True):
- return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"}
- if not _hmac.compare_digest(want, str(token or "")):
- return 403, {"error": "bad_token", "message": "that token is not valid"}
- # D-41: map the payload onto a record FIRST, so the flow that fires next walks it.
- row_id, mapped, note = webhook_row(rt, defn, body)
- started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None)
- out = {"started": bool(started), "at": _iso(),
- # Answered even when zero, so a caller wiring a map up can see whether their paths
- # resolved. Silence here would make "my JSON is not landing" undebuggable from the
- # outside, which is the only side the caller is on.
- "rowId": row_id or None, "mapped": mapped}
- if note:
- out["note"] = note # a 200 that wrote no row SAYS which reason it was
- return 200, out
-
-
-def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None):
- """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a
- row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the
- gate drives this without a network; production leaves them None.
-
- Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per
- tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most
- `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write.
- """
- trg = defn.get("trigger") or {}
- if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True):
- return None
- import oauth_connect
- creator = str(defn.get("createdBy") or "").strip() or "admin"
- token, err = oauth_connect.google_creds(rt, creator)
- if err:
- if (defn.get("statusNote") or "") != err:
- def _note(cur):
- cur = cur if isinstance(cur, dict) else {}
- dd = cur.get(str(auto_id))
- if dd is not None:
- dd["statusNote"] = _s(err, 200)
- return cur
- rt.update(STORE_KEY, _note, flush="sync")
- return None
- lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n))
- reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid))
- ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT)
- if lerr:
- return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}",
- {}, True)
- seen = set((defn.get("state") or {}).get("emailSeen") or [])
- fresh = [m for m in ids if m not in seen]
- if not fresh:
- return None # nothing new is not a run — no history spam
- if len(fresh) > FLOOD_LIMIT:
- return _commit_run(rt, auto_id, "partial",
- f"{len(fresh)} new emails matched in one poll — more than the "
- f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow "
- f"the query, or press Run now after adjusting it",
- {"held": len(fresh)}, True)
- fresh = fresh[:EMAIL_MAX_PER_POLL]
- rows_in, notes = [], []
- for mid in fresh:
- row, rerr = reader(mid)
- if row:
- rows_in.append(row)
- elif rerr:
- notes.append(rerr)
- table_key = (defn.get("config") or {}).get("targetTable") or ""
- if not table_key:
- return _commit_run(rt, auto_id, "error",
- "the email trigger has nowhere to write — the automation names no "
- "target database", {}, False)
- ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox",
- EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key)
- existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
- merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key))
- ut_write_rows(rt, table_key, merged)
- new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:]
- set_state(rt, auto_id, {"emailSeen": new_seen})
- summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — "
- f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written")
- if notes:
- summary += f" — {notes[0][:100]}"
- entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts,
- True, affected=list(merged)[:200])
- trigger_fire(rt, tenant, auto_id, log=log)
- return entry
-
-
-def compose_sentence(defn):
- """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the
- definition so it cannot lie about what runs."""
- cfg = defn.get("config") or {}
- trg = defn.get("trigger") or {}
- sched = defn.get("schedule") or {}
- kind = defn.get("kind")
- if trg.get("key") == "event_field":
- # Law 4: no watched field any more, so the sentence stops naming one. It said
- # "When None changes on ut_x" the moment the key stopped being stored.
- head = f"When a record in {trg.get('table')} matches conditions"
- elif trg.get("key") == "record_updated":
- head = f"When a record in {trg.get('table')} is updated"
- elif trg.get("key") == "ig_profile_match":
- head = "When an Instagram profile fits the criteria"
- elif trg.get("key") == "record_created":
- head = f"When a record is created in {trg.get('table')}"
- elif trg.get("key") == "webhook":
- head = "When the webhook is called"
- elif trg.get("key") == "email":
- head = f"When an email matches {trg.get('query')}"
- elif sched.get("enabled"):
- head = f"{_cron_label(sched.get('cron'))}"
- else:
- head = "When you press Run now"
- if kind == "scrape_db":
- host = urlparse(str(cfg.get("url") or "")).hostname or "the page"
- body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}"
- elif kind == "field_instagram":
- # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts
- # first" / ", anonymous only" described a choice that no longer exists.
- body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}"
- elif kind == "discover_instagram":
- body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into "
- f"{cfg.get('targetTable') or DISCOVER_TABLE}")
- else:
- # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has.
- # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every
- # plain automation would have introduced itself as "search Instagram for up to 0 profiles
- # into ut_ig_candidates" — a sentence composed from a definition that says none of it,
- # on the one surface whose whole promise is that it "cannot lie about what runs".
- n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions")))
- tbl = cfg.get("targetTable") or trg.get("table") or ""
- body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else ""))
- if n else "do nothing yet — this automation has no actions")
- lanes = cfg.get("lanes") or []
- tail = f", then route each record across {len(lanes)} lanes" if lanes else ""
- return f"{head}, {body}{tail}."
-
-
-def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
- """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it
- directly. Returns the run entry, or None when it was already running (the 409)."""
- defn = all_definitions(rt).get(str(auto_id))
- if defn is None:
- return None
- if not _claim(tenant, auto_id):
- return None
- # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever
- # happened to press Run — and above all not to the scheduler, which is not a person and
- # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by
- # whether a human or a cron got to the first run first.
- owner = str(defn.get("createdBy") or "").strip()
- if owner and username in MACHINE_OWNERS:
- username = owner
- try:
- _step(tenant, auto_id, "running")
- # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it
- # first and do not start another profile scrape while it is outstanding: re-running the
- # action would buy duplicate engagement reads and reintroduce the timeout this handoff
- # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`.
- # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one
- # rung earlier: a profile snapshot the vendor is still building is paid work, and
- # starting a fresh scrape for the same handle would buy the identical row a second time.
- # Ahead of the metric collector because the profile IS the thing the run was asked for;
- # the engagement batches hang off it.
- if _pending_profile_tasks(defn):
- state, summary, counts, affected, steps = collect_pending_profile_snapshots(
- rt, defn, username=username, log=log,
- step=lambda text: _step(tenant, auto_id, text))
- return _commit_run(rt, auto_id, state, summary,
- {k: v for k, v in counts.items() if k != RUN_NOTES_KEY},
- state != "error", affected, steps,
- notes=counts.get(RUN_NOTES_KEY))
- if _pending_metric_tasks(defn):
- state, summary, counts, affected, steps = collect_pending_metric_snapshots(
- rt, defn, username=username, log=log,
- step=lambda text: _step(tenant, auto_id, text))
- return _commit_run(rt, auto_id, state, summary,
- {k: v for k, v in counts.items() if k != RUN_NOTES_KEY},
- state != "error", affected, steps,
- notes=counts.get(RUN_NOTES_KEY))
- runner = RUNNERS.get(defn.get("kind"))
- if runner is None:
- return _commit_run(rt, auto_id, "error",
- f"unknown automation kind {defn.get('kind')!r}", {}, False)
- try:
- # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run.
- # `status.step` was already on the wire and D's half renders it; measured against the
- # code, it was set exactly ONCE ("running") and never again, so the word would have
- # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a
- # constant as a progress indicator is worse than rendering nothing: it looks like an
- # answer. The runners move it now, and the 120 s Bright Data wait counts out loud.
- state, summary, counts, affected, steps = runner(
- rt, defn, username=username, log=log,
- step=lambda text: _step(tenant, auto_id, text), rows=rows)
- except Refused as e:
- return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}")
- return _commit_run(rt, auto_id, "error",
- f"{type(e).__name__}: {str(e)[:200]}", {}, False)
- # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is
- # committed, over the records this run actually touched. ONE call site rather than three
- # inside the runners: every kind gets actions and endings for free, and a fourth runner
- # cannot forget to opt in.
- #
- # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated,
- # wired to the wire and covered by twelve gate checks that all called `apply_actions`
- # DIRECTLY — so the whole feature was green and unreachable. A person would have built a
- # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`.
- #
- # A failing action must not fail the RUN: the machine steps already wrote their rows and
- # reporting that as an error would misdescribe what happened. It degrades to `partial`
- # with the reason in the summary — the cap_note discipline.
- # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`,
- # which now reads this name — an assignment only on the success path would turn any
- # action failure into a NameError inside the handler that exists to prevent exactly that.
- # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked
- # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the
- # run a person stares at wondering why the automation stopped doing anything.
- run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or [])
- try:
- a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [],
- username=username, log=log,
- step=lambda text: _step(tenant, auto_id, text))
- # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so
- # the runner contract keeps its shape, and they must leave before the merge or they
- # would be a "count" everywhere downstream.
- run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or [])
- counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}}
- if a_counts.get("enrichMetricBatchesPending"):
- batches = int(a_counts["enrichMetricBatchesPending"])
- state = "partial" if state != "error" else state
- summary += (f" — {batches} post-engagement batch"
- f"{'' if batches == 1 else 'es'} still building; Views and other "
- "metrics will be collected automatically without another paid read")
- if a_counts.get("enrichUnbound"):
- # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is
- # `partial` because it genuinely did part of its job — it walked the records — and
- # the sentence names the ONE thing that has to change, in the two places a person
- # can change it. The old behaviour was `ok` with an empty table.
- state = "partial" if state != "error" else state
- summary += (" — the Instagram step did not run: this database has no profile "
- "column. Name one on the step, or mark a text column as the "
- "Instagram profile")
- if a_counts.get("enrichProfileBatchesPending"):
- # ⭐ The paid profile the vendor is still building. Said out loud so a run that
- # looks like a failure is read as the handoff it is — the tick finishes it.
- n = int(a_counts["enrichProfileBatchesPending"])
- state = "partial" if state != "error" else state
- summary += (f" — {n} profile read{'' if n == 1 else 's'} took longer than the "
- "wait allows and will be collected automatically, at no extra cost")
- if a_counts.get("enrichBlocked"):
- state = "partial" if state != "error" else state
- # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile
- # read(s) were blocked" is the exact string the owner read three mornings running
- # before asking "wtf is going on"; it names a quantity and withholds the one
- # thing that would let anybody act. The first note is the vendor's own words.
- blocked_note = next((n for n in run_notes if ": " in n), "")
- summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked"
- + (f". {_s(blocked_note, 220)}" if blocked_note else ""))
- # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the
- # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it
- # logged the cap and rolled up `ok`, so a flow that had silently stopped writing
- # looked exactly like one that had nothing to write. Same rule, same sentence shape.
- if a_counts.get("createCapped"):
- state = "partial" if state != "error" else state
- summary = (f"{summary} — {a_counts['createCapped']} row(s) NOT created: a target "
- f"database is at its row cap")
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}")
- state = "partial" if state != "error" else state
- summary = f"{summary} — the actions did not finish ({type(e).__name__})"
- return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
- steps, notes=run_notes)
- finally:
- _release(tenant, auto_id)
-
-
-def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None):
- """Start a run on a background thread. False when one is already in flight (→ 409)."""
- if running(tenant, auto_id):
- return False
- th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows),
- daemon=True, name=f"automation-{tenant}-{auto_id}")
- th.start()
- return True
-
-
-# ---------------------------------------------------------------------------------------------
-# THE TICK + the in-process scheduler
-# ---------------------------------------------------------------------------------------------
-
-def pending_collect_ids(rt):
- """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building.
-
- ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking
- forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and
- stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is
- true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false),
- so `due_ids` never returns it, nothing ever collected it, and the results the account had
- already been charged for sat ready at the vendor forever. The sentence was not wrong; it was
- describing a run nobody had scheduled.
-
- A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it
- regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something
- new*, this says *collect what is already running*, and folding the second into the first would
- make an unscheduled automation's paid result depend on someone remembering to press a button.
- """
- out = []
- for aid, d in all_definitions(rt).items():
- if not isinstance(d, dict):
- continue
- if (d.get("trigger") or {}).get("paused"):
- continue
- pending_discovery = (d.get("kind") == "discover_instagram"
- and str((d.get("state") or {}).get("pendingSnapshot") or "").strip())
- # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile
- # deferral would be stored and never collected, which is the same defect it fixes wearing
- # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was
- # measured on is `trigger: manual`. A capability written down but never walked is what
- # this whole change is about, so it must not be reintroduced one function later.
- if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d):
- out.append(aid)
- return sorted(out)
-
-
-def due_ids(rt, now=None):
- """Which of this tenant's automations a tick at `now` should start. Pure over the store."""
- return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now))
-
-
-def tick(rt, tenant, now=None, log=print):
- """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the
- ids started. The email polls are bounded and fail-quiet per automation — one broken
- mailbox connection must not stop the tenant's schedules."""
- started = []
- # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the
- # tenant has already been charged for; it is collected whether or not this automation is on a
- # schedule. `_claim` makes the union safe — an id in both lists starts once.
- for aid in dict.fromkeys(list(pending_collect_ids(rt)) + list(due_ids(rt, now))):
- if run_async(rt, tenant, aid, username="scheduler", log=log):
- started.append(aid)
- for aid, d in all_definitions(rt).items():
- if (d.get("trigger") or {}).get("key") == "email":
- try:
- if email_poll(rt, tenant, aid, d, log=log) is not None:
- started.append(aid)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}")
- # C7's amendment: date-window metrics advance with the tick, so `today` is never staler
- # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan.
- try:
- refresh_metrics(rt, log=log)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}")
- # ⭐ 2026-08-07 — the relational pass, on the same tick and for the same reason. A rollup on
- # table A goes stale when table B gains a row, and A cannot know that happened; the tick is
- # the only place that sees both. ⚠ SEPARATE try/except from the metrics above deliberately —
- # one pass failing must not silently cancel the other, which a shared block would do.
- try:
- refresh_relations(rt, log=log)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] relation refresh {tenant} failed: {type(e).__name__}: {e}")
- # ⭐ THE READ-THROUGH ROLLUPS, on the same tick and for a stronger version of the same reason.
- # A linked rollup goes stale when the LINKED TABLE gains a row; a source-backed one goes stale
- # when ODOO does — and its window moves on its own besides (a `ytd` column is wrong on 1
- # January without anybody writing anything). The tick is the only place that sees either.
- #
- # ⛔ ITS OWN try/except, like the two passes above, and the reason is the same one stated
- # there: this pass REFUSES loudly on a truncated group set, and a shared block would let that
- # honest refusal silently cancel the relational pass that had already succeeded.
- #
- # ⚠ IT WRITES NOTHING WHEN IT REFUSES — see `rollup_sql`'s header. A half-applied rollup mixes
- # two vintages of one column and looks completely normal, which is why the refusal is total.
- try:
- import rollup_sql
- for _tk in list((rt.get(UT_STORE_KEY) or {})):
- if rollup_sql.source_fields((rt.get(UT_STORE_KEY) or {}).get(_tk) or {}):
- # ⚠ `today` is DEFAULTED INSIDE `compute`, not passed: this scope has no
- # such local, and `refresh_metrics` above resolves it the same way.
- n = rollup_sql.compute(rt, _tk)
- if n:
- log(f"[aios-auto] source rollups {tenant}/{_tk}: {n}")
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] source rollup {tenant} failed: {type(e).__name__}: {e}")
- if started:
- log(f"[aios-auto] tick {tenant}: started {', '.join(started)}")
- return started
-
-
-def tick_all(now=None, log=print):
- """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop
- the others' schedules."""
- from harness import runtime as _rt
- out = {}
- for slug in _rt.known_tenants():
- try:
- out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}")
- return out
-
-
-#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer
-#: would be a busy loop against a vocabulary that cannot express it.
-TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60)
-_SCHEDULER = [None]
-
-
-def scheduler_loop(log=print):
- """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work.
-
- Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and
- `verify_seam` import this module through `main.py`, run against fake stores in seconds and
- exit. A loop that ticked on entry would fire inside them.
- """
- while True:
- time.sleep(TICK_SECONDS)
- try:
- tick_all(log=log)
- except Exception as e: # noqa: BLE001
- log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}")
-
-
-def start_scheduler(log=print):
- """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.**
-
- ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave
- brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196
- seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires
- two or three times inside a gate that imports `main.py` — and while a tick over a fake store
- finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and
- mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread
- that can move a gate's subject is a flaky suite waiting to happen.
-
- So it takes the convention `main.py:353` already established for exactly this hazard —
- `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup
- event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the
- external EventBridge tick (R5) does not depend on it either way, since that POSTs the
- endpoint rather than riding this thread.
- """
- if os.environ.get("AIOS_AUTOMATIONS") != "1":
- return False
- if _SCHEDULER[0] is not None:
- return False
- th = threading.Thread(target=scheduler_loop, kwargs={"log": log},
- daemon=True, name="automation-scheduler")
- _SCHEDULER[0] = th
- th.start()
- return True
+"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME.
+
+THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant
+store and a RUN that lives in this process. The definition is durable, small and rarely written;
+the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file
+is built to avoid, and it has two halves:
+
+ * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the
+ store outlives the process that wrote it, so a container restart mid-run leaves an automation
+ that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives
+ in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written
+ exactly ONCE per run, at the end.
+ * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s
+ floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape
+ by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes
+ its whole result first and commits it in a single `update` callback.
+
+WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not
+deployed, so its extraction core is copied into this file (the wave doc's instruction), ported
+httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with
+it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point:
+the skill version takes a URL from a developer on a CLI; this takes one from a request body.
+"""
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import hmac
+import ipaddress
+import json
+import os
+import re
+import socket
+import threading
+import time
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability
+# router had exactly one consumer in this file — the views top-up inside the Bright Data rung —
+# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the
+# connector that routes, which is the shape the split was for: this file no longer knows that
+# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore
+# a signal, not a convenience — it means something in the automation runtime started making a
+# vendor decision, and that belongs one layer down.
+
+try: # the parser the extraction half needs
+ from bs4 import BeautifulSoup
+except Exception: # pragma: no cover — deploy lag; see mailbox ②
+ BeautifulSoup = None
+
+# ---------------------------------------------------------------------------------------------
+# THE BUCKET (C4-AUTO)
+# ---------------------------------------------------------------------------------------------
+
+#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern.
+STORE_KEY = "automations"
+#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through
+#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for
+#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the
+#: session-D mailbox as a finding against A's file rather than fixed from here.
+UT_STORE_KEY = "user_tables"
+UT_PREFIX = "ut_"
+
+MAX_AUTOMATIONS = 40
+MAX_RUNS = 20 # trimmed history per automation
+MAX_NAME = 80
+MAX_UT_ROWS = 5000 # mirrors core.user_tables.MAX_ROWS
+MAX_UT_TABLES = 40
+
+#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste.
+#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the
+#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull
+#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's
+#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour
+#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still
+#: report success. A time series that stops after five days without saying so is worse than one
+#: that was never built.
+#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE
+#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it:
+#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day
+#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts
+#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had
+#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the
+#: silent version would have flatlined a chart inside a fortnight with a green dot over it.
+#:
+#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises
+#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables,
+#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3
+#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a
+#: smaller number.
+#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.)
+MAX_UT_IG_ROWS = 200_000
+IG_TABLE_PREFIX = "ut_ig_"
+
+#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was
+#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table —
+#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY
+#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would
+#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket
+#: serialisation cost, purely by an accident of naming. Naming the append tables is the version
+#: that stays true when the next `ut_ig_*` table is not one.
+IG_SNAPSHOTS_TABLE = "ut_ig_snapshots"
+IG_POSTS_TABLE = "ut_ig_posts"
+IG_POST_SNAPSHOTS_TABLE = "ut_ig_post_snapshots"
+IG_COMMENTS_TABLE = "ut_ig_comments"
+
+# ---------------------------------------------------------------------------------------------
+# ⭐⭐ WAVE 29 (item 7, DEBT D-9, owner rulings R1 + R2) — TIKTOK, AS A PARALLEL FAMILY
+# ---------------------------------------------------------------------------------------------
+# R1: FULL PARITY — profile AND posts AND comments, not a profile tier.
+# R2: a PARALLEL `ut_tt_*` family. `ut_ig_*` is untouched: zero migration, zero risk to the live
+# Instagram rows, and the two schemas may DIVERGE where the vendors do.
+#
+# ⛔ WHY A SECOND FAMILY RATHER THAN A `platform` COLUMN ON THE FIRST, stated here because it is
+# the question every reader will ask. `PRESET_PROFILE_FIELDS` carries `platform` and its identity
+# is `(platform, handle)` — so a TikTok PROFILE row could already have lived in `ut_ig_profile`.
+# The other four tables carry NO discriminator at all: `ut_ig_posts`/`ut_ig_comments` key on
+# `shortcode` and the snapshot tables on `influencer_key`, so an Instagram post and a TikTok video
+# that happened to share a code would MERGE SILENTLY. Adding `platform` to four live append tables
+# holding hundreds of thousands of rows is a migration on production data to buy a shared grid
+# nobody asked for. R2 chose the version with no migration.
+#
+# ⚠ THE ACCEPTED COST, so it is not rediscovered as a defect: the engine, rollups and grids learn
+# two families, and a cross-platform view needs a union. In exchange the two schemas can be HONEST
+# about their vendors — which is why `ut_tt_posts` has no `plays` column and `ut_tt_profile` says
+# `tt_id` rather than inheriting a column labelled "Instagram id".
+TT_TABLE_PREFIX = "ut_tt_"
+TT_PROFILE_TABLE = "ut_tt_profile"
+TT_SNAPSHOTS_TABLE = "ut_tt_snapshots"
+TT_POSTS_TABLE = "ut_tt_posts"
+TT_POST_SNAPSHOTS_TABLE = "ut_tt_post_snapshots"
+TT_COMMENTS_TABLE = "ut_tt_comments"
+
+AUTOMATION_RECORD_MODE = "automation"
+#: ⚠ THE RAISED CEILING FOLLOWS THE APPEND SHAPE, NOT THE PLATFORM. `ut_tt_snapshots` and
+#: `ut_tt_post_snapshots` are append tables for exactly the reason their IG twins are — one row per
+#: profile per pull, `maxPosts` rows per pull — so they inherit the ceiling by JOINING THIS SET,
+#: which is the mechanism the note above says survives the next table that is not an append table.
+APPEND_TABLES = frozenset({IG_SNAPSHOTS_TABLE, IG_POST_SNAPSHOTS_TABLE,
+ TT_SNAPSHOTS_TABLE, TT_POST_SNAPSHOTS_TABLE})
+
+#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT.
+#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger
+#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram
+#: column, an Instagram search — and they survive on the automations that already use them
+#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG).
+#:
+#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth:
+#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind
+#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was
+#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole
+#: description: three machine nodes it does not have, and a one-line summary about searching
+#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`.
+#: ⭐ WAVE 29 (D-9 / R1) — `discover_tiktok` joins, reachable ONLY through the
+#: `tiktok_profile_match` trigger (the same law that makes `discover_instagram` reachable). It
+#: lands here IN THE SAME CHANGE as its `RUNNERS` entry and its flip law: a kind in this tuple
+#: with no runner is a control that must refuse.
+KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram", "discover_tiktok")
+#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores
+#: when the body names none, which after R6 is every create the client makes.
+DEFAULT_KIND = "plain"
+#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate —
+#: the ruling retires the door, not the two automations behind it (see `create`).
+#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger.
+RETIRED_KINDS = ("scrape_db", "field_instagram")
+#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were
+#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name
+#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence
+#: per kind, in one place, because `create` and `clean_definition` both say it.
+RETIRED_KIND_REPLACEMENT = {
+ "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead",
+ "scrape_db": "Add a scrape step to any automation instead",
+}
+#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the
+#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`.
+#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any
+#: flow can take, so refusing the kind costs a person nothing but a different click.
+#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared
+#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a
+#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch
+#: door open on purpose, and there is a gate check that says so in those words. A kind may only be
+#: walled off at a door once something else answers the same need.
+#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.)
+KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database",
+ "field_instagram": "Instagram profile column",
+ "discover_instagram": "Find Instagram profiles",
+ "discover_tiktok": "Find TikTok profiles"}
+EXTRACTS = ("table", "jsonld")
+STATES = ("idle", "running", "ok", "error", "partial")
+#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid).
+#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a
+#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle).
+TIERS = ("anonymous", "brightdata")
+#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap
+#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown
+#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation
+#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps
+#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile.
+TIER_ALIASES = {"hiker": "brightdata"}
+
+
+def clean_tier(raw):
+ """A stored/posted tier → a tier this engine runs, or '' when it is neither."""
+ t = str(raw or "").strip().lower()
+ t = TIER_ALIASES.get(t, t)
+ return t if t in TIERS else ""
+
+
+def row_cap(table_key):
+ """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`.
+
+ An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything
+ else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one.
+ """
+ return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS
+
+#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI
+#: shows and the vocabulary the parser accepts cannot drift.
+CRON_PRESETS = [
+ {"cron": "*/15 * * * *", "label": "Every 15 minutes"},
+ {"cron": "0 * * * *", "label": "Hourly"},
+ {"cron": "0 6 * * *", "label": "Daily at 06:00"},
+ {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"},
+ {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"},
+]
+
+UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)"
+
+
+def _now():
+ return _dt.datetime.now()
+
+
+def _stamp(dt=None):
+ return (dt or _now()).strftime("%Y-%m-%d %H:%M")
+
+
+def _iso(dt=None):
+ """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18).
+
+ ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables
+ and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A
+ naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC
+ minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is
+ not expressible at all. With the offset the same string is an instant, and relative rendering
+ becomes possible without migrating a single stored row.
+
+ ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in
+ this module is against a naive `_now()`, and mixing aware and naive datetimes raises
+ `TypeError` — so the offset rides on the WIRE and never enters the arithmetic.
+ """
+ dt = dt or _now()
+ if dt.tzinfo is None:
+ dt = dt.astimezone() # a naive stamp from this process IS local time
+ return dt.isoformat(timespec="seconds")
+
+
+#: ⭐ WAVE 26 · R3 — the DAY out of any stamp we have ever written, for the `date`-typed columns.
+#:
+#: ⛔ IT MUST READ EVERY SHAPE THE STORE HOLDS, which is the same trap `_parse_iso` documents one
+#: function down: `first_found` cells exist as post-D-18 offset stamps
+#: (`2026-08-05T14:03:11+07:00`), as pre-D-18 naive ones (`2026-08-05T14:03:11`), as `_stamp()`'s
+#: space-separated minute form (`2026-08-05 14:03`) and, after this wave, as bare days. A
+#: converter that understood only the newest shape would blank the oldest rows — and a migration
+#: that empties cells is indistinguishable from one that moved them.
+#: ⚠ Returns "" for anything it cannot read rather than guessing a day. The migration treats ""
+#: as LEAVE ALONE, never as a value to write, so an unparseable cell keeps its original text and
+#: shows up as itself instead of disappearing.
+def _day(s):
+ """`2026-08-05T14:03:11+07:00` → `2026-08-05`. "" when there is no day in there."""
+ raw = str(s or "").strip()
+ if not raw:
+ return ""
+ head = raw.replace("T", " ").split(" ")[0]
+ try:
+ _dt.date.fromisoformat(head)
+ except ValueError:
+ dt = _parse_iso(raw)
+ return dt.strftime("%Y-%m-%d") if dt else ""
+ return head
+
+
+#: ⭐ WAVE 26 · AMENDMENT C1-a — the vendor's 0–1 engagement fraction → this product's 0–100 `pct`.
+#:
+#: MEASURED on corpus rows: `0.0074`, `0.0656`, `0.0014`, `0.0148`, `0.0274`, `0.0091`. Our `pct`
+#: renderer prints the stored number and appends `%`, so storing the raw fraction would show every
+#: creator in the book as `0.0%` — a measurement replaced by a wrong measurement, which is worse
+#: than the blank cell it came from ([[analyst-chart-library]]: this repo already carries a 0–1
+#: dialect and a 0–100 dialect, and they meet here).
+#: ⚠ BLANK STAYS BLANK. `""` means the vendor did not send one — both scrape probe rows were null
+#: — and `0.0` would claim we measured zero engagement.
+def _pct100(v):
+ """A 0–1 engagement fraction → a 0–100 percentage. "" when there is nothing to convert."""
+ if v is None or (isinstance(v, str) and not v.strip()):
+ return ""
+ try:
+ return _s(round(float(v) * 100.0, 4))
+ except (TypeError, ValueError):
+ return ""
+
+
+def _parse_iso(s):
+ """A stamp → a NAIVE LOCAL datetime, with or without an offset.
+
+ Both shapes exist in the store simultaneously and always will: every run committed before
+ D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new
+ shape would silently return None for them — and `is_due` reads None as "never ran", which
+ would re-fire every schedule once. Reading both is what makes the change additive.
+ """
+ raw = str(s or "").strip().replace("Z", "+00:00")
+ dt = None
+ try:
+ dt = _dt.datetime.fromisoformat(raw)
+ except Exception: # noqa: BLE001
+ try:
+ dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S")
+ except Exception: # noqa: BLE001
+ return None
+ return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt
+
+
+# ---------------------------------------------------------------------------------------------
+# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher
+# ---------------------------------------------------------------------------------------------
+# The skill version guards the URL a developer typed. This one guards a URL that arrived in a
+# request body, which changes the threat model in two ways the original does not cover:
+#
+# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)`
+# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…`
+# sails straight past a guard that only ever saw the first URL. The negative control ("a
+# localhost URL is refused") would still pass while the rail was wide open. `fetch` below
+# therefore takes the hops MANUALLY and re-guards every one.
+# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the
+# host and checks EVERY answer, not just the literal.
+#
+# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the
+# record between the guard and the socket is not stopped by it. Closing that needs a pinned
+# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather
+# than implied away — the rail refuses the realistic cases and says what it does not cover.
+
+class Refused(ValueError):
+ """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error."""
+
+
+def _ip_public(ip):
+ return not (ip.is_private or ip.is_loopback or ip.is_link_local
+ or ip.is_reserved or ip.is_unspecified or ip.is_multicast)
+
+
+def guard(url, allowed=()):
+ """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers
+ checked too, plus the optional `--allowed` domain rail scrape.py carries."""
+ p = urlparse(url or "")
+ if p.scheme not in ("http", "https"):
+ raise Refused(f"only http(s) allowed, not {p.scheme!r}")
+ host = (p.hostname or "").strip()
+ if not host:
+ raise Refused("no host in the URL")
+ low = host.lower()
+ if (low in ("localhost", "localhost.localdomain")
+ or low.endswith(".local") or low.endswith(".internal")
+ or low.endswith(".localhost")):
+ raise Refused(f"refused non-public host {host!r} (public web only)")
+ try: # a bare-IP host is decided on the literal
+ ip = ipaddress.ip_address(low)
+ except ValueError:
+ ip = None
+ if ip is not None and not _ip_public(ip):
+ raise Refused(f"refused non-public IP {host} (public web only)")
+ if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed):
+ raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}")
+ if ip is None:
+ try:
+ infos = socket.getaddrinfo(host, None)
+ except Exception as e:
+ raise Refused(f"could not resolve {host!r}: {type(e).__name__}")
+ for info in infos:
+ addr = info[4][0]
+ try:
+ resolved = ipaddress.ip_address(addr.split("%")[0])
+ except ValueError:
+ continue
+ if not _ip_public(resolved):
+ raise Refused(
+ f"refused {host!r}: it resolves to the non-public address {resolved}")
+ return True
+
+
+def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None):
+ """GET `url`, taking redirects BY HAND so every hop passes `guard`.
+
+ Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal,
+ not a silent truncation — an endless redirect is indistinguishable from an attempt to walk
+ the fetcher somewhere it was told not to go.
+ """
+ current = url
+ hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"}
+ hdrs.update(headers or {})
+ for _hop in range(max_hops + 1):
+ guard(current, allowed)
+ r = requests.get(current, timeout=timeout, headers=hdrs,
+ allow_redirects=False, stream=True)
+ if r.status_code in (301, 302, 303, 307, 308):
+ loc = r.headers.get("Location")
+ r.close()
+ if not loc:
+ raise Refused(f"redirect {r.status_code} with no Location header")
+ current = urljoin(current, loc)
+ continue
+ body = r.raw.read(max_kb * 1024, decode_content=True) or b""
+ status, final = r.status_code, str(r.url)
+ r.close()
+ return status, final, body
+ raise Refused(f"more than {max_hops} redirects — refusing to follow further")
+
+
+def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None):
+ """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`.
+
+ ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by
+ hand because a redirected GET is ordinary. A redirected POST is not: re-sending a
+ credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail
+ exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever
+ re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request.
+ (The vendor calls below are the only POSTs this module makes, and they carry the API key.)
+ """
+ guard(url)
+ hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"}
+ hdrs.update(headers or {})
+ r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False,
+ data=json.dumps(body if body is not None else {}), stream=True)
+ try:
+ if r.status_code in (301, 302, 303, 307, 308):
+ raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, "
+ f"never re-sent to a location the server picked")
+ return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"")
+ finally:
+ r.close()
+
+
+# ---------------------------------------------------------------------------------------------
+# EXTRACTION — vendored from scrape.py, unchanged in behaviour
+# ---------------------------------------------------------------------------------------------
+
+def _soup(body):
+ if BeautifulSoup is None:
+ raise RuntimeError(
+ "beautifulsoup4 is not installed in this environment — the automation engine's "
+ "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to "
+ "aios-web/requirements.txt).")
+ try:
+ return BeautifulSoup(body, "lxml")
+ except Exception:
+ return BeautifulSoup(body, "html.parser")
+
+
+def tables(soup):
+ """Every HTML table as rows. Dict rows when the first row looks like a header."""
+ out = []
+ for t in soup.find_all("table"):
+ rows = []
+ for tr in t.find_all("tr"):
+ cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])]
+ if cells:
+ rows.append(cells)
+ if not rows:
+ continue
+ head, body = rows[0], rows[1:]
+ if body and len(head) == len(body[0]) and all(head):
+ out.append([dict(zip(head, r)) for r in body if len(r) == len(head)])
+ else:
+ out.append(rows)
+ return out
+
+
+def jsonld(soup):
+ out = []
+ for s in soup.find_all("script", attrs={"type": "application/ld+json"}):
+ try:
+ out.append(json.loads(s.string or s.get_text()))
+ except Exception:
+ pass
+ return out
+
+
+def meta(soup):
+ m = {}
+ if soup.title and soup.title.string:
+ m["title"] = soup.title.string.strip()
+ for tag in soup.find_all("meta"):
+ k = tag.get("name") or tag.get("property")
+ v = tag.get("content")
+ if k and v and (k in ("description", "keywords", "author")
+ or k.startswith(("og:", "twitter:"))):
+ m[k] = v.strip()
+ return m
+
+
+def preview(url, extract="table", table_index=0, allowed=()):
+ """The field-map preview the editor calls BEFORE anything is created: what columns does this
+ page actually offer, and what do the first rows look like? Never writes."""
+ status, final, body = fetch(url, allowed=allowed)
+ if not (200 <= status < 300):
+ return {"ok": False, "status": status, "url": final,
+ "note": f"the page answered {status} — it may be bot-gated or gone",
+ "columns": [], "sample": [], "rowCount": 0}
+ soup = _soup(body)
+ rows = []
+ if extract == "jsonld":
+ blocks = jsonld(soup)
+ flat = []
+ for b in blocks:
+ if isinstance(b, list):
+ flat.extend([x for x in b if isinstance(x, dict)])
+ elif isinstance(b, dict):
+ items = b.get("itemListElement")
+ flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list)
+ else [b])
+ rows = [{k: _scalar(v) for k, v in d.items()} for d in flat]
+ else:
+ found = tables(soup)
+ idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0
+ picked = found[idx] if found else []
+ rows = [r for r in picked if isinstance(r, dict)]
+ cols, seen = [], set()
+ for r in rows[:50]:
+ for k in r:
+ if k not in seen:
+ seen.add(k)
+ cols.append(k)
+ return {"ok": True, "status": status, "url": final, "columns": cols,
+ "sample": rows[:8], "rowCount": len(rows),
+ "tableCount": len(tables(soup)) if extract != "jsonld" else 0,
+ "title": (meta(soup) or {}).get("title", "")}
+
+
+def _scalar(v):
+ if isinstance(v, (str, int, float)) and not isinstance(v, bool):
+ return str(v)
+ if isinstance(v, bool):
+ return "1" if v else ""
+ if isinstance(v, dict):
+ return str(v.get("name") or v.get("@id") or "")
+ if isinstance(v, list):
+ return ", ".join(_scalar(x) for x in v[:8])
+ return ""
+
+
+# ---------------------------------------------------------------------------------------------
+# CRON — a 5-field parser and a "what was the last fire time" walk
+# ---------------------------------------------------------------------------------------------
+# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward
+# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute
+# between the last run and now?"* — and answering it that way is what makes a missed tick
+# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and
+# not never.
+
+_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)]
+
+
+def _parse_field(spec, lo, hi):
+ out = set()
+ for part in str(spec).split(","):
+ part = part.strip()
+ if not part:
+ raise ValueError("empty cron field part")
+ step = 1
+ if "/" in part:
+ part, _, s = part.partition("/")
+ step = int(s)
+ if step < 1:
+ raise ValueError("cron step must be >= 1")
+ if part in ("*", "?"):
+ a, b = lo, hi
+ elif "-" in part.lstrip("-"):
+ a_s, _, b_s = part.partition("-")
+ a, b = int(a_s), int(b_s)
+ else:
+ a = b = int(part)
+ if a < lo or b > hi or a > b:
+ raise ValueError(f"cron value {part!r} out of range {lo}..{hi}")
+ out.update(range(a, b + 1, step))
+ return out
+
+
+def parse_cron(expr):
+ """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`.
+
+ Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at
+ WRITE time, because a cron string nobody can evaluate is an automation that silently never
+ runs and reports no error anywhere.
+ """
+ parts = str(expr or "").split()
+ if len(parts) != 5:
+ raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday")
+ sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)]
+ dom_r = parts[2].strip() not in ("*", "?")
+ dow_r = parts[4].strip() not in ("*", "?")
+ return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()),
+ dom_r, dow_r)
+
+
+def _day_matches(day, doms, months, dows, dom_r, dow_r):
+ if day.month not in months:
+ return False
+ # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the
+ # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes
+ # a schedule that looks right fire almost never.
+ dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0
+ if dom_r and dow_r:
+ return day.day in doms or dow in dows
+ if dom_r:
+ return day.day in doms
+ if dow_r:
+ return dow in dows
+ return True
+
+
+def prev_fire(expr, now=None, lookback_days=400):
+ """The most recent scheduled minute at or before `now`, or None inside the lookback.
+
+ Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first,
+ and only a matching day needs its hours/minutes searched.
+ """
+ minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr)
+ now = (now or _now()).replace(second=0, microsecond=0)
+ for back in range(lookback_days + 1):
+ day = (now - _dt.timedelta(days=back)).date()
+ if not _day_matches(day, doms, months, dows, dom_r, dow_r):
+ continue
+ same_day = back == 0
+ for h in sorted(hours, reverse=True):
+ if same_day and h > now.hour:
+ continue
+ for m in sorted(minutes, reverse=True):
+ if same_day and h == now.hour and m > now.minute:
+ continue
+ return _dt.datetime(day.year, day.month, day.day, h, m)
+ return None
+
+
+def is_due(defn, now=None):
+ """Should the scheduler run this automation right now?
+
+ Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the
+ moment the schedule was enabled, else creation) and at or before now. Anchoring on
+ `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at
+ 14:00" from running immediately and looking like a bug.
+ """
+ if not isinstance(defn, dict):
+ return False
+ sched = defn.get("schedule") or {}
+ if not sched.get("enabled"):
+ return False
+ try:
+ fire = prev_fire(sched.get("cron"), now=now)
+ except ValueError:
+ return False
+ if fire is None:
+ return False
+ since = (_parse_iso((defn.get("status") or {}).get("lastRunAt"))
+ or _parse_iso(sched.get("enabledAt"))
+ or _parse_iso(defn.get("created")))
+ return since is None or fire > since
+
+
+def next_fire(expr, now=None, lookahead_days=400):
+ """The next scheduled minute strictly after `now` — display only (the rail is `is_due`)."""
+ try:
+ minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr)
+ except ValueError:
+ return None
+ now = (now or _now()).replace(second=0, microsecond=0)
+ for ahead in range(lookahead_days + 1):
+ day = (now + _dt.timedelta(days=ahead)).date()
+ if not _day_matches(day, doms, months, dows, dom_r, dow_r):
+ continue
+ same_day = ahead == 0
+ for h in sorted(hours):
+ if same_day and h < now.hour:
+ continue
+ for m in sorted(minutes):
+ if same_day and h == now.hour and m <= now.minute:
+ continue
+ return _dt.datetime(day.year, day.month, day.day, h, m)
+ return None
+
+
+# ---------------------------------------------------------------------------------------------
+# THE UPSERT — pure, so the arithmetic is testable without a store or a network
+# ---------------------------------------------------------------------------------------------
+
+def upsert_rows(existing, incoming, key_field, cap=None):
+ """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`.
+
+ THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped
+ appearing on the source page has not necessarily stopped existing — the page changed its
+ filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream
+ hiccup into silent data loss, and the row may be carrying user-typed overlay values in
+ columns the scrape never touches. So the run reports `orphans: N` and leaves them alone.
+
+ Only MAPPED keys are written: a re-run never clobbers a column a user added by hand.
+
+ ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on
+ entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to
+ call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS`
+ that same loop is hundreds of millions of dict copies and the automation simply never
+ finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.)
+
+ ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different
+ facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the
+ DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" —
+ a property of the SYSTEM, and never benign. One number for both meant a table hitting its
+ ceiling was indistinguishable from a page with a few blank cells, which is how a time series
+ stops silently. The runners turn any `capped` into a `partial` run that NAMES the table.
+ """
+ cap = MAX_UT_ROWS if cap is None else int(cap)
+ rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()}
+ counts = {"inserted": 0, "updated": 0, "unchanged": 0,
+ "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0}
+ by_key, dupe_ids = {}, set()
+ for rid, row in rows.items():
+ kv = str(row.get(key_field, "") or "").strip()
+ if not kv:
+ continue
+ if kv in by_key:
+ dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left
+ continue
+ by_key[kv] = rid
+ next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1
+ seen_keys, incoming_dupes = set(), 0
+ for src in incoming or []:
+ kv = str((src or {}).get(key_field, "") or "").strip()
+ if not kv:
+ counts["skipped"] += 1 # no key -> cannot be upserted; never guessed
+ continue
+ if kv in seen_keys:
+ incoming_dupes += 1 # the SOURCE listed it twice; first wins
+ continue
+ seen_keys.add(kv)
+ rid = by_key.get(kv)
+ if rid is None:
+ if len(rows) >= cap:
+ counts["capped"] += 1 # LOUD: the runner turns this into a partial run
+ continue
+ rid = str(next_id)
+ next_id += 1
+ rows[rid] = dict(src)
+ by_key[kv] = rid
+ counts["inserted"] += 1
+ continue
+ before = rows[rid]
+ changed = {k: v for k, v in src.items() if str(before.get(k, "")) != str(v)}
+ if changed:
+ before.update(src)
+ counts["updated"] += 1
+ else:
+ counts["unchanged"] += 1
+ counts["duplicates"] = incoming_dupes + len(dupe_ids)
+ counts["orphans"] = sum(
+ 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids)
+ return rows, counts
+
+
+def dedupe_canonical_rows(existing, key_field, newest_by=""):
+ """Collapse duplicate logical rows while preserving the lowest stable row id.
+
+ Canonical entity tables use this before every upsert. Snapshot tables deliberately do not:
+ repeated shortcodes there are new timestamped observations, not duplicate posts. Values are
+ taken newest-first and then filled from older rows, so a sparse fresh projection does not
+ erase a field an earlier row knew.
+ """
+ rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()}
+ groups = {}
+ for rid, row in rows.items():
+ identity = str(row.get(key_field) or "").strip()
+ if identity:
+ groups.setdefault(identity, []).append((rid, row))
+ removed = 0
+ for members in groups.values():
+ if len(members) < 2:
+ continue
+ keep = min((rid for rid, _row in members), key=lambda r: (not r.isdigit(), int(r) if r.isdigit() else r))
+ ordered = sorted(members, key=lambda item: str(item[1].get(newest_by) or ""), reverse=True) \
+ if newest_by else members
+ merged = {}
+ for _rid, row in ordered:
+ for key, value in row.items():
+ if key not in merged or str(merged.get(key) or "").strip() == "":
+ merged[key] = value
+ rows[keep] = merged
+ for rid, _row in members:
+ if rid != keep:
+ rows.pop(rid, None)
+ removed += 1
+ return rows, removed
+
+
+# ---------------------------------------------------------------------------------------------
+# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key)
+# ---------------------------------------------------------------------------------------------
+
+def _ut_slug(label):
+ s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_")
+ return (s or "table")[:40]
+
+
+def ut_all(rt):
+ try:
+ return dict(rt.get(UT_STORE_KEY) or {})
+ except Exception:
+ return {}
+
+
+def disable_for_table(rt, table_key, note="target database deleted"):
+ """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted.
+
+ The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still
+ shows what existed and why it stopped — silently deleting a user's automation because its
+ target died would read as data loss. Returns the ids it touched."""
+ key = str(table_key or "")
+ touched = []
+
+ def _up(cur):
+ for aid, d in (cur or {}).items():
+ if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key:
+ sch = d.get("schedule")
+ if not isinstance(sch, dict):
+ sch = d["schedule"] = {}
+ sch["enabled"] = False
+ d["statusNote"] = note
+ touched.append(str(aid))
+ return cur
+
+ if key:
+ rt.update(STORE_KEY, _up, flush="sync")
+ return touched
+
+
+def ut_get(rt, key):
+ return ut_all(rt).get(str(key))
+
+
+def retire_automation_stage_fields(rt, tables=None):
+ """Delete obsolete Board-only fields and their hidden cells, never user fields.
+
+ The authoritative selector is the engine's own ``automation.stageField`` / ``cyclesField``
+ metadata — labels such as “Stage” are ordinary user vocabulary and are not touched. Old
+ generated timestamp/cycle cells are cleared too, including rows whose field definition was
+ removed by an interrupted earlier migration. Re-running this migration is a no-op.
+
+ ⭐ WAVE 29 (W29-T01) — ``tables`` LETS A CALLER LEND ITS SNAPSHOT. The SCAN below is
+ O(all row-cells in the tenant) over a bucket whose documented ceiling is 35.8 MB / ~1.4 s to
+ deep-copy (see this module's header), and `GET /automations` was paying for THREE independent
+ copies of it per request. The scan is read-only, so borrowing the caller's copy is free; the
+ WRITE below still goes through `rt.update`, which re-reads under the store lock, so a lent
+ snapshot can never be the thing that gets written back.
+ """
+ tables = ut_all(rt) if tables is None else tables
+ stage_keys = set()
+ for table in tables.values():
+ if not isinstance(table, dict):
+ continue
+ for field in table.get("fields") or []:
+ auto = field.get("automation") if isinstance(field, dict) else None
+ if isinstance(auto, dict) and (auto.get("stageField") or auto.get("cyclesField")):
+ stage_keys.add(str(field.get("key") or ""))
+ for row in (table.get("rows") or {}).values():
+ for key in (row or {}):
+ text = str(key)
+ if text.startswith("stage_auto_"):
+ stage_keys.add(text.removesuffix("_at").removesuffix("_cycles"))
+ stage_keys.discard("")
+ if not stage_keys:
+ return {"tables": 0, "fields": 0, "cells": 0}
+
+ changed = {"tables": set(), "fields": 0, "cells": 0}
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for table_key, table in cur.items():
+ if not isinstance(table, dict):
+ continue
+ kept, removed = [], set()
+ for field in table.get("fields") or []:
+ auto = field.get("automation") if isinstance(field, dict) else None
+ key = str(field.get("key") or "") if isinstance(field, dict) else ""
+ if key in stage_keys and isinstance(auto, dict) and \
+ (auto.get("stageField") or auto.get("cyclesField")):
+ removed.add(key)
+ changed["fields"] += 1
+ continue
+ kept.append(field)
+ if removed:
+ table["fields"] = kept
+ changed["tables"].add(str(table_key))
+ for row in (table.get("rows") or {}).values():
+ if not isinstance(row, dict):
+ continue
+ for stage_key in stage_keys:
+ for key in (stage_key, stage_key + "_at", stage_key + "_cycles"):
+ if key in row:
+ row.pop(key, None)
+ changed["cells"] += 1
+ changed["tables"].add(str(table_key))
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ return {"tables": len(changed["tables"]), "fields": changed["fields"],
+ "cells": changed["cells"]}
+
+
+def bind_unbound_fields(rt):
+ """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId`
+ are bound to the definition that already writes them — matched on the (targetTable,
+ fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no
+ definition references is DISABLED with the reason on it rather than deleted or guessed:
+ the column was already dead (nothing runs a column no flow names), and now it says so.
+ Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate."""
+ by_binding = {}
+ for aid, d in all_definitions(rt).items():
+ cfg = d.get("config") or {}
+ if cfg.get("fieldKey") and cfg.get("targetTable"):
+ by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid)
+ plan = {}
+ for tk, t in ut_all(rt).items():
+ for f in (t.get("fields") or []):
+ a = f.get("automation")
+ # An already-DISABLED bag is a decision this migration made on a previous pass —
+ # re-planning it every call would turn the once-per-process sweep into a write
+ # per read.
+ if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \
+ and not a.get("disabled"):
+ plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key"))))
+ if not plan:
+ return 0, 0
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for (tk, fk), aid in plan.items():
+ for f in ((cur.get(tk) or {}).get("fields") or []):
+ if f.get("key") == fk and isinstance(f.get("automation"), dict):
+ if aid:
+ f["automation"]["flowId"] = aid
+ else:
+ f["automation"]["disabled"] = True
+ f["automation"]["statusNote"] = (
+ "not bound to any flow — create an automation for this column "
+ "or delete it")
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ bound = sum(1 for v in plan.values() if v)
+ return bound, len(plan) - bound
+
+
+def ut_key_for(label, key=None):
+ """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which
+ must not create anything) resolves the same key by construction rather than by copying the
+ derivation and drifting from it."""
+ k = str(key or (UT_PREFIX + _ut_slug(label)))
+ return k if k.startswith(UT_PREFIX) else UT_PREFIX + k
+
+
+#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same
+#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging
+#: `core` into the API's boot path — and the two are held in step by a gate check.
+MACHINE_OWNERS = ("automation", "scheduler")
+
+
+def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag="",
+ record_mode="", lock_fields=False):
+ """Create the table if it is missing; return its key. Idempotent — a re-run of an automation
+ that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given)
+ and an existing table with that key is adopted, not duplicated.
+
+ Fields are MERGED, never replaced: a user who added a column to an automation's table keeps
+ it, and a new source column joins on the next run.
+
+ `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped
+ `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their
+ provenance. Fields already on the table keep whatever tag they have (first flow wins;
+ shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not
+ ownership).
+
+ ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever
+ or WHATEVER ran the automation, so the same table belonged to a person if its first run was
+ manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open`
+ admits only the creator or an admin, so **whether you could open your own Instagram
+ snapshots depended on a race you never saw**. The owner is now the automation's creator, and
+ a table already stamped with a machine name is ADOPTED the next time a run knows a human
+ one. Adoption is a repair, not a widening: the automation's creator is the person who asked
+ for the table in the first place.
+ """
+ key = ut_key_for(label, key)
+ # ⭐ WAVE 26 — THE MIGRATION RIDES THE WRITE PATH, and that placement is the point.
+ #
+ # `ut_ensure` MERGES fields by key and never re-types an existing column, so on its own it
+ # would leave every table already in production on the old `text` schema forever while new
+ # ones got the honest types — the split schema R3's note warned about, arriving through the
+ # very function the note was written on. Migrating here means a table is brought forward
+ # immediately BEFORE anything appends to it, so no caller has to remember anything and no
+ # tenant is left behind by a script nobody ran.
+ # ⚠ Cheap by construction: `migrate_ig_tables` returns without a write when the table is
+ # already current, which after the first run is every call.
+ if key and {f.get("key") for f in (fields or [])} & set(PRESET_PROFILE_KEYS):
+ try:
+ migrate_ig_tables(rt, log=lambda *_a: None, only=key)
+ except Exception: # noqa: BLE001
+ # A migration that cannot run must not stop the automation from writing its rows.
+ # The old schema still reads; a refused write loses the pull we just paid for.
+ pass
+ wanted = []
+ for raw_field in (fields or []):
+ field = dict(raw_field)
+ if flow_tag or lock_fields:
+ automation = dict(field.get("automation") or {})
+ if flow_tag:
+ automation.setdefault("flowId", str(flow_tag))
+ if lock_fields:
+ automation["preset"] = True
+ field["automation"] = automation
+ wanted.append(field)
+ created = _iso()
+ human = username if username and username not in MACHINE_OWNERS else ""
+
+ # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store
+ # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo
+ # budget, an idempotent helper that always writes is the same defect as a per-row insert.
+ have = ut_get(rt, key)
+ have_fields = {str(f.get("key") or ""): f for f in ((have or {}).get("fields") or [])}
+ missing = [f for f in wanted if f.get("key") not in have_fields]
+ missing_locks = [f for f in wanted
+ if lock_fields and f.get("key") in have_fields
+ and (not isinstance(have_fields[f.get("key")].get("automation"), dict)
+ or have_fields[f.get("key")]["automation"].get("preset") is not True)]
+ if (have is not None and not missing and not missing_locks
+ and not (record_mode and have.get("recordMode") != record_mode)
+ and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)):
+ return key
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ t = cur.get(key)
+ if t is None:
+ if len(cur) >= MAX_UT_TABLES:
+ return cur
+ cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation",
+ "createdBy": username, "created": created,
+ "fields": wanted, "rows": {}}
+ if record_mode:
+ cur[key]["recordMode"] = record_mode
+ return cur
+ have = {f.get("key") for f in (t.get("fields") or [])}
+ for f in wanted:
+ if f.get("key") not in have:
+ t.setdefault("fields", []).append(f)
+ have.add(f.get("key"))
+ elif lock_fields:
+ stored = next((g for g in (t.get("fields") or [])
+ if g.get("key") == f.get("key")), None)
+ if stored is not None:
+ automation = dict(stored.get("automation") or {})
+ if flow_tag:
+ automation.setdefault("flowId", str(flow_tag))
+ automation["preset"] = True
+ stored["automation"] = automation
+ # ADOPTION: a machine name is not an owner. It never overwrites a human one.
+ if human and (t.get("createdBy") or "") in MACHINE_OWNERS:
+ t["createdBy"] = human
+ # An automation's table SAYS an automation owns it — the nav badge reads from this.
+ t["source"] = t.get("source") or "Automation"
+ if record_mode:
+ t["recordMode"] = record_mode
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ return key
+
+
+def ut_write_rows(rt, key, rows):
+ """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header)."""
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ t = cur.get(key)
+ if t is not None:
+ t["rows"] = rows
+ return cur
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+
+
+def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None):
+ """Write one automation-owned cell (+ optional whole extra tables) in ONE update."""
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ t = cur.get(key)
+ if t is not None:
+ t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value
+ for k, rws in (extra_rows or {}).items():
+ tt = cur.get(k)
+ if tt is not None:
+ tt["rows"] = rws
+ return cur
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+
+
+IG_FIELD_DESCRIPTIONS = {
+ "alt_text": "Accessibility text attached to the Instagram post.",
+ # ⚠ WIDENED 2026-08-10, because the column gained a second writer and the old sentence would
+ # have made it lie. It was written for the anonymous rung, whose numbers are genuinely ROUNDED
+ # ("10.4K followers"). A DISCOVERY observation is exact-looking and STALE instead — read off
+ # the vendor's pre-collected corpus at a collection time we are not told. Both mean the same
+ # thing to a reader ("do not treat this as an exact count taken at Pulled at"), and a
+ # description that named only the first would have quietly excluded the second.
+ "approx": "Checked when the counts are rounded, or were read from a pre-collected corpus "
+ "rather than measured at the time shown.",
+ "avg_comments_12": "Average comments across the 12 most recent captured posts.",
+ "avg_engagement": "Average engagement rate reported for the profile.",
+ "avg_likes_12": "Average likes across the 12 most recent captured posts.",
+ "avg_plays_12": "Average plays across the 12 most recent captured posts.",
+ "avg_views_12": "Average views across the 12 most recent captured posts.",
+ "views": "View count for the post, as Instagram displays it.",
+ "video_duration": "Length of the video in seconds.",
+ "comments_disabled": "Checked when the creator has turned comments off for this post.",
+ "plays": "Times the video started playing, including replays.",
+ "bio": "Biography shown on the Instagram profile.",
+ "bio_hashtags": "Hashtags listed in the profile biography.",
+ "business_category": "Instagram's business category for the account.",
+ "caption": "Caption published with the Instagram post.",
+ "category": "Instagram's category for the profile.",
+ "comment_key": "Unique ID for the captured comment record.",
+ "commented_at": "Date the comment was posted.",
+ "comments": "Comment count reported for the post.",
+ "comments_captured": "Number of distinct comment records linked to this profile.",
+ "comments_link": "Comment records linked to this record.",
+ "country_code": "Country code reported for the profile.",
+ "created_by": "User whose automation first added this lead.",
+ "enriched_at": "Date the profile was last enriched.",
+ # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number
+ # beside it is the only thing that says how much of one.
+ "location_guess": "Most common city tagged across this profile's captured posts - a guess, "
+ "not a stated location.",
+ "location_confidence": "Share of this profile's geotagged posts that agree on the guessed "
+ "city.",
+ "external_url": "Website linked from the Instagram biography.",
+ "external_url_title": "Title Instagram shows for the biography link.",
+ "fbid": "Facebook ID associated with the Instagram account.",
+ "first_found": "Date an automation first found this profile.",
+ "followers": "Latest reported follower count.",
+ "following": "Latest reported number of accounts followed.",
+ "found_count": "Number of times discovery automations found this profile.",
+ "full_name": "Display name shown on the Instagram profile.",
+ "handle": "Instagram username without the @ symbol.",
+ "has_channel": "Checked when the profile has an Instagram channel.",
+ "hashtags": "Hashtags extracted from the post caption.",
+ "highlights_count": "Total story highlight collections reported for the profile.",
+ "ig_id": "Instagram's internal ID for the account.",
+ "influencer_key": "Normalized handle linking this row to its profile.",
+ "is_business": "Checked when Instagram marks the account as a business.",
+ "is_joined_recently": "Checked when Instagram marks the account as recently joined.",
+ "is_private": "Checked when the Instagram profile is private.",
+ "is_professional": "Checked when Instagram marks the account as professional.",
+ "last_found": "Date an automation most recently found this profile.",
+ "likes": "Like count reported for this record.",
+ "measured_at": "Date engagement metrics on this post were last read.",
+ "measurements_captured": "Number of measurement rows stored for this post.",
+ "paid_partnership": "Checked when Instagram marks the post as a paid partnership.",
+ "partner": "Brand named in the post's paid partnership metadata.",
+ "partner_id": "Partner ID reported for the Instagram profile.",
+ "platform": "Social network for this profile.",
+ "plays": "Video plays, including repeat plays, reported for the post.",
+ "post_link": "Post record linked to this measurement or comment.",
+ "post_measurements_captured": "Number of distinct post measurements linked to this profile.",
+ "post_snapshot_key": "Unique ID for this post measurement.",
+ "post_snapshots_link": "Engagement measurement rows linked to this record.",
+ "posted_at": "Date the Instagram post was published.",
+ "posts_captured": "Number of distinct post records captured for this profile.",
+ "posts_count": "Total posts reported for the Instagram profile.",
+ "posts_link": "Post records linked to this profile.",
+ "profile_name": "Profile name returned by the data provider.",
+ "profile_reads": "Number of profile snapshots stored for this profile.",
+ "profile_snapshots_link": "Profile snapshot rows linked to this profile.",
+ "profile_url": "Direct URL to the Instagram profile.",
+ "pronouns": "Pronouns shown on the Instagram profile.",
+ "pulled_at": "Date this snapshot was collected.",
+ "related_accounts": "Accounts Instagram suggests alongside this profile.",
+ "replies": "Reply count reported for the comment.",
+ "shortcode": "Instagram shortcode that uniquely identifies the post.",
+ "source_payload": "Full source response for this record.",
+ "snapshot_key": "Unique ID for this profile snapshot.",
+ "source": "Method or provider used to read this profile.",
+ "tagged_location": "Location tagged on the post; it is not the creator's residence.",
+ "type": "Instagram media type: image, video, or carousel.",
+ "url": "Direct URL to the Instagram post.",
+ "verified": "Checked when Instagram marks the profile as verified.",
+}
+
+
+#: ⭐ WAVE 29 (R2) — TIKTOK'S OWN DESCRIPTIONS, and the reason this exists rather than reusing the
+#: map above is one line of `field_def`: it defaults `description` to
+#: `IG_FIELD_DESCRIPTIONS.get(key)`. Nine `ut_tt_*` columns share a KEY with an Instagram column
+#: (`shortcode`, `type`, `url`, `verified`, `caption`, `likes`, `comments`, `replies`,
+#: `tagged_location`), so a TikTok video's Type column would have shipped explaining "Instagram
+#: media type" — a wrong sentence in the header tooltip of a column nobody would think to check.
+#: ⚠ A KEY ABSENT HERE GETS NO DESCRIPTION AT ALL, deliberately: silence is honest, and inheriting
+#: the Instagram sentence is the failure this map exists to prevent.
+TT_FIELD_DESCRIPTIONS = {
+ "platform": "Network this handle is on.",
+ "handle": "TikTok @name; the unique account handle.",
+ "full_name": "Display name shown on the TikTok profile.",
+ "tt_id": "TikTok's own numeric id for the account.",
+ "profile_url": "Direct URL to the TikTok profile.",
+ "bio": "Profile biography text.",
+ "external_url": "Link in the TikTok bio.",
+ "verified": "Checked when TikTok marks the account as verified.",
+ "is_private": "Checked when the account is private.",
+ "followers": "Follower count at the time of the pull.",
+ "following": "Accounts this profile follows.",
+ "posts_count": "Videos published by this account.",
+ "likes_received": "Total likes this account's videos have received.",
+ "avg_engagement": "Average engagement rate, stored as a percentage.",
+ "comment_engagement": "Comment engagement rate, stored as a percentage.",
+ "like_engagement": "Like engagement rate, stored as a percentage.",
+ "is_business": "Approximate: set when TikTok flags the account as a commerce user.",
+ "country_code": "Two-letter country code reported for the account.",
+ "predicted_lang": "Language TikTok predicts for this account.",
+ "account_created_at": "When the TikTok account itself was created.",
+ "region": "Region reported for the account.",
+ "shortcode": "TikTok's numeric id for the video; unique per post.",
+ "type": "TikTok post type: video or image.",
+ "url": "Direct URL to the TikTok post.",
+ "caption": "Post description text.",
+ "posted_at": "When the creator published the post.",
+ "likes": "Likes reported for the post.",
+ "comments": "Comment count reported for the post.",
+ "views": "Play count TikTok reports for the video.",
+ "shares": "Times the post was shared.",
+ "saves": "Times the post was saved to a collection.",
+ "video_duration": "Video length in seconds.",
+ "hashtags": "Hashtags used in the post.",
+ "tagged_location": "Commerce location reported for the post; it is not the creator's home.",
+ "influencer_key": "Handle of the account this record belongs to.",
+ "comment_key": "Unique ID for this comment.",
+ "commented_at": "When the comment was posted.",
+ "replies": "Reply count reported for the comment.",
+ "snapshot_key": "Unique ID for this profile snapshot.",
+ "post_snapshot_key": "Unique ID for this post measurement.",
+ "pulled_at": "When this measurement was read.",
+ "enriched_at": "When this row was last enriched.",
+ "source": "Method or provider used to read this record.",
+ "source_payload": "Full source response for this record.",
+}
+
+
+def field_def(text_key, label, ftype="text", **extra):
+ """One machine-spawned column definition.
+
+ ⭐ 2026-08-07 — `**extra` carries the per-field DECLARATIONS the preset set needs (`pinned`,
+ `profile`), and it stays ONE constructor rather than growing a second for "special" fields.
+ ⛔ Every key passed here must survive `core.user_tables._clean_field` UNCHANGED — `verify_api`'s
+ W25-1 section pins the drift at ZERO, so a key the validator drops or rewrites turns the whole
+ preset list red rather than failing quietly ([[default-must-pass-its-own-guard]]).
+
+ ⭐ WAVE 25 — `editRole` IS DECLARED HERE, and adding it closes a bypass rather than adding a
+ feature. `ut_ensure` writes these dicts STRAIGHT into the `user_tables` bucket, so
+ `core.user_tables._clean_field` — the single validator every field created through the ordinary
+ door passes — has never judged a single field this engine spawned. The difference was exactly
+ one key: `_clean_field` emits `editRole: 'admins'` and this did not, so `_clean_field(f) != f`
+ for every automation column in the product.
+ ⚠ NOTHING CHANGES BEHAVIOURALLY — `may_edit_field` asks `editRole == 'everyone'`, and an
+ absent key was already not that, so the bypass was fail-closed and therefore silent. It is
+ stated now, and `verify_automation` asserts the equality field-by-field, so the next key
+ `_clean_field` grows cannot go unnoticed here ([[default-must-pass-its-own-guard]]).
+ """
+ description = " ".join(str(extra.pop(
+ "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split())
+ out = {"key": text_key, "label": label, "type": ftype, "source": "overlay",
+ "editRole": "admins"}
+ if description:
+ out["description"] = description
+ out.update(extra)
+ return out
+
+
+def tt_field_def(text_key, label, ftype="text", **extra):
+ """One `ut_tt_*` column. `field_def` with TikTok's description map bound (wave 29, R2).
+
+ ⛔ `description` IS ALWAYS SUPPLIED, even when blank, so `field_def`'s Instagram default can
+ never be reached from here. An explicit empty string makes `field_def` omit the key, which is
+ the same shape an undescribed column already has — the point is that the sentence a TikTok
+ column carries is one somebody wrote about TikTok, or none at all.
+ """
+ extra.setdefault("description", TT_FIELD_DESCRIPTIONS.get(text_key, ""))
+ return field_def(text_key, label, ftype, **extra)
+
+
+# ---------------------------------------------------------------------------------------------
+# DEFINITIONS — validation and the bucket's read/write half
+# ---------------------------------------------------------------------------------------------
+
+def _s(v, n=200):
+ return str(v if v is not None else "")[:n]
+
+
+#: ⭐ WAVE 26 · C5 / R2 — HOW MANY POSTS ONE PULL MAY KEEP, and the ceiling is the VENDOR'S.
+#:
+#: ⛔ MEASURED 2026-08-05 and recorded at `_bd_posts_count`: **a Profiles row carries the TOP 12
+#: posts — a cap, not a count.** Asking for more does not fetch more; it just makes the config
+#: disagree with what the run can possibly do.
+#: ⚠ THIS REPLACES A SILENT `max(1, min(n, 200))` AT TWO CALL SITES, and the clamp was the defect
+#: rather than the number: a person typing 50 got a stored 50, a UI that read back 50, and twelve
+#: posts — with nothing anywhere saying why. A control that accepts a value it cannot honour is
+#: worse than one that refuses it, because the refusal is the only place the ceiling can be
+#: taught. So this REFUSES, and the sentence names the cap and who set it.
+#:
+#: ⭐⭐ 2026-08-09 — RAISED TO 30 (owner: *"move the max limit to 30 posts per profile"*), and the
+#: paragraph above needed correcting to do it honestly: **12 was OUR ROUTE'S cap, not the
+#: vendor's.** Re-measured the same day — the PROFILE record still returns 12 however many you ask
+#: for (asked 40, got 12), but the documented discover-by-url route answered `num_of_posts: 30`
+#: with exactly 30 rows in 90 s (23 Reels + 7 Carousels, @theresalearns). The ceiling was a
+#: property of the call we happened to make.
+#: ⛔ SO THE NUMBER MOVED AND THE CAPTURE MUST FOLLOW. Until `bd_profile_posts` is wired ahead of
+#: the views top-up, a `maxPosts` above 12 is honoured by the CONFIG and bounded by the PROFILE
+#: read at run time — the exact accept-what-you-cannot-honour shape this constant exists to
+#: prevent, now surviving in one place instead of two. It is recorded rather than hidden, and it
+#: is why `clean_post_groups` bounds a group by `maxPosts` rather than by 12.
+MAX_POSTS_PER_PULL = 30
+DEFAULT_POSTS_PER_PULL = 10
+
+#: ⭐⭐ THE WINDOW THE `avg_*_12` PRESET ROLLUPS AVERAGE OVER — ITS OWN CONSTANT, deliberately
+#: NOT `MAX_POSTS_PER_PULL`.
+#:
+#: ⛔ THEY WERE THE SAME NUMBER AND THAT WAS A LATENT BUG, caught by the gate the moment the cap
+#: moved: four columns are NAMED `avg_views_12` and LABELLED "Avg views · last 12 posts", so
+#: raising the capture cap to 30 silently made every one of them average thirty posts under a
+#: label that says twelve. Nobody would have looked at those columns again to check.
+#: ⚠ Raising the CAPTURE ceiling and redefining an EXISTING named measure are two different
+#: decisions, and only the first one was made. To widen the average, change this constant AND the
+#: four keys and labels together — a column whose meaning changes underneath its own name is
+#: worse than one that is merely narrow.
+AVG_WINDOW_POSTS = 12
+
+
+#: The post kinds a group filter may name — exactly what `_bd_type` maps a vendor row onto, so a
+#: filter cannot ask for a category that can never match a stored row.
+POST_TYPES = ("video", "image", "carousel")
+#: ⭐ W29-T09 — the words a PERSON reads for those three keys, server-owned for the same reason
+#: `KIND_LABELS` and `TRIGGER_LABELS` are. The owner asked for *"the last 12 reels"*, and `video`
+#: is the stored key: a client that translated it locally would be a second copy of this
+#: vocabulary, free to drift the day a fourth kind appears or a name changes.
+#: ⚠ "Reels & videos", not "Reels": `_bd_type` maps every non-image, non-carousel post here, so a
+#: label naming only reels would over-promise on a plain video post.
+POST_TYPE_LABELS = {"video": "Reels & videos", "image": "Photos", "carousel": "Carousels"}
+
+
+def clean_post_groups(raw, max_posts):
+ """`(groups, error)` for `config.postGroups` — "last N reels, last M images".
+
+ ⭐ 2026-08-09 (owner: *"not just last 12 but by group also"*). Shape:
+ `[{"type": "video", "limit": 12}, {"type": "image", "limit": 6}]`.
+
+ ⛔ ABSENT IS OFF, and off must stay the default forever: an enrich action stored before today
+ carries no such key, and inventing a filter for it would silently start DROPPING posts those
+ automations have always captured. Empty list and missing are the same answer.
+
+ ⚠ A MALFORMED GROUP IS REFUSED, not dropped — the opposite of `tier`/`noFallback`, and the
+ difference is legitimate. Those are RETIRED keys that live in stored data, so refusing them
+ would 400 old automations forever (D-65). This key is NEW: nothing stored can carry a broken
+ one, so the only way to see one is a person typing it, and a filter that silently ignores the
+ type you asked for is how you end up paying for reels and storing carousels.
+ """
+ if raw in (None, "", [], {}):
+ return [], None
+ if not isinstance(raw, list):
+ return None, "the post groups have to be a list of {type, limit} entries"
+ out, seen = [], set()
+ for item in raw:
+ if not isinstance(item, dict):
+ return None, "each post group has to be a {type, limit} entry"
+ t = str(item.get("type") or "").strip().lower()
+ if t not in POST_TYPES:
+ return None, (f"'{t or 'blank'}' is not a post type — use one of "
+ f"{', '.join(POST_TYPES)}")
+ if t in seen:
+ return None, f"the post groups name '{t}' twice — one limit per type"
+ seen.add(t)
+ try:
+ lim = int(item.get("limit"))
+ except (TypeError, ValueError):
+ return None, f"how many {t} posts to keep has to be a whole number"
+ if lim < 1:
+ return None, f"a {t} group has to keep at least one post"
+ # ⛔ BOUNDED BY WHAT THE RUN ACTUALLY BUYS. A group asking for 30 when the pull captures
+ # 12 is not an error a person can act on — it is a promise the run cannot keep — so it is
+ # refused HERE, where the number they typed is still on the screen in front of them.
+ if lim > max_posts:
+ return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} "
+ f"group cannot keep {lim} — raise the post count first")
+ out.append({"type": t, "limit": lim})
+ return out, None
+
+
+def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True):
+ """`(maxPosts, error)` — refuses out-of-range rather than clamping into range.
+
+ ⛔ `submitted=False` CLAMPS INSTEAD OF REFUSING, and that asymmetry is the whole reason this
+ takes a flag. **Every automation stored before wave 26 carries `maxPosts: 24`** — the old
+ clamp's default, which this cap now forbids. If an inherited value were refused the same way
+ a typed one is, every one of those automations would 400 on its next Save **forever**, for a
+ number nobody on that screen chose or can see. That is D-65's lesson exactly: a validator
+ that refuses stored data does not protect the user from it, it locks them out of their own
+ automation.
+ So: a value the panel SENT is the user asking for it, and gets the sentence. A value merely
+ INHERITED gets silently brought inside the cap it was already effectively subject to at the
+ vendor — the run was returning 12 either way ([[default-must-pass-its-own-guard]]).
+ """
+ if raw is None or (isinstance(raw, str) and not raw.strip()):
+ return default, None
+ try:
+ n = int(raw)
+ except (TypeError, ValueError):
+ if not submitted:
+ return default, None
+ return None, "the post limit must be a whole number"
+ if not submitted:
+ return max(1, min(n, MAX_POSTS_PER_PULL)), None
+ if n < 1:
+ return None, "keep at least one post per profile, or turn the post capture off"
+ if n > MAX_POSTS_PER_PULL:
+ return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts — that is the "
+ f"vendor's cap, not ours, so {n} would store the same "
+ f"{MAX_POSTS_PER_PULL} and read as more")
+ return n, None
+
+
+def clean_config(kind, raw, previous=None):
+ """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence.
+
+ ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail
+ rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not
+ edit `postMetrics` / `commentMetrics` / `dryRun` — those are node SWITCHES. So a plain "Save"
+ from a panel that never knew about them would silently turn off the dry run, or turn ON a
+ per-post purchase: a save that quietly changes what the automation costs. Absent ⇒ keep;
+ present ⇒ take it, including `false`.
+ """
+ raw = raw if isinstance(raw, dict) else {}
+ prev = previous if isinstance(previous, dict) else {}
+
+ def flag(name):
+ return bool(raw[name]) if name in raw else bool(prev.get(name))
+ if kind == "plain":
+ # A plain automation has no machine step, so it only needs the database its records walk.
+ #
+ # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on
+ # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else-
+ # trigger — so requiring a second copy of that fact here would be a mandatory field with
+ # exactly one legal answer, and a Save that refuses until you retype what you just picked.
+ # There is no `dryRun`: dry-run means "run the machine step but do not write", and there
+ # is no machine step to run.
+ target = _s(raw.get("targetTable"), 60).strip()
+ if target and not target.startswith(UT_PREFIX):
+ target = UT_PREFIX + target
+ return {"targetTable": target,
+ "targetLabel": _s(raw.get("targetLabel"), 60).strip()}, None
+ if kind == "scrape_db":
+ url = _s(raw.get("url"), 2000).strip()
+ if not url:
+ return None, "a source URL is required"
+ try:
+ guard(url)
+ except Refused as e:
+ return None, str(e)
+ extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table"
+ fmap = {}
+ for k, v in list((raw.get("fieldMap") or {}).items())[:60]:
+ tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_")
+ if tk:
+ fmap[_s(k, 120)] = tk[:60]
+ if not fmap:
+ return None, "map at least one source column to a field"
+ key_field = _s(raw.get("keyField"), 60).strip()
+ if key_field not in fmap.values():
+ return None, "the key field must be one of the mapped fields"
+ target = _s(raw.get("targetTable"), 60).strip()
+ if target and not target.startswith(UT_PREFIX):
+ target = UT_PREFIX + target
+ return {"url": url, "extract": extract,
+ "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)),
+ "fieldMap": fmap, "keyField": key_field,
+ "targetTable": target,
+ "dryRun": flag("dryRun"),
+ "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None
+ if kind == "field_instagram":
+ table = _s(raw.get("targetTable"), 60).strip()
+ if not table.startswith(UT_PREFIX):
+ return None, "an Instagram automation runs against a blank database (ut_*)"
+ fkey = _s(raw.get("fieldKey"), 80).strip()
+ if not fkey:
+ return None, "pick the automation column this run writes into"
+ # ⛔⛔ `tier` AND `noFallback` ARE ACCEPTED AND IGNORED (wave 28 / R5, contract C2).
+ # They are read from nothing and written to nothing: a stored definition carrying either
+ # still SAVES — it simply loses them on the next write — and neither is ever a reason to
+ # refuse. That asymmetry is D-65's law and it is not squeamishness: refusing an unknown
+ # key would 400 every automation a tenant stored before this wave, forever, on a screen
+ # that gives them no way to remove it. Dropping a retired key is a migration; refusing it
+ # is an outage.
+ # ⚠ `clean_tier`/`TIERS`/`bd_ready` KEEP THEIR NAMES. They are VENDOR vocabulary
+ # (`brightdata` is a provider, and `verify_automation` fences the name), not the retired
+ # USER concept — renaming them would be a second, unrelated change wearing this one's
+ # justification.
+ # C5: absent ⇒ keep whatever is stored, like every other switch in this branch — a panel
+ # that does not edit the post count must not reset it to the default on Save. And an
+ # INHERITED value is clamped rather than refused; see `clean_max_posts`.
+ sent = "maxPosts" in raw
+ max_posts, perr = clean_max_posts(
+ raw.get("maxPosts") if sent else prev.get("maxPosts"), submitted=sent)
+ if perr:
+ return None, perr
+ return {"targetTable": table, "fieldKey": fkey,
+ "urlField": _s(raw.get("urlField"), 80).strip(),
+ # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked
+ # for: the Profiles row carries post IDENTITY for free but NO engagement
+ # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST.
+ "postMetrics": flag("postMetrics"),
+ # The full Comments dataset can bill many rows per post. It is an explicit,
+ # independent opt-in and never follows the per-post switch automatically.
+ "commentMetrics": flag("commentMetrics"),
+ "dryRun": flag("dryRun"),
+ "maxPosts": max_posts}, None
+ # ⭐ WAVE 29 (D-9 / R1) — BOTH discovery kinds share this branch, because they ask the vendor
+ # the same question of two different corpora: a record ceiling, a predicate list, a join word
+ # and where to write. ⛔ THE ONLY DIFFERENCE IS THE DEFAULT TABLE, and it is resolved from the
+ # kind rather than hard-coded — a TikTok search falling back to `ut_ig_profile` would write
+ # TikTok rows into the Instagram family, which is precisely what R2's parallel family exists
+ # to prevent, and it would do it silently.
+ if kind in ("discover_instagram", "discover_tiktok"):
+ tiktok = kind == "discover_tiktok"
+ limit = int(raw.get("recordsLimit") or 0)
+ if limit < 1:
+ return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the "
+ "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)")
+ if limit > BD_MAX_RECORDS:
+ return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles"
+ # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what
+ # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading
+ # the operator afterwards is how the OR hole survived: the guard was handed the branches
+ # and never told they were a union.
+ join = "or" if str(raw.get("operator") or "").lower() == "or" else "and"
+ preds, perr = clean_predicates(raw.get("predicates"), join)
+ if perr:
+ return None, perr
+ target = (_s(raw.get("targetTable"), 60).strip()
+ or (TT_PROFILE_TABLE if tiktok else DISCOVER_TABLE))
+ if not target.startswith(UT_PREFIX):
+ target = UT_PREFIX + target
+ seed, serr = clean_seed(raw.get("seed") if "seed" in raw else prev.get("seed"))
+ if serr:
+ return None, serr
+ return {"recordsLimit": limit, "predicates": preds,
+ "operator": join,
+ "targetTable": target,
+ "targetLabel": (_s(raw.get("targetLabel"), 60).strip()
+ or (TT_TABLE_LABELS[TT_PROFILE_TABLE] if tiktok
+ else DISCOVER_LABEL)),
+ # C6/R5: WHERE the conditions above came from, when they were derived. Stored so
+ # the surface can say "these were filled in from the 'Florists' view, over 12
+ # records" instead of presenting them as if somebody typed them.
+ "seed": seed,
+ "dryRun": flag("dryRun")}, None
+ return None, f"unknown automation kind {kind!r}"
+
+
+def clean_seed(raw):
+ """C6: validate `config.seed`. Returns `({}, None)` when there is none — a discovery filter
+ somebody typed by hand has no seed, and that is the ordinary case.
+
+ ⚠ `derived` AND `basis` ARE STORED AS PROVENANCE, NOT AS A SECOND FILTER. R5 is explicit that
+ the derived conditions are "written into `config.predicates` as ordinary conditions — it is a
+ filling-in, not a parallel filter", so the RUN never reads this bag: it reads `predicates`,
+ like every other search. Keeping it means the surface can say where those rows came from, and
+ a user editing them freely is exactly what is supposed to happen.
+ """
+ if raw in (None, "", {}):
+ return {}, None
+ if not isinstance(raw, dict):
+ return None, "the seed must be an object"
+ source = _s(raw.get("source"), 12).strip().lower()
+ if source not in SEED_SOURCES:
+ return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES)
+ table = _s(raw.get("table"), 60).strip()
+ if table and not table.startswith(UT_PREFIX):
+ return None, f"a seed reads a blank database (ut_*) — {table!r} is not one"
+ basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {}
+ derived, _err = clean_predicates(raw.get("derived") or [], "and")
+ return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(),
+ # ⛔ A MALFORMED `derived` IS DROPPED, NEVER A REFUSAL. This bag is a RECORD of what
+ # was suggested; the conditions that matter are already in `predicates` and were
+ # validated there. Refusing a Save because a stored provenance note aged badly would
+ # block the user from editing the very filter it describes.
+ "derived": derived or [],
+ "basis": {"rows": max(0, int(basis.get("rows") or 0)),
+ "fields": [f for f in (basis.get("fields") or [])
+ if isinstance(f, dict)][:SEED_MAX_PREDICATES],
+ "related": basis.get("related")
+ if isinstance(basis.get("related"), dict) else {},
+ "note": _s(basis.get("note"), 200)}}, None
+
+
+def clean_schedule(raw, previous=None):
+ """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due`
+ measures from — without it, enabling a daily job fires it immediately)."""
+ raw = raw if isinstance(raw, dict) else {}
+ cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *"
+ parse_cron(cron) # raises -> the route answers 400
+ enabled = bool(raw.get("enabled"))
+ prev = previous or {}
+ out = {"cron": cron, "enabled": enabled}
+ if enabled:
+ out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso()
+ return out
+
+
+# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ─────────────
+# The owner's words: "we can make this automation a loop, so the ending should always be defined,
+# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically,
+# or the user have to click a button to reset, or after a certain amount of time it can
+# automatically reset to first cycle."
+#
+# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition
+# that predates this field must not start moving records on its own the day the code ships. A
+# loop is a thing somebody turns on.
+def clean_flow(raw, previous=None):
+ """Validate the builder's ordered action list. `(flow, error)`."""
+ raw = raw if isinstance(raw, dict) else {}
+ prev = previous if isinstance(previous, dict) else {}
+ actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions"))
+ if err:
+ return None, err
+ return {"actions": actions}, None
+
+
+#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the
+#: definition carries none.
+#:
+#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at
+#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the
+#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly
+#: created automation. Two numbers for one fact, in the panel, before anybody had typed
+#: anything. **Read off the screenshot; every assertion in the battery was green.**
+#:
+#: This is the same species as the default-versus-guard split fixed the same day: a value
+#: decided in one file and a value decided in another, describing the same thing. Aligning the
+#: constants closes the only window in which they can disagree — the seed — because every later
+#: state comes from the stored config.
+DISCOVER_SEED_RECORDS = 25
+
+
+def _ensure_ig_action(flow_raw, table):
+ """Create record is PERMANENT step 1 on `ig_profile_match` — it cannot be deleted or moved.
+
+ ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once,
+ on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a
+ comment warning that re-seeding on every clean would be "a control that will not take no for
+ an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be
+ deleted, because the nature of that automation is that it needs to first create a record in a
+ database somewhere from the list of profiles to fetch."*
+
+ Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER
+ produces rows has nowhere to put them until something writes them, so an Instagram search
+ with no Create record is not a customised automation — it is a search whose results are
+ discarded. That is not a preference to respect.
+
+ ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and
+ the values it maps are theirs. An existing `create_record` further down is MOVED to the front
+ rather than duplicated — re-seeding a second one would quietly double every run's writes.
+ """
+ flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {}
+ actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)]
+ if actions and actions[0].get("kind") == "create_record":
+ return _pin_unique(_ensure_enrich_step(
+ flow_raw if isinstance(flow_raw, dict) else flow))
+ at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1)
+ if at > 0:
+ actions.insert(0, actions.pop(at))
+ else:
+ # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner
+ # and no card: the wave-23 header records four kinds that shipped with illegal seeds and
+ # did exactly that. `create_record` needs a ut_-prefixed table and at least one value.
+ actions.insert(0, {
+ "id": "act_1", "kind": "create_record", "enabled": True, "when": None,
+ # `config.label` follows the `review` action's precedent in `_clean_action_config` —
+ # an action naming itself, inside the untyped config bag, so no shared TS type changes.
+ "config": {"table": str(table or DISCOVER_TABLE),
+ "label": "Save the profile",
+ # C5: the picture and the real write finally agree — see `_pin_unique`.
+ "uniqueOn": "handle",
+ "values": {"handle": "{{handle}}"}},
+ })
+ flow["actions"] = actions
+ return _pin_unique(_ensure_enrich_step(flow))
+
+
+def _ensure_enrich_step(flow_raw):
+ """⭐ 2026-08-07 (owner ruling) — ENRICH IS STEP 2 ON AN INSTAGRAM SEARCH.
+
+ Owner: *"make it default that this enrichment action is Step 2 always, and under Config of
+ Step 2 … we can have a toggle on or off."* Which is the same shape as step 1's ruling and for
+ the same reason: a search that finds profiles and never reads them has done half a job. The
+ difference is the control — step 1 is permanent because a flow without it discards its
+ results, while this one is permanent because the TOGGLE is how you turn it off. Deleting and
+ disabling would be two ways to say one thing, and only one of them survives a re-save.
+
+ ⚠ SEEDED ON, AND ON THE FREE RUNG. `tier: "anonymous"` costs nothing, so a search that gains
+ this step by upgrading does not quietly start spending; switching the Source to the paid
+ provider is an explicit choice a person makes in front of the sentence that names the cost.
+ ⚠ THE COOLDOWN IS SEEDED ON TOO (30 days). On a table nobody has enriched it changes nothing —
+ a blank `enriched_at` is never "recent" — and the moment there IS history it stops the flow
+ re-buying the same profile nightly. Off-by-default would make the expensive behaviour the
+ accident.
+
+ ⛔ THE EXISTENCE TEST WALKS THE FORKS (`walk_actions`). A person who moved enrichment inside an
+ If/then branch has one; seeding a second at the top level would enrich twice and bill twice,
+ which is the duplication `apply_actions` already paid for once with the pinned step 1.
+ """
+ flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {}
+ actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)]
+ if any(a.get("kind") == "enrich_instagram" for a in walk_actions(actions)):
+ return flow_raw if isinstance(flow_raw, dict) else flow
+ seeded = list(actions)
+ # Index 1 — after the pinned Create record, because there is nothing to enrich until the
+ # profiles have been written as records. `insert` past the end is a plain append, so a flow
+ # with only step 1 lands this at the end, which IS step 2.
+ seeded.insert(1, {
+ "id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
+ "config": {"postMetrics": False, "commentMetrics": False,
+ "dryRun": False, "maxPosts": DEFAULT_POSTS_PER_PULL,
+ "fromView": "", "sortField": DEFAULT_ENRICH_SORT, "sortDir": "desc",
+ "limit": DEFAULT_ENRICH_LIMIT, "skipRecent": True,
+ "skipRecentDays": DEFAULT_ENRICH_COOLDOWN_DAYS},
+ })
+ return {**flow, "actions": seeded}
+
+
+#: The key the discovery writer really upserts on. Both discovery runners key candidates on
+#: `candidate_key(platform, handle)`; `handle` is the half a person can see and the half this action's
+#: values carry, which is why the PINNED card shows that rather than the pair — `platform` is
+#: supplied by the runner, never typed by a person.
+#: ⛔ DEBT D-73 (closed wave 29): this note used to name wave 22's C6 compound — the one that
+#: paired the handle with its finder — as the live key. Wave 26 · R4/R5 retired it: the identity is
+#: the pair above and the TENANT is the unit, while `created_by` survives as an informational
+#: "Found by" stamp that no run may branch on.
+#: ⚠ The retired pair is DESCRIBED here and not reproduced, deliberately: a gate asserts this note
+#: names the key `_ck` actually takes, and a verbatim quotation of the wrong one reads to that gate
+#: exactly like the defect. A stale comment on correct code is how the next session reintroduces a
+#: bug with a rationale attached.
+IG_PINNED_UNIQUE = "handle"
+
+
+def _pin_unique(flow):
+ """C5: keep `uniqueOn: "handle"` on the PINNED step 1 across every save.
+
+ ⭐ WHY IT IS RE-STAMPED RATHER THAN MERELY SEEDED. `_clean_action_config` has no `previous` —
+ the builder posts the whole action list on every save — so a client that omitted the key would
+ silently reset it to `""`, i.e. back to append. The pinned card is a PICTURE of the engine's
+ own upsert (`apply_actions` skips it at runtime, see the note there), so the picture claiming
+ "append" while the engine upserts is exactly the surface-disagrees-with-the-engine defect this
+ module refuses everywhere else.
+
+ ⛔ AND IT CANNOT MINT A CONFIG THE GUARD REFUSES — HARD RULE 11, which is the whole reason
+ this is a function and not one line. `_clean_action_config` refuses a `uniqueOn` the action
+ does not write, so the stamp is applied ONLY when `handle` is among the action's own values.
+ A user who remaps the card to write different columns keeps their edit and still saves; the
+ alternative — stamping unconditionally — is a product-seeded default that its own validator
+ would 400, which is precisely the post-W24 hotfix this rule exists because of.
+
+ ⚠ NON-MUTATING, and that is load-bearing rather than tidiness. `clean_definition` passes
+ `prev.get("flow")` here when a PATCH carries no flow of its own — that is the STORED
+ definition's own dict, so writing into it would edit the live store object in memory, before
+ (and regardless of) any commit. Every touched level is copied instead.
+ """
+ if not isinstance(flow, dict):
+ return flow
+ acts = flow.get("actions")
+ if not isinstance(acts, list) or not acts or not isinstance(acts[0], dict):
+ return flow
+ first = acts[0]
+ if first.get("kind") != "create_record":
+ return flow
+ cfg = first.get("config")
+ if not isinstance(cfg, dict) or IG_PINNED_UNIQUE not in (cfg.get("values") or {}):
+ return flow
+ if str(cfg.get("uniqueOn") or "").strip():
+ return flow
+ return {**flow,
+ "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]}
+
+
+#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so
+#: the seeder and the un-seeder cannot disagree about what "seeded" means.
+_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich"}
+
+
+def _is_untouched_ig_seed(action, table):
+ """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15)
+
+ ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and
+ that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build
+ the same dicts one screen above, so a step gaining a config key next wave gains it here too.
+ A remembered key list would quietly start calling every seeded action "edited".
+
+ ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The
+ stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the
+ kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW
+ dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person",
+ and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only
+ version where the two sides are the same kind of object.
+
+ ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the
+ TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about
+ a step they still want, and an off seed is still a seed.
+ """
+ if not isinstance(action, dict):
+ return False
+ kind = str(action.get("kind") or "")
+ if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""):
+ return False
+ seeded, _seed_err = clean_actions(
+ (_ensure_ig_action({"actions": []}, table).get("actions") or []))
+ fresh = {a.get("kind"): a for a in (seeded or [])}
+ seed = fresh.get(kind)
+ if not seed:
+ return False
+ cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {})
+ if kind == "enrich_instagram":
+ cfg.pop("enabled", None)
+ seed_cfg.pop("enabled", None)
+ return cfg == seed_cfg and (action.get("when") or None) is None
+
+
+def _drop_ig_seeds(flow_raw, table):
+ """Strip the trigger's own seeded steps, keeping any the person has since made theirs.
+
+ ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT
+ THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it
+ does not delete the seeded actions … a create_record the person has since re-pointed at their
+ own table with their own values is THEIR action now"*. That reasoning is still correct and is
+ exactly what this function preserves — but it was applied to EVERY seeded action, including
+ the ones nobody had ever opened, and the result is what the owner reported: switch an
+ Instagram search to Manual and you are left holding a "Save the profile" step writing
+ `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not
+ somebody's work being protected; it is the machine's own leftover, pointed at a table the
+ automation has nothing to do with any more.
+
+ ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as
+ `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a
+ branch that destroys something, so the conservative half is the load-bearing half.
+ """
+ flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {}
+ actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)]
+ kept = [a for a in actions if not _is_untouched_ig_seed(a, table)]
+ if len(kept) == len(actions):
+ return flow_raw
+ return {**flow, "actions": kept}
+
+
+def ig_action_pinned(defn, index):
+ """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned`
+ flag is a second copy of the rule, and the copy is what a hand-written PATCH omits."""
+ return (defn or {}).get("kind") == "discover_instagram" and index == 0
+
+
+def clean_definition(raw, previous=None, username=""):
+ """Whole-definition validation. Returns `(defn, error)`."""
+ raw = raw if isinstance(raw, dict) else {}
+ prev = previous or {}
+ # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
+ # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
+ # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
+ # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
+ # invalid in BOTH its trigger and its config now answers with the trigger's sentence.
+ trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw
+ else prev.get("trigger"), prev.get("trigger"))
+ if terr:
+ return None, terr
+ # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
+ # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a
+ # dead end the moment the wizard was deleted and the client's create body became {name}.
+ kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND
+ drop_ig_seeds = False
+ if (trigger or {}).get("key") == "ig_profile_match":
+ # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather
+ # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and
+ # whose kind says something else is not a preference to respect, it is two halves of one
+ # answer disagreeing, and the trigger is the half the person actually picked.
+ kind = "discover_instagram"
+ elif (trigger or {}).get("key") == "tiktok_profile_match":
+ # ⭐ WAVE 29 (D-9 / R1) — LAW 1, TIKTOK'S HALF. Same rule, same reason: the trigger is how
+ # `discover_tiktok` gets chosen, and it is unconditional for the same reason the Instagram
+ # arm is — a definition whose trigger says TikTok discovery and whose kind says something
+ # else is two halves of one answer disagreeing.
+ kind = "discover_tiktok"
+ elif (kind == "discover_tiktok"
+ and ((prev.get("trigger") or {}) or {}).get("key") == "tiktok_profile_match"
+ and (trigger or {}).get("key") != "tiktok_profile_match"):
+ # ⛔⛔ LAW 1'S INVERSE, AND ITS ABSENCE ON THE INSTAGRAM SIDE WAS A MONEY BUG (see the arm
+ # below): with nothing to flip the kind BACK, switching the trigger to Manual left
+ # `RUNNERS[kind]` pointing at the discovery runner, so **Run now fired a PAID corpus
+ # search on a flow the person had just made manual.** Shipped here in the same change as
+ # law 1 rather than discovered the same way twice.
+ # ⚠ THE TEST IS THAT THE TRIGGER **MOVED** — comparing the RESOLVED key against the
+ # PREVIOUS one. Asking `"trigger" in raw` would fire on every empty patch, because
+ # `patch()` builds its raw as `dict(prev)` plus the caller's keys.
+ kind = DEFAULT_KIND
+ elif (kind == "discover_instagram"
+ and ((prev.get("trigger") or {}) or {}).get("key") == "ig_profile_match"
+ and (trigger or {}).get("key") != "ig_profile_match"):
+ # ⭐⭐ 2026-08-07 (owner report) — LAW 1 HAS AN INVERSE, AND ITS ABSENCE WAS A MONEY BUG.
+ #
+ # Law 1 above flips the kind TO `discover_instagram` when the trigger says Instagram
+ # discovery. Nothing flipped it BACK. So `kind` was inherited from `prev` forever: switch
+ # the trigger to Manual and the automation stayed `discover_instagram`, which means
+ #
+ # · `RUNNERS[kind]` is still `run_discover_instagram`, so pressing **Run now** on a flow
+ # the person had just made MANUAL fired a PAID Bright Data corpus search;
+ # · `ig_action_pinned` keys on the kind, so the seeded "Create record" stayed
+ # UNDELETABLE — the owner's report, verbatim: *"When a trigger change from the
+ # instagram trigger, the 'always first' create record just stuck there"*. The server
+ # would have accepted the delete; the CLIENT hid the control, because both halves ask
+ # the kind and the kind was lying.
+ #
+ # ⛔ THE TEST IS THAT THE TRIGGER **MOVED**, and the first version of this got it wrong in
+ # a way worth recording. It asked `"trigger" in raw` — but `patch()` builds its raw as
+ # `merged = dict(prev)` plus the caller's keys, so **`"trigger"` is present on EVERY
+ # patch**, and an empty `patch(rt, id, {})` flipped a discovery automation to `plain`.
+ # That turned five `section_bd` checks red and crashed the suite on a `StopIteration`
+ # three sections later. Comparing the RESOLVED key against the PREVIOUS one is the honest
+ # question: did this automation stop being an Instagram search?
+ #
+ # ⚠ NARROW ON PURPOSE, twice over. It fires only when the automation WAS on the Instagram
+ # trigger — a `discover_instagram` created with an explicit kind and some other trigger is
+ # somebody's deliberate state, not a mistake to correct. And only FROM
+ # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose
+ # triggers are their own business, and clobbering them would retire them by accident.
+ #
+ # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this
+ # replaces argued the other way, so here is why it was half right. It read: *"it does not
+ # delete the seeded actions … a create_record the person has since re-pointed at their own
+ # table with their own values is THEIR action now, and quietly destroying it is the silent
+ # data loss this module refuses everywhere else."* Every word of that is still true and is
+ # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the
+ # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to
+ # Manual still carrying a "Save the profile" step writing `{{handle}}` into
+ # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces
+ # handles, unpinned but still there, still runnable, and still pointed at a table that has
+ # nothing to do with this automation.
+ # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line.
+ kind = DEFAULT_KIND
+ drop_ig_seeds = True
+ if kind not in KINDS:
+ return None, f"unknown automation kind {kind!r}"
+ # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION.
+ # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain`
+ # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement.
+ # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way
+ # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two
+ # live automations save through this function every time their owner edits them. The gate's
+ # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved
+ # from `create` into `clean_definition` (the tempting simplification), this helper would go
+ # red across a dozen sections, which is the alarm that change deserves."* It did, and the
+ # alarm worked.
+ # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every
+ # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody
+ # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an
+ # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored
+ # ones are rewritten under their owner — and that is a ruling, not a refactor.
+ name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split())
+ if not name:
+ return None, "name the automation"
+ cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config")
+ if kind == "discover_instagram" and prev.get("kind") != "discover_instagram":
+ # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very
+ # first Save after picking this trigger 400s — against the stored-inert-with-
+ # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note).
+ # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard
+ # exists because an unbounded query is the one shape the vendor refuses outright, and
+ # coercing a blank into a number would be exactly the silent widening it protects against.
+ # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until
+ # a person presses Run now or arms the schedule.
+ cfg_raw = dict(cfg_raw or {})
+ if not _ig_int(cfg_raw.get("recordsLimit")):
+ cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS
+ config, err = clean_config(kind, cfg_raw, prev.get("config"))
+ if err:
+ return None, err
+ if trigger:
+ # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own
+ # configuration IS the config, and `clean_trigger` could not see it.
+ trigger["configured"] = _trigger_configured(trigger, config)
+ try:
+ schedule = clean_schedule(
+ raw.get("schedule") if isinstance(raw.get("schedule"), dict)
+ else prev.get("schedule"), prev.get("schedule"))
+ except ValueError as e:
+ return None, str(e)
+ # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.)
+ flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow")
+ # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only
+ # call was what made the action deletable; running it on every save is what makes step 1
+ # permanent, and it is deliberate rather than a widened condition nobody noticed.
+ if (trigger or {}).get("key") == "ig_profile_match":
+ flow_raw = _ensure_ig_action(flow_raw, config.get("targetTable") or DISCOVER_TABLE)
+ elif drop_ig_seeds:
+ # ITEM 15: the trigger just STOPPED being the Instagram one. Un-seed what the trigger
+ # planted and nobody has since made theirs — measured against the table the seeds were
+ # planted for, which is the PREVIOUS config's, not the one being saved.
+ flow_raw = _drop_ig_seeds(
+ flow_raw, (prev.get("config") or {}).get("targetTable") or DISCOVER_TABLE)
+ flow, ferr = clean_flow(flow_raw, prev.get("flow"))
+ if ferr:
+ return None, ferr
+ if (trigger or {}).get("key") == "ig_profile_match":
+ # ⭐ WAVE 25 · R2 — THE CREATE RECORD ACTION'S `config.table` IS AUTHORITATIVE, and
+ # `config.targetTable` FOLLOWS IT. Two fields have been naming the same database since
+ # wave 24: the pinned step 1 says where profiles are written, and the config says where
+ # the automation's records live — and for a discovery flow those are the same database by
+ # construction. When they disagreed, every surface picked a different winner (the canvas
+ # Write node read the config, the actual write read the action), which is a bug you can
+ # only see by comparing two panels.
+ #
+ # ⛔ THE ACTION WINS BECAUSE IT IS THE ONE A PERSON EDITS. R2b puts the preset list on the
+ # action's own configuration, so the table named there is the one they were looking at.
+ #
+ # ⚠ THIS IS ALSO THE MIGRATION, and it is a migration in the shape laws 4/5 established:
+ # a stored definition carrying two different values is reconciled on its next clean,
+ # silently and exactly once, because nothing writes the losing value back. A refusal here
+ # would 400 every Save of a definition that was legal yesterday.
+ # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the
+ # TIE-BREAK rather than the whole rule. The owner's report: change the database on the
+ # TRIGGER, save, and the picker springs back to the old one with no message. The cause is
+ # the branch below read unconditionally — the pinned action still named yesterday's table,
+ # so it overwrote a change the person had just made in front of it, silently, every time.
+ #
+ # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled
+ # "when the two disagree, whose value is real?" — the action's, because R2b puts the
+ # preset list on the action's own panel, so that is the one they were looking at. That
+ # answers a definition at REST. A save is different: it carries an INTENT, and the honest
+ # question is which side this particular save changed. So:
+ # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to
+ # follow it (leaving the action behind would just re-revert on the next save);
+ # · only the action moved, or neither did and they were already inconsistent -> R2's
+ # migration, unchanged, reconciling a stored definition exactly once.
+ # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is
+ # also the only reading that cannot lose a value: the action's table is the one with
+ # the field mapping attached to it.
+ first = (flow.get("actions") or [{}])[0]
+ pinned = str((first.get("config") or {}).get("table") or "") \
+ if first.get("kind") == "create_record" else ""
+ prev_target = str((prev.get("config") or {}).get("targetTable") or "")
+ prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0]
+ ).get("config") or {}).get("table") or "")
+ target_moved = bool(prev_target) and config.get("targetTable") != prev_target
+ action_moved = bool(prev_pinned) and pinned != prev_pinned
+ if pinned and target_moved and not action_moved:
+ # The person changed the trigger's database. Carry it INTO the pinned step so the two
+ # agree, instead of throwing their edit away to keep a stale copy consistent.
+ cfg1 = dict(first.get("config") or {})
+ cfg1["table"] = str(config.get("targetTable") or "")
+ flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]]
+ elif pinned and pinned != config.get("targetTable"):
+ config["targetTable"] = pinned
+ # The stored label described a DIFFERENT database, so keeping it would caption the new
+ # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure`
+ # never relabels a table that already exists — so an adopted hand-made database keeps
+ # the name its owner gave it.
+ config["targetLabel"] = ""
+ return {
+ "id": _s(prev.get("id") or raw.get("id"), 40),
+ "name": name, "kind": kind, "config": config, "schedule": schedule,
+ "trigger": trigger,
+ # The Builder's ordered actions. An absent flow is simply an automation with no actions.
+ "flow": flow,
+ "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""},
+ "runs": list(prev.get("runs") or [])[:MAX_RUNS],
+ # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20
+ # minutes to build, so a run persists its id here and the NEXT run collects it. It is
+ # carried through `patch` untouched because a user pressing Save must not silently orphan
+ # a set the vendor is already building (and already counting against the funds gate).
+ "state": dict(prev.get("state") or {}),
+ "created": prev.get("created") or _iso(),
+ "createdBy": prev.get("createdBy") or username,
+ }, None
+
+
+def set_state(rt, auto_id, patch_state):
+ """Merge into ONE automation's engine state. A key set to None is removed.
+
+ Its own tiny store write rather than a field on `_commit_run`, because the handoff must
+ survive a run that ends in `error` — a snapshot the vendor is building does not stop existing
+ because the run that started it failed afterwards.
+ """
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ d = cur.get(str(auto_id))
+ if d is None:
+ return cur
+ st = dict(d.get("state") or {})
+ for k, v in (patch_state or {}).items():
+ st.pop(k, None) if v is None else st.__setitem__(k, v)
+ d["state"] = st
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+
+
+def _without_retired_board(definition):
+ """Return a definition with retired Board-only state removed, without mutating the store.
+
+ This protects scheduled/manual execution before the next UI list request persists the
+ migration. It removes only the former Board configuration, actions, and audit trail; normal
+ actions and all user data remain intact.
+ """
+ if not isinstance(definition, dict):
+ return definition, False
+ out, changed = dict(definition), False
+ cfg = out.get("config")
+ if isinstance(cfg, dict) and "lanes" in cfg:
+ cfg = dict(cfg)
+ cfg.pop("lanes", None)
+ out["config"] = cfg
+ changed = True
+
+ def _actions(actions):
+ nonlocal changed
+ clean = []
+ for action in actions if isinstance(actions, list) else []:
+ if not isinstance(action, dict):
+ clean.append(action)
+ continue
+ if action.get("kind") == "review":
+ changed = True
+ continue
+ item = dict(action)
+ if item.get("kind") == "group" and isinstance(item.get("config"), dict):
+ group_cfg = dict(item["config"])
+ branches = group_cfg.get("branches")
+ if isinstance(branches, list):
+ kept = []
+ for branch in branches:
+ if not isinstance(branch, dict):
+ changed = True
+ continue
+ next_actions = _actions(branch.get("actions"))
+ if not next_actions:
+ changed = True
+ continue
+ kept.append({**branch, "actions": next_actions})
+ if not kept:
+ changed = True
+ continue
+ if kept != branches:
+ changed = True
+ group_cfg["branches"] = kept
+ item["config"] = group_cfg
+ elif isinstance(group_cfg.get("actions"), list):
+ nested = _actions(group_cfg["actions"])
+ if not nested:
+ changed = True
+ continue
+ if nested != group_cfg["actions"]:
+ group_cfg["actions"] = nested
+ item["config"] = group_cfg
+ clean.append(item)
+ return clean
+
+ flow = out.get("flow")
+ if isinstance(flow, dict):
+ next_flow = dict(flow)
+ actions = _actions(flow.get("actions"))
+ if actions != flow.get("actions"):
+ next_flow["actions"] = actions
+ if "ending" in next_flow:
+ next_flow.pop("ending", None)
+ changed = True
+ out["flow"] = next_flow
+ if "reviews" in out:
+ out.pop("reviews", None)
+ changed = True
+ return out, changed
+
+
+def retire_automation_board_state(rt, tables=None):
+ """Persist the idempotent Board retirement, then remove its generated table fields.
+
+ ``tables`` is the optional lent snapshot of the `user_tables` bucket (W29-T01) — it reaches
+ the stage scan and nothing else.
+ """
+ fields = retire_automation_stage_fields(rt, tables=tables)
+ try:
+ stored = dict(rt.get(STORE_KEY) or {})
+ except Exception:
+ return {"definitions": 0, **fields}
+ plan = {str(aid): cleaned for aid, raw in stored.items()
+ for cleaned, changed in [_without_retired_board(raw)] if changed}
+ if not plan:
+ return {"definitions": 0, **fields}
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for aid, cleaned in plan.items():
+ if aid in cur:
+ cur[aid] = cleaned
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+ return {"definitions": len(plan), **fields}
+
+
+def all_definitions(rt):
+ try:
+ raw = dict(rt.get(STORE_KEY) or {})
+ except Exception:
+ return {}
+ return {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()}
+
+
+def _new_id(existing):
+ n = 1
+ while f"auto_{n}" in existing:
+ n += 1
+ return f"auto_{n}"
+
+
+#: C2's three answers to "which database does this automation work on?" — the FIRST question the
+#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is).
+TARGET_MODES = ("existing", "new", "automated")
+
+
+def resolve_target(rt, raw, username=""):
+ """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`.
+
+ - `existing` — bind a database that is already there.
+ - `new` — mint a blank one now, so the automation has somewhere to write before its first
+ run instead of conjuring a table the person never agreed to.
+ - `automated` — leave it to the runner: a scraping automation MINTS its target on first run
+ (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database.
+ Nothing is created here, deliberately — an empty table created up front for a scrape that
+ never runs is litter nobody can explain.
+ """
+ if not isinstance(raw, dict) or not raw:
+ return {}, None # no target block = the pre-wizard shape
+ mode = _s(raw.get("mode"), 20).strip() or "existing"
+ if mode not in TARGET_MODES:
+ return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES)
+ if mode == "automated":
+ label = " ".join(_s(raw.get("label"), 60).split())
+ return ({"targetLabel": label} if label else {}), None
+ if mode == "existing":
+ key = _s(raw.get("table"), 60).strip()
+ if not key:
+ return None, "choose the database this automation works on"
+ t = ut_get(rt, key)
+ if t is None:
+ return None, f"{key!r} is not a database in this workspace"
+ return {"targetTable": key, "targetLabel": t.get("label") or key}, None
+ label = " ".join(_s(raw.get("label"), 60).split())
+ if not label:
+ return None, "name the new database"
+ import core.user_tables as _ut_new
+ key = _ut_new.create(label, username or "automation", st=rt)
+ if not key:
+ return None, ("the database could not be created — this workspace may be at its table "
+ "limit")
+ return {"targetTable": key, "targetLabel": label}, None
+
+
+def _unmint(rt, key):
+ """Delete a database this call minted moments ago, because the call is refusing (D-50).
+
+ ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user
+ needs to read; a rollback that raised would replace that sentence with a 500 and lose the
+ reason the create failed in the first place. The worst case of a swallowed failure here is
+ the orphan we had before — strictly no worse, and the refusal still reaches the person.
+ """
+ if not key:
+ return
+ try:
+ import core.user_tables as _ut_del
+ _ut_del.delete(key, st=rt)
+ except Exception: # noqa: BLE001
+ pass
+
+
+def create(rt, raw, username=""):
+ existing = all_definitions(rt)
+ if len(existing) >= MAX_AUTOMATIONS:
+ return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit"
+ raw = dict(raw or {})
+ # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation
+ # happens after it, so a correct refusal ("a source URL is required") used to leave an
+ # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close:
+ # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form
+ # twice silently accumulated empty databases in their nav.
+ #
+ # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It
+ # does not test for `mode == "new"`; it asks "did the table this call's target resolved to
+ # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and
+ # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target
+ # rather than "any table that appeared", so a concurrent create in another request is never
+ # collateral.
+ before = set(ut_all(rt) or {})
+ patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username)
+ if terr:
+ return None, terr
+ minted = str((patch_cfg or {}).get("targetTable") or "")
+ minted = minted if minted and minted not in before else ""
+ if patch_cfg:
+ raw["config"] = {**(raw.get("config") or {}), **patch_cfg}
+ # 2026-08-10 — point a targetless discovery flow at the profile database this tenant ALREADY
+ # has, instead of letting the pure validator mint a second empty one. See
+ # `discover_default_table`; a target the caller named is never rewritten.
+ raw = _apply_discover_default(rt, raw)
+ defn, err = clean_definition(raw, None, username)
+ if err:
+ _unmint(rt, minted)
+ return None, err
+ # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations
+ # that already use them and NO NEW ONE CAN BE CREATED."
+ #
+ # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live
+ # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing
+ # the kind there would make both of them unsaveable — the ruling retires the door, not the
+ # automations behind it. Enforced on the server rather than left to the deleted wizard: a
+ # wall that holds only because the client stopped asking is a convention, not a wall.
+ if defn["kind"] in RETIRED_KINDS:
+ _unmint(rt, minted)
+ return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no "
+ f"longer created — the ones already using it keep working. "
+ f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}")
+ defn["id"] = _new_id(existing)
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ cur[defn["id"]] = defn
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+ _presets_after_write(rt, defn, username) # R10: the columns exist before a run
+ if defn.get("trigger"):
+ _seed_event_state(rt, defn)
+ return defn, None
+
+
+def patch(rt, auto_id, raw, username=""):
+ existing = all_definitions(rt)
+ prev = existing.get(str(auto_id))
+ if prev is None:
+ return None, "no such automation"
+ merged = dict(prev)
+ # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a
+ # key missing here is accepted by the route, validated by `clean_definition`, and then
+ # DISCARDED — the client shows a saved flow that the store never received, and nothing goes
+ # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
+ # ordering that never has a window where the bug exists.
+ for k in ("name", "config", "schedule", "kind", "trigger", "flow"):
+ if k in (raw or {}):
+ merged[k] = raw[k]
+ # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
+ # Instagram trigger on an existing automation flips the kind, and the very next Save runs a
+ # config that has never carried a target through the pure validator, which mints the empty
+ # `ut_ig_candidates`. `create` alone would have left the commonest path untouched.
+ merged = _apply_discover_default(rt, merged)
+ defn, err = clean_definition(merged, prev, username)
+ if err:
+ return None, err
+ defn["id"] = str(auto_id)
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ cur[defn["id"]] = defn
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+ _presets_after_write(rt, defn, username) # R10: also on the save that RE-TARGETS
+ # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching
+ # DISARMED — enabling never fires for records already in the state.
+ if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}):
+ _seed_event_state(rt, defn)
+ return defn, None
+
+
+def _presets_after_write(rt, defn, username):
+ """⭐ WAVE 25 · R10 — THE PRESET COLUMNS APPEAR ON SAVE, not on the first run.
+
+ Owner ruling R10: choosing the trigger / naming the Create record target spawns the preset
+ columns immediately, "so R2b's list is real and the database is visible before anything is
+ spent". That last clause is the point — a person can look at the database, see the sixteen
+ columns, and decide whether to arm the schedule, instead of paying a vendor to find out what
+ they are agreeing to.
+
+ ⛔ THE RUN-TIME `ut_ensure` STAYS AS THE BACKSTOP (R10 says so explicitly), and it costs
+ nothing to keep: `ut_ensure` skips the write entirely when no field would change, so the first
+ run of a saved automation finds the table already correct and spends no commit.
+
+ ⚠ IT SPAWNS `CANDIDATE_FIELDS`, NOT `PRESET_PROFILE_FIELDS`. The discovery target needs the
+ search bookkeeping (`found_count`, `first_found`, `created_by`) as well as the
+ preset set, and spawning a subset here would mean the columns changed between Save and the
+ first run — two different answers to "what does this database look like", which is the exact
+ disagreement R10 exists to remove.
+
+ ⚠ A WORKSPACE AT ITS TABLE CEILING SPAWNS NOTHING AND THE SAVE STILL SUCCEEDS. `ut_ensure`
+ returns the key without creating when `MAX_UT_TABLES` is reached; refusing to save an
+ automation because the workspace is full would be a worse answer than saving one whose table
+ arrives later. The run-time path reports that condition LOUDLY (`ut_missing`, D-11), so the
+ honest failure still has exactly one home.
+ """
+ cfg = defn.get("config") or {}
+ target = str(_flow_table(defn) or cfg.get("targetTable") or "")
+ if not target:
+ return
+ try:
+ if (defn.get("trigger") or {}).get("key") == "ig_profile_match":
+ ut_ensure(rt, cfg.get("targetLabel") or DISCOVER_LABEL, CANDIDATE_FIELDS, username,
+ key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True)
+ return
+ enrich_actions = _actions_of_kind(((defn.get("flow") or {}).get("actions") or []),
+ "enrich_instagram")
+ table = ut_get(rt, target) or {}
+ named = next((str((a.get("config") or {}).get("profileField") or "").strip()
+ for a in enrich_actions
+ if str((a.get("config") or {}).get("profileField") or "").strip()), "")
+ bound = next((str(f.get("key") or "") for f in table.get("fields") or []
+ if isinstance(f.get("profile"), dict)), "") or named
+ if enrich_actions and bound:
+ ut_ensure(rt, table.get("label") or target, _profile_schema_for(bound), username,
+ key=target, flow_tag=str(defn.get("id") or ""), lock_fields=True)
+ except Exception as e: # noqa: BLE001
+ # ⛔ NEVER FAILS THE SAVE. The definition is already committed by the time this runs, so
+ # raising here would answer 500 for an automation that IS stored — the caller would retry
+ # and create a second one. The columns then arrive on the first run, which is precisely
+ # the behaviour this function is an improvement on rather than a replacement for.
+ print(f"[aios-auto] preset spawn deferred to the first run: {type(e).__name__}: {e}")
+
+
+def remove(rt, auto_id):
+ aid = str(auto_id)
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ cur.pop(aid, None)
+ return cur
+ rt.update(STORE_KEY, _up, flush="sync")
+ # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned
+ # silently: the column keeps its values and its config, and says why it stopped. Scanned
+ # first so a delete with no tagged fields costs no store commit.
+ hit = any(str((f.get("automation") or {}).get("flowId") or "") == aid
+ for t in ut_all(rt).values() for f in (t.get("fields") or []))
+ if not hit:
+ return
+
+ def _disable(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for t in cur.values():
+ for f in (t.get("fields") or []):
+ a = f.get("automation")
+ if isinstance(a, dict) and str(a.get("flowId") or "") == aid:
+ a["disabled"] = True
+ a["statusNote"] = f"its automation was deleted {_stamp()}"
+ return cur
+ rt.update(UT_STORE_KEY, _disable, flush="sync")
+
+
+# ---------------------------------------------------------------------------------------------
+# RUN STATE — process memory only (see the module header for why this may never be persisted)
+# ---------------------------------------------------------------------------------------------
+
+_RUN_LOCK = threading.RLock()
+_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'}
+
+
+def running(tenant, auto_id):
+ with _RUN_LOCK:
+ return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None
+
+
+def _claim(tenant, auto_id):
+ with _RUN_LOCK:
+ if (tenant, str(auto_id)) in _RUNNING:
+ return False
+ _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"}
+ return True
+
+
+def _step(tenant, auto_id, text):
+ with _RUN_LOCK:
+ cur = _RUNNING.get((tenant, str(auto_id)))
+ if cur is not None:
+ cur["step"] = text
+
+
+def _no_step(_text):
+ """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner
+ directly). A runner must never REQUIRE the slot — the slot is process state, and the runner
+ is the part a test is allowed to call on its own."""
+
+
+def _release(tenant, auto_id):
+ with _RUN_LOCK:
+ _RUNNING.pop((tenant, str(auto_id)), None)
+
+
+def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None, notes=None):
+ """THE ONE store write a run performs on the `automations` bucket.
+
+ `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here
+ because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to
+ invent a node status when this map is absent (every run stored before W19-C has no `steps`,
+ and painting those nodes green from the run's overall state would be a fabrication).
+
+ ⭐⭐ `notes` CLOSES DEBT D-103 (2026-08-09). A run's `counts` are integers and the
+ comprehension below drops everything else — so the per-record sentence a vendor gave us had
+ literally nowhere to live, and *"1 profile read(s) were blocked"* was the whole of what the
+ product could say. MEASURED on nurilab: the same record blocked on three separate runs and
+ the reason was unrecoverable from the store afterwards, so the diagnosis had to be rebuilt by
+ calling the vendors by hand. The note is the most valuable thing a run produces — a dead
+ handle, a vendor hiccup and an exhausted key are three different actions — and it was the one
+ thing thrown away.
+ ⚠ ON THE RUN, NOT ON THE RECORD. D-103's own prescription: no tenant table gains a column it
+ did not ask for, and W25/R4 already retired writing status STRINGS into people's grids.
+ """
+ entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400),
+ "counts": {k: int(v) for k, v in (counts or {}).items()
+ if isinstance(v, (int, float))},
+ "steps": {str(k): str(v) for k, v in (steps or {}).items()},
+ # Bounded on both axes: a 100-profile run must not put 100 sentences into a store
+ # entry that is kept 20 deep per automation.
+ "notes": [_s(n, 300) for n in (notes or []) if str(n or "").strip()][:25],
+ "affected": list(affected or [])[:200]}
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ d = cur.get(str(auto_id))
+ if d is None:
+ return cur
+ d["status"] = {"state": state, "lastRunAt": entry["ts"],
+ "lastSummary": entry["summary"]}
+ d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS]
+ # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with
+ # the reason on it — a dead credential must not burn quota (or a paid vendor's records)
+ # 96 times a day while a green toggle sits over a buried error log. Errors only:
+ # `partial` is the honest-progress state and pausing on it would punish honesty.
+ runs = d.get("runs") or []
+ if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all(
+ not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]):
+ sch = d.get("schedule")
+ if isinstance(sch, dict) and sch.get("enabled"):
+ sch["enabled"] = False
+ trg = d.get("trigger")
+ if isinstance(trg, dict):
+ trg["paused"] = True
+ d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive "
+ f"failures — fix the cause, then re-enable: "
+ f"{entry['summary'][:120]}")
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+ _notify_outcome(rt, auto_id, state, entry)
+ return entry
+
+
+def _notify_outcome(rt, auto_id, state, entry):
+ """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7).
+
+ Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is
+ written, so it is the only place this can go without a second definition of "what happened".
+
+ ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED.
+ 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the
+ MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a
+ background thread with no session, so an omitted `st` files nurilab's automation failures
+ in Royal Imports' inbox — D-16's class, silently, forever.
+ 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler
+ has no session at all, so the alternative is not "the wrong person" but "nobody", and
+ `notify` returns None on a blank owner — the notification would simply never exist, on
+ exactly the runs nobody was watching.
+ ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only
+ view-alert CREATION; `notify()` writes straight into the notifications bucket and
+ `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell.
+ ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that
+ SUCCEEDED into one that raised — the run entry is already committed one line above, and the
+ bell is a courtesy on top of it.
+ """
+ try:
+ defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {}
+ owner = str(defn.get("createdBy") or "").strip()
+ if not owner:
+ return
+ from core import alerts as _alerts
+ _alerts.notify(
+ owner=owner,
+ label=_s(defn.get("name") or "Automation", 80),
+ topic="automation",
+ key=str(auto_id),
+ detail=f"{state}: {entry.get('summary') or ''}"[:300],
+ st=rt)
+ except Exception: # noqa: BLE001
+ pass
+
+
+# ---------------------------------------------------------------------------------------------
+# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN
+# ---------------------------------------------------------------------------------------------
+# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and
+# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down)
+# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper:
+#
+# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie,
+# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the
+# supplier takes the scraping risk; it is never an Instagram credential and this module must
+# never be given one.
+#
+# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind).
+#
+# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress
+# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been
+# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and
+# the cell says which. A pull that quietly wrote zero posts and reported success would be worse
+# than no automation at all: the table would look maintained and be empty.
+#
+# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`,
+# so a run history answers "what still works anonymously?" from data rather than from memory.
+
+PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5)
+
+#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's
+#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number
+#: is a NOW value, so the series starts the day capture starts and a field we do not capture
+#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows
+#: has exactly the same cost as delaying capture — for that column — and is justified by exactly
+#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to
+#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices.
+#: Every returned provider field is retained in Source data. Promoted columns make commonly used
+#: values filterable; the raw source document prevents a new or uncommon field from being lost.
+#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose
+#: almost none of them, which is what the `source` column is for.
+SNAPSHOT_FIELDS = [
+ field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"),
+ field_def("pulled_at", "Pulled at", "date"), field_def("followers", "Followers", "int"),
+ field_def("following", "Following", "int"),
+ field_def("posts_count", "Post count", "int"),
+ field_def("full_name", "Name"), field_def("bio", "Bio"),
+ field_def("verified", "Verified", "checkbox"), field_def("source", "Read via"),
+ field_def("approx", "Counts are approximate", "checkbox"),
+ # The link in bio — the paid rung's field, and commercially the most useful single string an
+ # influencer row can carry (it is the shop/affiliate destination).
+ field_def("external_url", "Link in bio", "url"),
+ # --- the rest of the vendor's Profiles schema.
+ field_def("ig_id", "Instagram id"),
+ field_def("profile_url", "Profile", "url"),
+ field_def("avg_engagement", "Avg engagement", "pct"),
+ field_def("category", "Category"),
+ field_def("business_category", "Business category"),
+ field_def("is_business", "Business account", "checkbox"),
+ field_def("is_professional", "Professional account", "checkbox"),
+ field_def("is_private", "Private", "checkbox"),
+ field_def("highlights_count", "Story highlight count", "int"),
+ field_def("bio_hashtags", "Bio hashtags"),
+ field_def("pronouns", "Pronouns"),
+ # ⭐ 2026-08-07 — the rest of the vendor's Profiles schema (see `_bd_profile`). The snapshot
+ # row maps the WHOLE schema by design, so these belong here the moment the map reads them;
+ # leaving them out would be the "captured but unrecorded" half of the same loss the note at
+ # the top of this list is about.
+ field_def("profile_name", "Profile name"),
+ field_def("is_joined_recently", "Joined recently", "checkbox"),
+ field_def("has_channel", "Has channel", "checkbox"),
+ field_def("partner_id", "Partner id"),
+ field_def("external_url_title", "Link title"),
+ field_def("fbid", "Facebook id"),
+ field_def("related_accounts", "Related accounts"),
+ field_def("country_code", "Country"),
+ field_def("source_payload", "Source data", "json"),
+]
+POST_FIELDS = [
+ field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"),
+ field_def("posted_at", "Posted at", "date"),
+ field_def("type", "Type", "select", options=["image", "video", "carousel"]),
+ field_def("caption", "Caption"), field_def("url", "URL", "url"),
+ # A tagged place is useful content context and can be filtered as ordinary text. It is never
+ # presented as the creator's location: a creator can tag a holiday, venue or brand location.
+ field_def("tagged_location", "Tagged location"),
+ # The per-field map below gives operational fields first; this locked JSON document retains
+ # every other value Bright Data supplied, so a vendor schema addition is preserved
+ # instead of being silently discarded while the canonical field model catches up.
+ field_def("source_payload", "Source data", "json"),
+ # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous
+ # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached).
+ field_def("paid_partnership", "Paid partnership", "checkbox"),
+ field_def("partner", "Partner brand"),
+ field_def("hashtags", "Hashtags"),
+ field_def("alt_text", "Alt text"),
+ # ⭐⭐ 2026-08-07 — THE LATEST-VALUE ENGAGEMENT COLUMNS, AND THEY EXIST TO KEEP A ROLLUP AT
+ # ONE HOP.
+ #
+ # "Average views over the last N posts" is naturally TWO hops: profile → posts → each post's
+ # most recent snapshot → `views`. Airtable's rollup is one hop, and growing a second one is a
+ # much bigger build with a much worse failure mode (a rollup over a rollup, invalidated
+ # transitively). So the post row carries its own LATEST value and the rollup reads it
+ # directly.
+ #
+ # ⛔ R3's "ONE STORE FOR ONE SERIES" IS UNTOUCHED, and this is exactly the pattern the profile
+ # row already uses one level up: LATEST on the row (+ `enriched_at` to date it), the SERIES in
+ # `ut_ig_post_snapshots`. These three cells are rewritten on every pull from the snapshot that
+ # was just appended — they are a projection of that store, never a second copy of it, and
+ # deleting them would cost a convenience rather than a measurement.
+ # ⚠ BLANK, NEVER ZERO — and `_ig_zero_is_blank` is what finally ENFORCES it. `postMetrics` is
+ # off by default (it buys one vendor record per post), so on most pulls this stays empty, and
+ # empty means "not read", which is what makes an honest average possible at all. A 0 here
+ # would claim a post nobody watched. ⚠ The rule is scoped to this paid rung: a zero LIKE or
+ # COMMENT count is a real measurement and must survive.
+ #
+ # ⛔⛔ THERE IS NO `views` COLUMN HERE, AND THAT IS A RULING, NOT AN OVERSIGHT (2026-08-08,
+ # instagram-capture.md §4e/§4f — owner-approved after the evidence was bought).
+ # Bright Data's `views` is delivered at ACCOUNT grain: one value across up to 12 distinct
+ # reels, on 18 of 19 creators, through BOTH collection modes, while `likes` varies richly in
+ # the very same rows. Mapping it here is what put one number on many unrelated posts. We
+ # cannot say what it measures at ANY grain (`sriyynntt` returns 0 at 11,102 followers;
+ # `reviewby_ayyaa` returns null at 20k–328k likes), so it is not promoted anywhere — it stays
+ # in Source data, unlabelled, making no claim. Nothing is lost: the payloads are retained
+ # whole, so if the vendor ever populates it per-reel the history is re-derivable.
+ # ⚠ ENUMERATED, so nobody re-opens this hoping for a differently-named field: across all 30
+ # fields of a Reels row, `views` and `video_play_count` are the ONLY view-shaped keys, and the
+ # Posts dataset carries none at all. `plays` below IS the per-reel equivalent, correctly named
+ # and correctly mapped. It is blank because Meta retired the Plays metric on 2025-04-10
+ # (folded into a single "Views"), not because we are reading the wrong key.
+ # ⭐⭐ VIEWS IS BACK (2026-08-08), AND FROM A DIFFERENT SOURCE THAN THE ONE THAT WAS RETIRED.
+ # The retired column was fed by Bright Data's account-grain `views`. This one is fed by the
+ # `ig_post_views` CAPABILITY (providers.py), which resolves to Apify's `videoPlayCount` —
+ # measured against a browser-read ground truth to the digit. Same column name, same type, same
+ # rollup; the engine underneath is swappable and the preset database never moved. That is the
+ # owner's schema ruling working exactly as intended.
+ # ⚠ STILL BLANK, NEVER ZERO, and still only on the paid rung.
+ field_def("views", "Views", "int"),
+ field_def("plays", "Plays", "int"),
+ field_def("likes", "Likes", "int"),
+ field_def("comments", "Comments", "int"),
+ # ⭐⭐ 2026-08-09 (owner: *"whatever APIfy has more than BD pls use it and fix the post data
+ # further"*). Two fields the primary source does not return AT ALL, already paid for inside
+ # the same engagement response — so capturing them costs nothing extra.
+ # ⛔ A COLUMN MUST EXIST BEFORE A CELL CAN BE WRITTEN. `user_tables` filters an unknown key on
+ # every write door, so a normaliser that emits `video_duration` without this line writes
+ # nothing and reports success — the wave-28 defect that swallowed nine preset cells.
+ field_def("video_duration", "Video length (s)", "int"),
+ field_def("comments_disabled", "Comments off", "checkbox"),
+ field_def("measured_at", "Engagement read at", "date"),
+ # ⭐ 2026-08-07 — the second half of "spawn relevant Post/Comment database that is LINKED":
+ # a post reaches its own comment rows the same way a profile reaches its posts. Derived from
+ # `shortcode`, so it is correct the moment a comment row exists and needs no maintenance.
+ # ⚠ It resolves to nothing until comment capture is switched on, which is the honest state
+ # for a relation whose far side is empty — the same standing `ut_ig_post_snapshots` has when
+ # `postMetrics` is off.
+ field_def("comments_link", "Comment rows", "link",
+ description="Comment records linked to this post.",
+ link={"table": IG_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}),
+ field_def("post_snapshots_link", "Measurement rows", "link",
+ description="Engagement measurements linked to this post.",
+ link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}),
+ field_def("measurements_captured", "Measurements captured", "rollup",
+ rollup={"link": "post_snapshots_link", "fn": "countall"}),
+]
+
+#: ⭐⭐ 2026-08-07 (owner instruction) — THE COMMENT DATABASE.
+#:
+#: Owner: *"an enrichment automation should spawn relevant Post/Comment database that is linked to
+#: the profile automatically."* So the schema and the LINK ship; what does NOT ship on by default
+#: is the capture.
+#:
+#: ⛔ THIS REVERSES A STANDING RULING (D-22 / R11) AND THE REVERSAL IS DELIBERATE AND BOUNDED.
+#: Comment capture was refused on two grounds, and BOTH ARE STILL TRUE:
+#: 1. COST — comments are ~98% of a full-history bill (~$56,000 at 37.5M comments, §2a of
+#: `instagram-capture.md`). They are the single most expensive thing this product can buy.
+#: 2. PRIVACY — Bright Data flags `comment_user` and `post_user` as PII, and the append law has
+#: no erasure path for third parties who never appeared in the tracked set (D-24). A comment
+#: thread ingests identifiable people who never entered anybody's influencer list.
+#: ⛔ SO WHAT SHIPS HERE IS THE SCHEMA AND THE LINK. **NOTHING CAPTURES COMMENTS TODAY** — no code
+#: calls Bright Data's Comments dataset (`gd_ltppn085pokosxh13`), there is no `config.comments`
+#: switch, and this table stays EMPTY until somebody builds one. Stated in the present tense on
+#: purpose: an earlier draft of this note described the opt-in switch as if it existed, which is
+#: exactly the doc that "reads as authority and silently ages" that §0 of `instagram-capture.md`
+#: is written against.
+#: ⚠ WHEN IT IS BUILT it should be an OPT-IN defaulted OFF, the same posture as
+#: `config.postMetrics` and the Bright Data money switch W26/R15 kept — the relation costs
+#: nothing; the spend and the third-party ingest are a click somebody makes knowingly.
+#: Embedded comment payloads delivered with a Profile/Post/Reel response are retained because
+#: they are part of a record already paid for. The separate full Comments dataset remains OFF
+#: unless `commentMetrics` is explicitly enabled on the enrichment action.
+COMMENT_FIELDS = [
+ field_def("comment_key", "Comment"), field_def("shortcode", "Shortcode"),
+ field_def("influencer_key", "Influencer"),
+ field_def("commented_at", "Commented at", "date"),
+ field_def("likes", "Likes", "int"),
+ field_def("replies", "Replies", "int"),
+ field_def("source_payload", "Source data", "json"),
+ field_def("post_link", "Post", "link",
+ description="Post record linked to this comment.",
+ link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
+ "single": True}),
+ # Author/text and every provider-specific value stay intact in Source data. The promoted
+ # columns intentionally keep the grid concise while Likes and Replies make comment engagement
+ # directly filterable and rollup-ready.
+]
+POST_SNAPSHOT_FIELDS = [
+ field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"),
+ field_def("influencer_key", "Influencer"),
+ field_def("pulled_at", "Pulled at", "date"), field_def("likes", "Likes", "int"),
+ field_def("comments", "Comments", "int"),
+ # ⭐⭐ 2026-08-10 — TWO DENORMALISED POST FACTS, and they are worth having on their own merits
+ # before any rollup argument is made.
+ #
+ # `pulled_at` says when we LOOKED; `posted_at` says when the creator POSTED. A fact table with
+ # only the first can describe a measurement but not its AGE, so the single most useful thing
+ # this series can compute — views at N days old, i.e. velocity — is not expressible over it at
+ # all. `type` is the same shape one column over: "reels only" is the default lens on this data
+ # and without it every question has to go back through `ut_ig_posts` to ask what kind of post
+ # this was.
+ #
+ # ⚠ THE WRITER HAS BOTH IN HAND (`capture_rows` builds the identity row and the measurement row
+ # from the same vendor record), so this costs one dict key each and no extra call.
+ #
+ # ⛔ `options` IS NOT OPTIONAL ON A `select`. A select declaring none is the wave-26 item-24
+ # shape: the filter panel receives an empty list as an ANSWER, `if ([])` is truthy, and the
+ # control renders dead with nothing to say. Same three values `POST_FIELDS` declares — copied
+ # from it deliberately rather than shared, because these are two different tables' columns that
+ # happen to agree today, and `verify_automation` asserts the agreement.
+ # ⛔⛔ AND THEY DO NOT MAKE "THE LAST 10 REELS" EXPRESSIBLE OVER THIS TABLE. That takes TWO
+ # orderings (newest measurement per post, then newest posts), the rollup bag has one `sortBy`,
+ # and `ut_ig_posts` already performs the first of them. See the verdict in
+ # `.claude/wiki/research/entity-vs-series.md` — these columns are for AGE and KIND, not for
+ # moving the window here.
+ field_def("posted_at", "Posted at", "date"),
+ field_def("type", "Type", "select", options=["image", "video", "carousel"]),
+ # Plays move independently of likes on video, so it is its own series rather than a thing to
+ # derive. Paid rung only; blank means not read.
+ # The series regains Views alongside the latest-value projection on the Post row — one store
+ # for one series (R3) is untouched; this IS that store.
+ field_def("views", "Views", "int"),
+ field_def("plays", "Plays", "int"),
+ field_def("source_payload", "Source data", "json"),
+ field_def("post_link", "Post", "link",
+ description="Post record linked to this measurement.",
+ link={"table": IG_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
+ "single": True}),
+]
+
+
+# ---------------------------------------------------------------------------------------------
+# ⭐⭐ TIKTOK — THE FIVE `ut_tt_*` SCHEMAS (wave 29 · item 7 · D-9 · rulings R1 + R2)
+# ---------------------------------------------------------------------------------------------
+# ⛔ EVERY VENDOR FIELD NAME BELOW IS FROM ONE PROBED SOURCE — `waves/wave29/proto/tiktok-schema.md`,
+# 40 profile / 43 post / 17 comment fields read live from `GET /datasets/{id}/metadata` for $0.00,
+# each carrying the vendor's own type, description and `pii` flag. NOTHING HERE IS GUESSED, and the
+# names live in `connectors_tt.py` (the map), not here (the schema). This file declares what a
+# COLUMN is; the connector declares what the wire calls it.
+#
+# ⭐ THE SAME LAW AS THE INSTAGRAM LISTS ABOVE: map the schema, not just what was populated on the
+# probe. HISTORY IS UNBUYABLE — no vendor sells "followers on 1 January" — so a field we do not
+# capture today is permanently lost for today, and an occasionally-empty column costs a blank cell
+# while a missing one costs the months before somebody notices.
+#
+# ⚠ TWO DELIBERATE DIVERGENCES FROM THE INSTAGRAM SCHEMA, both licensed by R2 ("the two schemas may
+# diverge where the vendors do") and both recorded so neither reads as an omission:
+#
+# 1. NO `plays` COLUMN. TikTok returns ONE number, `play_count`, and it is the count TikTok
+# itself displays under a video — i.e. our `views`. Instagram has two columns because Meta
+# once had two metrics (`plays` retired 2025-04-10). Writing one vendor number into two of our
+# columns would manufacture a second measurement that a rollup could average or double-count,
+# and blank-vs-zero discipline says an unmeasured column must be blank, not a copy.
+# ⇒ `play_count` -> `views`, and `plays` does not exist on this family.
+# 2. `tt_id`, NOT `ig_id`. The probe doc flags this: our profile key is literally NAMED `ig_id`.
+# A TikTok row carrying a column labelled "Instagram id" is a header that lies, and this is a
+# NEW table family with no stored rows to migrate — so the honest name costs nothing.
+#
+# ⚠ THE COLUMNS THAT STAY BLANK ARE NAMED RATHER THAN DROPPED SILENTLY. TikTok has no equivalent of
+# `category`, `business_category`, `is_professional`, `highlights_count`, `bio_hashtags`,
+# `pronouns`, `profile_name`, `is_joined_recently`, `has_channel`, `partner_id`,
+# `external_url_title`, `fbid` or `related_accounts` (profile), nor of `alt_text`,
+# `comments_disabled`, `paid_partnership` or `partner` (post) — `commerce_info` is a LOCATION, not
+# a paid-partnership flag, and TikTok's `comment_setting` is profile-level rather than per-post.
+# Those columns are ABSENT here rather than declared and permanently blank — a column nothing can
+# ever write is a promise the grid keeps making that the vendor cannot keep.
+
+#: ⭐ WAVE 29 (contract C2) — the value the `profile` FLAG carries on a TikTok handle column: the
+#: twin of `PROFILE_SOURCE_IG`, which is declared further down beside the `PLATFORM_*` vocabulary
+#: with the note on why those two families of string are NOT the same thing. It sits here rather
+#: than there because the lists below are evaluated at IMPORT and would raise a NameError otherwise.
+#: ⛔ THIS CONSTANT IS ONLY HALF THE CONTRACT. The other half is one line in
+#: `platform/core/user_tables.py` — `PROFILE_SOURCES` must gain `'tiktok'` — which lives in another
+#: session's fence and is asked for in B's mailbox. Until that line lands, `_clean_profile` REFUSES
+#: this flag and the handle column of a `ut_tt_profile` spawn is dropped. Fail-closed and loud, by
+#: design: the alternative is a profile database whose profile column silently is not one.
+PROFILE_SOURCE_TT = "tiktok"
+
+#: The profile row: LATEST values plus `enriched_at`. The SERIES lives in `ut_tt_snapshots` —
+#: R3's "one store for one series" carries over unchanged, because the reason for it does (no
+#: vendor sells history, so the append table is the only history there will ever be).
+TT_PROFILE_FIELDS = [
+ # `platform` FIRST, exactly as the Instagram preset set has it — and on all five tables here,
+ # not just this one. The IG family carries it on the profile table alone, which is why a TikTok
+ # POST could never have lived beside an Instagram post; declaring it everywhere is what makes a
+ # future cross-platform union view a UNION rather than a guess.
+ tt_field_def("platform", "Platform"),
+ tt_field_def("handle", "Handle", pinned=True,
+ profile={"source": PROFILE_SOURCE_TT}),
+ tt_field_def("full_name", "Name"),
+ tt_field_def("tt_id", "TikTok id"),
+ tt_field_def("profile_url", "Profile", "url"),
+ tt_field_def("bio", "Bio"),
+ tt_field_def("external_url", "Link in bio", "url"),
+ tt_field_def("verified", "Verified", "checkbox"),
+ tt_field_def("is_private", "Private", "checkbox"),
+ tt_field_def("is_business", "Business account", "checkbox"),
+ tt_field_def("followers", "Followers", "int"),
+ tt_field_def("following", "Following", "int"),
+ tt_field_def("posts_count", "Video count", "int"),
+ # TikTok gives THREE engagement rates where Instagram gives one. All three are stored ×100 for
+ # the same reason `avg_engagement` is on the IG side (C1-a): the vendor sends a 0–1 fraction and
+ # our `pct` renderer appends the sign to the stored number, so the raw fraction would print a
+ # 6.6% creator as `0.0%`.
+ tt_field_def("avg_engagement", "Avg engagement", "pct"),
+ tt_field_def("like_engagement", "Like engagement", "pct"),
+ tt_field_def("comment_engagement", "Comment engagement", "pct"),
+ # Total likes RECEIVED across the account's videos — a TikTok-only number with no Instagram
+ # equivalent, and one of the few profile-level engagement facts a vendor gives away.
+ tt_field_def("likes_received", "Likes received", "int"),
+ tt_field_def("country_code", "Country"),
+ tt_field_def("region", "Region"),
+ tt_field_def("predicted_lang", "Language"),
+ # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP. The vendor's `create_time` on a profile is when the
+ # ACCOUNT was created; it is emphatically not "when `followers` was true". TikTok carries no
+ # measurement timestamp at all, exactly like Instagram — which is why the append law below
+ # (`@`) is the only thing that can date a number.
+ tt_field_def("account_created_at", "Account created", "date"),
+ tt_field_def("first_found", "First found", "date"),
+ tt_field_def("last_found", "Last found", "date"),
+ tt_field_def("found_count", "Times found", "int"),
+ tt_field_def("created_by", "Found by"),
+ tt_field_def("enriched_at", "Enriched at", "date"),
+ tt_field_def("source", "Read via"),
+ tt_field_def("source_payload", "Source data", "json"),
+ tt_field_def("profile_snapshots_link", "Measurement rows", "link",
+ description="Profile measurements linked to this account.",
+ link={"table": TT_SNAPSHOTS_TABLE, "on": "influencer_key", "from": "handle"}),
+ tt_field_def("posts_link", "Post rows", "link",
+ description="Post records linked to this account.",
+ link={"table": TT_POSTS_TABLE, "on": "influencer_key", "from": "handle"}),
+]
+
+#: The profile SERIES. One row per profile per pull, keyed `@` — the append law
+#: from `instagram-capture.md` §3, unchanged, because its cause is unchanged: the vendor stamps
+#: nothing, so the only honest date a number can carry is the moment WE read it.
+TT_SNAPSHOT_FIELDS = [
+ tt_field_def("platform", "Platform"),
+ tt_field_def("snapshot_key", "Snapshot"),
+ tt_field_def("influencer_key", "Influencer"),
+ tt_field_def("pulled_at", "Pulled at", "date"),
+ tt_field_def("followers", "Followers", "int"),
+ tt_field_def("following", "Following", "int"),
+ tt_field_def("posts_count", "Video count", "int"),
+ tt_field_def("likes_received", "Likes received", "int"),
+ tt_field_def("full_name", "Name"),
+ tt_field_def("bio", "Bio"),
+ tt_field_def("verified", "Verified", "checkbox"),
+ tt_field_def("is_private", "Private", "checkbox"),
+ tt_field_def("is_business", "Business account", "checkbox"),
+ tt_field_def("avg_engagement", "Avg engagement", "pct"),
+ tt_field_def("like_engagement", "Like engagement", "pct"),
+ tt_field_def("comment_engagement", "Comment engagement", "pct"),
+ tt_field_def("external_url", "Link in bio", "url"),
+ tt_field_def("tt_id", "TikTok id"),
+ tt_field_def("profile_url", "Profile", "url"),
+ tt_field_def("country_code", "Country"),
+ tt_field_def("region", "Region"),
+ tt_field_def("predicted_lang", "Language"),
+ tt_field_def("account_created_at", "Account created", "date"),
+ tt_field_def("source", "Read via"),
+ tt_field_def("approx", "Counts are approximate", "checkbox",
+ description="Checked when the counts on this row are rounded, not exact."),
+ tt_field_def("source_payload", "Source data", "json"),
+]
+
+#: One row per TikTok post. Identity is `shortcode`, and on TikTok that is a 19-digit numeric id —
+#: ✅ the SAME shape as the Comments dataset's `post_id`, so the comments→posts link joins on
+#: equality with NO normaliser (measured in the probe doc; Instagram needed one).
+TT_POST_FIELDS = [
+ tt_field_def("platform", "Platform"),
+ tt_field_def("shortcode", "Post id"),
+ tt_field_def("influencer_key", "Influencer"),
+ tt_field_def("posted_at", "Posted at", "date"),
+ # ⛔ THE VENDOR'S VOCABULARY IS `"video"` / `"content"` AND OURS HAS NO `"content"`. Declaring
+ # the vendor's word would put an untranslated API token in front of a user; declaring an option
+ # the mapper can emit but the select does not list would fail `_clean_field`. So the OPTIONS
+ # stay this product's words and `connectors_tt.normalize_post` does the translation — the same
+ # posture every other vendor value here takes. `carousel` is listed because a TikTok photo post
+ # carrying more than one image IS one, and the mapper decides from `carousel_images` rather
+ # than from the type token, which cannot express it.
+ tt_field_def("type", "Type", "select", options=["image", "video", "carousel"]),
+ tt_field_def("caption", "Caption"),
+ tt_field_def("url", "URL", "url"),
+ tt_field_def("hashtags", "Hashtags"),
+ tt_field_def("tagged_location", "Commerce location"),
+ # ⛔ `views` AND NO `plays` — see the divergence note at the top of this section. One vendor
+ # number, one column.
+ tt_field_def("views", "Views", "int"),
+ tt_field_def("likes", "Likes", "int"),
+ tt_field_def("comments", "Comments", "int"),
+ tt_field_def("shares", "Shares", "int"),
+ tt_field_def("saves", "Saves", "int"),
+ tt_field_def("video_duration", "Video length (s)", "int"),
+ tt_field_def("measured_at", "Engagement read at", "date",
+ description="When the engagement numbers on this row were read."),
+ tt_field_def("source_payload", "Source data", "json"),
+ tt_field_def("comments_link", "Comment rows", "link",
+ description="Comment records linked to this post.",
+ link={"table": TT_COMMENTS_TABLE, "on": "shortcode", "from": "shortcode"}),
+ tt_field_def("post_snapshots_link", "Measurement rows", "link",
+ description="Engagement measurements linked to this post.",
+ link={"table": TT_POST_SNAPSHOTS_TABLE, "on": "shortcode", "from": "shortcode"}),
+ tt_field_def("measurements_captured", "Measurements captured", "rollup",
+ description="How many measurements this post has.",
+ rollup={"link": "post_snapshots_link", "fn": "countall"}),
+]
+
+#: The POST series. ⚠ D-117 is live on the Instagram twin — `posted_at`/`type` are denormalised onto
+#: snapshot rows there and never reconcile with the post row afterwards. That pattern is NOT
+#: reproduced: this table carries the measurement and its identity, and asks `ut_tt_posts` for what
+#: kind of post it was. A fact that can disagree with its own dimension table is a fact nobody can
+#: trust, and the convenience it buys (one fewer hop in a rollup) is not worth a column that can be
+#: wrong.
+TT_POST_SNAPSHOT_FIELDS = [
+ tt_field_def("platform", "Platform"),
+ tt_field_def("post_snapshot_key", "Snapshot"),
+ tt_field_def("shortcode", "Post id"),
+ tt_field_def("influencer_key", "Influencer"),
+ tt_field_def("pulled_at", "Pulled at", "date"),
+ tt_field_def("views", "Views", "int"),
+ tt_field_def("likes", "Likes", "int"),
+ tt_field_def("comments", "Comments", "int"),
+ tt_field_def("shares", "Shares", "int"),
+ tt_field_def("saves", "Saves", "int"),
+ tt_field_def("source_payload", "Source data", "json"),
+ tt_field_def("post_link", "Post", "link",
+ description="Post record linked to this measurement.",
+ link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
+ "single": True}),
+]
+
+#: Comments. Same posture as Instagram's: the SCHEMA and the LINK ship, the CAPTURE is opt-in and
+#: defaults OFF (`commentMetrics`), because comments are the single most expensive thing this
+#: product can buy and they ingest identifiable third parties who never entered anybody's list
+#: (D-24). ⚠ TikTok's `replies` is an ARRAY where ours is an INT count — a name collision, resolved
+#: in the mapper by storing `num_replies` and leaving the array in Source data.
+TT_COMMENT_FIELDS = [
+ tt_field_def("platform", "Platform"),
+ tt_field_def("comment_key", "Comment"),
+ tt_field_def("shortcode", "Post id"),
+ tt_field_def("influencer_key", "Influencer"),
+ tt_field_def("commented_at", "Commented at", "date"),
+ tt_field_def("likes", "Likes", "int"),
+ tt_field_def("replies", "Replies", "int"),
+ tt_field_def("source_payload", "Source data", "json"),
+ tt_field_def("post_link", "Post", "link",
+ description="Post record linked to this comment.",
+ link={"table": TT_POSTS_TABLE, "on": "shortcode", "from": "shortcode",
+ "single": True}),
+]
+
+#: table key -> the field list that defines it. ⭐ DERIVED CONSUMPTION IS THE POINT: `ut_ensure`,
+#: the gates and every future runner read THIS rather than naming five constants, so adding a sixth
+#: `ut_tt_*` table is one entry instead of a sweep. The Instagram side has no equivalent map, which
+#: is exactly why its table names are scattered across the module.
+TT_TABLE_FIELDS = {
+ TT_PROFILE_TABLE: TT_PROFILE_FIELDS,
+ TT_SNAPSHOTS_TABLE: TT_SNAPSHOT_FIELDS,
+ TT_POSTS_TABLE: TT_POST_FIELDS,
+ TT_POST_SNAPSHOTS_TABLE: TT_POST_SNAPSHOT_FIELDS,
+ TT_COMMENTS_TABLE: TT_COMMENT_FIELDS,
+}
+#: The human labels a spawned `ut_tt_*` table wears in the database list.
+TT_TABLE_LABELS = {
+ TT_PROFILE_TABLE: "TikTok profiles",
+ TT_SNAPSHOTS_TABLE: "TikTok profile measurements",
+ TT_POSTS_TABLE: "TikTok posts",
+ TT_POST_SNAPSHOTS_TABLE: "TikTok post measurements",
+ TT_COMMENTS_TABLE: "TikTok comments",
+}
+
+
+# ---------------------------------------------------------------------------------------------
+# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`.
+# ---------------------------------------------------------------------------------------------
+# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE.
+# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation
+# runtime needs that, and the runtime is what this file is for.
+#
+# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation
+# runtime still reaches for after the split — twenty, down from the seventy-one that used to be
+# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor
+# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and
+# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that
+# true without those files having to learn where the wire went.
+# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked
+# that would be green whether or not anything moved. `verify_automation`'s `section_split`
+# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING
+# system instead of trusting this import line ([[gate-answers-the-wrong-question]]).
+#
+# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back
+# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily,
+# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail
+# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a
+# module-level import, the names it wants already exist and the cycle still resolves.
+from connectors_ig import ( # noqa: E402
+ BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_PROFILES, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT,
+ BD_RECORD_PRICE_SPEC,
+ _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_profile, _bd_rows,
+ _bd_tagged_location,
+ _first, _ig_int,
+ DEFERRED_MARK,
+ apify_profile,
+ bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_profiles_batch, bd_ready,
+ bd_snapshot_progress, depth_refusal,
+ ig_handle, public_source, pull_profile, select_post_groups, top_up_views,
+)
+
+
+# ---------------------------------------------------------------------------------------------
+# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23)
+# ---------------------------------------------------------------------------------------------
+# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's
+# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns
+# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND
+# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower
+# Georgia/Florida floral-design company — Fisch Floral's actual market.
+#
+# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE:
+#
+# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/`
+# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is
+# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the
+# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other.
+# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104,
+# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH,
+# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still
+# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a
+# snapshot that is not ready when it looks.
+# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery
+# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time`
+# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is
+# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the
+# next run collects it. A `partial` that says "still building, the next run collects it" is
+# the honest report of exactly what happened.
+# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and
+# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into
+# a wider profile set would silently multiply the enrichment bill AND pollute the
+# snapshot series with rows nobody chose. This path never decides WHICH rows to enrich.
+# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important
+# test died on it.
+#
+# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE
+# the API returns a number, `price: 0` means "not priced" rather than "free", and
+# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to
+# report. Every estimate below is derived from the PRICING PAGE and says so.
+
+#: The table a discovery run writes when the tenant has no Instagram profile database yet — the
+#: FIRST one, never a second (`discover_default_table` elects an existing one before this is
+#: reached). Upsert by handle, so re-finding a profile is free.
+#:
+#: ⭐ 2026-08-10 — RENAMED FROM `ut_ig_candidates` / "IG candidates" (owner: *"We only need ONE IG
+#: profile so it's not confusing"*). The old pair was two kinds of wrong at once. The KEY said
+#: `candidates` while every other table in the graph says what it holds (`ut_ig_posts`,
+#: `ut_ig_comments`, `ut_ig_snapshots`, `ut_ig_post_snapshots`) — and "candidate" was a concept
+#: W26/R6 retired when it deleted `tracked`: a row here is a PROFILE, and whether anyone wants it
+#: is a stage column's business, not the table's name. The LABEL then disagreed with the key on
+#: any tenant who renamed the table, which is exactly the mismatch the owner hit.
+#:
+#: ⚠ SAFE TO MOVE ONLY BECAUSE NOTHING HOLDS THE OLD KEY. Census 2026-08-10 across all three
+#: tenant stores: royal-imports ABSENT, gtmlab has no tables at all, and nurilab's was deleted the
+#: same day (0 rows, 0 views, 0 dependent automations). An automation that stored the old key
+#: explicitly still resolves to it — an explicit target is never rewritten — so a legacy tenant
+#: would keep working; there simply is not one.
+#: ⛔ A future rename is NOT this cheap. Once rows exist under a key, moving it is a migration
+#: touching rows, `link.table`, every stored `targetTable` and the `ut__*` bucket family.
+DISCOVER_TABLE = "ut_ig_profile"
+#: The label that key is CREATED with. `ut_ensure` sets a label only on create and never relabels,
+#: so changing this can rename nothing that already exists. One constant because it was three
+#: copies of the same string literal, and a default spelled three times is one edit from drifting.
+DISCOVER_LABEL = "IG profile"
+#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at
+#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed
+#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real
+#: headroom. ⚠ If the row size grows, this number is the thing to re-derive.
+BD_MAX_RECORDS = 500
+#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above).
+BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120)
+BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15)
+
+#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error
+#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact.
+BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes",
+ "array_includes", "not_array_includes", "is_null", "is_not_null")
+#: Operators that take NO value (everything else requires one).
+BD_NULLARY_OPS = ("is_null", "is_not_null")
+
+#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED**
+#: by the API and is therefore absent rather than offered-and-broken).
+#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY
+#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier
+#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the
+#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by
+#: accident; adding them back is a decision, not a typo.
+BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography",
+ "category_name", "business_category_name", "is_business_account",
+ "is_professional_account", "is_verified", "account", "full_name",
+ "external_url", "bio_hashtags", "post_hashtags", "profile_url",
+ "related_accounts", "profile_name", "id", "fbid", "highlights_count")
+#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste:
+#: these are the fields seen carrying values on real CORPUS rows. `category_name` and
+#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter
+#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist".
+BD_FILTER_LEAD = ("followers", "biography", "avg_engagement")
+
+#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count.
+#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still
+#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not
+#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a
+#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the
+#: scanner still has to walk, so they refine a search and cannot BE one.
+BD_NARROWING_FIELDS = frozenset({
+ "biography", "account", "full_name", "profile_name", "category_name",
+ "business_category_name", "external_url", "bio_hashtags", "post_hashtags",
+ "related_accounts", "id", "fbid", "profile_url"})
+#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the
+#: whole corpus, which is breadth wearing a condition's clothes.
+BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"})
+#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed
+#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60%
+#: populated: below it a filter misses more corpus than it matches, and the surface should
+#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%,
+#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never
+#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked.
+#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding
+#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true.
+BD_POPULATED_FIELDS = frozenset({
+ "account", "followers", "following", "posts_count", "avg_engagement", "biography",
+ "external_url", "is_business_account", "is_professional_account", "is_verified",
+ "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"})
+BD_MIN_NARROWING = 1
+
+# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ──────────────────
+# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY
+# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what
+# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner:
+# *"look at each damn schema and only provide toggles operator that make sense"*.
+#
+# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx`
+# defended raw names on the grounds that a refusal names the field exactly as the row does — and
+# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI
+# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio".
+BD_FIELD_LABELS = {
+ "followers": "Followers", "following": "Following", "posts_count": "Post count",
+ "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category",
+ "business_category_name": "Business category", "is_business_account": "Business account",
+ "is_professional_account": "Professional account", "is_verified": "Verified",
+ "account": "Handle", "full_name": "Name", "profile_name": "Profile name",
+ "external_url": "Link in bio", "profile_url": "Profile link",
+ "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts",
+ "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID",
+ "highlights_count": "Story highlight count",
+}
+#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident;
+#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson).
+BD_FIELD_HINTS = {
+ "account": "The @username",
+ "avg_engagement": "Likes and comments as a share of followers",
+ "id": "Instagram's internal number for the account — for matching a list you already have",
+ "fbid": "The linked Facebook id — for matching a list you already have",
+ "profile_name": "The name shown above the bio, when it differs from the account name",
+ "related_accounts": "Accounts Instagram suggests alongside this one",
+}
+#: The KIND decides the comparisons offered and the control drawn for the value.
+BD_FIELD_KINDS = {
+ **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement",
+ "highlights_count")},
+ **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")},
+ **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")},
+ **{n: "choice" for n in ("category_name", "business_category_name")},
+ **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url",
+ "profile_url", "id", "fbid")},
+}
+#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of
+#: each list is what `default_operator` must agree with.
+BD_OPS_BY_KIND = {
+ # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second
+ # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one
+ # idea — and the second one is the one written in vendor grammar.
+ "text": ("includes", "not_includes", "=", "is_not_null", "is_null"),
+ "choice": ("=", "!=", "is_not_null", "is_null"),
+ "number": (">=", "<=", "=", ">", "<"),
+ # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean
+ # is `=` with the other value, written the confusing way.
+ "boolean": ("=",),
+ "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"),
+}
+#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire
+#: (it is what the API accepts); nobody has to read it.
+BD_OP_LABELS = {
+ "includes": "contains", "not_includes": "does not contain",
+ "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of",
+ ">=": "at least", "<=": "at most", ">": "more than", "<": "less than",
+ "array_includes": "has any of", "not_array_includes": "does not have",
+ "is_null": "is empty", "is_not_null": "is not empty",
+}
+#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into.
+BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}]
+
+#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many
+#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start`
+#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing
+#: the whole search to "match any" (which would also drag Followers into the union — the hole the
+#: OR guard closed the same day).
+#:
+#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not
+#: contain floral OR does not contain flower" matches nearly every account alive — a negative over
+#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a
+#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs
+#: them enough to write the AND branch.
+BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"})
+MAX_PREDICATE_VALUES = 12
+
+
+def field_label(name):
+ """The human name for a searchable field — used by the SURFACE and by every REFUSAL."""
+ return BD_FIELD_LABELS.get(name) or str(name or "that field")
+
+
+def field_kind(name):
+ return BD_FIELD_KINDS.get(name, "text")
+
+
+def ops_for(name):
+ """The comparisons this field may be asked. A tuple, in offer order."""
+ return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"])
+
+#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing
+#: law, because the two drifted apart and the drift cost a user their first automation.
+#:
+#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no
+#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its
+#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS`
+#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door
+#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in —
+#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one
+#: sentence of drift, and nothing red anywhere at any point.
+#:
+#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field
+#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks
+#: one itself — a client-side default is a second copy of this table, free to disagree with the
+#: guard that judges it, which is exactly what happened.
+BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"})
+BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"})
+#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a
+#: worse question than an equality, and both narrow.
+BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"})
+
+
+def default_operator(name):
+ """The operator a NEW condition on `name` starts with.
+
+ Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from
+ saveable rather than two (change the comparison, then type a value) — with the second step
+ being one the surface never told anyone to take.
+ """
+ kind = field_kind(name)
+ if kind == "tags":
+ return "array_includes"
+ if kind in ("boolean", "choice"):
+ # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through
+ # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so
+ # the default would have been an operator the same module refuses to offer — the exact
+ # default-versus-guard split this function was written to close, reintroduced one commit
+ # later by adding a kind. Asserted per field in `verify_automation.py`.
+ return "="
+ if kind == "number":
+ # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person
+ # reaching for Followers means, and the guard's sentence is the honest answer when it is
+ # the ONLY condition present.
+ return ">="
+ return "=" if name in BD_EXACT_FIELDS else "includes"
+
+
+def predicate_narrows(p):
+ """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else."""
+ return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS
+ and str((p or {}).get("operator") or "") in BD_NARROWING_OPS)
+
+
+def narrowing_refusal(preds, operator="and"):
+ """C4's server law: the sentence when a predicate set does not narrow, else ''. One function
+ so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law)
+ refuse in the same words.
+
+ ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY
+ predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a
+ UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR
+ followers >= 10000` saved clean and asked the vendor for every account over ten thousand
+ followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop.
+ A money wall that a dropdown three rows above it can walk through is not a wall.
+
+ So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows
+ either way — `all([])` is True, which would have made "no conditions at all" the widest legal
+ query of the lot.
+ """
+ preds = list(preds or [])
+ if str(operator or "").lower() == "or":
+ if preds and all(predicate_narrows(p) for p in preds):
+ return ""
+ return ("with Match set to any, every condition has to describe the account itself "
+ "(Bio, Handle, Name, a hashtag). Matching any means the results are added "
+ "together, so one follower or yes/no condition widens the whole search. Narrow "
+ "every condition, or set Match to all")
+ if any(predicate_narrows(p) for p in preds):
+ return ""
+ return ("add a condition that describes the account itself — Bio contains, Handle is, a "
+ "hashtag. Follower counts and yes/no conditions alone match too much of Instagram "
+ "to search")
+
+
+def filter_meta():
+ """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]`
+ over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent,
+ R3).
+
+ `defaultOperator` rides here rather than being a client constant for the reason written over
+ `default_operator`: the client's own default was `is_not_null` for every field, which the
+ narrowing guard refuses on every field.
+ """
+ return [{"name": n,
+ "label": field_label(n),
+ "hint": BD_FIELD_HINTS.get(n, ""),
+ "kind": field_kind(n),
+ "populated": n in BD_POPULATED_FIELDS,
+ "narrowing": n in BD_NARROWING_FIELDS,
+ "defaultOperator": default_operator(n),
+ # The comparisons THIS field may be asked, already in plain English and already in
+ # offer order. A client that filtered a global list by field kind would be a second
+ # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale.
+ "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o),
+ "nullary": o in BD_NULLARY_OPS,
+ # `in`/`is any of` and the tag operators take a LIST — the control
+ # has to know that, and deriving it from the token in the client is
+ # the same second-copy mistake one level down.
+ "multi": o in BD_MULTI_VALUE_OPS}
+ for o in ops_for(n)],
+ "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []}
+ for n in BD_FILTER_FIELDS]
+
+#: ⭐ WAVE 25 · CONTRACT C1 — THE UNIFIED, CROSS-TENANT INSTAGRAM PRESET SET.
+#:
+#: ONE server-owned list. Every tenant gets the same keys and the same labels, which is what
+#: "unified across tenant" means and what makes the pooled master series joinable at all — two
+#: tenants calling the same number `followers` and `follower_count` is a pool you cannot query.
+#: ⛔ THERE IS NO CLIENT COPY OF THIS LIST. C and D read it off the wire (`GET
+#: /automations/presets`); a client-side copy is the wave-9 silent-drop failure in a new hat.
+#:
+#: ⭐ R3 — THESE HOLD **LATEST**, AND NOTHING ELSE. The full time series stays as append rows in
+#: `ut_ig_snapshots`. ONE STORE FOR ONE SERIES: no per-record `json` history column, no second copy
+#: of a number that already has a home.
+#: ⛔ W29-T34 — THIS LINE USED TO ADD *"which the metric fields already read"*, AND THAT WAS WRONG.
+#: A `metric` field reads the PLATFORM-WIDE MASTER, not this tenant's snapshot table:
+#: `compute_metric_cells` imports `ig_master` and calls `ig_master.series_for(...)` — a
+#: cross-tenant pooled repo in a different HF dataset, which is the whole point of C6's pooling and
+#: which no rollup kind can reach. The metric header below says exactly that, so the module was
+#: contradicting itself on which store answers the question.
+#: ⚠ And the practical finding behind the correction, recorded because it is surprising: a live
+#: census of all four tenants (13 tables / 228 fields / 64 store files) found **ZERO** metric fields
+#: in existence, while 47 rollups are live. The vocabulary stays by owner ruling; nothing uses it.
+#:
+#: ⭐⭐ 2026-08-07 — THIS IS NOW *EVERY* PROFILE FIELD THE VENDOR RETURNS, BY OWNER INSTRUCTION,
+#: AND THAT REVERSES D-25's FILL-RATE RULE. Owner, verbatim: *"make sure to have ALL Fields
+#: available to us from Bright Data to be pre-set Fields for us and populated."*
+#:
+#: ⛔ THE RULE IT REPLACES IS WRITTEN DOWN HERE RATHER THAN DELETED, because it was a good rule
+#: and the next reader will otherwise re-derive it and prune these columns back. It said: a
+#: LATEST-value column costs something a snapshot row does not — it is a column every user sees on
+#: their grid forever — so the split followed D-25's MEASURED fill-rates (50 mixed profiles,
+#: snapshot `snap_msfrlbmt7qvej8uby`) and a field earned a column only at ≥60% populated. That is
+#: why `business_category` (34%), `bio_hashtags` (10%), `post_hashtags` (18%) and `pronouns` were
+#: captured into the snapshots and were NOT preset columns.
+#:
+#: ⚠ THE MEASUREMENT IS UNCHANGED AND STILL WORTH KNOWING — several of these columns WILL be
+#: mostly blank on the scrape path, and D-25's central finding is why: **the scrape path and the
+#: corpus path are different data.** `avg_engagement` is 0% populated on scrape and 88% on corpus;
+#: `category` is 0% on scrape and 70% on corpus. So a blank here is very often "this ROUTE does
+#: not carry it", not "this account does not have it" — which is exactly what `enriched_at` and
+#: the blank-never-zero law exist to keep honest. What changed is the ANSWER to "does a
+#: sometimes-blank column earn a place on the grid", and that was always the owner's call to make.
+#: Source data retains every field the provider returns. The only separate decision is whether to
+#: request the paid full Comments dataset (`commentMetrics`, default OFF).
+#:
+#: ⭐ WAVE 26 · R5 — THE NETWORK VOCABULARY. A handle is only unique WITHIN a network, so this is
+#: half of the candidate identity (contract C3: the upsert key is `(platform, handle)`).
+#:
+#: ⚠ THE VALUES ARE DISPLAY STRINGS ON PURPOSE. They land in an ordinary `text` cell that a person
+#: reads, filters and groups by, so `"Instagram"` beats `"ig"` — and the set is small, closed and
+#: written down here rather than inferred from a runner's module name, because the day a second
+#: runner spells it `"instagram"` is the day the dedup key stops working and nothing errors: the
+#: profile simply appears twice.
+#: ⭐ 2026-08-07 — WHERE A PROFILE HANDLE POINTS, as `core.user_tables.PROFILE_SOURCES` spells it.
+#: A local literal for the same boot-path reason `MACHINE_OWNERS` and `UT_FIELD_TYPES` are locals
+#: here, and held in step by a gate rather than an import. ⛔ NOT the same vocabulary as
+#: `PLATFORM_*` below: this names the FLAG's source (which validator reads the cell), that names
+#: the NETWORK a row belongs to (half of the identity). They read alike and mean different things,
+#: which is exactly why both are written down instead of inferred.
+PROFILE_SOURCE_IG = "instagram"
+#: ⚠ Its TikTok twin, `PROFILE_SOURCE_TT`, is declared UP with the `ut_tt_*` schemas (wave 29): the
+#: field lists there are evaluated at import and would not see a constant defined here.
+
+PLATFORM_INSTAGRAM = "Instagram"
+#: ⭐ WAVE 29 — the `ut_tt_*` family stamps this on every row of all five of its tables. (It really
+#: was written by nothing until D-9 landed; the note that said so is kept in the log, not here.)
+PLATFORM_TIKTOK = "TikTok"
+PLATFORM_FACEBOOK = "Facebook"
+PLATFORMS = (PLATFORM_INSTAGRAM, PLATFORM_TIKTOK, PLATFORM_FACEBOOK)
+
+#: ⭐ WAVE 26 · R3 — THE TYPES ARE HONEST NOW, AND THE MIGRATION IS THE PRICE OF THAT.
+#:
+#: This block used to say the `text` types were deliberate, and the reasoning was sound as far as
+#: it went: `ut_ensure` merges by KEY and never re-types an existing column, so re-declaring
+#: `followers` as `int` gives NEW tables a schema every table already in production does not have
+#: — one vocabulary with two shapes, the exact drift this list exists to prevent. That argument
+#: was never wrong; it was an argument for doing the MIGRATION, and the note used it as a reason
+#: not to. Owner, 2026-08-06: *"Why is everything that instagram field found is in a text format?
+#: change this."*
+#: ⛔ SO THE TWO HALVES ARE INSEPARABLE AND SHIP TOGETHER. Declaring a type here without
+#: `migrate_ig_field_types()` reintroduces exactly the split-schema the old note feared — and it
+#: would do it silently, because a merged-by-key field list produces no error when two tables
+#: disagree about what a column IS.
+#:
+#: ⚠ `avg_engagement` CARRIES A UNIT CONVERSION, NOT JUST A TYPE (amendment C1-a). The vendor
+#: sends a 0–1 fraction (MEASURED: 0.0074 / 0.0656 / 0.0014 / 0.0148 / 0.0274 / 0.0091) and this
+#: product's `pct` renders the stored number with a `%` appended — so storing the raw fraction
+#: would print every creator in the book as `0.0%`. It is stored ×100 from here on, at the
+#: mapping and in the migration both. See `_pct100`.
+PRESET_PROFILE_FIELDS = [
+ # ⭐ WAVE 26 · R5 — WHICH NETWORK THIS HANDLE IS ON, and it is half of the identity now.
+ # `@inayma` on Instagram and `@inayma` on TikTok are two accounts owned by two different
+ # people as often as not, so the dedup key is (platform, handle) and never handle alone
+ # (C3). ⛔ NOT called `source`: `SNAPSHOT_FIELDS` already spends that word on *which rung
+ # answered* (`field_def("source", "Read via")`), and one word meaning two things across two
+ # tables in one product is the drift this whole list exists to prevent.
+ field_def("platform", "Platform"),
+ # ⭐⭐ 2026-08-07 (owner ruling) — THE HANDLE IS THE PRIMARY COLUMN *AND* THE ENRICH BINDING.
+ # Owner: *"if a user use this automation for instagram scraping, the Unique ID will always be
+ # REPLACED or made as the Handle. in any case the user can add their own record right and edit
+ # those records."* Two declarations, one field, and they are the same sentence read twice:
+ #
+ # `pinned` — the grid's identity column is `find(f => f.pinned) ?? fields[0]`, so WITHOUT
+ # this the primary is an accident of creation order. It is a DECLARATION, not a
+ # reorder: the stored field order is left alone (saved view orders keep working)
+ # and the client pins the column to position 0 wherever it sits. That is why this
+ # fixes tables that already exist without moving a single stored field.
+ # `profile` — C3/R7's flag. It makes `handle` the column `enrich_instagram` BINDS to
+ # (`profile_field_key` step 2), which is what turns "I added a profile to my
+ # database, how do I enrich it?" from an unanswerable question into a step. It
+ # also validates the cell (`@Name`, a pasted instagram.com link and a bare handle
+ # all normalise to the bare handle) and routes a typed value to the SHARED
+ # definition row rather than one user's overlay (C3-A1) — which is precisely what
+ # makes the owner's second clause true: a record you add BY HAND is a record the
+ # engine can read and enrich.
+ #
+ # ⛔ THE TWO KEYS MUST SURVIVE `_clean_field` UNCHANGED or `verify_api` W25-1 goes red. The flag
+ # is also why this field can never be retyped: `_clean_field` REFUSES a profile flag on
+ # anything but `text`.
+ field_def("handle", "Handle", pinned=True, profile={"source": PROFILE_SOURCE_IG}),
+ field_def("profile_url", "Profile", "url"),
+ field_def("full_name", "Name"), field_def("followers", "Followers", "int"),
+ field_def("following", "Following", "int"),
+ field_def("avg_engagement", "Avg engagement", "pct"),
+ field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"),
+ field_def("verified", "Verified", "checkbox"), field_def("category", "Category"),
+ # --- the enrichment half: what `run_field_instagram` already pulls, kept to the fields
+ # MEASURED ≥60% populated (see the note above).
+ field_def("posts_count", "Post count", "int"),
+ field_def("highlights_count", "Story highlight count", "int"),
+ field_def("is_business", "Business account", "checkbox"),
+ field_def("is_professional", "Professional account", "checkbox"),
+ # ⚠ STAYS `text`. It is a 17-digit opaque identifier, not a quantity — typing it `int` would
+ # invite a grid to sum it, and some of them exceed 2^53 so a JS reader would round it.
+ field_def("ig_id", "Instagram id"),
+ # --- ⭐ 2026-08-07: the rest of the vendor's profile schema, promoted by owner instruction
+ # (see the block comment above for what that reverses). Types are HONEST from birth, which
+ # costs nothing here: no table has ever carried these columns, so there is no stored `text`
+ # definition for the migration to convert — R3's conversion rule applies to the columns that
+ # already exist, and these are new everywhere.
+ field_def("business_category", "Business category"),
+ field_def("is_private", "Private account", "checkbox"),
+ field_def("bio_hashtags", "Bio hashtags"),
+ field_def("pronouns", "Pronouns"),
+ field_def("profile_name", "Profile name"),
+ field_def("is_joined_recently", "Joined recently", "checkbox"),
+ field_def("has_channel", "Has channel", "checkbox"),
+ field_def("partner_id", "Partner id"),
+ field_def("external_url_title", "Link title"),
+ # ⚠ `text` for `ig_id`'s reason, one identifier over: it is a name, not a quantity.
+ field_def("fbid", "Facebook id"),
+ field_def("related_accounts", "Related accounts"),
+ field_def("country_code", "Country"),
+ field_def("source_payload", "Source data", "json"),
+ # ⭐⭐ 2026-08-07 (owner instruction) — THE RELATION AND THE ROLLUPS OVER IT.
+ #
+ # Owner: *"an enrichment automation should spawn relevant Post/Comment database that is
+ # linked to the profile automatically… and this rollup needs to have formula that we can use
+ # to calculate things like average Views over last N posts."* Both halves are these five
+ # columns, and they are PRESETS rather than something a person assembles, because "linked
+ # automatically" is the instruction — a relation you have to wire up by hand is the feature
+ # not existing.
+ #
+ # ⛔ `from` IS NOT DECLARED, ON PURPOSE. `_link_from_key` falls back to the PROFILE-flagged
+ # column and then the pinned one, both of which are `handle` on every preset table — so this
+ # links correctly on a database where the handle column was renamed, and on one where the
+ # flag sits somewhere unexpected. Naming `handle` here would be the hard-coded subject
+ # [[gate-answers-the-wrong-question]] warns about, one layer down.
+ # ⚠ "Post rows", NOT "Post count" — one opens records and one reports the account total.
+ # Two count-like columns with the same label would be unreadable. Caught by the label-collision
+ # gate rather than on screen, which is what that gate is for.
+ field_def("posts_link", "Post rows", "link",
+ description="Post records linked to this profile.",
+ link={"table": IG_POSTS_TABLE, "on": "influencer_key"}),
+ field_def("profile_snapshots_link", "Profile history", "link",
+ description="Profile snapshots linked to this profile.",
+ link={"table": IG_SNAPSHOTS_TABLE, "on": "influencer_key"}),
+ field_def("post_snapshots_link", "Post measurement rows", "link",
+ description="Post engagement measurements linked to this profile.",
+ link={"table": IG_POST_SNAPSHOTS_TABLE, "on": "influencer_key"}),
+ field_def("comments_link", "Comment rows", "link",
+ description="Comment records linked to this profile.",
+ link={"table": IG_COMMENTS_TABLE, "on": "influencer_key"}),
+ # ⚠ THE ROLLUPS READ `ut_ig_posts`' OWN LATEST COLUMNS, which is why those exist — a rollup
+ # is ONE HOP (Airtable's rule and ours), and the engagement SERIES lives one table further
+ # out in `ut_ig_post_snapshots`.
+ # ⚠ 12 IS THIS MEASURE'S OWN WINDOW (`AVG_WINDOW_POSTS`), NOT THE CAPTURE CAP. It used to be
+ # `MAX_POSTS_PER_PULL` and read "the vendor's ceiling" — both halves are now wrong: the cap is 30
+ # (owner, 2026-08-09) and 30 is not the vendor's limit either. Binding these four to the cap meant
+ # raising it silently redefined every column named `_12`. `sortBy` is mandatory alongside a
+ # `limit`, and this is why — "the last 12" has to name what makes one post later than another.
+ # ⭐ UN-RETIRED 2026-08-08 together with the column it averages. It was retired for four hours
+ # because its input was an account-grain constant; with the input now a real per-reel
+ # measurement the average means what its label says again.
+ field_def("avg_views_12", "Avg views · last 12 posts", "rollup",
+ rollup={"link": "posts_link", "field": "views", "fn": "average",
+ "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
+ "distinctBy": "shortcode"}),
+ field_def("avg_plays_12", "Avg plays - last 12 posts", "rollup",
+ rollup={"link": "posts_link", "field": "plays", "fn": "average",
+ "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
+ "distinctBy": "shortcode"}),
+ field_def("avg_likes_12", "Avg likes · last 12 posts", "rollup",
+ rollup={"link": "posts_link", "field": "likes", "fn": "average",
+ "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
+ "distinctBy": "shortcode"}),
+ field_def("avg_comments_12", "Avg comments · last 12 posts", "rollup",
+ rollup={"link": "posts_link", "field": "comments", "fn": "average",
+ "limit": AVG_WINDOW_POSTS, "sortBy": "posted_at", "sortDir": "desc",
+ "distinctBy": "shortcode"}),
+ # ⭐ THE ONE HONEST POST COUNT WE HAVE. D-82: the vendor's `posts_count` is a FABRICATED ZERO
+ # on the paid rung (49/49 rows measured), so it is discarded and that column reads blank after
+ # a paid enrich. This counts the post rows actually captured — a different number and a true
+ # one, which is why it gets its own column and its own label rather than quietly filling
+ # `posts_count` with something that is not what that field means.
+ field_def("posts_captured", "Posts captured", "rollup",
+ rollup={"link": "posts_link", "fn": "countall", "distinctBy": "shortcode"}),
+ field_def("profile_reads", "Profile reads", "rollup",
+ rollup={"link": "profile_snapshots_link", "fn": "countall",
+ "distinctBy": "snapshot_key"}),
+ field_def("post_measurements_captured", "Post measurements captured", "rollup",
+ rollup={"link": "post_snapshots_link", "fn": "countall",
+ "distinctBy": "post_snapshot_key"}),
+ field_def("comments_captured", "Comments captured", "rollup",
+ rollup={"link": "comments_link", "fn": "countall",
+ "distinctBy": "comment_key"}),
+ # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING.
+ #
+ # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we
+ # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one
+ # thing a buyer most wants and the one thing the profile row cannot answer. The competitor
+ # teardown found the same gap solved by GUESSING from bio words, two buckets deep
+ # ([[janney-ai-teardown]]).
+ #
+ # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place
+ # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the
+ # post row, plus the vendor's fuller `location`/`location_details` inside the paid
+ # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a
+ # creator's own posts is a far better residency signal than a word in a bio, and it costs a
+ # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1.
+ #
+ # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator
+ # posting from twelve cities gets a low confidence rather than a confident wrong answer, and
+ # `location_confidence` is the number a person filters on before trusting the guess. A single
+ # geotagged post produces NO guess at all — n=1 dressed as a pattern is what
+ # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain.
+ field_def("location_guess", "Location (guess)"),
+ field_def("location_confidence", "Location confidence", "pct"),
+ # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never
+ # enriched" and a stale one means "enriched in March", and no cell on the row can tell them
+ # apart. It is the single field that turns the other fifteen from numbers into measurements.
+ field_def("enriched_at", "Enriched at", "date"),
+ # ⭐ WAVE 26 · R1 — THE LAST-N POST WINDOW, AND IT IS A VIEW RATHER THAN A STORE.
+ # ⛔ READ THE R3 NOTE ABOVE BEFORE CHANGING THIS. "One store for one series" still holds: the
+ # authoritative post record is `ut_ig_posts` (keyed by shortcode, so it ACCUMULATES) and the
+ # authoritative engagement series is `ut_ig_post_snapshots` (append-per-pull, carrying
+ # views/likes/comments at a `pulled_at`). This cell is a DERIVED window over those two,
+ # rewritten each run, so a person reading the profile row can see the recent posts without a
+ # join — and deleting it would cost a convenience, never a measurement. Shape: contract C2.
+]
+#: The keys the preset set owns — derived, so a field added above cannot be forgotten here.
+PRESET_PROFILE_KEYS = tuple(f["key"] for f in PRESET_PROFILE_FIELDS)
+
+#: ⭐ 2026-08-07 — WHICH preset field declares the primary column, and WHICH declares the profile
+#: flag. DERIVED for the same reason `PRESET_PROFILE_KEYS` is, and the negative control is what
+#: argued for it: with `"handle"` hard-coded in the migration, stripping the declaration left the
+#: migration cheerfully stamping a column the product no longer claimed. Moving a declaration now
+#: moves the migration with it, and REMOVING one stops the migration rather than leaving it to
+#: enforce a rule nobody declares any more.
+#: ⚠ Both are `""` when nothing declares them, and every reader treats `""` as "do nothing" —
+#: never as "field number zero".
+PRESET_PINNED_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("pinned")), "")
+PRESET_FLAG_KEY = next((f["key"] for f in PRESET_PROFILE_FIELDS if f.get("profile")), "")
+
+#: Discovery's table is the preset set PLUS its own bookkeeping. ⛔ DERIVED, NEVER RE-TYPED: the
+#: alternative is two lists that describe the same columns and drift one label at a time, which is
+#: the failure C1 exists to prevent. The five below are facts about the SEARCH (how often we found
+#: them, who for, and a human's decision) rather than about the profile, so they are discovery's
+#: and not part of the cross-tenant set.
+CANDIDATE_FIELDS = [
+ *PRESET_PROFILE_FIELDS,
+ # ⭐ WAVE 26 · R3 — `first_found` / `last_found` ARE DATES, not ISO strings in a text cell.
+ # They were written by `_iso()`, so the cell read `2026-08-05T14:03:11+07:00` and the owner
+ # called that format "extremely confusing" — correctly: it is a machine timestamp shown to a
+ # person, unsortable as a date and unfilterable by "last 7 days". The STAMP still carries its
+ # offset everywhere it is a time axis (`ut_ig_snapshots.pulled_at` is untouched); what
+ # changed is that "when did we first see this account" is a DAY, and a day is all anybody
+ # asks it for. `migrate_ig_field_types()` converts the stored strings.
+ field_def("found_count", "Times found", "int"),
+ field_def("first_found", "First found", "date"),
+ field_def("last_found", "Last found", "date"),
+ # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it
+ # describes, which is the first place anybody looks up what this column means. It read: *"the
+ # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two
+ # people discovering the same profile each get their own row, their own found_count and their
+ # own review card."* Wave 26 · R4/R5 retired every clause of that.
+ #
+ # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two
+ # people in one workspace who discover the same profile share ONE row, one `found_count` and
+ # one history — because they are looking at one company's leads, not two private lists.
+ # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it
+ # first. It is no part of the dedup key, and a run must never branch on it.
+ # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged,
+ # and the one clause of the old note that was still true.
+ field_def("created_by", "Found by"),
+ # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have
+ # a Field called stage to track the progress of the Automation per Record. We don't need
+ # another checkbox for this."* Correct, and it had been true since wave 22 shipped the stage
+ # column: every candidate carried BOTH a `stage_` select saying where it was AND a
+ # boolean saying whether it was kept — two progress fields that could disagree, with no rule
+ # about which one won. The stage field is the one progress column. Nothing replaces this.
+]
+
+
+def _profile_backlink_field(table_key, table_label, profile_key):
+ """A deterministic reciprocal link from one canonical IG table to one profile database.
+
+ The key includes the target table identity, so ten separate Profile databases can all point
+ into the same canonical Posts/Comments/history tables without one relation overwriting the
+ next. The join is derived in both directions and therefore needs no cross-table fan-out write.
+ """
+ digest = hashlib.sha1(str(table_key).encode("utf-8")).hexdigest()[:10]
+ return field_def(
+ f"profiles_{digest}", f"Profiles - {str(table_label or table_key)[:48]}", "link",
+ description=f"Profile records from {str(table_label or table_key)[:48]} linked to this row.",
+ link={"table": str(table_key), "on": str(profile_key), "from": "influencer_key"},
+ )
+
+
+def _profile_schema_for(bound_key):
+ """Preset fields for an existing Profile table without inventing a second identity column."""
+ out = []
+ for field in PRESET_PROFILE_FIELDS:
+ item = dict(field)
+ if item.get("key") == PRESET_FLAG_KEY and bound_key != PRESET_FLAG_KEY:
+ item.pop("profile", None)
+ out.append(item)
+ return out
+
+
+def _locked_ig_field(field):
+ """One canonical Instagram field with the immutable preset declaration attached."""
+ item = dict(field)
+ automation = dict(item.get("automation") or {})
+ automation["preset"] = True
+ item["automation"] = automation
+ return item
+
+
+def _profile_binding(table):
+ """The declared Profile identity field, falling back only to the canonical handle."""
+ fields = list((table or {}).get("fields") or [])
+ flagged = next((str(f.get("key") or "") for f in fields
+ if isinstance(f.get("profile"), dict)), "")
+ if flagged:
+ return flagged
+ return PRESET_FLAG_KEY if any(f.get("key") == PRESET_FLAG_KEY for f in fields) else ""
+
+
+def _ig_schema_contract(rt, profile_tables=None):
+ """The complete per-tenant Instagram graph contract, including every Profile backlink.
+
+ The fixed child tables are shared. Profile tables are discovered structurally, so an older
+ user-named target and the built-in candidate database receive the same preset columns, Links,
+ Rollups, locks, and reciprocal fields. Retired machine keys are the only columns deleted.
+ """
+ tables = ut_all(rt)
+ profile_keys = sorted(set(profile_tables if profile_tables is not None
+ else _ig_profile_tables(rt)))
+ backlinks, wanted, drops = [], {}, {
+ IG_SNAPSHOTS_TABLE: {"post_hashtags"},
+ # ⚠ `views` IS DELIBERATELY ABSENT FROM THIS DROP SET AGAIN. It was dropped earlier today
+ # while it carried Bright Data's account-grain number; it is now declared in POST_FIELDS
+ # and fed by the `ig_post_views` capability. Leaving it here would have made the migration
+ # delete, on every authenticated read, the column the same release just added — the
+ # drop set and the field list are two halves of ONE contract and must move together.
+ }
+ for table_key in profile_keys:
+ table = tables.get(table_key) or {}
+ bound = _profile_binding(table)
+ if not bound:
+ continue
+ label = str(table.get("label") or table_key)
+ backlinks.append(_profile_backlink_field(table_key, label, bound))
+ schema = _profile_schema_for(bound)
+ # ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram
+ # database work with correct Rollup and Link fields"*) — THE DERIVED OVERLAY, AND IT
+ # BELONGS HERE RATHER THAN IN A SEPARATE MIGRATION.
+ #
+ # ⛔ A standalone converter would LOSE. `_reconcile_ig_graph_fields` overwrites every
+ # contract key it owns — `rollup` included, by its own note — so a column converted beside
+ # the contract is re-typed back to `int` on the next reconcile, silently, hours later.
+ # Making the CONTRACT itself say "this column is derived" leaves exactly one writer of the
+ # schema instead of two that disagree.
+ #
+ # ⚠ THE SET IS PER TENANT AND IS A PROOF, NOT A LIST. `_derived_profile_columns` converts a
+ # column only where this tenant's own rows show the fold already equals the stored value —
+ # measured, because a named list would have blanked 221 live cells on nurilab alone
+ # (`avg_engagement` and `category`, 62 each). A tenant whose series is thinner converts
+ # fewer columns and loses nothing; as its history fills in, later passes convert more.
+ derived = set(_derived_profile_columns(table, tables.get(IG_SNAPSHOTS_TABLE) or {}))
+ wanted[table_key] = [_as_derived_field(f) if str(f.get("key")) in derived else f
+ for f in schema]
+ # `tracked` was the pre-Stage progress checkbox. Keeping both allows two progress states
+ # to disagree, so it is retired by key just like the old nested Posts JSON.
+ drops[table_key] = {"posts", "post_hashtags", "tracked"}
+ wanted.update({
+ IG_SNAPSHOTS_TABLE: [*SNAPSHOT_FIELDS, *backlinks],
+ IG_POSTS_TABLE: [*POST_FIELDS, *backlinks],
+ IG_POST_SNAPSHOTS_TABLE: [*POST_SNAPSHOT_FIELDS, *backlinks],
+ IG_COMMENTS_TABLE: [*COMMENT_FIELDS, *backlinks],
+ })
+ return wanted, drops
+
+
+# Only a unit-changing retype needs a cell conversion. Integer/checkbox/date cells already use
+# the scalar strings their renderers expect; engagement is the exception because Bright Data's
+# 0-1 fraction becomes this product's 0-100 percentage.
+_IG_RETYPE_CONVERTERS = {
+ (IG_SNAPSHOTS_TABLE, "avg_engagement"): _pct100,
+}
+
+
+def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None):
+ """Repair machine-owned IG field declarations in one coalesced store update.
+
+ `ut_ensure` deliberately merges only missing keys. That is correct for user columns but not
+ sufficient for a canonical relation: a stale `link.table`, rollup function, or type would
+ survive forever. This pass overwrites the contract keys for the fields we own, preserves
+ unrelated/user fields, and removes only explicitly retired preset columns plus their cells.
+ """
+ drop_by_table = drop_by_table or {}
+ wanted_by_table = {
+ key: [_locked_ig_field(f) for f in fields]
+ for key, fields in wanted_by_table.items()
+ }
+ changed = False
+
+ def _up(cur):
+ nonlocal changed
+ cur = cur if isinstance(cur, dict) else {}
+ for table_key, wanted_fields in wanted_by_table.items():
+ table = cur.get(table_key)
+ if table is None:
+ continue
+ wanted = {str(f.get("key")): dict(f) for f in wanted_fields}
+ drops = set(drop_by_table.get(table_key) or ())
+ fields, seen = [], set()
+ drops = _guarded_drops(table, drops)
+ for stored in table.get("fields") or []:
+ key = str(stored.get("key") or "")
+ if key in drops:
+ changed = True
+ continue
+ desired = wanted.get(key)
+ if desired is None:
+ fields.append(stored)
+ continue
+ # ⭐⭐ 2026-08-09 (owner: *"everything is custom and changeable always"*) — A
+ # COLUMN A HUMAN HAS TAKEN OVER IS LEFT ALONE. Every key below is overwritten
+ # from the shipped contract, `rollup` included, so without this the newly
+ # unlocked "edit a preset rollup" would save, render, recompute and then revert
+ # at the next enrichment run — an edit that looks like it worked and undoes
+ # itself hours later. `user_tables.user_edited` is the ONE reader of the stamp.
+ if _ut().user_edited(stored):
+ fields.append(stored)
+ seen.add(key)
+ continue
+ repaired = dict(stored)
+ for dkey, value in desired.items():
+ if dkey != "automation":
+ repaired[dkey] = value
+ automation = dict(stored.get("automation") or {})
+ automation.update(desired.get("automation") or {})
+ repaired["automation"] = automation
+ # These bags define behaviour, not decoration. If the desired field does not use
+ # one, a stale bag from an earlier type must not remain attached to it.
+ for bag in ("link", "rollup", "profile", "pinned"):
+ if bag not in desired:
+ repaired.pop(bag, None)
+ if desired.get("type") not in ("select", "multiselect"):
+ repaired.pop("options", None)
+ converter = _IG_RETYPE_CONVERTERS.get((table_key, key))
+ if (converter and stored.get("type") in ("text", "", None)
+ and repaired.get("type") != stored.get("type")):
+ for row in (table.get("rows") or {}).values():
+ if str(row.get(key) or "").strip():
+ converted = converter(row.get(key))
+ if converted:
+ row[key] = converted
+ changed = True
+ if repaired != stored:
+ changed = True
+ fields.append(repaired)
+ seen.add(key)
+ for key, desired in wanted.items():
+ if key not in seen and not any(str(f.get("key") or "") == key for f in fields):
+ fields.append(desired)
+ changed = True
+ if drops:
+ for row in (table.get("rows") or {}).values():
+ for key in drops:
+ if key in row:
+ row.pop(key, None)
+ changed = True
+ table["fields"] = fields
+ return cur
+
+ # Avoid a commit when the declarations are already exact.
+ snapshot = ut_all(rt)
+ needs = False
+ for table_key, wanted_fields in wanted_by_table.items():
+ if table_key not in snapshot:
+ continue
+ table = snapshot.get(table_key) or {}
+ by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []}
+ # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip
+ # the commit when nothing would change; a spared `tracked` column left in the drop set
+ # answers "yes, work to do" on every single call, forever, and the schema pass would
+ # rewrite the table on every authenticated read without ever changing a byte.
+ drops = _guarded_drops(table, drop_by_table.get(table_key) or ())
+ if drops & set(by_key):
+ needs = True
+ break
+ for desired in wanted_fields:
+ stored = by_key.get(str(desired.get("key") or ""))
+ mismatch = stored is None
+ if stored is not None:
+ for key, value in desired.items():
+ if key == "automation":
+ if any((stored.get("automation") or {}).get(k) != v
+ for k, v in value.items()):
+ mismatch = True
+ break
+ elif stored.get(key) != value:
+ mismatch = True
+ break
+ if not mismatch:
+ stale_bags = {"link", "rollup", "profile", "pinned"} - set(desired)
+ mismatch = any(k in stored for k in stale_bags)
+ if (not mismatch and desired.get("type") not in ("select", "multiselect")
+ and "options" in stored):
+ mismatch = True
+ if mismatch:
+ needs = True
+ break
+ if needs:
+ break
+ if needs:
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ return changed
+
+
+def ensure_ig_graph(rt, username="automation", flow_tag="", profile_table="", profile_field=""):
+ """Ensure the ONE per-tenant Instagram relational graph and return its canonical keys.
+
+ Every enrichment path calls this function. Fixed table keys prevent a flow aimed at a new
+ Profile database from spawning `IG posts 2`; deterministic reciprocal link fields connect
+ each Profile database to the same Posts, Comments, profile-history, and post-history stores.
+ """
+ profile = ut_get(rt, profile_table) if profile_table else None
+ bound = str(profile_field or "").strip()
+ if profile is not None and not bound:
+ bound = next((str(f.get("key") or "") for f in profile.get("fields") or []
+ if isinstance(f.get("profile"), dict)), "")
+ profile_label = str((profile or {}).get("label") or profile_table)
+ backlink = [_profile_backlink_field(profile_table, profile_label, bound)] \
+ if profile_table and profile is not None and bound else []
+
+ graph = {
+ IG_SNAPSHOTS_TABLE: ("IG snapshots", [*SNAPSHOT_FIELDS, *backlink]),
+ IG_POSTS_TABLE: ("IG posts", [*POST_FIELDS, *backlink]),
+ IG_POST_SNAPSHOTS_TABLE: ("IG post snapshots", [*POST_SNAPSHOT_FIELDS, *backlink]),
+ IG_COMMENTS_TABLE: ("IG comments", [*COMMENT_FIELDS, *backlink]),
+ }
+ keys = {}
+ for key, (label, fields) in graph.items():
+ keys[key] = ut_ensure(rt, label, fields, username, key=key, flow_tag=flow_tag,
+ record_mode=AUTOMATION_RECORD_MODE, lock_fields=True)
+
+ if profile_table and profile is not None and bound:
+ profile_fields = _profile_schema_for(bound)
+ ut_ensure(rt, profile_label or profile_table, profile_fields, username,
+ key=profile_table, flow_tag=flow_tag, lock_fields=True)
+ # Reconcile the WHOLE tenant graph, not just this run's target. That is what keeps an older
+ # Profile database and IG candidates on the same universal schema while all of them point to
+ # the same four children. Existing flow provenance is merged, never replaced.
+ wanted, drops = _ig_schema_contract(rt)
+ _reconcile_ig_graph_fields(rt, wanted, drops)
+ return keys
+
+
+# ── ⭐ WAVE 26 · THE MIGRATION (owner rulings R3 + R4/R5, contracts C1-a and C3) ───────────────
+#
+# Two changes land on data that already exists, and neither is optional once the field defs move:
+#
+# R3 the preset columns take honest types, so the CELLS have to match them — an ISO stamp in a
+# `date` column and a "20872" string in an `int` column are the split-schema the old
+# "types are text on purpose" note correctly feared. Re-typing without converting is worse
+# than not re-typing.
+# R4 the candidate identity becomes `(platform, handle)`, so rows that exist today as
+# `(handle, created_by)` DUPLICATES must be merged rather than left to collide.
+#
+# ⛔ IDEMPOTENCY IS THE WHOLE DESIGN, and the key is the STORED FIELD TYPE — never a flag, never a
+# marker column. Run twice, an `avg_engagement` of 0.0074 would go 0.74 then 74, and nothing would
+# look wrong until somebody read a 74% engagement rate off a creator with 3,000 followers. So a
+# column is converted only while its STORED definition still says `text`; the moment it says `pct`
+# the work is done and re-running is a no-op. That makes the migration safe to call on every boot,
+# which is what it is for.
+#
+# ⚠ THE TABLE LIST IS DERIVED, NOT NAMED — D-71's lesson, one wave old and already paid for twice.
+# `observed_categories` named its two source tables and went silently empty the day wave 25 let a
+# user point an automation somewhere else. Naming tables here would leave exactly those user-named
+# databases on the old schema, which is the same bug wearing a migration's clothes.
+
+#: A table is an Instagram profile table if it declares `handle` plus a real slice of the preset
+#: vocabulary. Deliberately structural: it finds `ut_beauty_influencer_leads` without being told.
+MIGRATE_MIN_PRESET_FIELDS = 3
+#: key -> the converter its new type needs. `checkbox` needs none (the writers already emit '1'/'').
+_MIGRATE_CONVERT = {
+ "first_found": _day, "last_found": _day, "enriched_at": _day,
+ "avg_engagement": _pct100,
+}
+
+
+def _ig_profile_tables(rt, tables=None):
+ """Every `ut_*` table in this tenant that looks like an Instagram profile table.
+
+ ``tables`` is the optional lent snapshot (W29-T01): read-only walk, so a caller that already
+ holds the bucket must not pay for a second deep copy of it.
+ """
+ out = []
+ for key, tbl in sorted(((ut_all(rt) if tables is None else tables) or {}).items()):
+ keys = {f.get("key") for f in (tbl or {}).get("fields") or []}
+ if "handle" in keys and len(keys & set(PRESET_PROFILE_KEYS)) >= MIGRATE_MIN_PRESET_FIELDS:
+ out.append(key)
+ return out
+
+
+def discover_default_table(rt, tables=None):
+ """Which database should a discovery automation write to when nobody has said? (2026-08-10)
+
+ Owner: *"the damn database is supposed to be dynamic for whatever instagram profile is there.
+ We only need ONE IG profile so it's not confusing."*
+
+ ⛔ THE COMPLAINT WAS ABOUT A SECOND EMPTY DATABASE, NOT A HARDCODED KEY. `ut_beauty_influencer_
+ leads` appears NOWHERE in this product as a literal — it is a slug `ut_ensure` minted from the
+ label the tenant typed, and `_ig_profile_tables` finds it structurally (that function's own
+ note says so). What IS hardcoded is `DISCOVER_TABLE`, the FALLBACK target — and because it is
+ a constant rather than a question, a tenant that already had an Instagram profile database got
+ a SECOND, empty one the first time a discovery automation was saved without a target.
+ MEASURED on nurilab: `ut_ig_candidates` (0 rows, 44 preset columns) sitting beside
+ `ut_beauty_influencer_leads` (105 rows), both matching the profile predicate, one of them
+ pure confusion.
+
+ So the default becomes a question asked of the tenant:
+
+ exactly one profile database (excluding the fallback) -> that one
+ several, exactly one of which holds rows -> the one in use
+ several in use, or none at all -> "" (the caller keeps DISCOVER_TABLE)
+
+ ⚠ RETURNS "" RATHER THAN GUESSING. Two populated profile databases is a real ambiguity and
+ picking one silently would write a paid discovery run into a database the user did not name —
+ worse than the empty table this exists to prevent. "" means "no opinion", and the caller's
+ existing fallback stands, which is exactly today's behaviour for that case.
+ ⚠ THE FALLBACK IS EXCLUDED FROM ITS OWN ELECTION. `ut_ig_candidates` carries the full preset
+ schema, so it matches `_ig_profile_tables` — without this line a tenant that already has the
+ empty table would keep re-electing it and nothing would ever change.
+
+ ⭐ WAVE 29 (W29-T01) — ONE READ, NOT TWO. This function read the whole `user_tables` bucket
+ here and `_ig_profile_tables` read it AGAIN one line below: two 35.8 MB-ceiling deep copies to
+ answer one question, on a route the automation surface calls on every open. The election is
+ now computed from a single snapshot, which is also the only way it can be internally
+ consistent — the two reads could disagree with each other under a concurrent write.
+ ``tables`` lets `GET /automations` lend the copy it already holds; the answer is still derived
+ fresh on every call, so there is no memo to invalidate when a database is created or deleted.
+ """
+ try:
+ tables = (ut_all(rt) if tables is None else tables) or {}
+ found = [k for k in _ig_profile_tables(rt, tables=tables) if k != DISCOVER_TABLE]
+ except Exception: # noqa: BLE001
+ return ""
+ if len(found) == 1:
+ return found[0]
+ if len(found) > 1:
+ used = [k for k in found if (tables.get(k) or {}).get("rows")]
+ if len(used) == 1:
+ return used[0]
+ return ""
+
+
+def _apply_discover_default(rt, raw):
+ """Fill in a discovery automation's target BEFORE the pure validator invents one.
+
+ ⛔ IT HAS TO HAPPEN HERE, AND THE REASON IS THE WHOLE DESIGN. `clean_config` is the thing that
+ turns a missing target into `DISCOVER_TABLE` — and it takes no `rt`, by design (it is a pure
+ validator with a two-argument contract every gate and route depends on). Resolving only at the
+ RUN sites would be cosmetic: the literal is written into the STORED config at save time, so
+ `cfg.get("targetTable")` is truthy forever after and no runtime resolver is ever consulted.
+ `create`/`patch` are the one pair that both know the tenant and sit above that validator.
+ ⚠ ONLY WHEN THE CALLER SAID NOTHING. An explicit target — including one a previous save
+ stored — is the user's choice and is never rewritten.
+ """
+ cfg = raw.get("config")
+ if not isinstance(cfg, dict) or str(cfg.get("targetTable") or "").strip():
+ return raw
+ default = discover_default_table(rt)
+ if not default:
+ return raw
+ return {**raw, "config": {**cfg, "targetTable": default}}
+
+
+def _vestigial_name_field(fields, rows):
+ """⭐ 2026-08-07 — the born-blank `Name` column, or None. The owner's *"REPLACED"* half.
+
+ `user_tables.create()` mints EXACTLY ONE column on a hand-made database —
+ `{key:'name', label:'Name', type:'text', default:True}` — and `ut_ensure` then APPENDS the
+ automation's columns after it, never reordering. So on every Instagram database somebody
+ created before pointing an automation at it, column 1 is a `Name` nothing will ever write.
+
+ ⛔ IT IS ONLY VESTIGIAL IF NOBODY EVER USED IT, and the test is deliberately conservative
+ because this is the one branch in the migration that DESTROYS something. A column carrying any
+ declaration (an automation binding, a metric, a profile flag) is somebody's work; a column with
+ a value in any row is somebody's data. Either way it survives as an ordinary column and merely
+ stops being the locked primary, which the `pinned` half achieves on its own.
+
+ ⚠ `key == 'name'` IS ALREADY DECISIVE and the rest is belt: a user-created column is minted
+ `custom__` (`buildOverlayField`) and every machine one comes from a `field_def`
+ list, so nothing but `create()`'s default can hold this key. The label and type are checked
+ anyway — a renamed or retyped column is a column someone touched on purpose.
+ """
+ f = next((f for f in fields if f.get("key") == "name"), None)
+ if f is None or f.get("label") != "Name" or f.get("type") != "text":
+ return None
+ if any(f.get(k) for k in ("automation", "metric", "profile", "measure", "formula")):
+ return None
+ if any(str(r.get("name") or "").strip() for r in rows.values()):
+ return None
+ return f
+
+
+def _merge_candidates(rows):
+ """C3's merge rule, applied to rows now colliding on `(platform, handle)`.
+
+ Earliest `first_found` wins - latest `last_found` wins - `found_count` SUMS - every other cell
+ takes the first non-blank, preferring the most recently enriched row.
+
+ ⚠ THE SURVIVING ROW KEEPS THE LOWEST ID. Row ids are referenced by saved views, comments and
+ board cards; minting a new one would orphan all of them, so a merge picks a survivor rather
+ than creating one.
+ """
+ groups, keepers = {}, {}
+ for rid in sorted(rows, key=lambda r: (len(str(r)), str(r))):
+ r = rows[rid]
+ h = str(r.get("handle") or "").strip()
+ if not h:
+ # ⛔⛔ THESE USED TO BE `continue`d, AND THE OUTPUT REPLACES THE TABLE'S ROWS — so a
+ # row with no handle was SILENTLY DELETED. Owner, 2026-08-07: *"How come when I add a
+ # manual handle in my 'Beauty influencer leads' database, after an automation run, it
+ # gets deleted?"*
+ #
+ # A row you add by hand is born blank and stays blank until you type into it, so the
+ # window is not narrow — it is every row, from creation until the cell is committed.
+ # And before the profile flag shipped (834decd) a typed handle landed in the TYPIST'S
+ # OVERLAY stratum, leaving the definition row's `handle` empty forever: the row was
+ # dropped even after it looked filled in on screen.
+ #
+ # ⚠ THE FUNCTION'S JOB IS TO MERGE DUPLICATE CANDIDATES, and a row with no handle is
+ # not a candidate — it is somebody's row. It cannot collide with anything (there is
+ # nothing to key it on), so it is carried through UNTOUCHED rather than judged. A
+ # merge pass that deletes what it cannot classify is not a merge, it is a filter.
+ keepers[rid] = r
+ continue
+ groups.setdefault((str(r.get("platform") or PLATFORM_INSTAGRAM).strip(), h),
+ []).append((rid, r))
+ out, merged = dict(keepers), 0
+ for _key, members in groups.items():
+ if len(members) == 1:
+ out[members[0][0]] = members[0][1]
+ continue
+ merged += len(members) - 1
+ # Most-recently-enriched first, so "first non-blank" prefers the freshest measurement.
+ ordered = sorted(members, key=lambda m: str(m[1].get("enriched_at") or ""), reverse=True)
+ keep_id = sorted(rid for rid, _r in members)[0]
+ acc = {}
+ for _rid, r in ordered:
+ for k, v in r.items():
+ if k not in acc and str(v or "").strip():
+ acc[k] = v
+ firsts = [str(r.get("first_found") or "").strip() for _i, r in members if r.get("first_found")]
+ lasts = [str(r.get("last_found") or "").strip() for _i, r in members if r.get("last_found")]
+ if firsts:
+ acc["first_found"] = min(firsts)
+ if lasts:
+ acc["last_found"] = max(lasts)
+ total = sum(_ig_int(r.get("found_count")) or 0 for _i, r in members)
+ if total:
+ acc["found_count"] = str(total)
+ out[keep_id] = acc
+ return out, merged
+
+
+#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER
+#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites
+#: *"we already have a Field called stage… we don't need another checkbox for this"* — and
+#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still
+#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing
+#: writes, nothing reads, and anybody can still click.
+#:
+#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the
+#: record of who approved what before the wave, or leave them and let the next hand-made `tracked`
+#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving
+#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT
+#: as a column nobody declared is the worse half.
+TRACKED_KEY = "tracked"
+
+
+def _retired_tracked_field(fields):
+ """The retired `tracked` column when it is still the MACHINE's own, else None.
+
+ ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that
+ DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a
+ column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED,
+ or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as
+ an ordinary column and merely stops being fed by anything, which was already true.
+
+ ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are
+ dropped unguarded: both are machine-derived cells with no history of anyone editing them.
+ `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test.
+ """
+ f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None)
+ if f is None or f.get("type") not in ("checkbox", "", None):
+ return None
+ if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")):
+ return None
+ return f
+
+
+def _guarded_drops(table, drops):
+ """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81).
+
+ ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check.
+ `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks
+ `_retired_tracked_field` whether the column is still the machine's own, and the schema
+ reconciliation, which drops by KEY with no question asked. Guarding only the first was
+ decorative — the schema pass runs immediately afterwards on the same table and deleted the
+ retyped column the guard had just spared. Two paths to one deletion with one of them checked
+ is the shape where a control looks present and enforces nothing
+ ([[defects-that-mask-each-other]]).
+ """
+ drops = set(drops or ())
+ if TRACKED_KEY in drops:
+ stored = [f for f in (table or {}).get("fields") or []
+ if str(f.get("key") or "") == TRACKED_KEY]
+ if stored and _retired_tracked_field(stored) is None:
+ drops.discard(TRACKED_KEY)
+ return drops
+
+
+def _orphan_tracked_rows(fields, rows):
+ """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject.
+
+ ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE
+ BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose
+ declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every
+ surface that renders from `fields` — and a migration that only looks at `fields` finds nothing
+ to do and `continue`s past the table. Storage, export and any future re-declaration of the key
+ all still see them.
+ """
+ if any(str(f.get("key") or "") == TRACKED_KEY for f in fields):
+ return [] # the column above owns its own cells
+ return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})]
+
+
+def backfill_corpus_snapshots(rt, log=print):
+ """The BACKFILL half: every profile row holding a measurement with no series behind it gets
+ its corpus observation. Returns how many were written.
+
+ ⛔ ONE BUILDER, TWO CALLERS — `run_discover_instagram` for rows arriving now, this for rows
+ that arrived before the forward fix existed. A second row-shape here would be two ideas of
+ what a corpus observation is, and they would disagree on the next promoted column.
+
+ ⚠ THE CONDITION IS "NO SERIES AT ALL", not "no series for this day", and the narrowness is
+ deliberate. A profile that HAS been enriched already owns real observations; synthesising a
+ corpus row dated `last_found` for it could out-rank the exact read in a `latest` rollup and
+ replace a measured number with a corpus one. The gap this closes is a row with NOTHING behind
+ it, which is the only case where a corpus observation can only improve matters.
+
+ ⚠ IDEMPOTENT BY THE SNAPSHOT KEY (`@`), never by a flag — so this is safe on
+ every write, which is what `migrate_ig_tables`' placement requires. After one pass the row has
+ a series, so it stops matching and the scan costs a dict comparison.
+ """
+ snaps = ut_get(rt, IG_SNAPSHOTS_TABLE)
+ if snaps is None:
+ return 0 # no series store — see `append_ig_snapshots` on why we never mint one
+ known = {str((r or {}).get("influencer_key") or "").strip().lower()
+ for r in (snaps.get("rows") or {}).values()}
+ known.discard("")
+ #: A cell counts as a MEASUREMENT if the snapshot schema has somewhere to put it — derived
+ #: from `SNAPSHOT_FIELDS` rather than named, for `_ig_profile_tables`' own reason (D-71: a
+ #: named list goes silently empty the day the schema moves).
+ measured = tuple(fd["key"] for fd in SNAPSHOT_FIELDS
+ if fd["key"] not in _SNAPSHOT_OWN_KEYS
+ and fd.get("type") in ("int", "pct"))
+ incoming = []
+ for table_key in _ig_profile_tables(rt):
+ for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values():
+ handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower()
+ if not handle or handle in known:
+ continue
+ if not any(str((row or {}).get(k) or "").strip() for k in measured):
+ continue # nothing was ever read about this row — there is no observation to record
+ built = corpus_snapshot_row(row)
+ if built:
+ incoming.append(built)
+ known.add(handle) # two profile tables holding one handle write ONE observation
+ return append_ig_snapshots(rt, incoming, log=log)
+
+
+def backfill_post_snapshot_grain(rt, log=print):
+ """Copy `posted_at`/`type` off the POST row onto every measurement missing them.
+
+ ⛔ STRUCTURAL, NEVER A FLAG: the row matches when the snapshot's cell is blank AND the post's
+ is not, so a pass that has already run matches nothing and a post that genuinely has no date
+ is never re-visited on a promise it cannot keep. Same idempotency discipline as the migration
+ around it.
+
+ ⚠ IT FILLS BLANKS AND NEVER OVERWRITES. A snapshot's `posted_at` is what the vendor said WHEN
+ THE MEASUREMENT WAS TAKEN; if a later pull disagrees with the post row, the measurement's own
+ record of the moment is the one to keep — a fact table that lets a dimension rewrite its
+ history is not a fact table.
+ """
+ psnaps = ut_get(rt, IG_POST_SNAPSHOTS_TABLE)
+ posts = ut_get(rt, IG_POSTS_TABLE)
+ if psnaps is None or posts is None:
+ return 0
+ have = {f.get("key") for f in (psnaps.get("fields") or [])}
+ carried = [k for k in ("posted_at", "type") if k in have]
+ if not carried:
+ return 0 # the columns are not declared yet — `ut_ensure` adds them first
+ by_shortcode = {}
+ for row in (posts.get("rows") or {}).values():
+ code = str((row or {}).get("shortcode") or "").strip()
+ if code:
+ by_shortcode[code] = row or {}
+ changes = {}
+ for rid, row in (psnaps.get("rows") or {}).items():
+ post = by_shortcode.get(str((row or {}).get("shortcode") or "").strip())
+ if not post:
+ continue
+ for key in carried:
+ want = str(post.get(key) or "").strip()
+ if want and not str((row or {}).get(key) or "").strip():
+ changes.setdefault(str(rid), {})[key] = _s(want, 200)
+ if not changes:
+ return 0
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ table = cur.get(IG_POST_SNAPSHOTS_TABLE)
+ if table is not None:
+ for rid, values in changes.items():
+ table.setdefault("rows", {}).setdefault(rid, {}).update(values)
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ log(f"[ig-migrate] post-measurement grain: {len(changes)} row(s) gained a post date/kind")
+ return len(changes)
+
+
+#: ⭐⭐ 2026-08-10 (owner: *"retire the old hardcoded method and replace the Instagram database work
+#: with correct Rollup and Link fields"*) — THE PROFILE COLUMNS THAT MAY BECOME ROLLUPS.
+#:
+#: A column qualifies when the SERIES has somewhere to put it: `SNAPSHOT_FIELDS` carries the same
+#: key, so `latest` over `profile_snapshots_link` can re-derive it. That is 27 of the 44 preset
+#: columns — the other 17 are identity/bookkeeping (`handle`, `platform`, `enriched_at`), already
+#: relational (the 12 links and rollups), or have no observation behind them at all
+#: (`location_guess`/`location_confidence`, which WE infer rather than read).
+#:
+#: ⛔⛔ AND "DERIVABLE" IS NOT "SAFE", WHICH IS THE WHOLE REASON THIS IS A PREDICATE AND NOT A LIST.
+#: MEASURED on nurilab before any conversion: a bare `latest` would BLANK 221 filled cells —
+#: `avg_engagement` and `category` lose 62 each — because the discovery reads that produced those
+#: values were never recorded as observations (the hole this same wave closed going forward, in its
+#: historical form: 62 profiles hold exactly ONE observation, the enrichment scrape, dated three
+#: days AFTER the discovery run whose numbers are on their row). A migration that converts on a
+#: NAMED LIST would do that damage on any tenant whose series is thinner than the list's author
+#: assumed.
+#:
+#: ⇒ So the migration converts a column only where it can PROVE, on this tenant's own rows, that
+#: the derived value equals the stored one everywhere. See `_convertible_profile_columns`.
+#: ⚠ Re-runnable by design: as the series fills in, later passes convert more. A column that is not
+#: safe today is not refused forever, it is simply not converted yet.
+IG_DERIVABLE_KEYS = tuple(
+ fd["key"] for fd in PRESET_PROFILE_FIELDS
+ if fd.get("type") not in ("link", "rollup")
+ and fd["key"] not in ("platform", "handle", "enriched_at", "first_found", "last_found",
+ "found_count", "created_by")
+ and fd["key"] in {s["key"] for s in SNAPSHOT_FIELDS})
+
+
+def _derived_profile_bag(key):
+ """The rollup that replaces the mapper's write of ONE profile column.
+
+ ⚠ `where: is_not_empty` IS NOT DECORATION. A bare `latest` returns the newest observation's
+ value INCLUDING ITS BLANK, so a cheap pull that did not read the field would erase what an
+ expensive one learned — the exact asymmetry `upsert_rows`' merge exists to prevent on the
+ scalar side. `where` runs BEFORE the ranking (contract C1), so this reads "the newest
+ observation that ACTUALLY READ this field", which is what the materialised cell meant.
+ MEASURED: it rescues 5 of the 12 columns a bare `latest` would damage; the other 7 need the
+ observation itself, which is why the guard below is a proof and not a hope.
+ """
+ return {"link": "profile_snapshots_link", "field": key, "fn": "latest",
+ "sortBy": "pulled_at", "sortDir": "desc",
+ "where": [{"field": key, "op": "is_not_empty"}], "whereConj": "and"}
+
+
+def _derived_profile_columns(table, snapshots):
+ """Which of `IG_DERIVABLE_KEYS` should be DECLARED as rollups on THIS table.
+
+ = the ones already converted, PLUS the ones whose derived value equals the stored value on
+ every row. Returns keys.
+
+ ⛔⛔ IT MUST RE-ASSERT THE ONES ALREADY CONVERTED, and forgetting that is a one-line revert with
+ a several-hour fuse. `_reconcile_ig_graph_fields` overwrites every contract key it owns —
+ `rollup` included, by design — so a converted column that this function stopped naming would be
+ re-typed back to `int`/`text` on the next reconcile, its bag dropped, and its value re-written
+ by the mapper. The edit would look applied and undo itself later, which is exactly
+ [[preset-unlock-needs-a-custody-stamp]]'s shape. An already-converted column is therefore
+ included unconditionally rather than re-proved: its cells ARE the derived values, so a proof
+ would be comparing the fold against itself.
+
+ ⛔ THE PROOF RUNS PER TENANT, PER COLUMN, AT MIGRATION TIME — never against a list somebody
+ measured on one workspace. `followers` derives exactly on nurilab and could be thin somewhere
+ else; `category` is damaged on nurilab and might be perfect elsewhere. The only honest
+ authority is the data in front of the migration.
+ ⚠ A column with NO stored values anywhere converts too, and that is deliberate rather than an
+ oversight: there is nothing to lose, and leaving it materialised would mean the schema differs
+ between two tenants for no reason a reader could discover.
+ """
+ fields = {str(f.get("key")): f for f in (table or {}).get("fields") or []}
+ rows = (table or {}).get("rows") or {}
+ by_subject = {}
+ for snap in ((snapshots or {}).get("rows") or {}).values():
+ key = str((snap or {}).get("influencer_key") or "").strip().lower()
+ if key:
+ by_subject.setdefault(key, []).append(snap or {})
+ # The engine's own ordering, once per subject — newest first, unrankable rows last.
+ ordered = {}
+ for subject, observations in by_subject.items():
+ keyed = [(o, _sort_key(o.get("pulled_at"), "date")) for o in observations]
+ rankable = [(o, k) for o, k in keyed if k is not None]
+ rankable.sort(key=lambda ok: ok[1], reverse=True)
+ ordered[subject] = [o for o, _k in rankable] + [o for o, k in keyed if k is None]
+ # ⛔ A rollup needs the LINK it folds. A profile table that has not been through the graph
+ # reconcile has no `profile_snapshots_link`, and declaring one there would mint a column whose
+ # link resolves to nothing — a blank cell wearing a configured column's clothes.
+ if "profile_snapshots_link" not in fields:
+ return []
+ out = []
+ for key in IG_DERIVABLE_KEYS:
+ field = fields.get(key)
+ if not field:
+ continue
+ if field.get("type") == "rollup":
+ out.append(key) # already converted — see the note above on why
+ continue
+ declared = str(field.get("type") or "text")
+ safe = True
+ for row in rows.values():
+ stored = str((row or {}).get(key) or "").strip()
+ window = [o for o in ordered.get(
+ str((row or {}).get("handle") or "").strip().lower(), [])
+ if str(o.get(key) or "").strip() != ""] # the `where` guard, applied here
+ derived = str(_rollup_fold("latest", [o.get(key) for o in window], declared)
+ or "").strip()
+ if stored != derived:
+ safe = False
+ break
+ if safe:
+ out.append(key)
+ return out
+
+
+def _as_derived_field(field):
+ """One preset scalar declaration → the same column declared as a rollup over the series."""
+ key = str(field.get("key") or "")
+ return {**{k: v for k, v in field.items() if k not in ("agg",)},
+ "type": "rollup", "rollup": _derived_profile_bag(key)}
+
+
+def migrate_ig_tables(rt, log=print, only=""):
+ """Bring every Instagram Profile and canonical child table onto one current contract.
+
+ Returns counted changes rather than a reassuring line. Profile cells are converted before
+ their type declarations change; then the graph reconciliation adds any missing preset
+ Links/Rollups, repairs child types, stamps locks, adds all reciprocal backlinks, and deletes
+ only retired machine fields.
+
+ `only` narrows to a single table, which is how `ut_ensure` calls it: the migration then rides
+ the WRITE PATH, so a table is brought forward immediately before anything appends to it and no
+ caller has to remember to run anything.
+ """
+ stats = {"tables": 0, "retyped": 0, "converted": 0, "stamped": 0, "merged": 0,
+ # ⭐ 2026-08-07 (owner ruling) — the primary-column half. Counted separately from
+ # `stamped` because they answer different questions: that one is "how many ROWS got a
+ # platform", these are "how many TABLES changed shape".
+ "pinned": 0, "flagged": 0, "droppedName": 0,
+ "droppedRecentPosts": 0, "droppedPostHashtags": 0,
+ "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0}
+ want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS}
+ # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red
+ # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a
+ # first run the table does not exist yet: with the detection bypassed it fell through to the
+ # writer and stamped a stub table with empty fields, clobbering the creation that was the whole
+ # point of the call (the owner stamp and the flow tag went with it). A table that does not
+ # exist, or that is not an Instagram profile table, has nothing to migrate.
+ detected = set(_ig_profile_tables(rt))
+ targets = ([only] if only in detected else []) if only else sorted(detected)
+ for table_key in targets:
+ tbl = ut_get(rt, table_key) or {}
+ fields = [dict(f) for f in tbl.get("fields") or []]
+ rows = {rid: dict(r) for rid, r in (tbl.get("rows") or {}).items()}
+ # ⭐ THE IDEMPOTENCY KEY. Only a column whose STORED type is still the old one is touched,
+ # so the cell conversions below can never run twice on the same value.
+ stale = [f for f in fields
+ if f.get("key") in want and f.get("type") != want[f["key"]]
+ and f.get("type") in ("text", "", None)]
+ stale_keys = {f["key"] for f in stale}
+ has_platform = any(f.get("key") == "platform" for f in fields)
+ needs_stamp = [rid for rid, r in rows.items() if not str(r.get("platform") or "").strip()]
+ # ⭐⭐ 2026-08-07 (owner ruling) — MAKE THE HANDLE THE PRIMARY COLUMN AND THE BINDING.
+ # `ut_ensure` merges by KEY and never updates a field that already exists, so declaring the
+ # two keys on `PRESET_PROFILE_FIELDS` reaches NEW tables only. Every table already in
+ # production needs them stamped here, and that is the whole reason this branch exists.
+ #
+ # ⚠ IDEMPOTENT BY READING THE STORED DECLARATION, never by a marker column — W26's rule,
+ # and it is free here: re-stamping a boolean and a two-key dict is a no-op by nature, which
+ # is exactly why these are safe to run on every write where a value conversion would not be.
+ pin_f = next((f for f in fields if f.get("key") == PRESET_PINNED_KEY), None) \
+ if PRESET_PINNED_KEY else None
+ flag_f = next((f for f in fields if f.get("key") == PRESET_FLAG_KEY), None) \
+ if PRESET_FLAG_KEY else None
+ # ⛔ AT MOST ONE PROFILE COLUMN PER TABLE is the law `user_tables` enforces at both write
+ # doors, and a migration is not exempt from it. If somebody has already flagged another
+ # column on this table, stamping the handle too would leave the table in a state its own
+ # validator refuses — and `_profile_of` returns the FIRST match, so the enrich action would
+ # silently bind to whichever came first in the field list. Leave it alone and pin only.
+ other_profile = next((f for f in fields if f is not flag_f
+ and isinstance(f.get("profile"), dict)), None)
+ want_pin = pin_f is not None and pin_f.get("pinned") is not True
+ want_flag = (flag_f is not None and other_profile is None
+ and not isinstance(flag_f.get("profile"), dict))
+ vestigial = _vestigial_name_field(fields, rows)
+ retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"}
+ if _retired_tracked_field(fields) is not None:
+ retired.add(TRACKED_KEY)
+ orphan_tracked = _orphan_tracked_rows(fields, rows)
+ if (not stale_keys and has_platform and not needs_stamp
+ and not want_pin and not want_flag and vestigial is None and not retired
+ and not orphan_tracked):
+ continue
+ stats["tables"] += 1
+ if want_pin:
+ pin_f["pinned"] = True
+ stats["pinned"] += 1
+ if want_flag:
+ flag_f["profile"] = {"source": PROFILE_SOURCE_IG}
+ stats["flagged"] += 1
+ if vestigial is not None:
+ # ⚠ THE CELLS GO WITH THE COLUMN. A row dict keeping a `name` key whose field no longer
+ # exists is invisible everywhere except the next export, where it reappears as a column
+ # nobody declared. `_vestigial_name_field` has already proven every one of them blank.
+ fields = [f for f in fields if f is not vestigial]
+ for r in rows.values():
+ r.pop("name", None)
+ stats["droppedName"] += 1
+ if retired:
+ fields = [f for f in fields if f.get("key") not in retired]
+ for r in rows.values():
+ for key in retired:
+ r.pop(key, None)
+ stats["droppedRecentPosts"] += int("posts" in retired)
+ stats["droppedPostHashtags"] += int("post_hashtags" in retired)
+ stats["droppedTracked"] += int(TRACKED_KEY in retired)
+ for rid in orphan_tracked:
+ # D-74: a tick whose column was already gone. Counted apart from the column drop
+ # because they answer different questions — "how many tables still declared it" and
+ # "how many rows were still carrying it after the declaration went".
+ rows[rid].pop(TRACKED_KEY, None)
+ stats["droppedTrackedCells"] += 1
+
+ for key in stale_keys:
+ conv = _MIGRATE_CONVERT.get(key)
+ if not conv:
+ continue # int/checkbox already store the right text shape
+ for r in rows.values():
+ if str(r.get(key) or "").strip():
+ new = conv(r[key])
+ # "" from a converter means UNREADABLE, and the cell keeps its original text.
+ # A migration that empties a cell is indistinguishable from one that moved it.
+ if new:
+ r[key], stats["converted"] = new, stats["converted"] + 1
+ for f in stale:
+ f["type"] = want[f["key"]]
+ stats["retyped"] += 1
+ for rid in needs_stamp:
+ rows[rid]["platform"] = PLATFORM_INSTAGRAM
+ stats["stamped"] += 1
+ if not has_platform:
+ fields = [dict(f) for f in CANDIDATE_FIELDS if f["key"] == "platform"] + fields
+ rows, merged = _merge_candidates(rows)
+ stats["merged"] += merged
+
+ def _up(cur, _f=fields, _r=rows):
+ cur = cur or {}
+ t = dict(cur.get(table_key) or {})
+ t["fields"], t["rows"] = _f, _r
+ cur[table_key] = t
+ return cur
+
+ rt.update(UT_STORE_KEY, _up)
+ # The profile loop owns value conversions and deduplication. The schema pass owns the
+ # declarations across ALL related databases and is safe only after those conversions, because
+ # the stored old type is the idempotency key for unit-changing values such as engagement.
+ wanted, drops = _ig_schema_contract(rt)
+ stats["schema"] = int(_reconcile_ig_graph_fields(rt, wanted, drops))
+ # ⛔⛔ A SECOND CONTRACT PASS, AND IT IS NOT BELT-AND-BRACES — WITHOUT IT THIS FUNCTION IS NOT
+ # IDEMPOTENT, which is the one property everything else here is built on.
+ #
+ # The derived overlay (`_derived_profile_columns`) decides by COMPARING the profile cell to the
+ # fold of the series. The pass above REPAIRS THE SERIES — it retypes the child columns and runs
+ # `_IG_RETYPE_CONVERTERS`, which rewrites `avg_engagement` from the vendor's 0-1 fraction to our
+ # 0-100 percent. So on a tenant whose children are stale, the first proof compares a repaired
+ # profile cell against an UNREPAIRED observation, finds them different, and declines to convert
+ # a column that is in fact perfectly derivable. The next call then converts it — and
+ # `verify_automation`'s "a SECOND pass is a no-op" check went red, which is exactly what that
+ # check is for. MEASURED on the wave-26 fixture: 24 columns converted on pass 2, none on pass 1.
+ #
+ # ⚠ The module already knew this shape one level up — the profile loop's own note says the
+ # schema pass "is safe only after those conversions, because the stored old type is the
+ # idempotency key". Same argument, one table further down.
+ # ⚠ CHEAP WHEN THERE IS NOTHING TO DO: on a current tenant the contract recomputes to the same
+ # declarations and `_reconcile_ig_graph_fields` writes nothing. Twice, not looped: the repair is
+ # what moves the data, and it has already run.
+ wanted2, drops2 = _ig_schema_contract(rt)
+ stats["schema"] = int(bool(stats["schema"]
+ | int(_reconcile_ig_graph_fields(rt, wanted2, drops2))))
+ # ⭐⭐ 2026-08-10 — THE TWO VALUE BACKFILLS, and they run AFTER the schema pass on purpose:
+ # `backfill_post_snapshot_grain` writes into columns that pass has just declared, and running
+ # it first would find nothing to fill and report a truthful zero about the wrong moment.
+ # Both are guarded on "is there anything to do" and cost a dict scan in the steady state, which
+ # is what makes them safe on a function that rides every `ut_ensure`.
+ stats["series"] = backfill_corpus_snapshots(rt, log=log)
+ stats["postGrain"] = backfill_post_snapshot_grain(rt, log=log)
+ if stats["tables"] or stats["schema"] or stats["series"] or stats["postGrain"]:
+ log(f"[ig-migrate] {stats}")
+ return stats
+
+
+# ── WAVE 25 · C6 (owner ruling R5) — SEEDING A SEARCH FROM A VIEW OR COHORT. ──────────────────
+# "Find me more accounts like the ones in this view." The server reads the seed rows, ranks their
+# SHARED characteristics, and FILLS IN the discovery conditions — visible and editable, never
+# hidden (R5).
+#
+# ⛔ THE RANKING IS RESTRICTED TWICE, AND BOTH RESTRICTIONS ARE MEASURED RATHER THAN CHOSEN:
+# 1. R5 restricts it to `BD_POPULATED_FIELDS` — a characteristic extracted from a field the
+# corpus barely populates produces a filter that returns nothing and looks exactly like
+# "no such accounts exist" (the trap `BD_FILTER_LEAD` already exists to warn about).
+# 2. A derived predicate must NARROW, or `narrowing_refusal` refuses to save it. The
+# intersection of the two sets is MEASURED at exactly eight fields — `account`, `biography`,
+# `external_url`, `fbid`, `full_name`, `id`, `profile_name`, `profile_url` — and each of them
+# narrows under its own `default_operator`, asserted in the gate rather than assumed.
+#
+# ⭐ AND FOUR OF THOSE EIGHT CANNOT GENERALISE, which is the part worth stating: `account`, `id`,
+# `fbid` and `profile_url` are IDENTITIES. A filter derived from them re-finds the seed rows and
+# nothing else — a search that returns what you gave it, at full price. So the ranker reads only
+# the three that describe a KIND of account: shared words in the bio, a shared domain in the link,
+# and shared words in the name.
+SEED_SOURCES = ("view", "cohort")
+SEED_MAX_ROWS = 200 # bounded and DISCLOSED in `basis.rows` — never a silent [:N]
+SEED_MAX_PREDICATES = 3 # a filter of ten derived guesses is not a filter
+SEED_MIN_COVERAGE = 0.5 # a "shared" characteristic under half the rows share is not shared
+SEED_MIN_ROWS = 2 # one row has nothing to share WITH
+#: ⛔ AND A COVERAGE BAR ALONE IS NOT ENOUGH, which the gate caught rather than review: at two
+#: seed rows, `coverage >= 0.5` is satisfied by a word appearing in exactly ONE of them. So every
+#: bio word in a two-row seed qualified as "shared", and the surface would have offered a
+#: characteristic derived from a single record as the thing those records have in common. A
+#: shared characteristic needs at least two rows to be shared BY — n=1 dressed as a pattern is
+#: the fabrication this module refuses everywhere else.
+SEED_MIN_ROWS_SHARING = 2
+#: Which ROW column feeds which VENDOR field. The row keys are the C1 preset set's; the vendor
+#: names are `BD_FILTER_FIELDS`'. Two vocabularies for one thing, so the mapping is written down.
+SEED_FROM_ROW = (("bio", "biography"), ("external_url", "external_url"),
+ ("full_name", "full_name"))
+#: Words that are shared by everything and therefore discriminate nothing. Deliberately SHORT —
+#: an aggressive list would silently drop a real signal, and the coverage bar already removes most
+#: noise. Anything ≤2 characters is dropped by length instead of by enumeration.
+SEED_STOPWORDS = frozenset({
+ "the", "and", "for", "with", "you", "your", "our", "are", "not", "all", "any", "from",
+ "this", "that", "have", "has", "was", "com", "www", "http", "https", "out", "new", "more",
+ "who", "how", "その", "инст"} | {"official", "welcome", "contact", "email", "dm", "link"})
+
+
+def _seed_tokens(text):
+ """One cell → the distinct words worth matching on. Lowercased, de-punctuated, ≥3 chars."""
+ words = re.findall(r"[a-z0-9]{3,}", str(text or "").lower())
+ return {w for w in words if w not in SEED_STOPWORDS}
+
+
+def _seed_domain(url):
+ """A link-in-bio → its registrable-ish host, or ''. `linktr.ee/x` and `linktr.ee/y` share a
+ domain and that IS the shared characteristic; the path is the thing that differs."""
+ raw = str(url or "").strip()
+ if not raw:
+ return ""
+ host = (urlparse(raw if "//" in raw else "https://" + raw).hostname or "").lower()
+ return host[4:] if host.startswith("www.") else host
+
+
+def seed_predicates(rows):
+ """R5: seed rows → `(derived_predicates, basis)`. PURE — no store, no vendor, no network.
+
+ `basis` is what makes the offer honest rather than magic: how many rows were read, which
+ characteristics were extracted, and **the MEASURED coverage of each**, so a weak signal reads
+ as weak instead of being presented with the same confidence as a strong one.
+
+ ⛔ EVERY DERIVED PREDICATE IS ASSERTED AGAINST `narrowing_refusal` BEFORE IT IS OFFERED
+ (HARD RULE 11). A suggestion the product's own save door then refuses is the post-W24
+ default-versus-guard defect rebuilt — and it would be worse here, because the user did not
+ type it: they would be told their own product's suggestion is invalid.
+
+ ⚠ AN EMPTY ANSWER IS A LEGITIMATE ANSWER. When the seed rows share nothing above the coverage
+ bar, `derived` is `[]` and `basis` says why. An honest nothing beats a filter that looks
+ plausible, returns zero rows, and reads as "no such accounts exist".
+ """
+ rows = [r for r in (rows or []) if isinstance(r, dict)]
+ total = len(rows)
+ basis = {"rows": total, "fields": [],
+ # R5's second lane, reported as the MEASUREMENT it is rather than sold as a feature.
+ # `related_accounts` is 22% populated corpus-wide (D-25) and is not part of the C1
+ # preset set, so on our own seed rows it is usually simply absent — which the surface
+ # must SAY, or a lane that adds nothing reads as a lane that failed.
+ "related": {"tried": total,
+ "found": sum(1 for r in rows
+ if str(r.get("related_accounts") or "").strip())},
+ "note": ""}
+ if total < SEED_MIN_ROWS:
+ basis["note"] = ("a seed needs at least two records — one record has nothing to share "
+ "with anything")
+ return [], basis
+ ranked = []
+ for row_key, vendor_field in SEED_FROM_ROW:
+ counts = {}
+ for r in rows:
+ vals = ({_seed_domain(r.get(row_key))} if vendor_field == "external_url"
+ else _seed_tokens(r.get(row_key)))
+ for v in vals:
+ if v:
+ counts[v] = counts.get(v, 0) + 1
+ for value, n in counts.items():
+ cov = n / total
+ if cov >= SEED_MIN_COVERAGE and n >= SEED_MIN_ROWS_SHARING:
+ ranked.append({"name": vendor_field, "value": value, "coverage": round(cov, 3),
+ "rows": n})
+ # Strongest first; ties broken by the LONGER value, which is the more specific one.
+ ranked.sort(key=lambda f: (-f["coverage"], -len(f["value"]), f["name"]))
+ derived, seen_fields = [], set()
+ for f in ranked:
+ if len(derived) >= SEED_MAX_PREDICATES:
+ break
+ # One predicate per FIELD: two `biography includes` rows AND-ed together narrow to the
+ # accounts carrying both words, which is a much smaller search than the seed implies.
+ if f["name"] in seen_fields:
+ continue
+ p = {"name": f["name"], "operator": default_operator(f["name"]), "value": f["value"]}
+ if not predicate_narrows(p):
+ continue # cannot be offered; it would be refused at the save door
+ seen_fields.add(f["name"])
+ derived.append(p)
+ basis["fields"].append({"name": f["name"], "label": field_label(f["name"]),
+ "value": f["value"], "coverage": f["coverage"]})
+ if derived and narrowing_refusal(derived, "and"):
+ # Belt AND braces: the per-predicate check above should make this unreachable, and it is
+ # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest
+ # empty rather than offering something the save door will reject.
+ basis["note"] = ("the shared characteristics found are not specific enough to search on "
+ "— add a condition of your own")
+ return [], basis
+ if not derived:
+ basis["note"] = basis["note"] or (
+ "these records share no bio words, link domain or name in common — nothing could be "
+ "derived, so the conditions are yours to write")
+ return derived, basis
+
+
+def seed_rows(rt, table_key, view_id=""):
+ """The rows a seed source selects. `(rows, problem)` — bounded, and the bound is DISCLOSED by
+ `basis.rows` rather than silently applied ([[no-unverifiable-aggregates]])."""
+ t = ut_get(rt, str(table_key or ""))
+ if t is None:
+ return [], f"{table_key!r} is not a database in this workspace"
+ rows = list((t.get("rows") or {}).values())
+ if str(view_id or "").strip():
+ # ⛔ THE SAME RESOLVER `enters_view` USES, not a second one. A seed that selected a
+ # different row set from the view it names would be describing a view nobody has.
+ tree, _fields, problem = view_filter(rt, table_key, view_id)
+ if problem:
+ return [], problem
+ rows = [r for r in rows if lane_match(tree, r)]
+ return rows[:SEED_MAX_ROWS], ""
+
+
+def automation_tables(defn):
+ """Every `ut_*` table THIS automation writes into — its config target plus every
+ `create_record` action's table, deduped, order preserved.
+
+ ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition.
+ Split out rather than parameterised because the two questions are different: that one asks
+ "where could a Category value be, anywhere in this tenant" and is deliberately generous; this
+ one asks "which rows has THIS automation already found" and must not reach into a table it
+ does not write, or a nightly search would start excluding another automation's finds.
+ """
+ keys, seen = [], set()
+ cfg = (defn or {}).get("config") or {}
+ for raw in [cfg.get("targetTable"),
+ *(((a or {}).get("config") or {}).get("table")
+ for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]:
+ k = str(raw or "").strip()
+ if k.startswith(UT_PREFIX) and k not in seen:
+ seen.add(k)
+ keys.append(k)
+ return keys
+
+
+def candidate_key(platform, handle):
+ """The discovery upsert identity: `(platform, handle)`, as one string.
+
+ ⭐ WAVE 29 — LIFTED OUT OF `run_discover_instagram`, where it was a closure, because a second
+ runner now needs the same identity. Two closures computing "the same" key is how one of them
+ grows a different default for a blank `platform` and the two networks quietly start sharing
+ rows — which is the exact data-loss shape wave 26's R4 added `platform` to prevent.
+
+ ⚠ A BLANK PLATFORM DEFAULTS TO INSTAGRAM, and that is a FACT about the stored data rather than
+ an assumption: every row written before wave 26 came from the Instagram runner. Blank-keyed
+ rows would fail to match their own re-find and duplicate the whole table on the next run.
+ """
+ return f"{str(platform or PLATFORM_INSTAGRAM).strip()}\n{str(handle or '').strip()}"
+
+
+def already_found_handles(rt, defn, cap=None):
+ """The handles this automation has already written, for the vendor-side exclusion (item 7).
+
+ ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a
+ profile has always been harmless — and never free: the vendor bills for every record it
+ returns, so a nightly search over stable keywords pays again, in full, for the same accounts
+ every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty
+ scout re-found its whole result set on every run. This is the list that stops it, sent as one
+ `not_in` the vendor evaluates BEFORE billing.
+
+ ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would
+ hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords
+ are two questions, and one must not answer with "somebody already looked at that".
+ """
+ cap = BD_EXCLUDE_MAX if cap is None else int(cap)
+ out, seen = [], set()
+ for table_key in automation_tables(defn):
+ for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values():
+ h = str((row or {}).get("handle") or "").strip().lstrip("@").lower()
+ if h and h not in seen:
+ seen.add(h)
+ out.append(h)
+ if len(out) >= cap:
+ return out
+ return out
+
+
+def observed_category_tables(rt):
+ """Every table a Category value could have been written into, DERIVED from the automations
+ this tenant actually has — the default discovery table and the snapshot table are the FLOOR,
+ not the list.
+
+ ⭐ Two authorities, because wave 24 and wave 25 each moved where the target is declared:
+ `config.targetTable` (the discovery runner's own write) and any `create_record` action's
+ `config.table` (R2 — authoritative since wave 25, and the field the Builder's Database picker
+ writes). Both are read, deduped, order preserved so the defaults come first.
+ """
+ keys, seen = [], set()
+
+ def _add(k):
+ k = str(k or "").strip()
+ if k and k.startswith("ut_") and k not in seen:
+ seen.add(k)
+ keys.append(k)
+
+ _add(DISCOVER_TABLE)
+ _add("ut_ig_snapshots")
+ try:
+ for defn in (all_definitions(rt) or {}).values():
+ cfg = (defn or {}).get("config") or {}
+ _add(cfg.get("targetTable"))
+ for act in walk_actions(((defn or {}).get("flow") or {}).get("actions")):
+ _add(((act or {}).get("config") or {}).get("table"))
+ except Exception: # noqa: BLE001
+ # A definition set that cannot be read costs the DERIVED lanes and keeps the defaults —
+ # the same posture as the master lane below: degrade to less vocabulary, never to an error.
+ pass
+ return keys
+
+
+def observed_categories(rt, limit=60):
+ """⭐ DEBT D-59 — the Category values WE HAVE ACTUALLY SEEN, for a combobox that still accepts
+ free text.
+
+ D-59's exit condition, verbatim: *"EITHER derive the options from values we have actually seen
+ (the `category` column on `ut_ig_candidates` + the master store, offered as a combobox that
+ still accepts free text), OR buy a sample large enough to enumerate the real vocabulary. **Not**:
+ transcribe a published taxonomy and hope it lines up."* This is the first branch.
+
+ ⛔ WHY A PUBLISHED TAXONOMY WOULD HAVE BEEN WORSE THAN NO DROPDOWN. A filter on a value the
+ corpus does not use returns zero rows and looks exactly like an honest "no such accounts
+ exist" — so a picker built from Instagram's own category list would mislead precisely when it
+ looked most authoritative. Every option here has been observed on a real row, and each carries
+ its COUNT so a value seen once reads differently from one seen forty times.
+
+ ⚠ IT STAYS A COMBOBOX. These are the values we have seen, not the values that exist — a
+ control that refused anything else would be a second, quieter version of the same lie.
+ """
+ seen = {}
+
+ def _eat(rows, key):
+ for r in rows or []:
+ v = " ".join(str((r or {}).get(key) or "").split())[:60]
+ if v:
+ seen[v] = seen.get(v, 0) + 1
+
+ # ⛔ THE TABLES ARE DERIVED, NOT NAMED — and this is a REGRESSION BY OMISSION that shipped
+ # green. The two constants below were the whole list, which was correct until **wave 25 R2
+ # made the Create record action's `config.table` AUTHORITATIVE**: from that ruling on, a
+ # discovery automation writes wherever the user pointed it, and `ut_ig_candidates` is merely
+ # the DEFAULT nobody keeps. MEASURED on nurilab 2026-08-06 — `ut_ig_candidates` held 0 rows
+ # while the tenant's two real target tables held 20 each, so this function honestly reported
+ # `{options: []}` and the combobox it feeds had nothing to offer. A vocabulary harvester that
+ # names its sources goes quietly empty the moment the product lets a user choose one.
+ for key in observed_category_tables(rt):
+ _eat(((ut_get(rt, key) or {}).get("rows") or {}).values(), "category")
+ try:
+ # ...and the PLATFORM master, which is the whole point of pooling it: a tenant with three
+ # candidates still gets a vocabulary drawn from every profile the platform has captured.
+ import ig_master
+ if ig_master.configured():
+ # ⚠ `_handle().get(SNAP_BUCKET)` — a dict of rows keyed by id. NOT `_upsert()`, which
+ # returns the upsert FUNCTION; calling that and reading `.get(...)` off it answers
+ # None, so the master lane would have degraded to nothing INSIDE the except below and
+ # this would have shipped as a silent no-op with the gate green. Verified against
+ # `ig_master.series_for`, which reads the same bucket the same way.
+ _eat((ig_master._handle().get(ig_master.SNAP_BUCKET) or {}).values(), "category")
+ except Exception: # noqa: BLE001
+ # A master that cannot be read costs the tenant its own values and nothing else.
+ pass
+ return [{"value": v, "count": n}
+ for v, n in sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))[:limit]]
+
+
+def preset_plan(rt, table_key):
+ """C1 / R2b: the preset set diffed against ONE database's CURRENT fields.
+
+ `{table, fields: [{key, label, type, present}], willUse: [...], willCreate: [...]}`
+
+ ⛔ COMPOSED HERE, NOT IN THE CLIENT. The owner's ask is a config panel that says which columns
+ an automation will USE and which it will CREATE — and the only thing that can answer it is
+ whatever holds both lists. A client that diffed the preset set against a table's fields would
+ be a second implementation of `ut_ensure`'s merge rule, free to disagree with the merge that
+ actually runs; it would be wrong precisely when the two lists differ, which is the only case
+ anybody is asking about.
+
+ ⚠ A TABLE THAT DOES NOT EXIST IS NOT AN ERROR — it is the ordinary state at R10's spawn-on-save
+ moment, and the honest answer is "all of them will be created". `present` is a fact about the
+ table, so it reads False for every field rather than the call refusing.
+
+ ⛔ 2026-08-07 — THE PROJECTION IS FOUR KEYS AND DELIBERATELY DROPS EVERY DECLARATION BAG
+ (`link`, `rollup`, `metric`, `profile`, `pinned`). Measured off the live Space the day the
+ relational pair shipped: `posts_link` and the four rollups arrive here with their bags EMPTY.
+ That is CORRECT for this payload — it answers "which columns will be used vs created", where
+ a name and a type are the whole question — and it is safe today because nothing RENDERS a
+ column from this list: the grid reads full field dicts from `/tables` (`scoped_pool` passes
+ `dict(f)` straight through), and the column itself is spawned server-side by `ut_ensure` from
+ `PRESET_PROFILE_FIELDS`, which carries the bag.
+ ⚠ **It stops being safe the moment somebody previews a rollup from this payload** — they would
+ render "Avg views - last 12 posts" configured by nothing. Read the field off `/tables`, or
+ widen this projection deliberately; do not assume a bag is here because the field has one.
+ """
+ have = {str(f.get("key")) for f in ((ut_get(rt, str(table_key or "")) or {}).get("fields")
+ or [])}
+ fields = [{"key": f["key"], "label": f["label"], "type": f.get("type") or "text",
+ "present": f["key"] in have}
+ for f in PRESET_PROFILE_FIELDS]
+ return {"table": str(table_key or ""),
+ "exists": ut_get(rt, str(table_key or "")) is not None,
+ "fields": fields,
+ "willUse": [f for f in fields if f["present"]],
+ "willCreate": [f for f in fields if not f["present"]]}
+
+
+def clean_predicates(raw, operator="and"):
+ """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`.
+
+ Refuses rather than coerces: a predicate naming a field the API rejects comes back as a
+ 400 with the field named, because the alternative — dropping it — turns "find me verified
+ accounts in Georgia" into "find me any account" and bills for the difference.
+ """
+ out = []
+ for p in (raw or [])[:12]:
+ if not isinstance(p, dict):
+ continue
+ name = _s(p.get("name") or p.get("field"), 60).strip()
+ op = _s(p.get("operator") or p.get("op"), 20).strip()
+ if name not in BD_FILTER_FIELDS:
+ return None, ("that field cannot be searched — the searchable ones are: "
+ + ", ".join(field_label(f) for f in BD_FILTER_FIELDS))
+ # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly
+ # because every operator was legal on every field; a yes/no column offering "at least"
+ # is a question with no meaning that the vendor is nevertheless asked.
+ allowed = ops_for(name)
+ if op not in allowed:
+ return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r} — "
+ "it takes: "
+ + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed))
+ entry = {"name": name, "operator": op}
+ if op not in BD_NULLARY_OPS:
+ raw_v = p.get("value")
+ # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and
+ # it is stored as a list rather than as N sibling predicates so the row a person sees
+ # and the row that is stored are the same thing.
+ vals = raw_v if isinstance(raw_v, list) else [raw_v]
+ vals = [v.strip() if isinstance(v, str) else v for v in vals]
+ vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))]
+ if not vals:
+ return None, f"give {field_label(name)} something to compare against"
+ if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS:
+ return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value — "
+ f"{field_label(name)} can only be given a list with "
+ + ", ".join(BD_OP_LABELS[o] for o in ops_for(name)
+ if o in BD_MULTI_VALUE_OPS))
+ if len(vals) > MAX_PREDICATE_VALUES:
+ return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values "
+ "in one condition")
+ if field_kind(name) == "boolean":
+ ok = {o["value"] for o in BD_BOOLEAN_OPTIONS}
+ bad = [v for v in vals if str(v).lower() not in ok]
+ if bad:
+ return None, f"{field_label(name)} is answered Yes or No"
+ # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers
+ # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and
+ # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field
+ # was a free-text box until this wave, so the Yes/No dropdown that makes it easy
+ # to ask is also the thing that would have made every boolean condition fail.
+ # The dropdown's option VALUES stay "true"/"false" (a carries strings);
+ # the conversion belongs here, at the edge that talks to the vendor.
+ vals = [str(v).lower() == "true" for v in vals]
+ if field_kind(name) == "number":
+ try:
+ vals = [float(v) if "." in str(v) else int(v) for v in vals]
+ except (TypeError, ValueError):
+ return None, f"{field_label(name)} takes a number"
+ # A single value stays a SCALAR on the wire. The vendor has only ever been sent
+ # scalars for these operators; the list form is expanded at send time and the
+ # one-value case must not quietly start exercising an untested shape.
+ entry["value"] = vals[0] if len(vals) == 1 else vals
+ out.append(entry)
+ if not out:
+ # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES.
+ # This used to refuse, which was right while a wizard collected the filters before the
+ # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger
+ # CREATES the automation, and the filters are typed afterwards — so a save-time refusal
+ # here meant the trigger could never be picked at all. It is the A3
+ # stored-inert-with-`configured:false` pattern this module already runs on.
+ #
+ # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient:
+ # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a
+ # search, deliberately, because "a stored config can predate the law, and the vendor
+ # bills for breadth whether the filter was saved yesterday or last month". The RUN is the
+ # wall. Every MALFORMED predicate above is still refused here, and the narrowing law
+ # below still refuses a non-empty filter that narrows nothing — only EMPTY changed.
+ return [], None
+ # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the
+ # measured hang this guards against. Same sentence at create/patch and at run.
+ guard_err = narrowing_refusal(out, operator) or depth_refusal(out, operator)
+ if guard_err:
+ return None, guard_err
+ return out, None
+
+
+def discover_estimate(records_limit):
+ """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the
+ section header. Returned as structured data so the UI cannot accidentally drop the caveat."""
+ n = max(0, int(records_limit or 0))
+ return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4),
+ "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC",
+ "note": "estimated from the published rate — an exact price is only known after a run"
+ "a price before a run, and this account's token cannot read a balance"}
+
+
+def _candidate_row(row, stamp):
+ """One corpus row → a `ut_ig_candidates` row. None when it has no handle.
+
+ ⛔ NEVER EMITS `found_count` — it is arithmetic the runner does against what is already
+ stored, so writing it here would reset the counter to 1 on every re-find. (It never emitted
+ `tracked` either; R6 deleted that column in wave 26.)
+ """
+ if not isinstance(row, dict):
+ return None
+ handle = str(_first(row, "account", "username", default="") or "").strip()
+ if not handle:
+ return None
+ return {
+ # ⭐ R5 — half the identity. See `PLATFORM_INSTAGRAM`: this runner only ever reads
+ # Instagram, so it is a constant here rather than something derived from the row; the
+ # TikTok runner will stamp its own and the (platform, handle) key keeps the two apart.
+ "platform": PLATFORM_INSTAGRAM,
+ "handle": handle,
+ "profile_url": str(_first(row, "profile_url", "url",
+ default=f"https://www.instagram.com/{handle}/")),
+ "full_name": str(_first(row, "full_name", "profile_name", default="") or ""),
+ "followers": _s(_ig_int(_first(row, "followers"))),
+ "following": _s(_ig_int(_first(row, "following"))),
+ # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is
+ # the path where it carries values (measured on all five of the 2026-08-04 result set).
+ # ⚠ ×100 since wave 26 (amendment C1-a): the vendor's fraction is not our `pct`.
+ "avg_engagement": _pct100(_first(row, "avg_engagement", default="")),
+ "bio": _s(_first(row, "biography", "bio", default=""), 500),
+ "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300),
+ "verified": "1" if _first(row, "is_verified", default=False) else "",
+ "category": _s(_first(row, "category_name", "business_category_name", default="")),
+ # R3: a DAY, matching the `date` type the column now declares. The full-precision stamp
+ # still exists on the snapshot series, which is where a time axis belongs.
+ "last_found": _day(stamp),
+ }
+
+
+# ── ⭐⭐ 2026-08-10 — DISCOVERY'S MISSING OBSERVATION ──────────────────────────────────────────
+#
+# ⛔ THE DEFECT, MEASURED ON NURILAB BEFORE ANY OF THIS WAS WRITTEN: 105 profiles, 104 carrying a
+# `followers` number, and only 97 with a single row of history behind it. Seven accounts had a
+# measurement nothing could re-derive, date or audit — `19,448 followers`, as of never, read via
+# nothing. `_candidate_row` above writes six MEASUREMENTS onto a profile row (followers,
+# following, engagement, verified, category, bio) and the discovery runner wrote no snapshot at
+# all, so discovery was the one rung in this module that read numbers and recorded no observation.
+#
+# ⛔ IT IS NOT A "MISSING ENRICHMENT". Those seven rows are not waiting to be enriched — they hold
+# real corpus numbers that are already on screen and already filterable. The gap is that the
+# ENTITY row is the only copy, which is the exact arrangement R3's "one store for one series" law
+# exists to forbid one level up.
+#
+# ⚠ A CORPUS ROW IS AN HONEST OBSERVATION, NOT A FAKE MEASUREMENT, and the two columns that make
+# it honest already existed: `source` says **Discovery** (this was read off the vendor's
+# pre-collected corpus, not measured for you) and `approx` is checked. Together they say "do not
+# read this as an exact count taken at `pulled_at`" — which is the whole difference between
+# recording what we know and inventing what we do not.
+#
+# ⭐ AND THE DATE GRAIN MAKES THE TIE-BREAK COME OUT RIGHT, which is worth stating because it
+# looks like an accident: `pulled_at` here is a DAY (`last_found`), so `_sort_key` reads it as
+# MIDNIGHT, while an enrichment on that same day carries a real timestamp. A `latest` rollup over
+# the series therefore prefers the exact read over the corpus read whenever both happened on one
+# day — the ordering you would have to hand-write, falling out of the grain.
+
+#: The `via` a DISCOVERY read reports. `PUBLIC_SOURCE` maps it to the word in the cell.
+IG_VIA_DISCOVERY = "brightdata:discovery"
+
+#: The five keys a snapshot row owns about ITSELF. Everything else it carries is a measurement
+#: copied off the profile row, and the list of those is DERIVED from `SNAPSHOT_FIELDS` rather
+#: than typed out — a hand-list is what silently stops carrying the next promoted column.
+_SNAPSHOT_OWN_KEYS = ("snapshot_key", "influencer_key", "pulled_at", "source", "approx")
+
+
+def corpus_snapshot_row(row, day=""):
+ """ONE profile row read from the corpus → its `ut_ig_snapshots` observation. None if unusable.
+
+ `day` overrides the row's own `last_found`, which is what the BACKFILL passes when a row's
+ only date is `first_found`.
+
+ ⛔ IDEMPOTENT BY ITS KEY, never by a flag or a marker column. `snapshot_key` is
+ `@`, so re-running this over the same rows on the same day rewrites the same key
+ and `upsert_rows` merges it — the migration is safe to call on every write, which is the only
+ way it can be safe to call at all (W26's rule, and the same reason `migrate_ig_tables` keys
+ idempotency on the stored TYPE).
+
+ ⚠ BLANKS ARE OMITTED RATHER THAN WRITTEN. `capture_rows` writes every key including the empty
+ ones because a paid pull's blank means "this rung did not read it" and the row is append-keyed.
+ Here the merge is the hazard instead: a second discovery on the same day returning a thinner
+ corpus record would otherwise ERASE what the first one learned. An absent key renders exactly
+ as an empty cell, so nothing is lost by leaving it out.
+ """
+ handle = str((row or {}).get("handle") or "").strip().lstrip("@").lower()
+ if not handle:
+ return None
+ when = _day(day or (row or {}).get("last_found") or (row or {}).get("first_found") or "")
+ if not when:
+ return None
+ out = {"snapshot_key": f"{handle}@{when}", "influencer_key": handle, "pulled_at": when,
+ "source": _s(public_source(IG_VIA_DISCOVERY)), "approx": "1"}
+ for fd in SNAPSHOT_FIELDS:
+ key = fd["key"]
+ if key in _SNAPSHOT_OWN_KEYS:
+ continue
+ value = (row or {}).get(key)
+ if str(value or "").strip():
+ out[key] = str(value) if key == "source_payload" else _s(value, 400)
+ return out
+
+
+def append_ig_snapshots(rt, incoming, snap_key=IG_SNAPSHOTS_TABLE, log=print):
+ """Merge corpus observations into an EXISTING `ut_ig_snapshots`. Returns rows written.
+
+ ⛔ IT NEVER CREATES THE TABLE, and that refusal is structural rather than cautious: the only
+ honest way to create it is `ensure_ig_graph`, which calls `ut_ensure`, which calls
+ `migrate_ig_tables` — and the BACKFILL caller is inside `migrate_ig_tables`. Reaching for the
+ creator from there is unbounded recursion. A tenant with no series store has nothing to
+ back-fill INTO; the forward path (`run_ig_discovery`) creates the graph properly and this
+ function then has somewhere to write.
+ """
+ rows_in = [r for r in (incoming or []) if r]
+ if not rows_in:
+ return 0
+ existing = ut_get(rt, snap_key)
+ if existing is None:
+ return 0
+ merged, counts = upsert_rows(dict(existing.get("rows") or {}), rows_in, "snapshot_key",
+ cap=row_cap(snap_key))
+ if counts["capped"]:
+ # D-11's law: a full append table means the SERIES has stopped growing, which is the one
+ # failure a chart cannot show you.
+ log(f"[aios-auto] corpus series: {counts['capped']} observation(s) refused by "
+ f"{snap_key}'s row cap")
+ written = counts["inserted"] + counts["updated"]
+ if not written:
+ return 0
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ if cur.get(snap_key) is not None:
+ cur[snap_key]["rows"] = merged
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ return written
+
+
+# ---------------------------------------------------------------------------------------------
+# RUNNERS
+# ---------------------------------------------------------------------------------------------
+
+def cap_note(capped, missing=()):
+ """The sentence(s) a cap breach adds to a run summary.
+
+ `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit.
+ `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not
+ be created at all.
+
+ ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the
+ ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that
+ does not say WHERE is only marginally better than a silent one.
+
+ ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused
+ SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the
+ runners' `if tgt is not None` write guard then skipped that bucket, and the run reported
+ success over a database that does not exist. Exactly the silent-stop failure the row cap
+ already had, one level up — and harder to spot, because the table is not there to look at.
+ """
+ out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} "
+ f"from this run were NOT written" for k, n in (capped or []) if n]
+ out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its "
+ f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it"
+ for k in (missing or [])]
+ return "; ".join(out)
+
+
+def ut_missing(rt, *keys):
+ """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11)."""
+ have = ut_all(rt)
+ return [k for k in keys if k and k not in have]
+
+
+def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None):
+ """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key."""
+ cfg = defn.get("config") or {}
+ fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField")
+ dry = bool(cfg.get("dryRun"))
+ # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from
+ # the rollup — inferring is exactly how a green dot ended up over an empty table (see the
+ # rollup comment in `run_field_instagram`).
+ steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"}
+ step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}")
+ status, final, body = fetch(cfg["url"])
+ if not (200 <= status < 300):
+ steps["fetch"] = "error"
+ return ("error", f"{cfg['url']} answered {status} — nothing was written",
+ {"status": status}, [], steps)
+ steps["fetch"] = "ok"
+ soup = _soup(body)
+ if cfg.get("extract") == "jsonld":
+ blocks = jsonld(soup)
+ flat = []
+ for b in blocks:
+ if isinstance(b, list):
+ flat.extend([x for x in b if isinstance(x, dict)])
+ elif isinstance(b, dict):
+ items = b.get("itemListElement")
+ flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list)
+ else [b])
+ raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat]
+ else:
+ found = tables(soup)
+ idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0
+ raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)]
+ if not raw_rows:
+ steps["extract"] = "error"
+ return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [],
+ steps)
+ steps["extract"] = "ok"
+ step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}")
+
+ incoming = []
+ for r in raw_rows:
+ mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()}
+ if any(v for v in mapped.values()):
+ incoming.append(mapped)
+
+ fields = [field_def(tgt, src if len(src) <= 60 else src[:60])
+ for src, tgt in fmap.items()]
+ # the key column leads, so the table opens on the identity it is upserted by
+ fields.sort(key=lambda f: 0 if f["key"] == key_field else 1)
+ label = cfg.get("targetLabel") or defn.get("name") or "Scraped table"
+ if dry:
+ # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`,
+ # which would create the table: a dry run that leaves a new empty database behind is not
+ # a dry run.
+ table_key = ut_key_for(label, cfg.get("targetTable") or None)
+ else:
+ table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None)
+ t = ut_get(rt, table_key) or {}
+ rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field,
+ cap=row_cap(table_key))
+ touched = [rid for rid, row in rows.items()
+ if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}]
+ if not dry:
+ ut_write_rows(rt, table_key, rows)
+ counts["source_rows"] = len(incoming)
+ affected = touched[:200]
+ summary = (f"{counts['inserted']} new, {counts['updated']} updated, "
+ f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page "
+ f"(kept)")
+ capped = [(table_key, counts.get("capped", 0))]
+ missing = [] if dry else ut_missing(rt, table_key)
+ counts["missing_tables"] = len(missing)
+ note = cap_note(capped, missing)
+ if note:
+ summary += f" — {note}"
+ steps["write"] = "partial"
+ if dry:
+ summary = f"Test run — nothing saved. Would have written: {summary}"
+ steps["write"] = "skipped"
+ elif not note:
+ steps["write"] = "ok"
+ state = "partial" if (counts.get("capped") or counts.get("skipped") or missing
+ or dry) else "ok"
+ log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}")
+ return (state, summary, counts, affected, steps)
+
+
+def capture_rows(res, pulled):
+ """ONE pull → `(snapshot_row, post_identity_rows, post_metric_rows, comment_rows)`.
+
+ ⭐ WAVE 25 (C4) — FACTORED OUT OF `run_field_instagram` SO THE ENRICH ACTION CAN REUSE IT
+ RATHER THAN FORK IT. C4's instruction is literal: "Reuse `pull_profile` and the write path —
+ do not fork them." A second copy of this mapping is the shape that goes wrong invisibly,
+ because the two copies would each be *plausible* and would disagree only on the rows a
+ particular rung happened to return.
+
+ ⚠ THE TWO WRITE DISCIPLINES ARE OPPOSITE HERE, AND BOTH ARE DELIBERATE:
+ * A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written every
+ time and a blank is unambiguous — it means THIS PULL did not read it.
+ * AN IDENTITY ROW OMITS ITS BLANKS, because it is UPSERTED: `posted_at`/`caption`/`type` are
+ readable on some rungs and not others (the Bright Data PROFILE row carries post identity
+ with `datetime: None`, measured 24/24, while the engagement rung knows `date_posted`), and
+ `upsert_rows` writes exactly the keys it is handed — so sending "" would let a cheap run
+ ERASE what an expensive one learned.
+ * AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The post series is
+ the append table that FILLS (maxPosts rows per profile per pull); with the paid engagement
+ rung OFF every one of those rows would carry three blanks — pure noise, eating a 200k
+ ceiling and drawing a chart of nothing. A row in a time series should mean "this was true
+ then"; a row meaning "nobody looked" belongs nowhere.
+ """
+ prof = (res or {}).get("profile") or {}
+ snap_row = {
+ "snapshot_key": f"{prof.get('username')}@{pulled}",
+ "influencer_key": prof.get("username"), "pulled_at": pulled,
+ # ⛔ THE PUBLIC WORD, NOT THE VENDOR KEY — this cell is rendered in a grid column called
+ # "Read via". `res["via"]` keeps the real key for the call graph and the server log.
+ "source": _s(public_source((res or {}).get("via"))), "approx": _s(prof.get("approx")),
+ "bio": _s(prof.get("bio"), 500),
+ "external_url": _s(prof.get("external_url"), 300),
+ }
+ for fd in SNAPSHOT_FIELDS:
+ k = fd["key"]
+ if k not in snap_row:
+ value = prof.get(k)
+ # Profile history declares the same percentage dialect as the current Profile row.
+ # Converting only the latest projection left the append series at 0-1 while its field
+ # now said pct (0-100), so the two views of one measurement disagreed by 100x.
+ if k == "source_payload":
+ snap_row[k] = str(value or "")
+ else:
+ snap_row[k] = _s(_pct100(value) if k == "avg_engagement" else value, 400)
+ idents, metrics_rows, comment_rows = [], [], []
+ for p in (res or {}).get("posts") or []:
+ ident = {"shortcode": p["shortcode"], "influencer_key": prof.get("username"),
+ "url": _s(p.get("url"), 300)}
+ for k, n in (("posted_at", 200), ("type", 200), ("caption", 800),
+ ("paid_partnership", 8), ("partner", 120), ("hashtags", 400),
+ ("alt_text", 400), ("tagged_location", 400)):
+ if p.get(k):
+ ident[k] = _s(p.get(k), n)
+ if p.get("source_payload"):
+ ident["source_payload"] = str(p["source_payload"])
+ metrics = {k: p.get(k) for k in ("likes", "comments", "views", "plays")}
+ # ⭐⭐ 2026-08-07 — THE LATEST ENGAGEMENT VALUES, ONTO THE POST ROW ITSELF.
+ #
+ # This is what keeps a rollup at ONE HOP (Airtable's rule and ours): without it, "average
+ # views over the last 12 posts" would have to walk profile → posts → each post's most
+ # recent snapshot, and a two-hop rollup is a much larger feature with a much worse
+ # invalidation story. The post row now carries its own latest, exactly as the profile row
+ # carries LATEST + `enriched_at` while its series lives in `ut_ig_snapshots` (R3).
+ #
+ # ⛔ ONE STORE FOR ONE SERIES IS UNTOUCHED: `ut_ig_post_snapshots` below is still the
+ # authoritative engagement series and still gets its appended row. These cells are a
+ # projection of the row being appended in the same breath, never a second source.
+ # ⛔ A KEY IS WRITTEN ONLY WHEN THE VENDOR ANSWERED. `upsert_rows` merges, so an absent
+ # key leaves the previous pull's value standing — which is the correct behaviour for a
+ # LATEST column and the reason this cannot be a blanket `_s(...)` over all three: writing
+ # "" on a run that did not buy metrics would ERASE what an earlier paid run learned, the
+ # same failure the snapshot-append rule above is written against.
+ measured = {k: v for k, v in metrics.items() if v is not None and str(v).strip() != ""}
+ if measured:
+ ident.update({k: _s(v, 40) for k, v in measured.items()})
+ # The stamp is what makes the numbers readable: a blank `plays` beside a
+ # `measured_at` of last week means "we looked and the vendor had nothing", and with
+ # no stamp it means that AND "we never looked", indistinguishably.
+ ident["measured_at"] = _day(pulled)
+ idents.append(ident)
+ if any(v is not None for v in metrics.values()):
+ metrics_rows.append({
+ "post_snapshot_key": f"{p['shortcode']}@{pulled}",
+ "shortcode": p["shortcode"], "influencer_key": prof.get("username"),
+ "pulled_at": pulled,
+ # The two denormalised post facts (see `POST_SNAPSHOT_FIELDS`). Written from the
+ # SAME vendor record the identity row above is built from, so they cannot disagree
+ # with it on this pull.
+ # ⚠ BLANK IS EXPECTED AND IS NOT A BUG on the profile rung: `wave20-split`
+ # measured `datetime` as None on 24/24 posts from the Profiles dataset, so a pull
+ # that learns a post's engagement often does not learn its date in the same
+ # breath. `backfill_post_snapshot_grain` fills those from the post row, which by
+ # then may have learned it from a different rung.
+ "posted_at": _s(p.get("posted_at"), 200), "type": _s(p.get("type"), 200),
+ "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]),
+ "views": _s(metrics["views"]), "plays": _s(metrics["plays"]),
+ "source_payload": str(p.get("source_payload") or "")})
+ for comment in p.get("embedded_comments") or []:
+ # The embedded shape often names only the commenter. Its parent Post is the
+ # authoritative owner, so bind every preview to this profile and post explicitly.
+ comment_rows.append({**comment, "shortcode": p["shortcode"],
+ "influencer_key": prof.get("username") or ""})
+ comment_rows.extend((res or {}).get("comments") or [])
+ # The same comment can be returned as both `latest_comments` and `top_comments`, or by the
+ # optional full endpoint after an embedded preview. The canonical key is the one row truth.
+ unique_comments = {}
+ for row in comment_rows:
+ key = str((row or {}).get("comment_key") or "")
+ if key:
+ unique_comments[key] = {**(unique_comments.get(key) or {}), **row}
+ return snap_row, idents, metrics_rows, list(unique_comments.values())
+
+
+#: C4 — how ONE pulled profile becomes the C1 preset CELLS on the record being enriched.
+#: `profile key -> preset column key`, so the mapping is a table rather than sixteen lines of
+#: `row[...] = prof.get(...)` that a future field addition can silently miss.
+#: ⚠ Keys absent from a pull are simply not written (R3: a blank preset cell means "this pull did
+#: not read it", and `enriched_at` is what distinguishes that from "never enriched").
+PRESET_FROM_PROFILE = {
+ "username": "handle", "profile_url": "profile_url", "full_name": "full_name",
+ "followers": "followers", "following": "following", "avg_engagement": "avg_engagement",
+ "bio": "bio", "external_url": "external_url", "verified": "verified",
+ "category": "category", "posts_count": "posts_count",
+ "highlights_count": "highlights_count", "is_business": "is_business",
+ "is_professional": "is_professional", "ig_id": "ig_id",
+ # ⭐ 2026-08-07 — the promoted fields. A preset column with no row in this table is a column
+ # that can only ever be blank, so "make the preset fields populated" is THIS half of the
+ # owner's instruction and the field list is only the other half.
+ # ⚠ The two lists are held in step by a derived gate rather than by care: every
+ # `PRESET_PROFILE_KEYS` entry must either be written by this map or be explicitly declared
+ # as written elsewhere (`platform`, `enriched_at`, `posts` are stamped by `preset_cells`).
+ "business_category": "business_category", "is_private": "is_private",
+ "bio_hashtags": "bio_hashtags",
+ "pronouns": "pronouns", "profile_name": "profile_name",
+ "is_joined_recently": "is_joined_recently", "has_channel": "has_channel",
+ "partner_id": "partner_id", "external_url_title": "external_url_title",
+ "fbid": "fbid", "related_accounts": "related_accounts",
+ "country_code": "country_code", "source_payload": "source_payload",
+}
+
+#: ⭐ 2026-08-07 — the preset keys `PRESET_FROM_PROFILE` deliberately does NOT carry, because a
+#: different writer stamps them. DERIVED gates compare the two lists, and without this the gate
+#: could only be written as "these three are fine" — a hard-coded exception list, which is the
+#: shape [[gate-answers-the-wrong-question]] warns about.
+#: ⚠ The five relational columns are written by `compute_relation_cells` on the tick, NOT by an
+#: enrichment run — which is the whole point of them: they stay true when the LINKED table
+#: changes, and a pull that touched no profile still updates a profile's post count.
+PRESET_WRITTEN_ELSEWHERE = (
+ "platform", "handle", "enriched_at",
+ "posts_link", "profile_snapshots_link", "post_snapshots_link", "comments_link",
+ "avg_views_12", "avg_plays_12", "avg_likes_12", "avg_comments_12",
+ "posts_captured", "profile_reads", "post_measurements_captured", "comments_captured",
+)
+
+
+#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a
+#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING`
+#: refuses on the discovery side for exactly the same reason.
+LOCATION_MIN_POSTS = 2
+
+
+def _post_place(post):
+ """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload
+ second.
+
+ ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the
+ vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the
+ `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details`
+ shape. The complete vendor row is retained in `source_payload` on the same record, already
+ paid for, so a post whose normalised field came back blank can still be read here.
+ ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region,
+ country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to
+ "where is this person"; "Indonesia" mostly is not.
+ """
+ if not isinstance(post, dict):
+ return ""
+ raw = str(post.get("tagged_location") or "").strip()
+ if not raw:
+ try:
+ payload = json.loads(post.get("source_payload") or "{}")
+ except Exception: # noqa: BLE001
+ payload = {}
+ raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else ""
+ return raw.split(",")[0].strip()
+
+
+def location_guess(posts):
+ """`(city, confidence_pct)` from the places a creator's own posts were tagged in.
+
+ `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not
+ the share of ALL posts, because a creator who geotags three of twelve is telling us about
+ three, and dividing by twelve would report a real signal as weak. How thin the evidence is
+ stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two.
+ """
+ places = [p for p in (_post_place(x) for x in posts or []) if p]
+ if len(places) < LOCATION_MIN_POSTS:
+ return "", None
+ counts = {}
+ for p in places:
+ counts[p] = counts.get(p, 0) + 1
+ # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be
+ # resolved alphabetically, which is an arbitrary answer wearing a confident number.
+ best = max(places, key=lambda p: (counts[p], -places.index(p)))
+ return best, round(100.0 * counts[best] / len(places), 1)
+
+
+def preset_cells(res, pulled):
+ """C4/R3: one pull → the LATEST-value cells written onto the enriched record.
+
+ ⛔ `enriched_at` IS ALWAYS WRITTEN when a pull succeeded, and it is the field that makes the
+ other fifteen readable: without it a blank `followers` means both "never enriched" and
+ "enriched in March and the vendor had nothing", and no cell on the row can tell them apart.
+ """
+ prof = (res or {}).get("profile") or {}
+ cells = {}
+ for src, dest in PRESET_FROM_PROFILE.items():
+ v = prof.get(src)
+ if v is not None and v != "":
+ # ⚠ WAVE 26 / C1-a — the ONE key whose value is not carried straight across. The
+ # vendor's engagement is a 0–1 fraction and our `pct` column is 0–100, so a
+ # straight-through copy here would write the same wrong number the discovery path
+ # used to write. Both writers convert; neither is the exception.
+ cells[dest] = (_pct100(v) if dest == "avg_engagement" else
+ (str(v) if dest == "source_payload" else _s(v, 500)))
+ # ⭐ R5 — stamped by the writer, not carried from the pull: this function only ever sees an
+ # Instagram profile. It matters on an ENRICH of a row somebody typed by hand, which may have
+ # arrived with no platform at all — and a blank half of the dedup key is how one account
+ # becomes two rows.
+ cells["platform"] = PLATFORM_INSTAGRAM
+ # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when
+ # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose
+ # twelve posts happened to carry no geotag would make the column worse the more it ran.
+ place, confidence = location_guess((res or {}).get("posts"))
+ if place:
+ cells["location_guess"] = _s(place, 120)
+ cells["location_confidence"] = str(confidence)
+ # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a
+ # question in days; the full stamp keeps its precision on the snapshot series.
+ cells["enriched_at"] = _day(pulled)
+ return cells
+
+
+def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step,
+ rows=None):
+ """Automation #2 (R7): for every row of a database that carries a profile URL, pull the public
+ profile through the paid capability chain, write a status string into the automation column,
+ and append a timestamped row to each of the three IG tables.
+
+ ⭐ WAVE 28 / R5 — there is no tier and no rung choice here any more. `pull_profile` routes per
+ CAPABILITY and reports `blocked` rather than downgrading to an approximate row, so the two
+ "which rung answered" counters this function used to keep have nothing left to distinguish."""
+ cfg = defn.get("config") or {}
+ table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey")
+ post_metrics = bool(cfg.get("postMetrics"))
+ comment_metrics = bool(cfg.get("commentMetrics"))
+ dry = bool(cfg.get("dryRun"))
+ # ⚠ THE STEP KEYS ARE THE CANVAS NODE IDS (contract C3) and the two must move together — a
+ # status written under a node id `graph()` no longer emits is a dot nothing renders, which is
+ # indistinguishable from a step that never ran.
+ steps = {"trigger": "ok", "source": "idle",
+ "capture_posts": "idle" if post_metrics else "skipped",
+ "capture_comments": "idle" if comment_metrics else "skipped",
+ "write": "idle"}
+ t = ut_get(rt, table_key)
+ if not t:
+ steps["source"] = "error"
+ return ("error", f"{table_key} is not a database in this workspace", {}, [], steps)
+ url_field = cfg.get("urlField") or _auto_url_field(t, fkey)
+ if not url_field:
+ steps["source"] = "error"
+ return ("error", "the automation column has no URL field bound to it", {}, [], steps)
+ steps["source"] = "ok"
+ rows = dict(t.get("rows") or {})
+ # the relational tables the pull lands in (R7): every row timestamped for time-range filters
+ if dry:
+ # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.)
+ snap_key, post_key, ps_key, comment_key = (IG_SNAPSHOTS_TABLE, IG_POSTS_TABLE,
+ IG_POST_SNAPSHOTS_TABLE, IG_COMMENTS_TABLE)
+ else:
+ graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
+ profile_table=table_key)
+ snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
+ graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
+ missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key, comment_key)
+ snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {})
+ posts = dict((ut_get(rt, post_key) or {}).get("rows") or {})
+ psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {})
+ comments = dict((ut_get(rt, comment_key) or {}).get("rows") or {})
+
+ counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0,
+ "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, "comment_rows": 0,
+ "missing_tables": len(missing)}
+ cells, affected, notes = {}, [], []
+ pending_metrics = []
+ # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows`
+ # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows.
+ # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation
+ # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.)
+ in_snaps, in_posts, in_psnaps, in_comments = [], [], [], []
+ targets = [(rid, str(r.get(url_field, "") or "").strip())
+ for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()]
+ for i, (rid, url) in enumerate(targets):
+ if i:
+ time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7)
+ # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile
+ # column automation blocks for minutes by design. A counting step is the difference
+ # between "working through them" and "stuck".
+ step(f"Capturing profile {i + 1} of {len(targets)}")
+ counts["profiles"] += 1
+ res = pull_profile(url, max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL, log=log,
+ post_metrics=post_metrics, comment_metrics=comment_metrics,
+ pending_metrics=(pending_metrics if not dry else None))
+ pulled = _iso()
+ via = res.get("via") or ""
+ read_ok = res["state"] in ("ok", "partial")
+ # ⚠ `counts["paid"]` SURVIVES R5 AND IT IS NOT THE RETIRED TIER. It counts profiles a PAID
+ # vendor actually answered, which is the run's spend report — the thing the owner reads to
+ # reconcile a bill. What died is the free rung it used to be contrasted with, so the test
+ # is now simply "did a vendor answer" rather than "did the vendor we were told to try".
+ if read_ok and via in ("brightdata", "apify"):
+ counts["paid"] += 1
+ if read_ok:
+ counts["ok" if res["state"] == "ok" else "partial"] += 1
+ prof = res["profile"]
+ snap_row, ident_rows, metric_rows, captured_comments = capture_rows(res, pulled)
+ in_snaps.append(snap_row)
+ in_posts.extend(ident_rows)
+ in_psnaps.extend(metric_rows)
+ in_comments.extend(captured_comments)
+ counts["metrics"] += len(metric_rows)
+ counts["comment_rows"] += len(captured_comments)
+ counts["posts"] += len(res.get("posts") or [])
+ detail = f"{prof.get('followers') or '?'} followers"
+ if prof.get("approx"):
+ detail += " (approx)"
+ if res.get("posts"):
+ detail += f", {len(res['posts'])} posts"
+ elif res["state"] == "partial":
+ detail += ", posts not readable on the rung that answered"
+ cells[rid] = f"{res['state']} · {_stamp()} · {detail}"
+ elif res["state"] == "blocked":
+ counts["blocked"] += 1
+ cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}"
+ notes.append(res.get("note") or "blocked")
+ else:
+ counts["error"] += 1
+ cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}"
+ notes.append(res.get("note") or "error")
+ affected.append(rid)
+
+ # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows.
+ snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key))
+ posts, collapsed_posts = dedupe_canonical_rows(posts, "shortcode", newest_by="measured_at")
+ posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key))
+ c_post["duplicates"] += collapsed_posts
+ psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key))
+ comments, collapsed_comments = dedupe_canonical_rows(comments, "comment_key")
+ comments, c_comments = upsert_rows(comments, in_comments, "comment_key", cap=row_cap(comment_key))
+ c_comments["duplicates"] += collapsed_comments
+ counts["new_posts"] = c_post["inserted"]
+ capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]),
+ (ps_key, c_ps["capped"]), (comment_key, c_comments["capped"])]
+ counts["capped"] = sum(n for _k, n in capped)
+
+ if not dry:
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ tt = cur.get(table_key)
+ if tt is not None:
+ for rid, val in cells.items():
+ tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val
+ for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
+ (comment_key, comments)):
+ tgt = cur.get(k)
+ if tgt is not None:
+ tgt["rows"] = rws
+ _refresh_relations_inplace(cur, log=log)
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables
+ queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""), pending_metrics)
+ if queued:
+ counts["metric_batches_pending"] = queued
+
+ # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated:
+ # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment),
+ # `error` a LOUD partial — a pooled history with silent holes is worse than none.
+ master_note = ""
+ if not dry and (in_snaps or in_posts or in_psnaps):
+ import ig_master
+ m_status, m_note = ig_master.append_run(getattr(rt, "key", ""),
+ in_snaps, in_posts, in_psnaps)
+ if m_status == "error":
+ master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \
+ f"is complete and the next run re-appends"
+ counts["master_failed"] = 1
+ elif m_status == "ok":
+ counts["master"] = len(in_snaps) + len(in_psnaps)
+
+ # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while
+ # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail
+ # showed a green dot over a table with no posts in it. A summary that disagrees with the
+ # cells it summarises is the failure this whole module's honest-status rule exists to
+ # prevent, so `partial` now propagates. Measured 2026-08-04.
+ read = counts["ok"] + counts["partial"]
+ if not read:
+ state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok"
+ elif counts["blocked"] or counts["error"] or counts["partial"]:
+ state = "partial"
+ else:
+ state = "ok"
+ if counts.get("metric_batches_pending"):
+ state = "partial"
+ # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`)
+ if not targets:
+ cap_state = "idle"
+ elif not read:
+ cap_state = "error" if counts["error"] else "blocked"
+ else:
+ cap_state = "partial" if (counts["blocked"] or counts["error"]
+ or counts["partial"]) else "ok"
+ # ⭐ C3 — THE CAPTURE FORK IS GONE, SO ITS OUTCOME LANDS ON `source`. There is one way to
+ # read a profile now, and the node that names the profile set is the honest owner of "did
+ # reading them work". The old `capture`/`capture_paid`/`capture_anon` trio described a branch
+ # that no longer exists in behaviour OR on screen.
+ steps["source"] = cap_state if targets else steps["source"]
+ # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually
+ # appended. "It was switched on" is not the same fact as "it answered", and painting the
+ # second from the first is the fabrication `node_status` refuses to make.
+ # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A
+ # run that found no posts to enrich did not have a rung refuse it — nothing was requested —
+ # so that reads `skipped`, the same word an off switch earns.
+ steps["capture_posts"] = ("ok" if counts["metrics"]
+ else "partial" if counts.get("metric_batches_pending")
+ else "blocked" if (post_metrics and counts["posts"])
+ else "skipped")
+ steps["capture_comments"] = ("ok" if counts.get("comments")
+ else "blocked" if (comment_metrics and counts["posts"])
+ else "skipped")
+
+ summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts "
+ f"({counts['new_posts']} new)")
+ if counts.get("metric_batches_pending"):
+ summary += (f", {counts['metric_batches_pending']} post-engagement batch"
+ f"{'' if counts['metric_batches_pending'] == 1 else 'es'} still building "
+ "(collected automatically)")
+ if counts["paid"]:
+ summary += f", {counts['paid']} with exact counts"
+ if counts["metrics"]:
+ summary += f", {counts['metrics']} post engagement snapshots"
+ elif post_metrics and counts["posts"] and not counts.get("metric_batches_pending"):
+ # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there
+ # was no engagement", which is a claim about Instagram rather than about our run.
+ summary += ", no post engagement was readable"
+ if counts["comment_rows"]:
+ summary += f", {counts['comment_rows']} comment rows captured"
+ elif comment_metrics and counts["posts"]:
+ summary += ", no comment engagement was readable"
+ if counts["partial"]:
+ summary += f", {counts['partial']} profile-only (posts not readable)"
+ if counts["blocked"]:
+ summary += f", {counts['blocked']} blocked"
+ note = cap_note(capped, missing)
+ if note:
+ summary += f" — {note}"
+ state = "partial" if state != "error" else state
+ steps["write"] = "partial"
+ elif not dry:
+ steps["write"] = "ok"
+ if master_note:
+ summary += f" — {master_note}"
+ state = "partial" if state != "error" else state
+ steps["write"] = "partial"
+ if dry:
+ summary = f"Test run — nothing saved. Would have written: {summary}"
+ steps["write"] = "skipped"
+ state = "partial" if state != "error" else state
+ if notes:
+ summary += f" — {notes[0][:120]}"
+ if not dry:
+ # C7: the fresh master rows are exactly what the table's metric cells summarise.
+ try:
+ compute_metric_cells(rt, table_key)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}")
+ return (state, summary, counts, affected, steps)
+
+
+def _auto_url_field(table, fkey):
+ """The URL field an automation column is bound to: its own `automation.urlField`, else the
+ first url-typed column on the table. Never guessed silently — the caller reports which."""
+ for f in table.get("fields") or []:
+ if f.get("key") == fkey:
+ bound = ((f.get("automation") or {}).get("urlField") or "").strip()
+ if bound:
+ return bound
+ for f in table.get("fields") or []:
+ if f.get("type") == "url":
+ return f.get("key")
+ return ""
+
+
+def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_step,
+ rows=None):
+ """Automation #3 (owner ruling R7 / D-23): FIND handles nobody here has typed in.
+
+ Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus
+ query and hands its snapshot id to the next run, or COLLECTS one a previous run started.
+ Every outcome is a sentence about what actually happened; none of them is a green zero.
+ """
+ cfg = defn.get("config") or {}
+ dry = bool(cfg.get("dryRun"))
+ limit = int(cfg.get("recordsLimit") or 5)
+ preds = list(cfg.get("predicates") or [])
+ auto_id = str(defn.get("id") or "")
+ steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"}
+ counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0,
+ "missing_tables": 0}
+
+ if not bd_ready():
+ # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is
+ # the vendor's — so this is where the run stops, saying which env var is missing.
+ steps["find"] = "blocked"
+ return ("partial", "Profile search is not set up yet — nothing was searched and nothing "
+ "was charged", counts, [], steps)
+
+ pending = str((defn.get("state") or {}).get("pendingSnapshot") or "")
+ if not pending:
+ # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and
+ # the vendor bills for breadth whether the filter was saved yesterday or last month.
+ # D-68: the SHAPE guard rides beside the breadth guard, at save AND at run - a config
+ # stored before this check existed must not reach the vendor either.
+ guard_err = (narrowing_refusal(preds, cfg.get("operator"))
+ or depth_refusal(preds, cfg.get("operator")))
+ if guard_err:
+ steps["find"] = "blocked"
+ return ("error", f"the search was not started — {guard_err}", counts, [], steps)
+ if dry:
+ est = discover_estimate(limit)
+ steps["find"] = steps["write"] = "skipped"
+ return ("partial",
+ f"Test run — nothing saved. Would look for up to {limit} "
+ f"profile{'' if limit == 1 else 's'} matching "
+ f"{_predicate_sentence(preds, cfg.get('operator'))}, for about "
+ f"${est['usd']}", counts, [], steps)
+ step("Starting the search")
+ # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never
+ # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the
+ # vendor refuses. Reported either way, because "why did this run find fewer" and "why did
+ # this run cost the same as last night" are both questions this line answers.
+ excl = {}
+ sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit,
+ exclude_handles=already_found_handles(rt, defn),
+ applied=excl)
+ if excl.get("excluded"):
+ step(f"excluded {excl['excluded']} already-found "
+ f"profile{'' if excl['excluded'] == 1 else 's'} at the provider")
+ elif excl.get("dropped"):
+ step(f"searching without the already-found list — {excl['dropped']}")
+ if note:
+ steps["find"] = "blocked"
+ return ("error", f"the search was not started — {note}", counts, [], steps)
+ pending = sid
+ # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes
+ # through `rt.update`, which can RAISE on an unavailable store — and that exception
+ # becomes an error run, so a `log()` placed after it never executes. The snapshot would
+ # then exist, be billed, and have its id in neither the store nor the logs. Logging first
+ # means the worst case is still recoverable by a human reading the Space output.
+ log(f"[aios-auto] discover {auto_id}: started {sid}")
+ # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop
+ # existing because this process dies thirty seconds later, and a lost id is a set we
+ # paid attention to and can never collect.
+ set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()})
+ steps["find"] = "ok"
+
+ waited, status, size, note = 0.0, "", 0, ""
+ while True:
+ # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to
+ # BD_FILTER_WAIT (120 s) at the vendor, and D measured that this wait IS the whole of
+ # "Run once now is laggy / looks stuck". A counter here is what makes a legitimate
+ # two-minute wait distinguishable from a hung thread; without it the rendered step reads
+ # the same word for both, which is a progress indicator that cannot indicate progress.
+ step(f"Searching — {int(waited)}s of "
+ f"{int(BD_FILTER_WAIT)}s")
+ status, size, note = bd_filter_status(pending)
+ if note:
+ steps["collect"] = "error"
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ return ("error", note, counts, [], steps)
+ if status != "building" or waited >= BD_FILTER_WAIT:
+ break
+ time.sleep(BD_FILTER_POLL)
+ waited += BD_FILTER_POLL
+ step("Reading the profiles the search found")
+
+ if status == "building":
+ # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20
+ # minutes; holding a worker thread that long on a free-tier container to poll would be
+ # the wrong shape. Say plainly that it is still running and who collects it.
+ steps["collect"] = "partial"
+ # ⚠ THE SENTENCE NAMES WHO COLLECTS IT, because the old one did not and that is what made
+ # this read as "stuck": *"The next run picks up the results"* is a promise about a run
+ # that, on a MANUAL automation, nobody had scheduled. `pending_collect_ids` makes the tick
+ # finish it, so the promise is now kept by something rather than by the reader.
+ mins = 0
+ try:
+ since = (defn.get("state") or {}).get("pendingSince")
+ if since:
+ mins = max(0, int((_dt.datetime.now(_dt.timezone.utc)
+ - _dt.datetime.fromisoformat(str(since))).total_seconds()
+ // 60))
+ except Exception: # noqa: BLE001
+ mins = 0
+ been = f" — {mins} min so far" if mins else ""
+ # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by
+ # naming the corpus size: *"the wait is the SEARCH over 620 million profiles, not the 10
+ # rows"*. It was TRUE, and it was answering a question the person had not asked. Somebody
+ # watching a run that has taken eighteen minutes wants to know whether it is broken and
+ # when it ends; how big the index is, is our fact about our supplier. Keeping it also
+ # meant this line was the one place a vendor's catalogue size was quoted to a customer, so
+ # it would have gone stale the day the vendor grew — a number nobody would think to
+ # re-measure because nothing depends on it.
+ # ⚠ THE SHAPE OF THE ANSWER SURVIVES and is the part that mattered: the wait does not
+ # shrink when you ask for fewer records, and a person who does not know that reads
+ # "20 minutes for 10 rows" as a fault.
+ return ("partial",
+ f"Still searching at the provider{been}. A corpus search takes about 20 minutes "
+ f"however few records you asked for. Nothing is lost, you are not charged twice, "
+ f"and the results are collected automatically as soon as they are ready",
+ counts, [], steps)
+ if status == "empty":
+ # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword
+ # that is too specific, so it says what to do about it instead of reporting a fault.
+ steps["collect"] = "ok"
+ steps["write"] = "skipped"
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ return ("partial", "No profiles matched. Try a shorter or more common keyword — "
+ "'floral' finds more accounts than 'floral design studio'",
+ counts, [], steps)
+ if status != "ready":
+ steps["collect"] = "error"
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ return ("error", f"the search ended as {status or 'unreadable'} and returned nothing",
+ counts, [], steps)
+
+ rows, dnote = bd_filter_rows(pending)
+ if dnote:
+ # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it.
+ steps["collect"] = "partial"
+ return ("partial", f"the search finished with {size} match"
+ f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps)
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ steps["collect"] = "ok"
+
+ stamp = _iso()
+ incoming = [c for c in (_candidate_row(r, stamp) for r in rows) if c]
+ counts["found"] = len(incoming)
+ # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and
+ # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to
+ # "the vendor delivered 3". It also catches the shape where an error document comes back
+ # down the rows path and parses as one unusable row instead of an error.
+ counts["dropped"] = max(0, len(rows) - len(incoming))
+ label = cfg.get("targetLabel") or DISCOVER_LABEL
+ table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry
+ else ut_ensure(rt, label, CANDIDATE_FIELDS, username,
+ key=cfg.get("targetTable") or DISCOVER_TABLE,
+ flow_tag=str(defn.get("id") or ""), lock_fields=True))
+ existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
+ # --- ⭐ WAVE 26 · C3 / owner ruling R4 — THE UPSERT KEY IS `(platform, handle)`.
+ #
+ # ⛔ THIS RETIRES WAVE 22's C6 COMPOUND `(handle, created_by)`, deliberately, and the reasoning
+ # that built it is worth stating before it is discarded rather than after: per-user rows gave
+ # each person their own `found_count` and their own review card, so two colleagues scouting the
+ # same market never edited each other's work. Owner, 2026-08-06: one profile is ONE row. The
+ # tenant is the unit, not the user. `created_by` survives as "Found by" — informational, and no
+ # longer part of the identity.
+ #
+ # ⚠ AND THE OWNER ADDED THE GUARD THAT MAKES IT SAFE, WHICH IS THE HALF THAT WOULD HAVE BEEN
+ # MISSED: a handle is only unique WITHIN a network. `@inayma` on Instagram and `@inayma` on
+ # TikTok are routinely different people, so keying on the handle ALONE would have silently
+ # merged two accounts into one row the day the TikTok runner ships (D-9) — a data-loss bug with
+ # no error, discovered months later by someone wondering why a creator's followers halved.
+ # Hence `platform`, and hence it being part of the key rather than a label beside it.
+ _ck = candidate_key
+
+ # ⚠ A PRE-WAVE-26 ROW HAS NO `platform` CELL, and `_ck` defaults it to Instagram rather than to
+ # blank on purpose: every row that exists today was written by the Instagram runner, so that
+ # default is a FACT about the stored data, not an assumption. Blank-keyed rows would fail to
+ # match their own re-find and duplicate the whole table on the next run.
+ for r in existing.values():
+ if not str(r.get("platform") or "").strip():
+ r["platform"] = PLATFORM_INSTAGRAM
+ existing2 = {rid: {**r, "_ckey": _ck(r.get("platform"), r.get("handle"))}
+ for rid, r in existing.items()}
+ # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding
+ # a profile is the signal that it keeps matching, so the counter grows instead of
+ # resetting. `first_found` is written only when the pair is new, so it never moves.
+ seen = {r["_ckey"]: r for r in existing2.values()}
+ for c in incoming:
+ # "Found by" — the first finder is recorded and later finders do not overwrite them, which
+ # is what the column means now that it is no longer part of the identity.
+ c["created_by"] = username
+ c["_ckey"] = _ck(c.get("platform"), c["handle"])
+ prev_row = seen.get(c["_ckey"])
+ if prev_row and str(prev_row.get("created_by") or "").strip():
+ c["created_by"] = prev_row["created_by"]
+ if prev_row:
+ counts["seen_again"] += 1
+ c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1)
+ else:
+ counts["new"] += 1
+ c["found_count"] = "1"
+ c["first_found"] = stamp
+ # ⭐ WAVE 26 · ITEM 1 — THE STAGE STAMP IS PER-AUTOMATION, SO IT CANNOT HANG OFF
+ # "is this row new to the TABLE".
+ #
+ # ⛔ MEASURED DEFECT, two discovery automations pointed at one database: the second one's
+ # board was EMPTY. It reported "2 profiles found — 0 new, 2 seen before", which is true,
+ # and then drew nothing, because this stamp lived inside the `else` above. `skey` is
+ # `stage_`; the branch it sat in asks whether some OTHER automation had
+ # already created the row. So the first automation to reach a handle claimed it, and every
+ # later automation sharing that database silently had no cards — with a summary saying it
+ # had found them. Nothing errored, and both boards are green surfaces.
+ #
+ # ⚠ THE ORIGINAL LAW IS PRESERVED, and it is the reason this is a condition rather than an
+ # unconditional write: a re-find must never pull a card somebody has already moved back
+ # into Review. So the question is "has THIS automation placed this record yet?" — not "is
+ # this record new?" A blank stage cell for this automation means unplaced, which is
+ # exactly the state a new card is in.
+ merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key))
+ merged = {rid: {k: v for k, v in r.items() if k != "_ckey"}
+ for rid, r in merged.items()}
+ counts["capped"] = mc["capped"]
+ missing = [] if dry else ut_missing(rt, table_key)
+ counts["missing_tables"] = len(missing)
+ if not dry and not missing:
+ ut_write_rows(rt, table_key, merged)
+ steps["write"] = "ok"
+ # ⭐⭐ THE FORWARD HALF (2026-08-10). Every candidate this run wrote now also leaves an
+ # OBSERVATION behind — see `corpus_snapshot_row`. Without it a backfill is theatre: the
+ # next discovery run re-opens exactly the gap the backfill just closed, which is how a
+ # data defect becomes a recurring one.
+ #
+ # ⚠ `ensure_ig_graph`, NOT a snapshot-only `ut_ensure`, and it is a real behaviour change
+ # worth owning: a tenant that has never enriched now gets the four canonical child
+ # databases the first time discovery runs (four of `MAX_UT_TABLES = 40`). The alternative
+ # writes a series into a store the profile table has no LINK to — the numbers would be
+ # recorded and unreachable, which is half a feature wearing a whole one's clothes. A
+ # tenant running discovery is an Instagram tenant; this moves the spawn earlier, it does
+ # not add one.
+ # ⚠ `incoming`, not `merged`: the observation belongs to the profiles THIS RUN read, not
+ # to every row the table happens to hold.
+ try:
+ graph = ensure_ig_graph(rt, username, auto_id, profile_table=table_key)
+ counts["series"] = append_ig_snapshots(
+ rt, [corpus_snapshot_row(c, day=stamp) for c in incoming],
+ graph[IG_SNAPSHOTS_TABLE], log=log)
+ except Exception as e: # noqa: BLE001
+ # The candidates already landed. A series write that fails must not throw away the
+ # rows we just paid the vendor for — the same posture the preset top-up takes.
+ log(f"[aios-auto] discover {auto_id}: corpus series write failed: "
+ f"{type(e).__name__}: {e}")
+ elif dry:
+ steps["write"] = "skipped"
+
+ affected = [rid for rid, r in merged.items()
+ if str(r.get("handle")) in {c["handle"] for c in incoming}][:200]
+ est = discover_estimate(counts["found"])
+ summary = (f"{counts['found']} profile{'' if counts['found'] == 1 else 's'} found — "
+ f"{counts['new']} new, {counts['seen_again']} seen before. About "
+ f"${est['usd']}")
+ if counts["dropped"]:
+ summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no "
+ f"username and could not be saved")
+ cnote = cap_note([(table_key, counts["capped"])], missing)
+ if cnote:
+ summary += f" — {cnote}"
+ steps["write"] = "partial"
+ if dry:
+ summary = f"Test run — nothing saved. Would have written: {summary}"
+ state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok"
+ if not dry and not missing:
+ try:
+ compute_metric_cells(rt, table_key) # C7 — see the field runner
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}")
+ log(f"[aios-auto] discover {auto_id} -> {table_key}: {summary}")
+ return (state, summary, counts, affected, steps)
+
+
+#: ⭐ W29-T04 — the discovery-run bookkeeping columns, so `run_discover_tiktok` and the D-116 merge
+#: rule agree on what discovery OWNS versus what enrichment owns. Anything not in this set is a
+#: MEASUREMENT, and a corpus re-find must never overwrite an enriched measurement with its own
+#: cheaper approximation.
+TT_DISCOVERY_KEYS = frozenset({"platform", "handle", "created_by", "found_count",
+ "first_found", "last_found", "source", "source_payload"})
+
+
+def _tt_candidate_row(row, stamp):
+ """One TikTok corpus row → a `ut_tt_profile` candidate row. None without a handle.
+
+ ⛔ NEVER EMITS `found_count` — the runner computes it against what is already stored, so
+ writing it here would reset the counter to 1 on every re-find. Same rule, same reason, as
+ `_candidate_row` on the Instagram side.
+ ⭐ THE FIELD MAP IS THE CONNECTOR'S, not a second one written here. `normalize_profile` is what
+ the enrich path will use too, so a corpus row and a scrape row cannot disagree about which
+ vendor key becomes which column.
+ """
+ import connectors_tt as _tt
+ cells = _tt.normalize_profile(row if isinstance(row, dict) else {})
+ if not str(cells.get("handle") or "").strip():
+ return None
+ return {**cells, "last_found": stamp, "source": "corpus"}
+
+
+def run_discover_tiktok(rt, defn, username="automation", log=print, step=_no_step, rows=None):
+ """⭐⭐ DEBT D-9, open since wave 19 — FIND TIKTOK HANDLES NOBODY HERE HAS TYPED IN.
+
+ Two-phase by construction, exactly like the Instagram runner: a run either STARTS a corpus
+ query and hands its snapshot id to the next run, or COLLECTS one a previous run started. Every
+ outcome is a sentence about what actually happened; none of them is a green zero.
+
+ ⛔ IT IS A SIBLING OF `run_discover_instagram`, NOT A REFACTOR OF IT, AND THAT IS A DECISION.
+ The two-phase scaffolding below is duplicated. Extracting it would be architecturally right —
+ the `SOURCES` registry says so in its own comment — and it would rewrite the one code path this
+ product bills money through, which `section_bd`, `section_w27`, NC33/NC34 and four derived
+ sweeps all pin. R2 already chose "parallel, zero risk to the live Instagram rows" for the data;
+ the same reasoning applies to the runner. The extraction is booked as a PENDING line with the
+ Bright Data transport, which is the second duplication pointing at the same seam — two of them
+ is a better argument for the next wave than a half-finished refactor is for this one.
+
+ ⛔ THE DISCOVERY HALVES BRIGHT DATA SELLS FOR TIKTOK BY KEYWORD AND BY PROFILE **404 FOR OUR
+ KEY** (probed: `gd_lj71gn6l68bz7y9hc`, `gd_lilwhto81z415d9mdl`). Discovery therefore routes
+ through the Profiles dataset's corpus filter, which is the same door Instagram uses.
+ """
+ import connectors_tt as _tt
+ cfg = defn.get("config") or {}
+ dry = bool(cfg.get("dryRun"))
+ limit = int(cfg.get("recordsLimit") or 5)
+ preds = list(cfg.get("predicates") or [])
+ auto_id = str(defn.get("id") or "")
+ steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"}
+ counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0,
+ "missing_tables": 0}
+
+ if not bd_ready():
+ # Fail-closed, and it says which credential is missing. Discovery has no free rung to drop
+ # to — the corpus is the vendor's.
+ steps["find"] = "blocked"
+ return ("partial", "Profile search is not set up yet — nothing was searched and nothing "
+ "was charged", counts, [], steps)
+
+ pending = str((defn.get("state") or {}).get("pendingSnapshot") or "")
+ if not pending:
+ guard_err = (narrowing_refusal(preds, cfg.get("operator"))
+ or depth_refusal(preds, cfg.get("operator")))
+ if guard_err:
+ steps["find"] = "blocked"
+ return ("error", f"the search was not started — {guard_err}", counts, [], steps)
+ if dry:
+ est = discover_estimate(limit)
+ steps["find"] = steps["write"] = "skipped"
+ return ("partial",
+ f"Test run — nothing saved. Would look for up to {limit} TikTok "
+ f"profile{'' if limit == 1 else 's'} matching "
+ f"{_predicate_sentence(preds, cfg.get('operator'))}, for about "
+ f"${est['usd']}", counts, [], steps)
+ step("Starting the search")
+ excl = {}
+ # ⚠ `already_found_handles` NEEDS NO TIKTOK BRANCH — it is scoped to THIS definition's own
+ # tables (`automation_tables`), so a TikTok automation targeting `ut_tt_profile` excludes
+ # exactly the handles it has itself found. Verified rather than assumed: the function reads
+ # `config.targetTable` plus each `create_record` action's table, and names no IG constant.
+ sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit,
+ dataset_id=_tt.TT_DS_PROFILES,
+ exclude_handles=already_found_handles(rt, defn),
+ applied=excl)
+ if excl.get("excluded"):
+ step(f"excluded {excl['excluded']} already-found "
+ f"profile{'' if excl['excluded'] == 1 else 's'} at the provider")
+ elif excl.get("dropped"):
+ step(f"searching without the already-found list — {excl['dropped']}")
+ if note:
+ steps["find"] = "blocked"
+ return ("error", f"the search was not started — {note}", counts, [], steps)
+ pending = sid
+ # LOG BEFORE PERSISTING: `set_state` can raise on an unavailable store, and a snapshot that
+ # exists, is billed and has its id in neither the store nor the logs is unrecoverable.
+ log(f"[aios-auto] discover-tt {auto_id}: started {sid}")
+ set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()})
+ steps["find"] = "ok"
+
+ waited, status, size, note = 0.0, "", 0, ""
+ while True:
+ step(f"Searching — {int(waited)}s of {int(BD_FILTER_WAIT)}s")
+ status, size, note = bd_filter_status(pending)
+ if note:
+ steps["collect"] = "error"
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ return ("error", note, counts, [], steps)
+ if status != "building" or waited >= BD_FILTER_WAIT:
+ break
+ time.sleep(BD_FILTER_POLL)
+ waited += BD_FILTER_POLL
+ step("Reading the profiles the search found")
+
+ if status == "building":
+ mins = 0
+ try:
+ since = (defn.get("state") or {}).get("pendingSince")
+ if since:
+ mins = max(0, int((_dt.datetime.now(_dt.timezone.utc)
+ - _dt.datetime.fromisoformat(str(since))).total_seconds()
+ // 60))
+ except Exception: # noqa: BLE001
+ mins = 0
+ been = f" — {mins} min so far" if mins else ""
+ steps["collect"] = "partial"
+ return ("partial",
+ f"Still searching at the provider{been}. A corpus search takes about 20 minutes "
+ f"however few records you asked for. Nothing is lost, you are not charged twice, "
+ f"and the results are collected automatically as soon as they are ready",
+ counts, [], steps)
+ if status == "empty":
+ steps["collect"] = "ok"
+ steps["write"] = "skipped"
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ return ("partial", "No profiles matched. Try a shorter or more common keyword — "
+ "'floral' finds more accounts than 'floral design studio'",
+ counts, [], steps)
+ if status != "ready":
+ steps["collect"] = "error"
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ return ("error", f"the search ended as {status or 'unreadable'} and returned nothing",
+ counts, [], steps)
+
+ rows, dnote = bd_filter_rows(pending)
+ if dnote:
+ steps["collect"] = "partial"
+ return ("partial", f"the search finished with {size} match"
+ f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps)
+ set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None})
+ steps["collect"] = "ok"
+
+ stamp = _iso()
+ incoming = [c for c in (_tt_candidate_row(r, stamp) for r in rows) if c]
+ counts["found"] = len(incoming)
+ # Count what we could NOT use: "the vendor delivered 50 and we kept 3" must not read the same
+ # as "the vendor delivered 3".
+ counts["dropped"] = max(0, len(rows) - len(incoming))
+ label = cfg.get("targetLabel") or TT_TABLE_LABELS[TT_PROFILE_TABLE]
+ table_key = (ut_key_for(label, cfg.get("targetTable") or TT_PROFILE_TABLE) if dry
+ else ut_ensure(rt, label, TT_PROFILE_FIELDS, username,
+ key=cfg.get("targetTable") or TT_PROFILE_TABLE,
+ flow_tag=auto_id, lock_fields=True))
+ existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
+ # ⚠ A ROW WITH NO `platform` CELL IN A `ut_tt_*` TABLE IS STAMPED TIKTOK, not Instagram. The
+ # shared `candidate_key` defaults a blank to Instagram — correct for the IG table, where every
+ # legacy row came from that runner, and wrong here, where every row came from this one.
+ for r in existing.values():
+ if not str(r.get("platform") or "").strip():
+ r["platform"] = PLATFORM_TIKTOK
+ existing2 = {rid: {**r, "_ckey": candidate_key(r.get("platform"), r.get("handle"))}
+ for rid, r in existing.items()}
+ seen = {r["_ckey"]: r for r in existing2.values()}
+ for c in incoming:
+ c["created_by"] = username
+ c["_ckey"] = candidate_key(c.get("platform"), c["handle"])
+ prev_row = seen.get(c["_ckey"])
+ if prev_row and str(prev_row.get("created_by") or "").strip():
+ c["created_by"] = prev_row["created_by"]
+ if prev_row:
+ counts["seen_again"] += 1
+ c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1)
+ # ⛔⛔ DEBT D-116, AND THIS RUNNER DELIBERATELY DOES NOT INHERIT IT. On the Instagram
+ # side a discovery re-find OVERWRITES an enriched `followers` with the corpus number —
+ # the corpus is cheaper, rounder and older than a scrape, so the row silently gets
+ # WORSE every night while the automation reports success. The fix is a precedence rule,
+ # not a blank check: a row that has been ENRICHED (`enriched_at` is stamped only by the
+ # enrich path) keeps its exact measurements, and discovery may only fill what is
+ # genuinely empty and update its own bookkeeping.
+ # ⚠ `TT_DISCOVERY_KEYS` is what discovery OWNS; everything else on a corpus row is a
+ # measurement, and an approximation must never replace an exact one.
+ if str(prev_row.get("enriched_at") or "").strip():
+ for k in list(c):
+ if (k not in TT_DISCOVERY_KEYS and k != "_ckey"
+ and str(prev_row.get(k) or "").strip()):
+ c.pop(k)
+ else:
+ counts["new"] += 1
+ c["found_count"] = "1"
+ c["first_found"] = stamp
+ merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key))
+ merged = {rid: {k: v for k, v in r.items() if k != "_ckey"}
+ for rid, r in merged.items()}
+ counts["capped"] = mc["capped"]
+ missing = [] if dry else ut_missing(rt, table_key)
+ counts["missing_tables"] = len(missing)
+ if not dry and not missing:
+ ut_write_rows(rt, table_key, merged)
+ steps["write"] = "ok"
+ elif dry:
+ steps["write"] = "skipped"
+
+ affected = [rid for rid, r in merged.items()
+ if str(r.get("handle")) in {c["handle"] for c in incoming}][:200]
+ est = discover_estimate(counts["found"])
+ summary = (f"{counts['found']} TikTok profile{'' if counts['found'] == 1 else 's'} found — "
+ f"{counts['new']} new, {counts['seen_again']} seen before. About "
+ f"${est['usd']}")
+ if counts["dropped"]:
+ summary += (f". {counts['dropped']} result{'' if counts['dropped'] == 1 else 's'} had no "
+ f"username and could not be saved")
+ cnote = cap_note([(table_key, counts["capped"])], missing)
+ if cnote:
+ summary += f" — {cnote}"
+ steps["write"] = "partial"
+ if dry:
+ summary = f"Test run — nothing saved. Would have written: {summary}"
+ state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok"
+ log(f"[aios-auto] discover-tt {auto_id} -> {table_key}: {summary}")
+ return (state, summary, counts, affected, steps)
+
+
+def _predicate_sentence(preds, operator="and"):
+ """A predicate list as something a person can read back. Used in summaries and the canvas.
+
+ Human words on BOTH halves — it used to render `biography includes floral`, the vendor's
+ column name beside the vendor's operator token, in a sentence shown to a customer.
+ """
+ joiner = " or " if str(operator or "").lower() == "or" else " and "
+ parts = []
+ for p in preds or []:
+ v = p.get("value")
+ shown = " or ".join(str(x) for x in v) if isinstance(v, list) else v
+ parts.append(f"{field_label(p.get('name'))} "
+ f"{BD_OP_LABELS.get(p.get('operator'), p.get('operator'))}"
+ + ("" if shown is None else f" {shown}"))
+ return joiner.join(parts) or "no conditions"
+
+
+#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather
+#: than a second, smaller number nobody could explain: a plain automation's records ARE its
+#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a
+#: flow could be pointed at and which would not finish.
+MAX_FLOW_RECORDS = MAX_UT_ROWS
+
+
+def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None):
+ """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole
+ automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special
+ case and `apply_actions` still runs at the ONE call site that already exists.
+
+ Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed
+ `error: unknown automation kind 'plain'` — on the kind every new automation now has.
+
+ ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here.
+ `apply_actions` returns immediately when there is no table, so a flow whose only action is
+ `create_record` into some other database does NOTHING — and a green dot over nothing is the
+ exact defect this module names in three other places. It reports the honest state and says
+ which fact is missing.
+ """
+ table = _flow_table(defn)
+ if not table:
+ return ("partial",
+ "no database is bound yet — a plain automation walks the records of its target "
+ "database, and this one names none (pick one on the trigger, or in Properties)",
+ {}, [], {})
+ all_rows = (ut_get(rt, table) or {}).get("rows") or {}
+ # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug
+ # on the kind every new automation now has. `plain` has no machine step, so it has nothing of
+ # its own to call "the records this run touched" — and walking the WHOLE table would mean
+ # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the
+ # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them
+ # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what
+ # those genuinely mean.
+ ids = ([r for r in (rows or []) if str(r) in all_rows] if rows
+ else sorted(all_rows, key=_rid_num))
+
+ # An enrichment-only flow is the one exception to the ordinary manual-run rule above. Its
+ # saved View and quota are not merely how the action decides whether to spend; they ARE the
+ # set of records the automation was asked to process. Walking every row first meant a run
+ # bound to a 10-record "Pending" view announced (and needlessly visited) all 61 table rows.
+ # Besides being misleading, that shape made the runtime scale with unrelated historical rows.
+ #
+ # Keep this deliberately narrow: any flow with another action must still hand that action the
+ # whole manual/scheduled scope. Only one direct, enabled enrich action has no other record
+ # semantics to preserve, so it can begin at its selected records safely.
+ flow_actions = list((defn.get("flow") or {}).get("actions") or [])
+ enabled_direct = [a for a in flow_actions if isinstance(a, dict) and a.get("enabled", True)]
+ enrich_scope_note = ""
+ if rows is None and len(enabled_direct) == 1 \
+ and enabled_direct[0].get("kind") == "enrich_instagram":
+ enrich_cfg = enabled_direct[0].get("config") or {}
+ profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField"))
+ if profile_key:
+ # ⛔⛔ THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES.
+ # `enrich_selection` is called TWICE per run — here, to decide which records the
+ # runner walks and announces, and again inside `_walk` to decide which the action
+ # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so
+ # the run announced *"(1 of the 30 asked for)"* in its summary while its own note
+ # from the second call said *"0 of the 30"* — one run, two answers, both printed.
+ # ⚠ `gone` is read from the definition here rather than passed in, because this
+ # function has the definition and `_walk` does the same read; a third source of the
+ # same fact is how the two would drift again ([[one-evaluator-per-question]]).
+ selected, enrich_scope_note = enrich_selection(
+ rt, table, enrich_cfg, profile_key,
+ gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}))
+ if enrich_scope_note.startswith("the enrich step names a view it cannot read"):
+ return ("partial", f"nothing walked in {table} — {enrich_scope_note}", {}, [], {})
+ ids = selected
+ step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}")
+ counts = {"records": len(ids)}
+ if len(ids) > MAX_FLOW_RECORDS:
+ # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names
+ # both numbers so "it only processed some of them" is readable rather than deducible.
+ total = len(ids)
+ ids = ids[:MAX_FLOW_RECORDS]
+ return ("partial",
+ f"{total} records in {table} — this run walked the first "
+ f"{MAX_FLOW_RECORDS}", counts, ids, {})
+ n = len(ids)
+ # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK.
+ # Owner: *"how come it says that its 51 records walked in ut_beauty_influencer_leads (all)?
+ # wth is even ut_beauty_influencer_leads (all)??"* — and both halves of that name were ours:
+ # · `ut_beauty_influencer_leads` is the storage KEY. Every other surface in the product
+ # shows the database's LABEL ("Beauty influencer leads"); this one leaked the key.
+ # · `(all)` meant "every record, because a manual run has no trigger records to narrow to",
+ # which is unguessable from the word. It reads as a filter nobody chose.
+ # ⚠ THE NUMBER WAS ALWAYS HONEST and is unchanged: a `plain` automation walks its whole
+ # database, and a per-STEP narrowing (an enrich step's `fromView`) is a different count that
+ # this sentence never claimed to report. What was wrong was that nothing said so.
+ label = str((ut_get(rt, table) or {}).get("label") or "").strip() or table
+ scope = ("the records that fired the trigger" if rows
+ else ("the records selected for Instagram enrichment"
+ if len(enabled_direct) == 1
+ and enabled_direct[0].get("kind") == "enrich_instagram"
+ else "every record — this run was started by hand, so nothing narrowed it"))
+ if enrich_scope_note:
+ scope += f" ({enrich_scope_note})"
+ # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400
+ # characters by `_commit_run`, and this note is the longest thing a run says — it names
+ # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING
+ # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy
+ # of this sentence is never produced. Without this line the explanation exists only in a
+ # string that is about to be truncated, on exactly the runs that look like the automation
+ # has stopped working.
+ counts[RUN_NOTES_KEY] = [enrich_scope_note]
+ return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}",
+ counts, ids, {})
+
+
+RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db,
+ "field_instagram": run_field_instagram,
+ "discover_instagram": run_discover_instagram,
+ "discover_tiktok": run_discover_tiktok}
+
+
+# ---------------------------------------------------------------------------------------------
+# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b)
+# ---------------------------------------------------------------------------------------------
+# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI →
+# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness
+# boolean on the wire, four client strings and two gate sections. That is what a hard-coded
+# vendor costs, and TikTok (D-9) would have paid it again from scratch.
+#
+# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the
+# three verbs it can do — and the surfaces read the declaration instead of naming a vendor:
+#
+# probe "is this configured?" -> the honest readiness bit the UI shows
+# capture enrich a handle we know -> `pull_profile`-shaped
+# discover find handles we do not -> the corpus query
+#
+# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in
+# later; it is the honest statement that this source has no discovery route, and a surface that
+# offers one anyway is offering a button that must refuse.
+SOURCES = {
+ "instagram": {
+ # ⛔ `vendor` NAMES NO COMPANY (owner instruction 2026-08-09). It reaches operator-facing
+ # copy, and which provider serves a capability is a routing decision the product owns —
+ # `providers.py` still holds the real keys, chains and costs.
+ "key": "instagram", "label": "Instagram", "vendor": "Scraper",
+ "probe": bd_ready,
+ "capture": pull_profile,
+ "discover": bd_filter_start,
+ "captureKind": "field_instagram",
+ "discoverKind": "discover_instagram",
+ "tiers": TIERS,
+ },
+ # ⭐⭐ WAVE 29 — D-9 LANDS, AND IT LANDED AS THE ONE ENTRY THIS REGISTRY PREDICTED IT WOULD.
+ # The canvas, the append tables, the caps and the honest-status contract were already in place;
+ # what TikTok added was a field map, a discovery runner and three dataset ids.
+ "tiktok": {
+ # `vendor` NAMES NO COMPANY (owner instruction 2026-08-09) — it reaches operator-facing
+ # copy, and which provider serves a capability is a routing decision `providers.py` owns.
+ "key": "tiktok", "label": "TikTok", "vendor": "Scraper",
+ "probe": bd_ready,
+ # ⚠ `capture` IS None UNTIL W29-T05 WIRES THE ENRICHMENT ACTION, and `source_status` does
+ # not read it — but a surface that offered a capture this source cannot do would be
+ # offering a button that must refuse, which is exactly what this registry's own note
+ # forbids. Declared as absent rather than pointed at the Instagram function.
+ "capture": None,
+ "discover": bd_filter_start,
+ "captureKind": None,
+ "discoverKind": "discover_tiktok",
+ # No tier ladder: R6/R7 retired the free rung from enrichment entirely, and TikTok never
+ # had one. An empty tuple is the honest statement, not a gap to fill in later.
+ "tiers": (),
+ },
+}
+
+
+def source_status():
+ """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source.
+
+ ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface
+ needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the
+ credential to a browser would put a billable secret in every user's devtools.
+ """
+ # ⛔ `vendor` IS NOT ON THE WIRE (2026-08-06). It stays in `SOURCES` as the record of which
+ # supplier this rung actually uses — that history is worth keeping in the code — but a
+ # customer has no use for it, and a server-supplied string rendered by a generic component
+ # is precisely the half a client-file scan cannot see.
+ return [{"key": s["key"], "label": s["label"],
+ "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))}
+ for s in SOURCES.values()]
+
+
+# ---------------------------------------------------------------------------------------------
+# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client
+# ---------------------------------------------------------------------------------------------
+# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are
+# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible —
+# a canvas that still shows a step the engine stopped running looks completely fine. So the node
+# list is derived from the DEFINITION by the same module that executes it, rides on the payload
+# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`.
+# The client owns layout (pixels, pan/zoom, hit targets) and nothing else.
+#
+# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is
+# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine
+# walks it linearly. Nothing here should be read as a general branching runtime.
+
+#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be
+#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw
+#: away the only information the dot is there to carry.
+NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped")
+
+#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a
+#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is
+#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones
+#: here are all REAL — each changes what the next run does:
+#: schedule the trigger fires on its cron, or only by hand
+#: postMetrics likes/comments per post are bought, or the engagement series does not grow —
+#: it costs a vendor record PER POST rather than per profile (measured: the
+#: profile row carries post identity and no engagement)
+#: commentMetrics the Comments dataset is bought, which can bill many rows per post
+#: write DRY RUN — read everything, compute the counts, write nothing anywhere
+#:
+#: ⭐⭐ WAVE 28 / R5+R6 (contract C3) — `paid` AND `fallback` ARE GONE, and what replaced them is
+#: the point of the ruling: the money switches used to be "which rung do we try" (a question about
+#: our plumbing), and they are now "what do you want captured" (a question about the user's data).
+#: Profile is always captured and has no switch — an automation that fetches nothing is not an
+#: automation with a step turned off, it is a broken one with a lie on it.
+#: ⚠ `trigger: "schedule"` IS NOT PART OF THAT COLLAPSE. It is the trigger card's own on/off and
+#: has its own branch in `toggle_node` (event triggers flip themselves, not the cron); C3's "the
+#: three toggles" names the CAPTURE ladder it is reshaping.
+NODE_TOGGLES = {"trigger": "schedule", "capture_posts": "postMetrics",
+ "capture_comments": "commentMetrics", "write": "write"}
+
+
+def _cron_label(cron):
+ return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "")
+
+
+def node_status(steps, nid):
+ """A node's dot — from the last run's MEASURED step outcome, or `idle`.
+
+ ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely
+ so a negative control can prove that. Every run stored before W19-C has no `steps` map at all,
+ and painting those nodes green because the run said `ok` would manufacture a measurement — the
+ same defect as the green dot over the empty posts table recorded in `run_field_instagram`.
+ No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI.
+ """
+ v = str((steps or {}).get(nid) or "")
+ return v if v in NODE_STATES else "idle"
+
+
+def graph(defn):
+ """One automation → `{nodes, edges}`, left to right. Pure over the definition."""
+ defn = defn if isinstance(defn, dict) else {}
+ cfg = defn.get("config") or {}
+ sched = defn.get("schedule") or {}
+ runs = defn.get("runs") or []
+ last = runs[0] if runs and isinstance(runs[0], dict) else {}
+ steps = last.get("steps") if isinstance(last.get("steps"), dict) else {}
+
+ def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True):
+ return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail,
+ "col": col, "row": row, "panel": panel or nid, "enabled": bool(on),
+ "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)}
+
+ on = bool(sched.get("enabled"))
+ trg = defn.get("trigger") or {}
+ if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"):
+ # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail
+ # read the same node, so an event-triggered flow must not read "Schedule". The membership
+ # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old
+ # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so
+ # the else-branch was reachable) — green, compiling, and wrong on screen.
+ t_on = bool(trg.get("enabled", True)) and not trg.get("paused")
+ sub = TRIGGER_LABELS.get(trg["key"], trg["key"])
+ watched = ", ".join(trg.get("fields") or []) or "any field"
+ # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the
+ # whole of it) and `record_updated` no longer names a condition (its watched fields are).
+ # These two lines are the migration's visible half: a node still describing a `.field`
+ # the validator has stopped storing would be the surface and the store disagreeing.
+ det = {"event_field": (f"{trg.get('table', '')}"
+ + (f" — {_lane_sentence(trg.get('when'))}"
+ if trg.get("when") else "")),
+ "record_updated": f"{trg.get('table', '')} — {watched}",
+ "record_created": str(trg.get("table") or ""),
+ "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}",
+ "form_submitted": str(trg.get("table") or ""),
+ "webhook": "POST the hook URL to fire it",
+ "ig_profile_match": (_predicate_sentence(cfg.get("predicates"),
+ cfg.get("operator"))[:80]
+ if cfg.get("predicates")
+ else "No filters yet — nothing to search for"),
+ "email": str(trg.get("query") or "")}.get(trg["key"], "")
+ if not trg.get("configured", True):
+ det = "Finish setting this trigger up before it can fire"
+ if trg.get("paused"):
+ det = str(defn.get("statusNote") or "paused")
+ nodes = [node("trigger", "trigger", sub,
+ "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)]
+ else:
+ # ⭐ 2026-08-07 (owner ruling) — THE CARD NAMES WHAT ACTUALLY FIRES IT. With the cron off
+ # this node is a MANUAL trigger, and titling it "Schedule" was the screen disagreeing with
+ # the user's own pick — the same class of defect as the Database picker that could not be
+ # reached. ⚠ The SUBTITLE is untouched ("Manual only" / the cron label): it is what the
+ # disabled-schedule gate asserts, and it was never the wrong half.
+ # ⚠ `panel="schedule"` STAYS whichever way it reads. Picking Manual says how this fires
+ # TODAY, not that it may never be scheduled — the cron has to stay one click away, and
+ # this is the only node that offers it.
+ nodes = [node("trigger", "trigger", "Schedule" if on else "Manual",
+ _cron_label(sched.get("cron")) if on else "Manual only", 0,
+ panel="schedule", on=on,
+ detail="" if on else "It still runs when you press Run now")]
+ edges = []
+
+ if defn.get("kind") == "field_instagram":
+ # ⭐⭐ WAVE 28 / CONTRACT C3 — FOUR NODES, THREE OF THEM SWITCHES OVER WHAT IS CAPTURED.
+ # This branch used to draw a FORK: `Capture` splitting into `Exact counts` (paid) and
+ # `Estimated counts` (the free anonymous ladder), rejoining at `Post engagement`. R5
+ # deleted the ladder, so the fork had one arm; keeping it would have drawn a decision the
+ # engine no longer makes, with a switch (`fallback`) flipping a config key the cleaners
+ # now discard. A canvas that offers a choice the runtime ignores is worse than no canvas.
+ # ⚠ `Profile set` STILL CARRIES NO SWITCH, and now that is the whole ruling rather than an
+ # implementation detail: the profile is always captured (R6), Posts and Comments are the
+ # opt-ins, and both are OFF until asked for.
+ metrics_on = bool(cfg.get("postMetrics"))
+ comments_on = bool(cfg.get("commentMetrics"))
+ max_posts = cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL
+ nodes += [
+ node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1,
+ panel="source",
+ detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField")
+ else "URL column: the one the field is bound to"),
+ # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY
+ # and no engagement (measured), so likes/comments are a SECOND vendor call per post.
+ # A switch that multiplies a run's cost by the post count deserves to be visible on
+ # the canvas rather than buried in a config panel.
+ node("capture_posts", "capture", "Post data",
+ "Likes and comments per post" if metrics_on else "Off",
+ 2, panel="capture", on=metrics_on,
+ detail=(f"Up to {max_posts} posts per profile" if metrics_on else
+ "ut_ig_post_snapshots only grows while this is on")),
+ node("capture_comments", "capture", "Comment data",
+ "Comments on those posts" if comments_on else "Off",
+ 3, panel="capture", on=comments_on,
+ detail=("The full comments dataset — many rows per post" if comments_on else
+ "Comments already embedded in a paid post row are kept either way")),
+ node("write", "write", "Write",
+ cfg.get("targetTable") or "No database", 4, panel="write",
+ on=not cfg.get("dryRun"),
+ detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"),
+ ]
+ edges = [{"from": "trigger", "to": "source", "label": ""},
+ {"from": "source", "to": "capture_posts", "label": ""},
+ {"from": "capture_posts", "to": "capture_comments", "label": ""},
+ {"from": "capture_comments", "to": "write", "label": ""}]
+ elif defn.get("kind") == "discover_instagram":
+ limit = int(cfg.get("recordsLimit") or 0)
+ est = discover_estimate(limit)
+ pending = str((defn.get("state") or {}).get("pendingSnapshot") or "")
+ nodes += [
+ node("find", "source", "Find profiles",
+ _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1,
+ panel="find",
+ detail=(f"Up to {limit} profiles · about ${est['usd']}"
+ if bd_ready()
+ else "Profile search is not set up yet")),
+ node("collect", "capture", "Collect results",
+ "Searching…" if pending else "Takes about 20 minutes", 2,
+ panel="find",
+ detail=""),
+ node("write", "write", "Save results",
+ cfg.get("targetTable") or DISCOVER_TABLE, 3, panel="write",
+ on=not cfg.get("dryRun"),
+ detail=""),
+ ]
+ edges = [{"from": "trigger", "to": "find", "label": ""},
+ {"from": "find", "to": "collect", "label": ""},
+ {"from": "collect", "to": "write", "label": ""}]
+ elif defn.get("kind") == "scrape_db":
+ url = str(cfg.get("url") or "")
+ host = (urlparse(url).hostname or url or "No URL") if url else "No URL"
+ which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld"
+ else f"HTML table #{int(cfg.get('tableIndex') or 0)}")
+ mapped = len(cfg.get("fieldMap") or {})
+ nodes += [
+ node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]),
+ node("extract", "capture", "Extract", which, 2, panel="columns",
+ detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: "
+ f"{cfg.get('keyField') or '—'}"),
+ node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3,
+ panel="write", on=not cfg.get("dryRun"),
+ detail=cfg.get("targetTable") or "a new database"),
+ ]
+ edges = [{"from": "trigger", "to": "fetch", "label": ""},
+ {"from": "fetch", "to": "extract", "label": ""},
+ {"from": "extract", "to": "write", "label": ""}]
+ # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point.
+ # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught
+ # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML
+ # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A
+ # surface that draws nothing is obviously incomplete; one that draws a fetch step that does
+ # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at
+ # once because all three read this.
+ #
+ # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE
+ # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node,
+ # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on
+ # that promise, so it is stated rather than left to be inferred from the control flow.
+ return {"nodes": nodes, "edges": edges}
+
+
+def toggle_node(rt, auto_id, node_id):
+ """Flip ONE node's switch. Returns `(defn, error)`.
+
+ The node → config mapping lives on the SERVER for the reason the whole graph does: a client
+ that knew which field each node writes would be a second copy of that knowledge, free to
+ drift. The canvas just says "this node was clicked".
+ """
+ prev = all_definitions(rt).get(str(auto_id))
+ if prev is None:
+ return None, "no such automation"
+ target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None)
+ if target is None:
+ return None, f"there is no {node_id!r} step in this automation"
+ if not target.get("toggle"):
+ return None, (f"the {target['title']} step cannot be turned off — it is what this "
+ f"automation is")
+ cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {})
+ which = target["toggle"]
+ trg_key = str((prev.get("trigger") or {}).get("key") or "")
+ if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS:
+ # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping
+ # the cron under a node labelled "When a field changes" would be a switch that lies.
+ # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that
+ # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so
+ # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so
+ # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half.
+ trg = dict(prev.get("trigger") or {})
+ trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused"))
+ trg["paused"] = False # a deliberate flip clears an auto-pause: human wins
+ return patch(rt, auto_id, {"trigger": trg})
+ if which == "schedule":
+ sched["enabled"] = not sched.get("enabled")
+ elif which == "postMetrics":
+ cfg["postMetrics"] = not cfg.get("postMetrics")
+ elif which == "commentMetrics":
+ cfg["commentMetrics"] = not cfg.get("commentMetrics")
+ elif which == "write":
+ cfg["dryRun"] = not cfg.get("dryRun")
+ return patch(rt, auto_id, {"config": cfg, "schedule": sched})
+
+
+# ---------------------------------------------------------------------------------------------
+# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed
+# ---------------------------------------------------------------------------------------------
+# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were
+# records of the target database, and each record's position lived in a single-select STAGE FIELD
+# the engine created on the customer's own database. Wave 26 deleted the board's built-in
+# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`,
+# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review
+# branch of the action walk, and the endings that sent a terminal card back round.
+#
+# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote
+# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles`
+# counter per automation — to render a view of state the RUN LOG already holds. The customer paid
+# for that in columns they did not ask for, in a store commit on every scheduled run, and in a
+# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we
+# had added. `migrate_ig_tables` now drops those columns wherever an automation created them.
+#
+# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE:
+# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the
+# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was
+# one of four callers. The name is the last thing the board left behind here.
+# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board`
+# — the MIGRATION. It must outlive the thing it retires, or a definition written before the
+# retirement walks into a runtime that no longer understands it.
+# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are
+# parked with no caller and say so at their own definitions.
+
+LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes",
+ "is_empty", "is_not_empty")
+LANE_NULLARY_OPS = ("is_empty", "is_not_empty")
+
+# ─────────────────────────────────────────────────────────────────────────────────────────────
+# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}`
+#
+# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why
+# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches
+# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc
+# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the
+# same function applied one level up, so the cheaper honest version is to evaluate the leaf where
+# it lies and never touch the bytes.)
+#
+# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An
+# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and
+# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep.
+# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an
+# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that
+# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine
+# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it.
+MAX_COND_DEPTH = 3
+MAX_COND_CHILDREN = 12
+COND_GROUP_KEYS = ("all", "any")
+
+
+def clean_cond(raw, depth=0, where=""):
+ """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no
+ condition", which is legal everywhere a condition is optional (the catch-all lane, an
+ unconditioned trigger, an action group that always runs)."""
+ if raw in (None, "", {}):
+ return None, None
+ at = f" on {where}" if where else ""
+ if not isinstance(raw, dict):
+ return None, f"the condition{at} must be an object"
+ conj = [k for k in COND_GROUP_KEYS if k in raw]
+ if len(conj) > 1:
+ return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the "
+ f"other")
+ if conj:
+ key = conj[0]
+ if depth + 1 > MAX_COND_DEPTH:
+ return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — "
+ f"the group{at} is deeper")
+ kids_raw = raw.get(key)
+ if not isinstance(kids_raw, list) or not kids_raw:
+ return None, f"the '{key}' group{at} needs at least one condition inside it"
+ if len(kids_raw) > MAX_COND_CHILDREN:
+ return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — "
+ f"the '{key}' group{at} has {len(kids_raw)}")
+ kids = []
+ for child in kids_raw:
+ c, err = clean_cond(child, depth + 1, where)
+ if err:
+ return None, err
+ if c is None:
+ return None, (f"an empty condition sits inside the '{key}' group{at} — "
+ f"finish it or remove it")
+ kids.append(c)
+ return {key: kids}, None
+ field = _s(raw.get("field"), 80).strip()
+ op = _s(raw.get("op") or raw.get("operator"), 20).strip()
+ if not field:
+ # ⚠ WAVE 26 — the owner met this sentence repeatedly and it told them nothing they could
+ # act on. It states a fact about the stored tree; it never said what to DO, and the usual
+ # cause is not a typo but an empty field picker (the flow had no walking record to offer
+ # columns from). The fix belongs in the message.
+ return None, (f"the condition{at} names no field — pick a column, or remove the "
+ f"condition")
+ if op not in LANE_OPS:
+ return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS)
+ cond = {"field": field, "op": op}
+ if op not in LANE_NULLARY_OPS:
+ v = raw.get("value")
+ if v is None or (isinstance(v, str) and not v.strip()):
+ return None, f"give a value to compare {field!r} against{at}"
+ cond["value"] = v.strip() if isinstance(v, str) else v
+ return cond, None
+
+
+def cond_fields(cond):
+ """Every field name a tree reads — the set a caller must have on the row image before the
+ answer means anything."""
+ if not isinstance(cond, dict):
+ return set()
+ for key in COND_GROUP_KEYS:
+ if key in cond:
+ out = set()
+ for child in cond.get(key) or []:
+ out |= cond_fields(child)
+ return out
+ f = cond.get("field")
+ return {f} if f else set()
+#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would
+#: collide with the stage the label is a choice FOR, and the board could no longer tell a
+#: routed card from a gated one.
+# ⚠ `tracked` / `declined` STAY RESERVED even though R6 deleted the stages that used them: the
+# labels still sit in boards stored before this wave, and a user lane named "Tracked" beside a
+# legacy stamped cell would read as the same lane while behaving as a different one.
+def _lane_num(v):
+ try:
+ return float(str(v).replace(",", ""))
+ except (TypeError, ValueError):
+ return None
+
+
+def lane_match(cond, row):
+ """Does `row` satisfy this condition TREE? None matches everything (the catch-all).
+
+ Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22
+ caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping
+ this name rather than growing a second entry point that could disagree with it.
+
+ ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse
+ as a number answers False — the record simply does not enter the lane — never "treat blank
+ as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank
+ cell is an unknown, and an unknown cannot be less than 10,000.
+ """
+ if cond is None:
+ return True
+ if isinstance(cond, dict):
+ if "all" in cond:
+ return all(lane_match(c, row) for c in cond.get("all") or [])
+ if "any" in cond:
+ return any(lane_match(c, row) for c in cond.get("any") or [])
+ raw = (row or {}).get(cond.get("field"))
+ op = cond.get("op")
+ if op == "is_empty":
+ return raw in (None, "")
+ if op == "is_not_empty":
+ return raw not in (None, "")
+ want = cond.get("value")
+ if op in (">", ">=", "<", "<="):
+ a, b = _lane_num(raw), _lane_num(want)
+ if a is None or b is None:
+ return False
+ return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op]
+ have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "")
+ if op == "includes":
+ return want_s.lower() in have_s.lower()
+ if op == "not_includes":
+ return want_s.lower() not in have_s.lower()
+ a, b = _lane_num(raw), _lane_num(want)
+ same = (a is not None and b is not None and a == b) or have_s == want_s
+ return same if op == "=" else (not same) if op == "!=" else False
+
+
+# ─────────────────────────────────────────────────────────────────────────────────────────────
+# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things.
+#
+# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run.
+# A `group` holds nested actions behind a condition — Airtable's "conditional action group",
+# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's
+# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this
+# builder is a column, not a canvas).
+#
+# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a
+# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack /
+# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty
+# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready
+# kind with a sentence, so the faded state is a wall and not a styling choice.
+MAX_ACTIONS = 20 # per automation, counting nested ones
+MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow
+MAX_BRANCHES = 6 # per If / then
+MAX_ACTION_VALUES = 20 # cells one update/create action may write
+FIND_LIMIT_MAX = 100
+#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder`
+#: and never by a literal list of group names: a client-side ordering is a second copy of this
+#: table, and the way it fails is that a group added here renders last, or not at all, with
+#: nothing red anywhere.
+ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4}
+ACTION_CATALOG = [
+ # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ────────
+ # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row
+ # is a WALL, not a styling choice — and each `detail` says the true reason rather than a
+ # placeholder, because "coming soon" on five rows is how a menu stops being believed.
+ #
+ # ⚠ WAVE 25 RETARGETED THE ONE STRING THAT NAMED A WAVE, and the reason generalises: W24
+ # scheduled this build for wave 25 and wave 25's six owner items do not include it, so
+ # "building next (wave 25)" became false the moment this wave shipped — a menu that dates its
+ # own promises has to be re-read by whoever misses the date, and a stale date is worse than
+ # none because it reads as a commitment somebody already broke. The replacement names the
+ # missing CAPABILITY, like its four siblings always did; D-51 carries the schedule.
+ {"kind": "web_goto", "label": "Open a page", "group": "Web action", "ready": False,
+ "detail": "Needs the browser job — a real browser running inside an ephemeral worker"},
+ {"kind": "web_fill", "label": "Fill a field", "group": "Web action", "ready": False,
+ "detail": "Needs the recorder that captures a selector instead of a coordinate"},
+ {"kind": "web_click", "label": "Click something", "group": "Web action", "ready": False,
+ "detail": "Needs the recorder that captures a selector instead of a coordinate"},
+ {"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": False,
+ "detail": "Needs the browser job — the scrape action reads static pages today"},
+ {"kind": "web_repair", "label": "Repair a broken step", "group": "Web action",
+ "ready": False,
+ "detail": "Self-healing selectors arrive with the browser engine (Scrapling)"},
+ # ── 2. DATABASE ──────────────────────────────────────────────────────────────────────────
+ {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True,
+ "detail": "Write values onto the record walking the flow"},
+ {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True,
+ "detail": "Add a row to another database"},
+ {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True,
+ "detail": "Look rows up by condition; the run log opens them"},
+ # ── 3. CONNECTED ─────────────────────────────────────────────────────────────────────────
+ # ⭐ WAVE 25 · C4 (owner rulings R3/R4) — THE ENRICH ACTION, and it REPLACES a whole KIND.
+ # `field_instagram` was an automation you created to fill one column; this is a step any flow
+ # can take, which is the shape it should always have had — enriching a profile is something
+ # you do TO a record, not a species of automation.
+ # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu,
+ # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`.
+ {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected",
+ "ready": True, "connector": "scraper",
+ "detail": "Fill this record's Instagram columns from its profile, and add a point to its "
+ "history"},
+ {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False,
+ "detail": "Needs a send scope on the Gmail connection"},
+ {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
+ "detail": "Needs the Slack connector"},
+ # ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
+ # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
+ # and R8 made it a FORK with lettered branches, which is a thing people already have a name
+ # for. The KIND is untouched — renaming it would orphan every stored action for a caption.
+ {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True,
+ "detail": "Send the record down one of several branches, by condition"},
+ {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic",
+ "ready": False, "detail": "Run the same actions on every item in a list"},
+ {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False,
+ "detail": "Not built — a sandbox is its own decision"},
+ {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic",
+ "ready": False, "detail": "Needs an AI action implementation"},
+]
+
+
+#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can
+#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an
+#: "Integrations" header, one row per connector, a chevron into that connector's own submenu).
+#:
+#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are
+#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the
+#: key and render the label, never join it against the directory.
+#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key.
+#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one
+#: connector wearing two names on two screens somebody sees within a second of each other, and a
+#: second literal is how that happens. This tuple says only WHICH connectors an action may name;
+#: what they are CALLED has exactly one source.
+ACTION_CONNECTORS = ("scraper",)
+
+
+def _connector_meta(key):
+ """`{key, label}` for a connector key, from the one place either picker declares it."""
+ key = str(key or "")
+ if key not in ACTION_CONNECTORS:
+ return None
+ return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None)
+
+
+def action_catalog():
+ """The catalog as the wire carries it — a copy, because a caller that mutated the module
+ constant would change every later reader's answer.
+
+ ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from
+ `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two
+ different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST
+ (not first) — a new group appearing above "Web action" because its order defaulted to 0 is
+ the failure that would look deliberate.
+
+ ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client
+ nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two
+ rows in one nest cannot disagree about what that nest is called. A row naming an unknown
+ connector loses the stamp instead of inventing a nest with a raw slug for a title.
+ """
+ last = max(ACTION_GROUP_ORDER.values()) + 1
+ out = []
+ for a in ACTION_CATALOG:
+ row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)}
+ conn = _connector_meta(a.get("connector"))
+ if conn:
+ row["connector"] = dict(conn)
+ else:
+ row.pop("connector", None)
+ out.append(row)
+ return out
+
+
+def _action_label(act):
+ for row in ACTION_CATALOG:
+ if row["kind"] == act.get("kind"):
+ return row["label"]
+ return str(act.get("kind") or "Action")
+
+
+def clean_actions(raw, depth=0, _seen=None, _count=None):
+ """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces.
+
+ Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder
+ survives a Save. A missing or colliding one is minted `act_`; minting on every clean would
+ move the selection under the person editing it.
+ """
+ if raw in (None, ""):
+ return [], None
+ if not isinstance(raw, list):
+ return None, "actions must be a list"
+ seen = _seen if _seen is not None else set()
+ count = _count if _count is not None else [0]
+ out = []
+ for entry in raw:
+ if not isinstance(entry, dict):
+ return None, "each action must be an object with a kind"
+ kind = _s(entry.get("kind"), 30).strip()
+ row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None)
+ if row is None:
+ return None, (f"{kind or 'that action'!r} is not one of: "
+ + ", ".join(a["kind"] for a in ACTION_CATALOG))
+ if not row["ready"]:
+ return None, (f"{row['label']!r} is on the menu but not built yet — "
+ f"{row['detail'][0].lower()}{row['detail'][1:]}")
+ count[0] += 1
+ if count[0] > MAX_ACTIONS:
+ return None, f"an automation runs at most {MAX_ACTIONS} actions"
+ aid = _s(entry.get("id"), 40).strip()
+ if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen:
+ n = 1
+ while f"act_{n}" in seen:
+ n += 1
+ aid = f"act_{n}"
+ seen.add(aid)
+ # ⭐ WAVE 26 · C4 / owner ruling R9 — A `create_record` CARRIES NO CONDITION, EVER.
+ #
+ # Owner, correcting their own earlier answer mid-grill: *"if it is Create Record, I don't
+ # think you can even add Conditions at all. That's not how the Create Record works."*
+ # Airtable agrees, and so does the shape: every other action operates ON the record the
+ # flow is walking, so "run this only when …" is a question about something
+ # that exists. Create record MAKES one. There is nothing to test yet, which is why the
+ # picker on this panel was empty and why every condition saved against it named no field —
+ # the owner's *"the condition on action act_1 names no field"*. The condition belongs on
+ # the trigger, or on a conditional group wrapping the action.
+ #
+ # ⛔ DROPPED, NOT REFUSED, AND THE DIFFERENCE IS D-65. Refusing would 400 every stored
+ # automation that already carries one — forever, with no way to edit it out, because the
+ # editor cannot save the automation it needs to fix. A drop is recoverable; a refusal is a
+ # locked door.
+ # ⚠ Dropped SILENTLY, and that is defensible only because the panel goes with it: wave 26
+ # removes the condition editor from this action kind entirely (C4), so there is no control
+ # whose value could appear to be ignored. If a `create_record` condition editor ever comes
+ # back, this needs a disclosure channel — `clean_actions` has none today.
+ if kind == "create_record":
+ when = None
+ else:
+ when, cerr = clean_cond(entry.get("when"), where=f"action {aid}")
+ if cerr:
+ return None, cerr
+ cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {}
+ cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count)
+ if cerr:
+ return None, cerr
+ out.append({"id": aid, "kind": kind,
+ "enabled": bool(entry.get("enabled", True)),
+ "when": when, "config": cfg})
+ return out, None
+
+
+def _clean_action_config(kind, cfg, depth, seen, count):
+ """One action's `config`, per kind. Returns `(config, error)`."""
+ if kind == "group":
+ # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`,
+ # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an
+ # if with no else, so expressing "otherwise" meant a second group carrying the negation
+ # by hand — two conditions a later edit could put out of step with each other.
+ if depth + 1 > MAX_GROUP_DEPTH:
+ return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — "
+ f"past that a flow is a flowchart and belongs on the board")
+ raw_branches = cfg.get("branches")
+ if not isinstance(raw_branches, list) or not raw_branches:
+ # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and
+ # the live automations carry it — refusing it would 400 every Save of a flow that
+ # was legal yesterday. Read-side, so nothing has to be rewritten in the store.
+ raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"),
+ "actions": cfg.get("actions")}]
+ if len(raw_branches) > MAX_BRANCHES:
+ return None, (f"an If / then has at most {MAX_BRANCHES} branches — past that the "
+ f"record is being routed, which is what the board's lanes are for")
+ out_branches, bseen = [], set()
+ for i, br in enumerate(raw_branches):
+ if not isinstance(br, dict):
+ return None, "each branch of an If / then is an object"
+ cond, cerr = clean_cond(br.get("cond"), where="the branch")
+ if cerr:
+ return None, cerr
+ if cond is None and i != len(raw_branches) - 1:
+ # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches
+ # and a null condition matches everything, so a catch-all above another branch
+ # would make every branch below it dead code — silently, and only for the
+ # records that reached it. Refused with the reason rather than reordered: this
+ # module does not quietly rewrite what somebody built.
+ return None, ("only the LAST branch of an If / then can be the Otherwise leg — "
+ "a catch-all above another branch makes the ones below it "
+ "unreachable")
+ kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count)
+ if kerr:
+ return None, kerr
+ if not kids:
+ return None, "a branch with no actions inside it does nothing"
+ bid = _s(br.get("id"), 40).strip()
+ if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen:
+ bid = f"b{i + 1}"
+ bseen.add(bid)
+ label = " ".join(_s(br.get("label"), 40).split()) or (
+ "Otherwise" if cond is None else _branch_letter(i))
+ out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids})
+ return {"branches": out_branches}, None
+ if kind in ("update_record", "create_record"):
+ values = cfg.get("values")
+ if not isinstance(values, dict) or not values:
+ return None, f"the {kind.replace('_', ' ')} action writes no values"
+ if len(values) > MAX_ACTION_VALUES:
+ return None, f"one action writes at most {MAX_ACTION_VALUES} cells"
+ clean_vals = {}
+ for k, v in values.items():
+ key = _s(k, 80).strip()
+ if not key:
+ return None, "a value is written to a field with no name"
+ if v is not None and not isinstance(v, (str, int, float, bool)):
+ return None, (f"the value for {key!r} must be text or a number — "
+ f"an automation writes cells, not objects")
+ clean_vals[key] = _s(v, 500) if isinstance(v, str) else v
+ out = {"values": clean_vals}
+ # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is
+ # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic
+ # "Create record". Follows the `review` arm below, which has carried `config.label` since
+ # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config`
+ # bag so no shared client type has to change for it.
+ lbl = " ".join(_s(cfg.get("label"), 60).split())
+ if lbl:
+ out["label"] = lbl
+ if kind == "create_record":
+ table = _s(cfg.get("table"), 60).strip()
+ if not table:
+ return None, "the create record action names no database"
+ if not table.startswith(UT_PREFIX):
+ return None, (f"actions write to blank databases (ut_*) this wave — "
+ f"{table!r} is not one")
+ out["table"] = table
+ # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS
+ # forever (`_commit_action_writes` mints `max+1`), so ANY scheduled flow carrying a
+ # Create record duplicates a row per run — silently, and worse every day.
+ #
+ # ⛔ `""` IS THE STORED DEFAULT AND IT MEANS TODAY'S APPEND. Not a migration, not a
+ # guess: a stored automation must not change meaning because this key arrived. The
+ # upsert is a thing somebody turns on, the same shape `clean_ending` gave `terminal`.
+ #
+ # ⛔ AND IT MUST NAME A FIELD THIS ACTION ACTUALLY WRITES. Upserting on a key the
+ # action never sets means every incoming row carries a BLANK key — `upsert_rows`
+ # counts those as `skipped` and writes nothing at all. That is a control that reads
+ # as "no duplicates" and delivers "no rows", which is the worse failure by a distance.
+ # Same shape as `scrape_db`'s "the key field must be one of the mapped fields"
+ # (`clean_config`, above) — one precedent, not a second mechanism.
+ unique = _s(cfg.get("uniqueOn"), 80).strip()
+ if unique and unique not in clean_vals:
+ return None, (f"this action does not write {unique!r}, so it cannot keep records "
+ f"unique on it — it writes: " + ", ".join(sorted(clean_vals)))
+ out["uniqueOn"] = unique
+ return out, None
+ if kind == "enrich_instagram":
+ # ⭐ WAVE 25 · C4. The switches are `field_instagram`'s, unchanged in meaning — the paid
+ # rung, its fallback, the per-post engagement buy and the dry run — because they are the
+ # things that decide what a run COSTS and R4 moves the capability, not the controls.
+ #
+ # ⚠ `profileField` MAY BE EMPTY, and that is the A3 stored-inert pattern rather than
+ # laxness: the C3 profile FLAG resolves it at run time, so the action can be added to a
+ # flow before anybody has flagged the column. An unresolvable binding fails CLOSED with a
+ # sentence at run (`profile_field_key`), which is the honest half — refusing the save
+ # would make the action unaddable on a database that has not been flagged yet.
+ # C5: ONE validator, shared with `clean_config`'s machine-kind branch. Two independent
+ # clamps on the same key is how the action and the legacy kind come to disagree about
+ # what "10 posts" means.
+ #
+ # ⚠ `submitted=False` HERE, AND IT IS THE OPPOSITE OF THE OTHER CALL SITE. The difference
+ # is not the value, it is what "present" MEANS on each path. `clean_config` gets a PATCH,
+ # so a key being there is a person having typed it. This function gets the whole action
+ # list re-posted on every save (`_pin_unique`'s note says so in as many words), so a
+ # `maxPosts` here is usually just the client echoing back what was already stored — and
+ # every action written before this wave stored the old default of 24. Refusing would
+ # therefore 400 those automations forever on a number nobody chose, which is D-65.
+ # The ceiling is still taught, in the place where a person can act on it: the panel's
+ # input is bounded at MAX_POSTS_PER_PULL (C5 tells B not to offer more).
+ max_posts, perr = clean_max_posts(cfg.get("maxPosts"), submitted=False)
+ if perr:
+ return None, perr
+ # ⭐ 2026-08-07 (owner ruling) — THE SELECTION, and every key here is OPTIONAL with a
+ # working default. That is D-65's lesson applied ahead of time rather than after: an enrich
+ # action stored before this wave carries none of these, and it must keep saving and keep
+ # running. Absent `fromView` = the whole database; absent `limit` = DEFAULT_ENRICH_LIMIT;
+ # `skipRecent` absent = OFF, because a cooldown nobody asked for silently stops enriching.
+ # ⚠ `limit` is CLAMPED here and again in `enrich_selection`. Not belt-and-braces: this door
+ # sees a person's typed number and can refuse politely, while the selection sees whatever
+ # is STORED — including configs written before the cap existed. Clamping only here would
+ # let an old 5000 through at run time.
+ try:
+ e_limit = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT)
+ except (TypeError, ValueError):
+ return None, "how many records to enrich has to be a number"
+ if e_limit < 1:
+ return None, "an enrich step has to be allowed at least one record"
+ try:
+ e_days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
+ except (TypeError, ValueError):
+ return None, "the number of days has to be a number"
+ groups, gerr = clean_post_groups(cfg.get("postGroups"), max_posts)
+ if gerr:
+ return None, gerr
+ return {"profileField": _s(cfg.get("profileField"), 80).strip(),
+ "fromView": _s(cfg.get("fromView"), 80).strip(),
+ "sortField": _s(cfg.get("sortField"), 60).strip() or DEFAULT_ENRICH_SORT,
+ # Anything that is not "asc" is newest-first, so a typo cannot invert the order a
+ # person is spending against — it lands on the documented default instead.
+ "sortDir": "asc" if str(cfg.get("sortDir") or "").strip().lower() == "asc"
+ else "desc",
+ "limit": min(e_limit, MAX_ENRICH_PER_RUN),
+ "skipRecent": bool(cfg.get("skipRecent")),
+ "skipRecentDays": max(1, e_days),
+ # ⛔ `tier`/`noFallback` are ACCEPTED AND IGNORED here for the same reason as in
+ # `clean_config` (R5 / C2): a stored action carrying them still saves and simply
+ # loses them, because refusing a retired key 400s every definition written before
+ # this wave (D-65).
+ # ⛔ THE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked for.
+ "postMetrics": bool(cfg.get("postMetrics")),
+ # The separate Comments dataset is more granular than a post read and stays off
+ # until the user intentionally enables it.
+ "commentMetrics": bool(cfg.get("commentMetrics")),
+ "dryRun": bool(cfg.get("dryRun")),
+ # ⭐ 2026-08-09 (owner) — "last 12 reels / 12 videos", per GROUP. Absent = OFF, so
+ # every action stored before today keeps saving and keeps running unchanged.
+ "postGroups": groups,
+ "maxPosts": max_posts}, None
+ if kind == "find_records":
+ table = _s(cfg.get("table"), 60).strip()
+ if not table:
+ return None, "the find records action names no database"
+ cond, cerr = clean_cond(cfg.get("cond"), where="the find")
+ if cerr:
+ return None, cerr
+ try:
+ limit = int(cfg.get("limit") or 25)
+ except (TypeError, ValueError):
+ return None, "the find records limit must be a whole number"
+ if limit < 1 or limit > FIND_LIMIT_MAX:
+ return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}"
+ return {"table": table, "cond": cond, "limit": limit}, None
+ return {}, None
+
+
+def _branch_letter(i):
+ """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a
+ label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is
+ worse than an ugly name."""
+ return chr(65 + i % 26) * (1 + i // 26)
+
+
+def group_branches(act):
+ """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape.
+
+ ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying
+ for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored
+ group might carry either shape, and the one that forgot would silently skip a live
+ automation's actions. A definition cleaned since wave 24 always carries `branches`; one read
+ straight out of the store (the runner reads definitions, not payloads) may not.
+ """
+ cfg = (act or {}).get("config") or {}
+ brs = cfg.get("branches")
+ if isinstance(brs, list) and brs:
+ return [b for b in brs if isinstance(b, dict)]
+ return [{"id": "b1", "label": "A", "cond": cfg.get("cond"),
+ "actions": cfg.get("actions") or []}]
+
+
+def walk_actions(actions):
+ """Every action in the tree, depth-first, groups included. One walker, so "how many actions
+ does this flow have" and "which review stages exist" cannot answer differently.
+
+ ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action
+ COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version
+ that still read `config.actions` would report a fork's contents as zero actions, hide every
+ review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing.
+ """
+ for act in actions or []:
+ yield act
+ if act.get("kind") == "group":
+ for br in group_branches(act):
+ for kid in walk_actions(br.get("actions")):
+ yield kid
+
+
+def interpolate(text, row):
+ """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an
+ action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would
+ put template syntax in front of a customer, and a blank is the visible failure."""
+ if not isinstance(text, str) or "{{" not in text:
+ return text
+ def _sub(m):
+ v = (row or {}).get(m.group(1).strip())
+ return "" if v is None else str(v)
+ return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text)
+
+
+# ─────────────────────────────────────────────────────────────────────────────────────────────
+# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order.
+#
+# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits
+# them once at the end (the 256-commits/hr budget every writer in this module respects). A
+# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget
+# on one busy flow.
+# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the
+# runner stamps its stage and stops walking THAT record — later actions belong to the branch a
+# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause.
+# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the
+# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write
+# cannot fire an event trigger, its own or a sibling's.
+
+def _act_row_patch(patches, table, rid, values):
+ patches.setdefault(table, {}).setdefault(str(rid), {}).update(values)
+
+
+def profile_field_key(table, named=""):
+ """C3/C4: WHICH COLUMN on this database carries the Instagram handle. '' when none does.
+
+ Resolution order, and the missing branch is the important one:
+ 1. the column the action NAMES — the person picked it, so it is not a guess;
+ 2. otherwise the column carrying C3's flag `profile: {source: "instagram"}`;
+ 3. otherwise **NOTHING**, and the run says so.
+
+ ⛔ THERE IS DELIBERATELY NO FALL-BACK TO `_auto_url_field`. That helper finds "the first `url`
+ column" and is exactly right for `field_instagram`, whose whole configuration was a URL column
+ — but here it would silently bind the enrich step to whatever URL column happens to be first,
+ on a database where nobody has said which column is a profile. The failure would be a run that
+ reports success having enriched from the wrong column, which is worse than one that refuses:
+ a wrong number is harder to notice than a missing one, and it would be written into the
+ permanent history as if it had been measured.
+ """
+ fields = (table or {}).get("fields") or []
+ want = str(named or "").strip()
+ if want:
+ return want if any(str(f.get("key")) == want for f in fields) else ""
+ for f in fields:
+ p = f.get("profile")
+ if isinstance(p, dict) and str(p.get("source") or "") == PROFILE_SOURCE_IG:
+ return str(f.get("key") or "")
+ return ""
+
+
+def _flow_table(defn):
+ """The database a flow's actions and ending operate on: the automation's own target, or the
+ table its event trigger watches when the flow has no target of its own."""
+ cfg = (defn or {}).get("config") or {}
+ if defn.get("kind") == "discover_instagram":
+ return cfg.get("targetTable") or DISCOVER_TABLE
+ return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "")
+
+
+#: ⭐ 2026-08-07 (owner ruling) — WHAT ONE ENRICH STEP MAY SPEND IN A SINGLE RUN.
+#: `limit` is the number the panel offers; this is the ceiling the module enforces whatever the
+#: panel sends, and it is DISCLOSED when it bites rather than applied as a silent `[:N]`
+#: ([[no-unverifiable-aggregates]]). 100 profiles is ~4 minutes of vendor pacing on its own.
+MAX_ENRICH_PER_RUN = 100
+DEFAULT_ENRICH_LIMIT = 25
+DEFAULT_ENRICH_COOLDOWN_DAYS = 30
+#: Which column the selection orders by when nobody has said. `first_found` newest-first is the
+#: owner's own example ("top N enrichment sorted by date") and the useful default: it enriches what
+#: discovery just found rather than re-walking the oldest leads in the book.
+DEFAULT_ENRICH_SORT = "first_found"
+
+#: ⭐⭐ 2026-08-09 (DEBT D-103) — WHERE A RUN'S PER-RECORD REASONS TRAVEL.
+#: A run's `counts` are integers and `_commit_run` drops everything non-numeric, so the sentence
+#: a vendor gave us had nowhere to ride and was thrown away at three separate layers. This key is
+#: a deliberate passenger IN the counts dict, popped by `run_now` before the counts are stored.
+#: ⛔ A RESERVED KEY, NOT A COUNT: it must never be rendered as one, which is why it starts with
+#: an underscore — `COUNT_LABELS` on the client is an allow-list and cannot pick it up by
+#: accident, and `_commit_run`'s numeric filter is the second net under it.
+RUN_NOTES_KEY = "_notes"
+#: ⭐⭐ WAVE 28 / OWNER RULING R9 — `NOT_FOUND_RETRY_DAYS` IS RETIRED. A handle a VENDOR SAID DOES
+#: NOT EXIST is now a TOMBSTONE: never auto-retried, at any interval.
+#: ⛔ THE 30-DAY BACKOFF WAS THE DEFENSIBLE ANSWER AND IT WAS STILL WRONG, which is the sentence
+#: worth keeping. Its argument was that a handle can be renamed back or a suspension lifted, so
+#: the verdict should expire. But nothing about US changes on day 31 — the only new information
+#: is a guess that the world moved — so the timer buys one paid vendor call per dead handle per
+#: month, forever, on a row nobody is looking at, and reports it as "1 blocked". A tenant with a
+#: hundred stale handles pays a hundred times a month to be told the same thing.
+#: ⭐ WHAT RE-ARMS IT IS A HUMAN, and there are two doors:
+#: 1. the KEY is `(platform, handle)` — so correcting a typo re-arms IMMEDIATELY, with no
+#: wiring at all, because the corrected handle simply is not the one we recorded; and
+#: 2. `clear_gone()` — called when the profile CELL is written, so re-typing the SAME handle
+#: (a person saying "try it again") also re-arms.
+#: ⚠ The run keeps NAMING the skipped handles every time, not only on the run that discovered
+#: them: a tombstone the owner cannot see is just a disappearance.
+#: A queued profile snapshot the vendor never finishes is dropped after this, with a note. An
+#: unbounded pending list is the forever-loop this whole change exists to remove, wearing the
+#: opposite mask ([[gate-answers-the-wrong-question]]).
+PENDING_PROFILE_MAX_HOURS = 24
+
+
+def _order_key(value):
+ """Order one CELL. Numbers numerically, everything else as text.
+
+ ⚠ Cells are stored as STRINGS, so `"20872"` and `"9"` compare in the wrong order as text —
+ which on a `followers` sort is not a cosmetic wrong order, it is the wrong 25 accounts getting
+ paid for. Numbers are tried first and text is the fallback, never the other way round.
+ ⚠ Dates need no special case: R3 made these columns `date` and they store `YYYY-MM-DD`, where
+ lexicographic order IS chronological order. If that format ever changes, this comment is the
+ thing that was relied on.
+ """
+ s = str(value if value is not None else "").strip()
+ try:
+ return (0, float(s.replace(",", "")), "")
+ except ValueError:
+ return (1, 0.0, s.lower())
+
+
+def _days_since(day, today=None):
+ """Whole days between a `YYYY-MM-DD` stamp and today, or None when it cannot be read.
+
+ ⛔ None means UNKNOWN and every caller must treat it as "not measured", never as 0 or as a
+ large number — the cooldown below reads an unreadable stamp as "never enriched", which spends
+ money rather than skipping. That is the right way round: a skipped record is invisible, and a
+ stamp we cannot parse is our bug, not a reason to silently stop enriching somebody's data.
+ """
+ import datetime as _dt
+ s = str(day or "").strip()[:10]
+ if not s:
+ return None
+ try:
+ d = _dt.date.fromisoformat(s)
+ except ValueError:
+ return None
+ return ((today or _dt.date.today()) - d).days
+
+
+def _hours_since(stamp):
+ """Whole hours since a full ISO stamp, or None when it cannot be read.
+
+ ⚠ ISO, NOT `YYYY-MM-DD` — `_days_since` above answers a question in DAYS about a `date`
+ column and truncates to 10 characters. A pending snapshot's age is measured in hours and its
+ stamp carries a time, so reusing that function would read every entry as "queued at
+ midnight". Same reason its None means UNKNOWN: an unreadable stamp must not age a paid,
+ outstanding snapshot out of the queue.
+ """
+ import datetime as _dt
+ s = str(stamp or "").strip()
+ if not s:
+ return None
+ try:
+ t = _dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
+ except ValueError:
+ return None
+ now = _dt.datetime.now(_dt.timezone.utc)
+ if t.tzinfo is None:
+ t = t.replace(tzinfo=_dt.timezone.utc)
+ return max(0, int((now - t).total_seconds() // 3600))
+
+
+def _gone_key(row, handle):
+ """The identity a "this account does not exist" verdict is remembered under.
+
+ `(platform, handle)` — wave 26 R4's dedup key, not the handle alone, because `@x` on
+ Instagram and `@x` on TikTok are two accounts. Blank platform means Instagram, which is what
+ every row on a preset profile table is and what `preset_cells` stamps.
+ """
+ plat = str((row or {}).get("platform") or PLATFORM_INSTAGRAM).strip().lower()
+ return f"{plat}:{str(handle or '').strip().lstrip('@').lower()}"
+
+
+def clear_gone(rt, table_key, handle, row=None):
+ """R9's re-arm door: forget the "this account does not exist" verdict for ONE handle.
+
+ Returns the number of automations whose memory was changed — 0 is the ordinary answer and is
+ not an error, because most cell edits are not on a dead handle.
+
+ ⭐⭐ WHY THIS IS A PUBLIC FUNCTION IN THE ENGINE RATHER THAN A LOOKUP IN THE ROW WRITER.
+ The verdict lives on the AUTOMATION (`state.enrichNotFound`), not on the row — it has to,
+ because it is a fact about what a run learned and paid for. But the thing that re-arms it is a
+ ROW event, and the row writer must not need to know the shape of an automation's state to
+ trigger it. So the seam is one call with the three things the writer already has, and every
+ walk of `all_definitions` stays on this side of the fence.
+ ⚠ IT MATCHES THE SAME WAY THE SKIP DOES. `_gone_key` is the one implementation of "which
+ handle is this", so a verdict can never be recorded under a key this cannot find
+ ([[one-evaluator-per-question]]).
+
+ ⛔ WITHOUT ITS CALLER THIS IS INERT, AND THAT IS RECORDED RATHER THAN ASSUMED. R9's first
+ door — correcting a typo — works with no wiring at all, because the key IS the handle and a
+ different handle is simply not the one we recorded. This second door only opens when the row
+ write path calls it, which lives in `routes_tables.patch_row` (another session's fence).
+ Until that line lands, re-typing the SAME dead handle stays skipped
+ ([[flag-shipped-without-its-writer]] — named on purpose, so it is not discovered later).
+ """
+ key = _gone_key(row or {}, handle)
+ if not str(handle or "").strip():
+ return 0
+ changed = 0
+ for auto_id, defn in (all_definitions(rt) or {}).items():
+ if str((defn.get("config") or {}).get("targetTable") or "") != str(table_key):
+ continue
+ known = dict(((defn.get("state") or {}).get("enrichNotFound")) or {})
+ if key not in known:
+ continue
+ known.pop(key, None)
+ # ⚠ `or None` — an empty dict must clear the key rather than store `{}`, which is what
+ # every other state writer here does and what keeps a definition from growing a graveyard
+ # of empty maps.
+ set_state(rt, str(auto_id), {"enrichNotFound": known or None})
+ changed += 1
+ return changed
+
+
+def _prime_enrich_batch(chosen, rows, profile_key, table, enrich, prefetch,
+ step=_no_step, log=print):
+ """Buy the selection's profiles in ONE vendor call per chunk. Returns how many resolved.
+
+ ⭐⭐ 2026-08-09 — THE FIX FOR "WHY IS THIS ONE RECORD PER CALL". `bd_scrape` has always taken a
+ LIST of URLs and its own docstring names the failure mode ("…turning a 20-profile run into an
+ hour"); the only profile caller passed a single-element list from inside the per-record walk.
+ MEASURED on nurilab: a 25-record walk still running at 67 minutes, 25 billed snapshots, 25
+ vendor emails. This runs once, where the whole chosen set and the row bodies are both in hand.
+
+ ⛔ IT IS A FAST PATH AND MUST STAY ONE. Everything it fails to resolve is simply absent from
+ `prefetch`, and the per-record rung then behaves exactly as it does today — including its
+ corpus fallback and its Apify second opinion. A batch that cannot make the run WRONG is a
+ batch that can be shipped on a paid path; that is why the vendor call is wrapped and a failure
+ returns 0 instead of raising.
+
+ ⛔ A DEFERRED CHUNK IS FANNED OUT PER DESTINATION, NOT PER SNAPSHOT. One snapshot id now
+ serves many records, so each pending task keeps its OWN `influencer`/`table`/`rowId` and they
+ share the id — `_pending_profile_tasks` is explicit that a profile cannot be resolved from the
+ handle alone ("two databases may both hold `@x`"). The handle is ALSO written into `prefetch`
+ behind `DEFERRED_MARK` so the walk does not buy the same snapshot a second time.
+ """
+ want, dests, handles = [], {}, []
+ for rid in (chosen or []):
+ row = rows.get(str(rid)) or {}
+ h = str(row.get(profile_key, "") or "").strip().lstrip("@").lower()
+ if not h or h in prefetch:
+ continue
+ want.append((h, str(rid)))
+ if h not in dests:
+ handles.append(h)
+ dests.setdefault(h, []).append(str(rid))
+ if not handles:
+ return 0
+ step(f"Reading {len(handles)} profile{'' if len(handles) == 1 else 's'} from the source")
+ deferrals = []
+ try:
+ nodes, note = bd_profiles_batch(handles, deferred=deferrals)
+ except Exception as exc: # noqa: BLE001 — never fatal, see docstring
+ log(f"[aios-auto] profile batch failed, falling back to per-record reads: "
+ f"{type(exc).__name__}: {exc}")
+ return 0
+ prefetch.update(nodes)
+ if note:
+ log(f"[aios-auto] profile batch note: {_s(note, 300)}")
+ for d in deferrals:
+ sid = str(d.get("snapshotId") or "")
+ if not sid:
+ continue
+ for u in (d.get("urls") or []):
+ h = str(ig_handle(str(u)) or "").strip().lstrip("@").lower()
+ if not h or h in nodes:
+ continue
+ prefetch[h] = {DEFERRED_MARK: sid}
+ for rid in dests.get(h, []):
+ enrich["pendingProfiles"].append(
+ {"snapshotId": sid, "datasetId": str(d.get("datasetId") or ""),
+ "urls": [str(u)], "kind": "profile", "influencer": h,
+ "table": table, "rowId": rid, "requestedAt": _iso()})
+ return len(nodes)
+
+
+def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None):
+ """⭐ 2026-08-07 (owner ruling) — WHICH records this enrich step spends on, in order.
+
+ Returns `(ordered_row_ids, note)`. The note is the honest account of what the selection did
+ and is surfaced in the run summary; `""` means there is nothing worth saying.
+
+ ⛔ **`limit` IS A QUOTA OF WORK DONE, NOT A WINDOW OF ROWS EXAMINED**, and that is the owner's
+ ruling in as many words: *"if a user choose to enrich 30 and from that sorted list of 30, 20 is
+ enriched last 30 days, then it goes to next list like this"*. So the walk CONTINUES past every
+ skipped record until 30 have actually been enriched or the list runs out. The naive reading —
+ take the top 30, then filter — would have billed for 10 and reported success, and the number in
+ the box would have meant something different every run depending on how much of the top of the
+ list happened to be fresh. A quota is predictable; a filtered window is not.
+
+ The order of operations, each step narrowing the one above:
+ 1. the optional saved VIEW — resolved through `view_filter`, the SAME resolver `enters_view`
+ and `seed_rows` use. A view that has been deleted is a PROBLEM, never an empty tree: an
+ empty tree matches everything, so degrading would turn "enrich my shortlist" into "enrich
+ the entire database", at vendor prices.
+ 2. the SORT — the owner's "top N sorted by date", blanks always last in both directions
+ (a blank is unknown, not smallest).
+ 3. the COOLDOWN — skip anything enriched within N days, when the toggle is on.
+ 4. the QUOTA — stop at `limit`, itself clamped to `MAX_ENRICH_PER_RUN`.
+
+ ⚠ A record with a BLANK handle is skipped and never counted against the quota — there is
+ nothing to enrich and it must not consume a slot somebody paid for.
+ """
+ t = ut_get(rt, str(table_key or ""))
+ if t is None:
+ return [], f"{table_key!r} is not a database in this workspace"
+ rows = dict(t.get("rows") or {})
+ notes = []
+
+ view_id = str(cfg.get("fromView") or "").strip()
+ if view_id:
+ tree, fields, problem = view_filter(rt, table_key, view_id)
+ if problem:
+ # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet
+ # fallback costs real money.
+ return [], f"the enrich step names a view it cannot read — {problem}"
+ # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS
+ # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE.
+ #
+ # `view_filter` returns a SAVED VIEW's filter tree — `{"nodes": [{colId, op, value}],
+ # "conj"}` in the GRID's dialect, whose operators are `contains/eq/neq/gt/gte/lt/lte/
+ # isEmpty/isNotEmpty/between/within`. `lane_match` reads an AUTOMATION LANE condition —
+ # `{field, op, value}` with `=/!=/>/includes/is_empty/…` — and dispatches on
+ # `COND_GROUP_KEYS`. Handed a view tree it found no group key, read `raw.get("field")`,
+ # got `""`, and answered **False for every row**: MEASURED on the owner's own automation,
+ # a view matching exactly one record selected NOTHING, reported NO error, and the run
+ # committed `ok`. *"I just chose the View 'Enrichment test' where there is only 1 manual
+ # record... how come it says 51 records walked?"*
+ #
+ # ⛔ AND THE RIGHT EVALUATOR WAS ALREADY IN THE FILE. `_row_gate`'s `enters_view` branch
+ # resolves the SAME tree through `harness.filter_eval.matches(tree, row, fields)` and has
+ # always been correct. `_seed_event_state`'s own docstring states the law this line broke:
+ # *"Two implementations of 'does this row match' is how a seed disagrees with the edge it
+ # is supposed to arm."* There were three. Now there are two callers of one function.
+ # ⚠ `fields` is no longer discarded — `matches` needs the column TYPES to compare a date
+ # as a date and a number as a number, which is the half `lane_match` could not have had.
+ import harness.filter_eval as filter_eval
+ rows = {rid: r for rid, r in rows.items() if filter_eval.matches(tree, r, fields)}
+
+ sort_field = str(cfg.get("sortField") or DEFAULT_ENRICH_SORT).strip() or DEFAULT_ENRICH_SORT
+ newest_first = str(cfg.get("sortDir") or "desc").strip().lower() != "asc"
+ # ⭐⭐ 2026-08-07 (owner ruling) — A MISSING DATE MEANS **NOW**, NOT "UNKNOWN".
+ #
+ # Owner: *"treat a missing first found as now, and manual entry should go first, instead of
+ # treated as last. I want to see my inayma manual entry works with automation enrichment."*
+ # And they are right about the semantics, not just the preference: on these tables the ONLY
+ # rows without a `first_found` are ones a PERSON typed, because the discovery runner stamps it
+ # on every row it writes. So a blank is not missing data — it is a row that was first found
+ # today, by the person sitting in front of it, and the one they most want enriched.
+ #
+ # The old rule sorted blanks LAST in both directions, so a hand-typed handle sat at position
+ # 41 of 41 and fell outside a limit of 25 — the row the whole feature exists for was the one
+ # it never reached.
+ #
+ # ⛔ DATE COLUMNS ONLY, and the narrowing is the honest half. "Blank means now" is a fact about
+ # a TIMESTAMP; a blank `followers` is not "the most followers", it is genuinely unknown, and
+ # treating it as the maximum would spend the budget on the rows we know least about. So a
+ # non-date sort keeps the old rule: unknown sorts last, in both directions.
+ #
+ # ⚠ NOTE THIS FALLS OUT OF THE KEY RATHER THAN BEING A SECOND PASS — `(0, value)` for a real
+ # date and `(1, "")` for a blank, with `reverse` doing the rest. Newest-first puts blanks at
+ # the front (they are "now"); oldest-first puts them at the back (they are still "now"). One
+ # rule, both directions, no branch that can disagree with itself.
+ ftype = str(((next((f for f in (t.get("fields") or [])
+ if str(f.get("key")) == sort_field), None)) or {}).get("type") or "")
+ if ftype == "date":
+ ordered = sorted(rows.items(),
+ key=lambda x: (0, str(x[1].get(sort_field) or "").strip())
+ if str(x[1].get(sort_field) or "").strip() else (1, ""),
+ reverse=newest_first)
+ else:
+ have = [(rid, r) for rid, r in rows.items() if str(r.get(sort_field) or "").strip()]
+ blank = [(rid, r) for rid, r in rows.items() if not str(r.get(sort_field) or "").strip()]
+ have.sort(key=lambda x: _order_key(x[1].get(sort_field)), reverse=newest_first)
+ blank.sort(key=lambda x: _rid_num(x[0]))
+ ordered = have + blank
+
+ try:
+ quota = int(cfg.get("limit") or DEFAULT_ENRICH_LIMIT)
+ except (TypeError, ValueError):
+ quota = DEFAULT_ENRICH_LIMIT
+ quota = max(1, min(quota, MAX_ENRICH_PER_RUN))
+ if cfg.get("limit") and quota != int(cfg.get("limit") or 0):
+ notes.append(f"the limit was capped at {MAX_ENRICH_PER_RUN} for one run")
+
+ cooling = bool(cfg.get("skipRecent"))
+ try:
+ days = int(cfg.get("skipRecentDays") or DEFAULT_ENRICH_COOLDOWN_DAYS)
+ except (TypeError, ValueError):
+ days = DEFAULT_ENRICH_COOLDOWN_DAYS
+ days = max(1, days)
+
+ picked, cooled, blank_handle = [], 0, 0
+ # ⭐⭐ 2026-08-09 — HANDLES A VENDOR HAS ALREADY SAID DO NOT EXIST.
+ #
+ # ⛔ THE DEFECT THIS CLOSES IS STRUCTURAL, AND IT IS THE WORD "AGAIN" IN THE OWNER'S REPORT.
+ # A blocked read writes NO cells, so `enriched_at` stays unset, so `_days_since(None)` is
+ # None, so the cooldown above can never exclude the row — while `Followers is empty` keeps it
+ # in the Pending cohort by construction. MEASURED on nurilab: one dead handle, re-bought at
+ # 06:00 on three consecutive days, reported each time as an opaque "1 blocked". No vendor fix
+ # removes that loop; only a memory of the verdict does.
+ #
+ # ⚠ THEY ARE **NAMED**, NOT SILENTLY DROPPED. The whole point is that the owner can act — the
+ # note goes into the run every single time, not only on the run that discovered it, because a
+ # run that quietly reports "0 records walked" tomorrow puts them straight back at "wtf".
+ skipped_gone = []
+ for rid, r in ordered:
+ if len(picked) >= quota:
+ break
+ raw_handle = str(r.get(profile_key) or "").strip()
+ if not raw_handle:
+ blank_handle += 1
+ continue
+ # ⛔ R9 — NO EXPIRY. There is deliberately no date arithmetic here any more: a verdict is
+ # a verdict until a human edits the cell. An `at` stamp is still STORED (it is what the
+ # owner reads to know when we last paid to be told this), it is simply not a clock.
+ if isinstance((gone or {}).get(_gone_key(r, raw_handle)), dict):
+ skipped_gone.append(raw_handle)
+ continue
+ if cooling:
+ since = _days_since(r.get("enriched_at"), today=today)
+ if since is not None and since < days:
+ cooled += 1
+ continue
+ picked.append(str(rid))
+
+ # ⛔ THE HONEST ACCOUNT. "10 enriched" reads as success whether the quota was 10 or 30, so the
+ # run says when it could NOT fill the quota and why — the same disclosure rule `cap_note` and
+ # `run_plain` follow. Silence here would make a shrinking selection invisible.
+ if cooled:
+ notes.append(f"{cooled} skipped as enriched in the last {days} days")
+ if skipped_gone:
+ shown = ", ".join(f"@{h}" for h in skipped_gone[:5])
+ # ⚠ THE SENTENCE IS THE FEATURE. It must name the handles AND the two things a person can
+ # do, because nothing else will ever retry them — under R9 this note is the only path
+ # back from a tombstone, so a vaguer version would strand the row permanently.
+ notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account "
+ f"({shown}{', …' if len(skipped_gone) > 5 else ''}) — delete the row or "
+ f"correct the handle; they are not retried automatically")
+ if blank_handle:
+ notes.append(f"{blank_handle} skipped with no handle")
+ if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered):
+ notes.append(f"{len(picked)} of the {quota} asked for — the list ran out")
+ return picked, "; ".join(notes)
+
+
+def _has_action(actions, kind):
+ """Does this flow contain `kind` ANYWHERE, forks included?
+
+ ⛔ FORKS ARE THE WHOLE REASON THIS IS A FUNCTION. A group's children live under
+ `config.branches[].actions` (C-FORK), so a flat scan of the top level answers False for an
+ enrich step somebody put inside an If — and the caller would then skip a schema top-up the
+ run genuinely needs. Same walk `mapTree` does on the client, and the same trap wave 24
+ recorded when four hand-rolled walks all forgot to descend.
+ """
+ for a in (actions or []):
+ if not isinstance(a, dict):
+ continue
+ if a.get("kind") == kind:
+ return True
+ for br in ((a.get("config") or {}).get("branches") or []):
+ if _has_action((br or {}).get("actions") or [], kind):
+ return True
+ return False
+
+
+def _actions_of_kind(actions, kind):
+ """Every action of `kind`, including actions nested inside If branches."""
+ out = []
+ for action in actions or []:
+ if not isinstance(action, dict):
+ continue
+ if action.get("kind") == kind:
+ out.append(action)
+ for branch in ((action.get("config") or {}).get("branches") or []):
+ out.extend(_actions_of_kind((branch or {}).get("actions") or [], kind))
+ return out
+
+
+def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print,
+ step=_no_step):
+ """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts.
+
+ ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery
+ automation's review stage comes from `stages_for`, not from actions,
+ so an early return on an empty action list would have silently disabled the owner's loop
+ feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in
+ review; the tests all happened to pass a non-empty action list, which is why it was green.
+ """
+ defn, _retired = _without_retired_board(defn)
+ flow = (defn or {}).get("flow") or {}
+ actions = flow.get("actions") or []
+ # ⛔ THE PINNED STEP IS A PICTURE OF THE ENGINE'S OWN WRITE, NOT A SECOND ONE.
+ #
+ # MEASURED 2026-08-06: a discovery run wrote its candidates through `ut_write_rows` (the
+ # "Save results" node) and THEN walked the flow over the rows it had just written — where
+ # the seeded `create_record` inserted every one of them AGAIN. Two rows became four, on
+ # every run, silently, since wave 24 seeded that action. Making step 1 permanent (owner
+ # ruling, same day) would have made the duplication permanent and unavoidable with it.
+ #
+ # ⚠ THE CARD STAYS. It is what the owner asked for and it is honest — "this search puts what
+ # it finds in a database" is exactly what the engine does. What it must not do is do it
+ # twice. Skipped HERE rather than not-seeded, so the builder still shows the step and the
+ # server still refuses to let it be removed.
+ if ig_action_pinned(defn, 0) and actions:
+ actions = actions[1:]
+ counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0}
+ # ⭐ C4 — THE ENRICH ACCUMULATOR. Every append row the enrich action produces is collected
+ # across the WHOLE walk and written once at the end, for the reason the module header states
+ # and `run_field_instagram` learned the hard way: `upsert_rows` rebuilds the row dict on
+ # entry, so calling it per record is O(existing) per record — survivable at 5,000 rows and an
+ # automation that never finishes at `MAX_UT_IG_ROWS`. One flush, three upserts.
+ enrich = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
+ # ⭐ 2026-08-09 — deferred PROFILE snapshots (paid, still building at the vendor)
+ # and handles a vendor has said do not exist. Both are collected across the whole
+ # walk and written once, for the same reason every other list here is.
+ "pendingProfiles": [], "gone": {},
+ "profiles": 0, "ok": 0, "blocked": 0, "notes": [], "dry": False}
+ # The verdicts this automation already holds, read ONCE — `enrich_selection` consults them
+ # per record and re-reading the definition per row would be a store read per record.
+ known_gone = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})
+ # ⭐ 2026-08-07 — THE SELECTION, RESOLVED ONCE PER ACTION AND NOT ONCE PER RECORD.
+ # `enrich_selection` sorts and scans the whole table; doing that inside the per-record walk
+ # would be O(rows²) and, worse, would re-answer "which 25 records" every time a record walked
+ # through — so the quota could never be honoured. Keyed by ACTION id, because two enrich steps
+ # in one flow are two independent budgets.
+ enrich_plan = {}
+ # ⭐⭐ 2026-08-09 — `{handle: node}` FOR THE WHOLE SELECTION, BOUGHT IN ONE CALL PER CHUNK.
+ # The vendor read used to happen inside the per-record walk, one URL per `/v3/scrape` and a
+ # `PACE_SECONDS` floor between records, so a 25-record walk was 25 billed snapshots and was
+ # MEASURED still running at 67 minutes. `bd_scrape` always accepted a list; nothing ever
+ # handed it one. Run-scoped rather than per-action: two enrich steps over the same table are
+ # two budgets (`enrich_plan`) but the same profiles, and a handle already bought is not worth
+ # buying twice.
+ enrich_prefetch = {}
+ if not table_key:
+ return counts
+ table = str(table_key)
+ # ⭐⭐ 2026-08-07 (owner report) — THE PRESET COLUMNS MUST EXIST BEFORE THE CELLS ARE WRITTEN.
+ #
+ # ⛔ MEASURED on the owner's own row: a successful enrich reported `enriched: 1` and filled
+ # NINE cells out of a pull that carried far more, because `_act_row_patch` can only write a
+ # cell whose COLUMN is declared — `user_tables` filters an unknown key on every write door.
+ # So every preset the target table did not already happen to have was silently dropped, and
+ # the run still said ok. Owner: *"UNTIL I SEE THAT ALL OF INAYMA'S INSTAGRAM FIELDS GET
+ # FILLED"* — this is the half that was swallowing them.
+ #
+ # ⚠ WHY IT WAS MISSING RATHER THAN BROKEN: the columns are spawned by `_presets_after_write`,
+ # which runs when a **Create record** action is SAVED (W25/R10). A `plain` automation whose
+ # only step is `enrich_instagram` never saves one, so nothing had ever declared them — the
+ # feature worked on discovery tables purely because discovery creates them for its own reasons.
+ #
+ # ⚠ ONCE PER RUN, not once per record, and only when an enrich step is actually present:
+ # `ut_ensure` MERGES by key and returns without a write when nothing is new, so the steady
+ # state is one dict comparison. Guarded by the same walk that would use it, so a flow with no
+ # enrich step never touches the schema of the database it walks.
+ # ⛔ ONLY ON A TABLE THAT IS ALREADY BOUND, and the narrowing is the careful half. The preset
+ # set declares `handle` with `pinned` AND `profile: {source: 'instagram'}` — so topping up an
+ # ARBITRARY database would (a) reshape someone's schema with 36 columns because they pointed a
+ # step at it, and (b) add a SECOND profile column to a table that already flagged a different
+ # one, which `user_tables` forbids at both write doors. A table with no profile column is an
+ # UNBOUND enrich: it must stay unbound and say so, which is a different defect (D-79) with its
+ # own honest refusal, not something to paper over by inventing the binding.
+ _tbl_now = ut_get(rt, table) or {}
+ _enrich_actions = _actions_of_kind(actions, "enrich_instagram")
+ _bound = next((f for f in (_tbl_now.get("fields") or [])
+ if isinstance(f.get("profile"), dict)), None)
+ if _bound is None:
+ _named = next((str((a.get("config") or {}).get("profileField") or "").strip()
+ for a in _enrich_actions
+ if str((a.get("config") or {}).get("profileField") or "").strip()), "")
+ _bound = next((f for f in (_tbl_now.get("fields") or [])
+ if str(f.get("key") or "") == _named), None)
+ if _bound and _enrich_actions:
+ # ⚠ AND THE DECLARATIONS ARE STRIPPED FROM ANYTHING NEW. The binding already exists —
+ # `_bound` is it — so a preset arriving now must carry DATA, never a second identity: a
+ # table whose profile column is `ig_handle` would otherwise gain a rival `handle`.
+ topup = [({k: v for k, v in f.items() if k not in ("profile", "pinned")}
+ if f.get("key") != _bound.get("key") else dict(f))
+ for f in PRESET_PROFILE_FIELDS]
+ try:
+ if any(not bool((a.get("config") or {}).get("dryRun")) for a in _enrich_actions):
+ ensure_ig_graph(rt, username, str(defn.get("id") or ""),
+ profile_table=table, profile_field=str(_bound.get("key") or ""))
+ else:
+ # Dry run keeps its original no-create contract; only the already-established
+ # Profile table is topped up, and no canonical related database is spawned.
+ ut_ensure(rt, _tbl_now.get("label") or table, topup, username, key=table,
+ flow_tag=str(defn.get("id") or ""), lock_fields=True)
+ except Exception as exc: # noqa: BLE001
+ # A schema top-up that cannot run must not stop the enrichment: the cells that DO
+ # have columns still land, which is strictly better than the pull being thrown away.
+ log(f"[aios-auto] preset top-up on {table} failed: {type(exc).__name__}: {exc}")
+ rows = dict((ut_get(rt, table) or {}).get("rows") or {})
+ if not rows:
+ return counts
+ # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The
+ # walk used to keep a `reached` map and stamp each record with the furthest action it made —
+ # a board column's worth of bookkeeping written into the tenant's own table on every run.
+ # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a
+ # cell the customer has to look at and a store commit we pay for on every scheduled run.
+ # The RUN LOG is the operational record of what happened now, and it is the only one.
+ patches, creates = {}, {}
+
+ def _walk(acts, row, rid, depth=0):
+ """Walk the actions in order for ONE record.
+
+ ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean
+ "this record was SUSPENDED by a review gate" — the one branch that could stop a walk
+ early. With review retired nothing suspends anything, so every record walks its whole
+ flow; the signature stays so a future gate-style action has an obvious place to say so.
+ """
+ for act in acts:
+ if not act.get("enabled", True):
+ continue
+ if not lane_match(act.get("when"), row):
+ continue
+ kind = act.get("kind")
+ cfg = act.get("config") or {}
+ counts["actionsRun"] += 1
+ if kind == "group":
+ # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs.
+ # Declaration order is priority order, the same rule `route_record` applies to
+ # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch
+ # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions`
+ # has already guaranteed a null condition can only be LAST.
+ for br in group_branches(act):
+ if lane_match(br.get("cond"), row):
+ if _walk(br.get("actions") or [], row, rid, depth + 1):
+ return True
+ break
+ elif kind == "update_record":
+ vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()}
+ row.update(vals) # later actions see the write, as they must
+ _act_row_patch(patches, table, rid, vals)
+ counts["updated"] += 1
+ elif kind == "create_record":
+ target = str(cfg.get("table") or "")
+ vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()}
+ # C5: keyed by (table, uniqueOn) rather than by table alone, because two actions
+ # may legitimately write to ONE database on different keys — collapsing them onto
+ # the table would silently apply one action's uniqueness rule to the other's rows.
+ creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals)
+ counts["created"] += 1
+ elif kind == "enrich_instagram":
+ # ⭐ C4 (R3/R4). Reuses `pull_profile` and `capture_rows` — the SAME functions
+ # `run_field_instagram` calls, not a second implementation of either.
+ pkey = profile_field_key(ut_get(rt, table), cfg.get("profileField"))
+ if not pkey:
+ # FAIL CLOSED, ONCE, WITH THE REASON. Not per record: the binding is a
+ # property of the flow, so a 100-record run would otherwise put the same
+ # sentence in the log a hundred times and bury everything else.
+ if "unbound" not in enrich["notes"]:
+ enrich["notes"].append("unbound")
+ log("[aios-auto] enrich_instagram: no profile column on "
+ f"{table} — name one on the action, or mark a text column as an "
+ "Instagram profile")
+ continue
+ # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET?
+ # Resolved once (see `enrich_plan`) and then a membership test. A record outside
+ # the selection is NOT an error and NOT a skip worth logging per row — it is simply
+ # not this run's work, and the selection's own note already accounts for it.
+ aid = str(act.get("id") or "")
+ if aid not in enrich_plan:
+ chosen, sel_note = enrich_selection(rt, table, cfg, pkey, gone=known_gone)
+ enrich_plan[aid] = set(chosen)
+ if sel_note:
+ enrich["notes"].append(sel_note)
+ # ⭐ ONE VENDOR CALL PER CHUNK FOR THE WHOLE SELECTION, here and nowhere else:
+ # this is the only place the full chosen set and the row bodies are both in
+ # hand. Anything it resolves the per-record rung below reads from memory.
+ _prime_enrich_batch(chosen, rows, pkey, table, enrich,
+ enrich_prefetch, step, log)
+ if str(rid) not in enrich_plan[aid]:
+ continue
+ handle_raw = str(row.get(pkey, "") or "").strip()
+ if not handle_raw:
+ continue # nothing to enrich on this record, not an error
+ # ⚠ THE PACE FLOOR IS THE VENDOR'S, SO IT IS PAID ONLY WHEN THE VENDOR IS CALLED.
+ # A handle already in `enrich_prefetch` was bought by the batch above and is read
+ # from memory; sleeping 2.5 s before a dictionary lookup would hand most of the
+ # batching win straight back (25 records = ~62 s of pure waiting).
+ if enrich["profiles"] and str(handle_raw).strip().lstrip("@").lower() \
+ not in enrich_prefetch:
+ time.sleep(PACE_SECONDS) # >=2 s between profiles (R7), as the runner does
+ enrich["profiles"] += 1
+ # ⭐ THE DEFERRED-PROFILE SINK IS PER RECORD so the snapshot can be stamped with
+ # the row it belongs to: the collector writes preset cells back onto THAT record,
+ # and a run-wide list would have no way to say which handle each snapshot was for.
+ pend_prof = []
+ res = pull_profile(handle_raw,
+ max_posts=cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL,
+ log=log,
+ post_metrics=bool(cfg.get("postMetrics")),
+ comment_metrics=bool(cfg.get("commentMetrics")),
+ pending_metrics=(enrich["pending"]
+ if not cfg.get("dryRun") else None),
+ pending_profile=(pend_prof
+ if not cfg.get("dryRun") else None),
+ prefetch=enrich_prefetch)
+ # ⭐ 2026-08-09 (owner: *"a Filter on our enrichment so we only get Last 12 reels /
+ # 12 videos etc … not just last 12 but by group also"*). `config.postGroups` is
+ # `[{"type": "video", "limit": 12}, …]`; a type nobody names is dropped.
+ # ⛔ DEFAULT OFF — with no `postGroups` the list is returned unchanged, so this
+ # costs nothing and changes nothing until somebody configures it.
+ # ⚠ IT FILTERS THE WINDOW WE ALREADY BOUGHT, it does not widen it. The profile
+ # read is capped at 12 posts (MEASURED: asked 40, got 12), so "12 reels AND 12
+ # images" needs `bd_profile_posts` — which is built and proven (30 rows in 90 s)
+ # but deliberately NOT wired here: the views top-up runs INSIDE `pull_profile`
+ # against the posts it captured, so swapping a wider list in afterwards would
+ # hand back posts with no view counts. Widening belongs ahead of that step, in
+ # the paid path, and that is its own change. See DEBT.
+ if cfg.get("postGroups"):
+ res = {**res, "posts": select_post_groups(res.get("posts") or [],
+ cfg.get("postGroups"))}
+ for _p in pend_prof:
+ _p["table"], _p["rowId"], _p["requestedAt"] = table, str(rid), _iso()
+ enrich["pendingProfiles"].extend(pend_prof)
+ pulled = _iso()
+ if res["state"] in ("ok", "partial"):
+ enrich["ok"] += 1
+ snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
+ enrich["snaps"].append(snap_row)
+ enrich["posts"].extend(idents)
+ enrich["psnaps"].extend(metric_rows)
+ enrich["comments"].extend(comment_rows)
+ # R3: LATEST onto the record itself. `row.update` too, so a LATER action in
+ # this same flow sees the enriched values — the rule `update_record` already
+ # follows, and without it "enrich, then route by follower count" would judge
+ # the record on the values it had before the pull.
+ cells = preset_cells(res, pulled)
+ if cfg.get("dryRun"):
+ enrich["dry"] = True
+ else:
+ row.update(cells)
+ _act_row_patch(patches, table, rid, cells)
+ else:
+ enrich["blocked"] += 1
+ # ⭐⭐ D-103 — THE NOTE IS NAMED AND KEPT AT FULL LENGTH.
+ # It used to be `_s(note, 90)` with no handle attached: three runs blocked
+ # the same record and the store could not say which record, let alone why.
+ # 90 characters also truncated the one measured sentence mid-clause. This is
+ # the most valuable thing the run produces and it now reaches the run entry.
+ enrich["notes"].append(
+ f"@{handle_raw}: {_s(res.get('note'), 300) or res['state']}")
+ # A vendor STATING that the account does not exist is remembered, so the next
+ # run stops paying to be told the same thing.
+ if res.get("gone"):
+ enrich["gone"][_gone_key(row, handle_raw)] = {
+ "at": _iso(), "handle": handle_raw,
+ "note": _s(res.get("note"), 300)}
+ elif kind == "find_records":
+ found = find_records(rt, cfg.get("table"), cfg.get("cond"),
+ int(cfg.get("limit") or 25))
+ counts["found"] += len(found)
+ # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had
+ # already stopped being reachable one wave earlier — which is the part worth reading.
+ # `_without_retired_board` strips every `review` action out of a definition on the
+ # READ path, so `all_definitions` and this function have not seen one since the board
+ # was retired. What was left behind was a branch referencing `skey`, `stamp` and
+ # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not
+ # dead code that merely wasted space: it was a `NameError` held back by a migration
+ # rather than by a guard, and any change that let one stored `review` action through
+ # would have crashed the whole action walk for every record in that flow.
+ # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps
+ # review "as an AI decision without lanes", and the cheap-first provider ladder behind
+ # it is real, working, measured work. They are PARKED, deliberately and in writing,
+ # not orphaned; see their own notes.
+ return False
+
+ if actions:
+ for rid in list(row_ids or [])[:FLOOD_LIMIT]:
+ row = dict(rows.get(str(rid)) or {})
+ if not row:
+ continue
+ _walk(actions, row, str(rid))
+ counts.update(_enrich_flush(rt, defn, username, enrich, log))
+ # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was
+ # incremented once per create the flow decided to make; what landed is what the store says,
+ # and with `uniqueOn` on they are routinely different (a re-run of a scheduled flow matches
+ # every row it made last time — which is the whole point of the feature).
+ counts.update(_commit_action_writes(rt, table, patches, creates, username, log))
+ return counts
+
+
+def migrate_field_instagram(rt, defn):
+ """⭐ WAVE 25 · R4 — one stored `field_instagram` definition → a `plain` one carrying the
+ `enrich_instagram` action. Returns `(definition, changed)`.
+
+ R4: "the ENRICH ACTION REPLACES the `field_instagram` KIND… migrate the one live
+ `field_instagram` automation to a plain flow carrying it; delete the kind and its label."
+
+ ⛔ THE BINDING IS RESOLVED HERE, NOT LEFT TO THE FLAG, and this is the line the migration
+ turns on. `run_field_instagram` reads `cfg.urlField` **or falls back to `_auto_url_field`** —
+ so a live automation whose `urlField` is blank has been working off that fallback for months.
+ The enrich action deliberately has no such fallback (see `profile_field_key`), so migrating a
+ blank `urlField` verbatim would produce an automation that USED to work and now refuses. The
+ fallback is therefore evaluated ONCE, here, and the answer is written down as an explicit
+ binding — which is also the honest outcome: the column stops being implicit.
+
+ ⚠ ONE BEHAVIOUR DOES CHANGE, AND IT IS THE POINT OF THE RULING RATHER THAN A REGRESSION. The
+ old kind wrote a STATUS STRING ("ok · 2026-08-06 · 12,400 followers") into `config.fieldKey`'s
+ column; the action writes the C1 PRESET CELLS instead. The status column is left in place and
+ simply stops being written — deleting somebody's column as part of a migration would be data
+ loss, and a stale cell beside a fresh `enriched_at` is readable for what it is.
+
+ ⚠ PURE OVER THE DEFINITION apart from the one table READ. It writes nothing, so a caller can
+ run it to INSPECT what a migration would do — which is exactly how R4's "must be PROVEN
+ against the real stored definition" is meant to be satisfied.
+ """
+ if (defn or {}).get("kind") != "field_instagram":
+ return defn, False
+ cfg = dict(defn.get("config") or {})
+ target = str(cfg.get("targetTable") or "")
+ bound = str(cfg.get("urlField") or "").strip()
+ if not bound and target:
+ bound = str(_auto_url_field(ut_get(rt, target) or {}, cfg.get("fieldKey")) or "")
+ try:
+ max_posts = int(cfg.get("maxPosts") or DEFAULT_POSTS_PER_PULL)
+ except (TypeError, ValueError):
+ max_posts = 24
+ act = {"id": "act_enrich", "kind": "enrich_instagram", "enabled": True, "when": None,
+ "config": {"profileField": bound,
+ "postMetrics": bool(cfg.get("postMetrics")),
+ "commentMetrics": bool(cfg.get("commentMetrics")),
+ "dryRun": bool(cfg.get("dryRun")),
+ "maxPosts": max(1, min(max_posts, 200))}}
+ flow = dict(defn.get("flow") or {})
+ out = dict(defn)
+ out["kind"] = DEFAULT_KIND
+ # The enrich step goes FIRST — it is what the automation was, and every action the owner
+ # added afterwards was written expecting the capture to have happened.
+ out["flow"] = {"actions": [act] + list(flow.get("actions") or [])}
+ out["config"] = {"targetTable": target,
+ "targetLabel": str(cfg.get("targetLabel") or "")}
+ return out, True
+
+
+def _enrich_flush(rt, defn, username, acc, log):
+ """C4: the enrich action's canonical IG tables, written ONCE for the whole run.
+
+ Returns the counts the run reports. Empty when no enrich action ran, so a flow without one
+ pays nothing and its run history is unchanged (`run_now` merges only non-zero keys).
+
+ ⛔ THE SAME WRITE PATH AS `run_field_instagram`, NOT A PARALLEL ONE: `ut_ensure` the three
+ tables, `upsert_rows` on their own keys with their own caps, one coalesced `rt.update`, then
+ the write-through to the platform master. Forking any of those would give the enrich action a
+ history that the metric fields could not see — and it is the LAST step, the write-through to
+ the platform master, that they read (W29-T34: `compute_metric_cells` → `ig_master.series_for`,
+ never this tenant's `ut_ig_snapshots`), so dropping that one line is the version of this fork
+ that would look harmless.
+ """
+ if "unbound" in (acc.get("notes") or []):
+ # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step
+ # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so
+ # this function used to return `{}`, `run_now` merged only non-zero keys, and the run
+ # committed **`ok` — "N records walked"** having written nothing at all. The sentence
+ # naming the fix went to `log()`, i.e. the server console, which no customer reads. That
+ # is "green over nothing" on the one path where somebody is waiting for data.
+ # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a
+ # special case — and so the run entry itself records that this happened.
+ return {"enrichUnbound": 1}
+ # ⭐⭐ 2026-08-09 — THE NOTES SURVIVE A RUN THAT READ NOTHING, and that is not a detail.
+ # `if not acc["profiles"]: return {}` is exactly the branch a run takes when EVERY candidate
+ # was skipped as a known-dead handle — so the sentence explaining why the automation appears
+ # to do nothing would have been dropped on precisely the runs that most need it, and the
+ # owner would be back at "0 records walked, wtf". The selection note is produced before any
+ # profile is read and must outlive that early return.
+ if not acc.get("profiles"):
+ notes = list(acc.get("notes") or [])
+ return {RUN_NOTES_KEY: notes} if notes else {}
+ out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]}
+ if acc.get("notes"):
+ out[RUN_NOTES_KEY] = list(acc["notes"])
+ # ⭐ D-103's own prescription: "the block note survives on the RUN … not a status column, so
+ # no table gains a column it did not ask for". `RUN_NOTES_KEY` is that channel.
+ if not acc.get("dry"):
+ # The vendor's not-found verdicts, merged into engine state (never a tenant column and
+ # never a status string — W25/R4 retired those). Merged rather than replaced: a run that
+ # walked one record must not forget what earlier runs learned about the others.
+ if acc.get("gone"):
+ merged = dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})
+ merged.update(acc["gone"])
+ set_state(rt, str(defn.get("id") or ""), {"enrichNotFound": merged})
+ pending_profiles = queue_pending_profile_snapshots(
+ rt, str(defn.get("id") or ""), acc.get("pendingProfiles") or [])
+ if pending_profiles:
+ out["enrichProfileBatchesPending"] = pending_profiles
+ queued = queue_pending_metric_snapshots(rt, str(defn.get("id") or ""),
+ acc.get("pending") or [])
+ if queued:
+ out["enrichMetricBatchesPending"] = queued
+ if acc.get("dry") or not (acc["snaps"] or acc["posts"] or acc["psnaps"] or acc["comments"]):
+ # A dry run resolves nothing and writes nothing — not even `ut_ensure`, which CREATES.
+ if acc.get("dry"):
+ out["enrichDryRun"] = acc["profiles"]
+ return out
+ tag = str(defn.get("id") or "")
+ profile_table = str(_flow_table(defn) or "")
+ graph = ensure_ig_graph(rt, username, tag, profile_table=profile_table)
+ snap_key, post_key, ps_key, comment_key = (graph[IG_SNAPSHOTS_TABLE], graph[IG_POSTS_TABLE],
+ graph[IG_POST_SNAPSHOTS_TABLE], graph[IG_COMMENTS_TABLE])
+ missing = ut_missing(rt, snap_key, post_key, ps_key, comment_key)
+ snaps, c_snap = upsert_rows(dict((ut_get(rt, snap_key) or {}).get("rows") or {}),
+ acc["snaps"], "snapshot_key", cap=row_cap(snap_key))
+ old_posts, collapsed_posts = dedupe_canonical_rows(
+ dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
+ posts, c_post = upsert_rows(old_posts, acc["posts"], "shortcode", cap=row_cap(post_key))
+ c_post["duplicates"] += collapsed_posts
+ psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}),
+ acc["psnaps"], "post_snapshot_key", cap=row_cap(ps_key))
+ old_comments, collapsed_comments = dedupe_canonical_rows(
+ dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key")
+ comments, c_comments = upsert_rows(old_comments, acc["comments"], "comment_key",
+ cap=row_cap(comment_key))
+ c_comments["duplicates"] += collapsed_comments
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps),
+ (comment_key, comments)):
+ if cur.get(k) is not None:
+ cur[k]["rows"] = rws
+ _refresh_relations_inplace(cur, log=log)
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all three tables
+ # LOUD, never silent (D-11): a full append table means the SERIES has stopped growing, which
+ # is the failure a chart cannot show you.
+ capped = c_snap["capped"] + c_post["capped"] + c_ps["capped"] + c_comments["capped"]
+ if capped:
+ out["enrichCapped"] = capped
+ log(f"[aios-auto] enrich_instagram: {capped} row(s) refused by a table's row cap")
+ if missing:
+ out["enrichMissingTables"] = len(missing)
+ log(f"[aios-auto] enrich_instagram: could not create {', '.join(missing)}")
+ out["enrichPoints"] = len(acc["snaps"])
+ out["enrichPosts"] = c_post["inserted"]
+ out["enrichComments"] = c_comments["inserted"]
+ # C6/R2's write-through to the PLATFORM MASTER. Three postures, never conflated — the same
+ # contract `run_field_instagram` follows, because a pooled history with silent holes is worse
+ # than none.
+ try:
+ import ig_master
+ m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), acc["snaps"],
+ acc["posts"], acc["psnaps"])
+ if m_status == "error":
+ out["enrichMasterFailed"] = 1
+ log(f"[aios-auto] enrich_instagram: the platform master copy FAILED — {m_note}; "
+ f"the tenant copy is complete and the next run re-appends")
+ elif m_status == "ok":
+ out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"])
+ except Exception as e: # noqa: BLE001
+ out["enrichMasterFailed"] = 1
+ log(f"[aios-auto] enrich_instagram: master write-through raised "
+ f"{type(e).__name__}: {e}")
+ return out
+
+
+_PENDING_METRIC_DATASETS = frozenset((BD_DS_POSTS, BD_DS_REELS, BD_DS_COMMENTS))
+_PENDING_METRIC_KINDS = frozenset(("posts", "comments"))
+
+
+def _pending_metric_tasks(defn):
+ """Read validated, deduplicated paid metric snapshots from automation continuation state.
+
+ A snapshot ID is a vendor-issued capability for work already paid for. It is intentionally
+ stored as engine state beside discovery's pending corpus snapshot, never in a Profile cell or
+ the user-editable flow. Invalid/stale shapes are ignored rather than sent back to a vendor
+ endpoint, and the collector never starts a second scrape request.
+ """
+ raw = ((defn or {}).get("state") or {}).get("pendingMetricSnapshots") or []
+ out, seen = [], set()
+ for item in raw if isinstance(raw, list) else []:
+ if not isinstance(item, dict):
+ continue
+ sid = str(item.get("snapshotId") or "").strip()
+ dataset = str(item.get("datasetId") or "").strip()
+ kind = str(item.get("kind") or "").strip()
+ handle = str(item.get("influencer") or "").strip().lstrip("@").lower()
+ if (not sid.startswith("sd_") or dataset not in _PENDING_METRIC_DATASETS
+ or kind not in _PENDING_METRIC_KINDS or not handle):
+ continue
+ key = (sid, dataset, kind, handle)
+ if key in seen:
+ continue
+ seen.add(key)
+ out.append({"snapshotId": sid, "datasetId": dataset, "kind": kind,
+ "influencer": handle, "requestedAt": str(item.get("requestedAt") or ""),
+ "lastChecked": str(item.get("lastChecked") or ""),
+ "lastNote": _s(item.get("lastNote"), 160)})
+ return out
+
+
+def queue_pending_metric_snapshots(rt, auto_id, pending):
+ """Durably retain new engagement snapshots without creating another paid provider request."""
+ aid = str(auto_id or "").strip()
+ if not aid:
+ return 0
+ incoming = _pending_metric_tasks({"state": {"pendingMetricSnapshots": pending}})
+ if not incoming:
+ return 0
+ added = [0]
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ definition = cur.get(aid)
+ if not isinstance(definition, dict):
+ return cur
+ state = definition.setdefault("state", {})
+ existing = _pending_metric_tasks(definition)
+ known = {(x["snapshotId"], x["datasetId"], x["kind"], x["influencer"])
+ for x in existing}
+ for task in incoming:
+ key = (task["snapshotId"], task["datasetId"], task["kind"], task["influencer"])
+ if key not in known:
+ existing.append(task)
+ known.add(key)
+ added[0] += 1
+ state["pendingMetricSnapshots"] = existing
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+ return added[0]
+
+
+def _match_profile_row(rows, handle):
+ """THIS handle's row out of a snapshot that may now hold several. `None` when it is not there.
+
+ ⭐⭐ 2026-08-09 — `rows[0]` WAS SAFE ONLY WHILE EVERY SNAPSHOT HELD EXACTLY ONE PROFILE. With
+ `_prime_enrich_batch` buying five URLs per call, one snapshot id serves up to five records, and
+ taking the first row would write ONE profile's followers onto all of them — a wrong number that
+ looks exactly like a right one. Identity comes off the row (`account`/`username`), which is the
+ same field `_bd_profile` reads first.
+
+ ⚠ THE SINGLE-ROW FALLBACK IS DELIBERATE AND NARROW. Every task queued before this change points
+ at a one-URL snapshot, and some of those rows have an unreadable identity; for exactly that
+ shape the task's own handle is still the best evidence. It applies only when the snapshot holds
+ ONE row, so it can never mis-assign inside a batch.
+ """
+ want = str(handle or "").strip().lstrip("@").lower()
+ usable = [n for n in (rows or []) if isinstance(n, dict)]
+ for node in usable:
+ got = str(_first(node, "account", "username", default="") or "").strip()
+ if got.lstrip("@").lower() == want and want:
+ return node
+ if len(usable) == 1:
+ return usable[0]
+ return None
+
+
+def _pending_profile_tasks(defn):
+ """Validated, deduplicated deferred PROFILE snapshots from automation state.
+
+ ⭐⭐ ITS OWN LIST, NOT `pendingMetricSnapshots`, and the separation is load-bearing rather
+ than tidy. `_pending_metric_tasks` filters on `datasetId in {posts, reels, comments}` and
+ `kind in {posts, comments}` — so a profile entry appended to that list is silently dropped to
+ zero by its own validator, and even if it survived, the collector would hand it to
+ `_write_collected_metric_rows`, a posts/comments writer that has nothing to do with a profile
+ row. Reusing the name would have shipped a green no-op of exactly the class this change
+ exists to remove.
+
+ ⚠ A PROFILE TASK CARRIES ITS DESTINATION (`table` + `rowId`). A collected profile is written
+ back as PRESET CELLS onto the record that asked for it, so unlike a metric batch it cannot be
+ resolved from the handle alone: two databases may both hold `@x`.
+ """
+ raw = ((defn or {}).get("state") or {}).get("pendingProfileSnapshots") or []
+ out, seen = [], set()
+ for item in raw if isinstance(raw, list) else []:
+ if not isinstance(item, dict):
+ continue
+ sid = str(item.get("snapshotId") or "").strip()
+ dataset = str(item.get("datasetId") or "").strip()
+ handle = str(item.get("influencer") or "").strip().lstrip("@").lower()
+ table = str(item.get("table") or "").strip()
+ row_id = str(item.get("rowId") or "").strip()
+ # ⛔ `sd_` ONLY. A `snap_…` corpus id sent to `/datasets/v3/…` is a flat 404 about a
+ # snapshot that is alive (§2c), and a malformed id must never be handed back to a vendor
+ # endpoint at all.
+ if (not sid.startswith("sd_") or dataset != BD_DS_PROFILES or not handle
+ or not table or not row_id):
+ continue
+ if sid in seen:
+ continue
+ seen.add(sid)
+ out.append({"snapshotId": sid, "datasetId": dataset, "kind": "profile",
+ "influencer": handle, "table": table, "rowId": row_id,
+ "requestedAt": str(item.get("requestedAt") or ""),
+ "lastChecked": str(item.get("lastChecked") or ""),
+ "lastNote": _s(item.get("lastNote"), 200)})
+ return out
+
+
+def queue_pending_profile_snapshots(rt, auto_id, pending):
+ """Durably retain deferred profile snapshots. Starts no new paid request. Returns how many
+ were newly added."""
+ aid = str(auto_id or "").strip()
+ if not aid:
+ return 0
+ incoming = _pending_profile_tasks({"state": {"pendingProfileSnapshots": pending}})
+ if not incoming:
+ return 0
+ added = [0]
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ definition = cur.get(aid)
+ if not isinstance(definition, dict):
+ return cur
+ state = definition.setdefault("state", {})
+ existing = _pending_profile_tasks(definition)
+ known = {x["snapshotId"] for x in existing}
+ for task in incoming:
+ if task["snapshotId"] not in known:
+ existing.append(task)
+ known.add(task["snapshotId"])
+ added[0] += 1
+ state["pendingProfileSnapshots"] = existing
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+ return added[0]
+
+
+def collect_pending_profile_snapshots(rt, defn, username="automation", log=print, step=_no_step):
+ """Finish deferred PROFILE reads the tenant has already paid for. Starts no new scrape.
+
+ ⭐⭐ 2026-08-09 — THE HALF THAT DID NOT EXIST. `bd_scrape` has always accepted a `deferred`
+ list, and every post/reel/comment call passed one; the PROFILE call did not, so a profile the
+ vendor took longer than `BD_SCRAPE_WAIT` to collect was billed and its snapshot id thrown
+ away — on every run, forever. MEASURED on nurilab: `collection_duration` 320 s against a
+ 180 s budget, two abandoned `sd_…` snapshots in two runs.
+
+ ⛔ IT CLOSES A TASK THAT FINISHED EMPTY. A pending entry that can never resolve is the same
+ forever-loop wearing a different mask, so `bd_snapshot_progress` deciding `done` with zero
+ records ends the task with the vendor's reason attached — and, when the vendor blames the
+ target rather than itself, records the not-found verdict so the handle stops being re-bought.
+ """
+ tasks = _pending_profile_tasks(defn)
+ if not tasks:
+ return ("ok", "No pending profile reads.", {}, [], {"source": "idle"})
+ step(f"Collecting {len(tasks)} deferred profile read{'' if len(tasks) == 1 else 's'}")
+ remaining, patches, affected, notes = [], {}, [], []
+ acc = {"snaps": [], "posts": [], "psnaps": [], "comments": [], "pending": [],
+ "pendingProfiles": [], "gone": {}, "profiles": 0, "ok": 0, "blocked": 0,
+ "notes": [], "dry": False}
+ ready = waiting = closed = 0
+ # ⭐ ONE SNAPSHOT, ONE STATUS CALL, ONE FETCH. A batched read gives several tasks the SAME
+ # `snapshotId`, and asking the vendor about it once per task would spend the round trips the
+ # batching just saved. Memoised for this collection pass only — a snapshot's state must be
+ # re-read on the NEXT run, which is a fresh call and a fresh dict.
+ _progress_memo, _rows_memo = {}, {}
+
+ def _progress_of(sid):
+ if sid not in _progress_memo:
+ _progress_memo[sid] = bd_snapshot_progress(sid)
+ return _progress_memo[sid]
+
+ def _store_profile(task, profile, via):
+ """Write ONE collected profile — snapshot row + preset cells on the record that asked.
+
+ ONE implementation, TWO callers (the primary snapshot and the backup rung below), so the
+ two paths cannot drift into writing different things — the same rule `top_up_views`
+ follows for the inline/deferred split.
+ """
+ res = {"state": "ok", "profile": profile, "posts": [], "comments": [],
+ "via": via, "note": ""}
+ pulled = _iso()
+ snap_row, idents, metric_rows, comment_rows = capture_rows(res, pulled)
+ acc["snaps"].append(snap_row)
+ acc["posts"].extend(idents)
+ acc["psnaps"].extend(metric_rows)
+ acc["comments"].extend(comment_rows)
+ _act_row_patch(patches, task["table"], task["rowId"], preset_cells(res, pulled))
+ affected.append(task["rowId"])
+
+ def _backup_profile(handle):
+ """The second rung, asked ONLY once the first has finished and delivered nothing.
+
+ ⭐ 2026-08-09 (owner: *"when it errors like this, just route to APIfy"*). THE GAP THIS
+ CLOSES: `pull_profile` has walked `ig_profile -> (primary, backup)` since 2026-08-08, but
+ a profile that DEFERRED never came back through `pull_profile` — it came back here, and
+ this collector had no second rung at all. So the one path where the primary is most
+ likely to have failed was the one path with no fallback.
+
+ ⛔ NEVER A TOP-UP. It runs only in the finished-and-empty branch, so a profile the primary
+ answered is never re-bought from a second vendor — the mistake the capability split exists
+ to prevent. A refusal returns None and the caller reports blocked exactly as before.
+ """
+ try:
+ import providers as _p
+ if not _p.PROVIDERS["apify"].can("ig_profile"):
+ return None
+ prof, note = apify_profile(str(handle))
+ except Exception as exc: # noqa: BLE001 — a backup must not raise
+ log(f"[aios-auto] backup profile rung failed for @{handle}: "
+ f"{type(exc).__name__}: {exc}")
+ return None
+ if prof and prof.get("followers") is not None:
+ return prof
+ return None
+
+ def _rows_of(sid):
+ if sid not in _rows_memo:
+ payload, err = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"})
+ got = _bd_rows(payload) if not err else []
+ if got and (_bd_deferral(payload) or
+ (len(got) == 1 and str(got[0].get("status") or "") in
+ ("running", "building", "collecting"))):
+ got = []
+ _rows_memo[sid] = got
+ return _rows_memo[sid]
+
+ for task in tasks:
+ stale_h = _hours_since(task.get("requestedAt"))
+ state, records, empty_note = _progress_of(task["snapshotId"])
+ if state in ("done", "failed") and not records:
+ acc["profiles"] += 1
+ # ⭐ ASK THE BACKUP BEFORE GIVING UP. The primary has FINISHED and delivered nothing,
+ # so there is nothing left to wait for and no risk of buying the same record twice.
+ backup = _backup_profile(task["influencer"])
+ if backup is not None:
+ ready += 1
+ acc["ok"] += 1
+ _store_profile(task, backup, "apify")
+ note = (f"@{task['influencer']}: the primary source returned nothing, so a backup "
+ f"source supplied the profile ({backup.get('followers')} followers)")
+ notes.append(note)
+ acc["notes"].append(note)
+ continue
+ closed += 1
+ acc["blocked"] += 1
+ note = f"@{task['influencer']}: {_s(empty_note, 240)}"
+ notes.append(note)
+ acc["notes"].append(note)
+ # ⭐ `failed` = the vendor finished, collected nothing, and blamed the TARGET. On a
+ # profile request that means the account could not be reached at all, so the verdict
+ # is remembered and the selection stops re-buying it (R9: permanently, until a
+ # human edits the handle cell — see `clear_gone`).
+ # ⚠ `done`-with-zero is NOT remembered: "we found no matches" is a statement about
+ # the query, and turning it into "this account does not exist" would silently retire
+ # live handles.
+ if state == "failed":
+ acc["gone"][_gone_key({}, task["influencer"])] = {
+ "at": _iso(), "handle": task["influencer"], "note": _s(empty_note, 300)}
+ continue
+ rows = _rows_of(task["snapshotId"]) if state != "running" else []
+ if not rows:
+ # ⚠ BOUNDED. A snapshot the vendor never finishes must not be polled until the end of
+ # time; after `PENDING_PROFILE_MAX_HOURS` it is dropped WITH a sentence, never
+ # silently. An unbounded queue is the forever-loop this change removes, inverted.
+ if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS:
+ closed += 1
+ note = (f"@{task['influencer']}: the source never finished the profile read "
+ f"queued {int(stale_h)}h ago ({task['snapshotId']}) — it was dropped; "
+ f"the next run will ask again")
+ notes.append(note)
+ acc["notes"].append(note)
+ continue
+ waiting += 1
+ remaining.append({**task, "lastChecked": _iso(),
+ "lastNote": _s("still building", 200)})
+ continue
+ # ⛔ THIS HANDLE'S ROW, NOT THE FIRST ONE — see `_match_profile_row`. A batched snapshot
+ # holds several profiles and `rows[0]` would write one creator's numbers onto every record
+ # in the chunk.
+ node = _match_profile_row(rows, task["influencer"])
+ if node is None:
+ closed += 1
+ acc["profiles"] += 1
+ acc["blocked"] += 1
+ note = (f"@{task['influencer']}: the source delivered {len(rows)} profile"
+ f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle — "
+ f"it was dropped from the batch and the next run will ask again")
+ notes.append(note)
+ acc["notes"].append(note)
+ continue
+ ready += 1
+ acc["profiles"] += 1
+ profile = _bd_profile(node, task["influencer"])
+ if profile.get("followers") is None and profile.get("following") is None:
+ acc["blocked"] += 1
+ note = (f"@{task['influencer']}: the source delivered the profile but no "
+ f"follower/following counts were readable in it")
+ notes.append(note)
+ acc["notes"].append(note)
+ continue
+ acc["ok"] += 1
+ _store_profile(task, profile, "brightdata:deferred")
+ notes.append(f"@{task['influencer']}: collected the profile the source had already been "
+ f"paid for ({profile.get('followers')} followers)")
+
+ set_state(rt, str(defn.get("id") or ""),
+ {"pendingProfileSnapshots": remaining or None})
+ counts = _enrich_flush(rt, defn, username, acc, log)
+ counts.pop(RUN_NOTES_KEY, None) # this function owns the note list below
+ counts.update(_commit_action_writes(rt, str(_flow_table(defn) or ""), patches, {},
+ username, log))
+ counts.update({"profileBatchesCollected": ready, "profileBatchesPending": waiting,
+ "profileBatchesEmpty": closed})
+ if notes:
+ counts[RUN_NOTES_KEY] = notes
+ head = (f"{ready} deferred profile read{'' if ready == 1 else 's'} collected"
+ if ready else "no deferred profile read was ready")
+ tail = "".join([f"; {closed} finished with nothing to collect" if closed else "",
+ f"; {waiting} still building" if waiting else ""])
+ state = "ok" if ready and not closed else "partial"
+ return (state, head + tail, counts, affected,
+ {"source": "ok" if ready else "partial", "write": "ok" if ready else "idle"})
+
+
+def _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log):
+ """Write a completed metric snapshot through the canonical Post/Comment graph once."""
+ if not (idents or snapshots or comments):
+ return {"posts": 0, "snapshots": 0, "comments": 0}
+ profile_table = str(_flow_table(defn) or "")
+ if not profile_table:
+ raise Refused("the pending engagement snapshot has no Profile database to link to")
+ graph = ensure_ig_graph(rt, username, str(defn.get("id") or ""),
+ profile_table=profile_table)
+ post_key, ps_key, comment_key = (graph[IG_POSTS_TABLE], graph[IG_POST_SNAPSHOTS_TABLE],
+ graph[IG_COMMENTS_TABLE])
+ old_posts, collapsed_posts = dedupe_canonical_rows(
+ dict((ut_get(rt, post_key) or {}).get("rows") or {}), "shortcode", newest_by="measured_at")
+ posts, c_post = upsert_rows(old_posts, idents, "shortcode", cap=row_cap(post_key))
+ c_post["duplicates"] += collapsed_posts
+ psnaps, c_ps = upsert_rows(dict((ut_get(rt, ps_key) or {}).get("rows") or {}), snapshots,
+ "post_snapshot_key", cap=row_cap(ps_key))
+ old_comments, collapsed_comments = dedupe_canonical_rows(
+ dict((ut_get(rt, comment_key) or {}).get("rows") or {}), "comment_key")
+ comments_rows, c_comments = upsert_rows(old_comments, comments, "comment_key",
+ cap=row_cap(comment_key))
+ c_comments["duplicates"] += collapsed_comments
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for key, rows in ((post_key, posts), (ps_key, psnaps), (comment_key, comments_rows)):
+ if cur.get(key) is not None:
+ cur[key]["rows"] = rows
+ _refresh_relations_inplace(cur, log=log)
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ try:
+ import ig_master
+ status, note = ig_master.append_run(getattr(rt, "key", ""), [], idents, snapshots)
+ if status == "error":
+ log(f"[aios-auto] deferred post metrics master copy FAILED: {note}")
+ except Exception as exc: # noqa: BLE001
+ log(f"[aios-auto] deferred post metrics master copy FAILED: {type(exc).__name__}: {exc}")
+ return {"posts": c_post["inserted"], "snapshots": c_ps["inserted"],
+ "comments": c_comments["inserted"]}
+
+
+def collect_pending_metric_snapshots(rt, defn, username="automation", log=print, step=_no_step):
+ """Collect deferred paid Post/Reel/Comment snapshots; never launches a new scrape request."""
+ tasks = _pending_metric_tasks(defn)
+ if not tasks:
+ return ("ok", "No pending post-engagement snapshots.", {}, [], {"capture_posts": "idle"})
+ step(f"Collecting {len(tasks)} post-engagement batch{'' if len(tasks) == 1 else 'es'}")
+ remaining, idents, snapshots, comments = [], [], [], []
+ ready, waiting, closed, run_notes = 0, 0, 0, []
+ for task in tasks:
+ # ⭐⭐ 2026-08-09 — ASK THE STATUS DOCUMENT, NOT THE ROWS. `building` used to be
+ # `not rows or …`, so a snapshot the vendor had FINISHED with zero records was re-queued
+ # as "still building" on every tick — forever, because nothing about it would ever
+ # change. That is the same forever-loop the profile path was measured in, one dataset
+ # over, and it was latent here the whole time.
+ state, records, empty_note = bd_snapshot_progress(task["snapshotId"])
+ if state in ("done", "failed") and not records:
+ # ⛔ CLOSED, NOT RE-QUEUED. The vendor is finished and there is nothing to collect;
+ # keeping the entry would be a pending task that can never resolve.
+ closed += 1
+ run_notes.append(f"{task['influencer']}: {_s(empty_note, 160)}")
+ continue
+ payload, note = bd_call(f"{BD_PATH_SNAPSHOT}/{task['snapshotId']}", {"format": "json"})
+ rows = _bd_rows(payload) if not note else []
+ building = (state == "running" or not rows or _bd_deferral(payload) or
+ (len(rows) == 1 and str(rows[0].get("status") or "") in
+ ("running", "building", "collecting")))
+ if building:
+ remaining.append({**task, "lastChecked": _iso(),
+ "lastNote": _s(note or "still building", 160)})
+ waiting += 1
+ continue
+ ready += 1
+ pulled = _iso()
+ if task["kind"] == "posts":
+ mapped_posts = []
+ for raw in rows:
+ post = _bd_post_metrics(raw)
+ if post:
+ # The snapshot was created from this Profile's canonical post URLs. Keep
+ # that explicit backlink even when a vendor response omits `user_posted`.
+ post["influencer_key"] = task["influencer"]
+ mapped_posts.append(post)
+ if mapped_posts:
+ # ⭐⭐ 2026-08-09 — THE VIEWS TOP-UP RUNS HERE TOO, and its absence is why the
+ # owner's `theresalearns` run filled every column except Views.
+ #
+ # ⛔ Bright Data is DECLARED INCAPABLE of `ig_post_views` (`providers.py`), so a
+ # Posts row physically cannot carry a view count — MEASURED on the stored
+ # payloads: 12 rows, `content_type: "Reel"`, and no view/play key in any of them.
+ # Views only ever comes from the Apify capability. That top-up lived INSIDE
+ # `pull_profile_bd`, so it ran only when the Posts scrape answered within the wait
+ # budget; when the batch deferred — which is routine, and what happened here — the
+ # rows came back through THIS function and Apify was never asked.
+ # ⇒ `top_up_views` is now one function with two callers rather than a copy, so
+ # the inline and deferred paths cannot answer this differently again.
+ # ⚠ It mutates `mapped_posts` in place and must run BEFORE `capture_rows`, which
+ # is what freezes the values into the post + snapshot rows.
+ v_note = top_up_views(mapped_posts, log=log)
+ if v_note:
+ run_notes.append(f"@{task['influencer']}: {v_note}")
+ _unused, post_rows, metric_rows, embedded = capture_rows(
+ {"state": "ok", "profile": {"username": task["influencer"]},
+ "posts": mapped_posts, "comments": [], "via": "brightdata:deferred"}, pulled)
+ idents.extend(post_rows)
+ snapshots.extend(metric_rows)
+ comments.extend(embedded)
+ else:
+ for raw in rows:
+ comment = _bd_comment(raw, influencer_key=task["influencer"])
+ if comment:
+ comments.append(comment)
+
+ set_state(rt, str(defn.get("id") or ""),
+ {"pendingMetricSnapshots": remaining or None})
+ written = _write_collected_metric_rows(rt, defn, username, idents, snapshots, comments, log)
+ counts = {"metricBatchesCollected": ready, "metricBatchesPending": waiting,
+ "metricBatchesEmpty": closed,
+ "postEngagementSnapshots": written["snapshots"],
+ "commentsCollected": written["comments"]}
+ # ⚠ THE RUNNER CONTRACT STAYS A 5-TUPLE and the notes ride in `counts` under the reserved
+ # `RUN_NOTES_KEY`, which `run_now` pops. Widening the tuple for one runner would make four
+ # other call sites disagree about the shape of a run — and `_commit_run` already drops
+ # non-numeric count values, so a pop that is ever missed degrades to today's behaviour rather
+ # than to a crash.
+ if run_notes:
+ counts[RUN_NOTES_KEY] = run_notes
+ # ⚠ THE EMPTY ONES ARE NAMED, not silently dropped. A batch that finished with no records is
+ # a real outcome the tenant paid for and it must read as an answer, not as a disappearance.
+ tail = (f"; {closed} finished with nothing to collect" if closed else "")
+ if waiting:
+ return ("partial",
+ f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected; "
+ f"{waiting} still building and will be collected automatically{tail}",
+ counts, [], {"capture_posts": "partial", "write": "ok"})
+ return ("ok", f"{ready} post-engagement batch{'' if ready == 1 else 'es'} collected{tail}",
+ counts, [], {"capture_posts": "ok", "write": "ok"})
+
+
+def ai_decide(rt, defn, act, row, row_id="", log=print):
+ """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or ""
+ to leave the card for a person.
+
+ ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed
+ answer, or a label the review does not offer all return "" — and "" means the card sits at
+ the review gate exactly as it would with no AI at all. The feature can be broken, absent or
+ wrong and the worst outcome is a human doing the work.
+
+ ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN
+ ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board
+ under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely
+ worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter →
+ anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is
+ MISSING is only the door — an action kind that asks a question and takes an answer, without a
+ stage column to write it into. **Do not delete this in a dead-code sweep without reading that
+ sentence first**; equally, do not treat it as shipped — nothing reaches it today.
+ """
+ cfg = act.get("config") or {}
+ options = list(cfg.get("next") or [])
+ if not options:
+ return ""
+ try:
+ import ai_review
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}")
+ return ""
+ fields = [f.get("key") for f in
+ (ut_get(rt, ((defn.get("config") or {}).get("targetTable")
+ or (defn.get("trigger") or {}).get("table") or "")) or {}
+ ).get("fields") or []]
+ choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options,
+ row=row, fields=[f for f in fields if f],
+ label=cfg.get("label") or "Review")
+ if not choice:
+ if meta.get("problem"):
+ log(f"[aios-auto] ai review declined to answer: {meta['problem']}")
+ return ""
+ review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review",
+ choice, meta.get("provider") or "ai", by="ai",
+ note=meta.get("reason") or "", model=meta.get("model") or "")
+ return choice
+
+
+def find_records(rt, table_key, cond, limit=25):
+ """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts
+ the count in the run log, where it drills — [[no-unverifiable-aggregates]])."""
+ rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {}
+ out = []
+ for rid, row in rows.items():
+ if lane_match(cond, row or {}):
+ out.append(str(rid))
+ if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)):
+ break
+ return out
+
+
+def _commit_action_writes(rt, table, patches, creates, username, log):
+ """The ONE write. Row patches merge into the target's rows; creates land in their own tables —
+ APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a).
+
+ Returns the REALIZED create counts, and returning them is the point rather than a
+ convenience. `apply_actions` counts an ATTEMPT per create while it walks the records; only
+ this function knows how many of those became rows, how many matched one that was already
+ there, and how many the cap refused. A run that reported the attempt as "created" would be
+ the summary-disagrees-with-what-happened defect this module names in three other places.
+
+ ⛔ THE UPSERT IS `upsert_rows`, NOT A MATCH LOOP WRITTEN HERE. D-6 closed on "ONE
+ implementation repo-wide" after a second one was deleted from `core/user_tables.py`; hand
+ rolling a third inside this function is that debt returning with a new name — and it would
+ quietly diverge on the two rules that took a wave each to get right (an orphan is COUNTED,
+ NEVER DELETED, and `capped` is its own count rather than folded into `skipped`).
+ """
+ out = {"created": 0, "createUpdated": 0, "createUnchanged": 0,
+ "createCapped": 0, "createSkipped": 0}
+ if not patches and not creates:
+ return out
+ # ⭐ PATCHES ARE STAGED, NOT WRITTEN YET (2026-08-06). They used to commit here, one
+ # `ut_write_rows` per patched table, and that was free while the only patcher was a review
+ # gate on a table no create action touched. Owner item 1 stamps EVERY walked record with the
+ # step it reached, so the flow's own table is now patched on essentially every run — and a
+ # flow that also creates into that same table would have committed it TWICE per run, against
+ # the 20 s flush floor and the 256-commits/hr repo budget this module is shaped around.
+ # Staged into `staged` and handed to the creates pass, which writes each table exactly once.
+ staged = {}
+ for tkey, rowpatch in (patches or {}).items():
+ cur = dict((ut_get(rt, tkey) or {}).get("rows") or {})
+ for rid, vals in rowpatch.items():
+ cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals}
+ staged[str(tkey)] = cur
+ # ⛔ ONE WRITE PER TABLE, even when several actions target it under different keys. The
+ # accumulator is keyed by (table, uniqueOn), so a naive loop would call `ut_write_rows` once
+ # per GROUP — two store commits for one table, against the 20 s flush floor and the 256/hr
+ # repo budget this whole module is shaped around.
+ by_table = {}
+ for (tkey, unique), new_rows in (creates or {}).items():
+ by_table.setdefault(str(tkey), []).append((str(unique or ""), new_rows))
+ for tkey, groups in by_table.items():
+ t = ut_get(rt, tkey)
+ if t is None:
+ n = sum(len(r) for _u, r in groups)
+ log(f"[aios-auto] create_record: {tkey} no longer exists — {n} skipped")
+ out["createSkipped"] += n
+ continue
+ # The PATCHED rows when this table was also stamped this run, so the creates land on top
+ # of the stamp rather than on a copy of the store that predates it.
+ cur = staged.pop(tkey, None)
+ cur = dict(t.get("rows") or {}) if cur is None else cur
+ cap = row_cap(tkey)
+ for unique, new_rows in groups:
+ if unique:
+ cur, c = upsert_rows(cur, new_rows, unique, cap=cap)
+ out["created"] += c["inserted"]
+ out["createUpdated"] += c["updated"]
+ out["createUnchanged"] += c["unchanged"]
+ out["createCapped"] += c["capped"]
+ # ⚠ `skipped` here means "this row had no value for the unique key", which for a
+ # create action is a mapped value that interpolated to nothing — worth surfacing,
+ # because the symptom is otherwise a run that says it created less than it walked.
+ out["createSkipped"] += c["skipped"]
+ if c["capped"]:
+ log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — "
+ f"{c['capped']} row(s) not written")
+ continue
+ nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1
+ for i, vals in enumerate(new_rows):
+ if len(cur) >= cap:
+ # THE [:N] HONESTY RULE ([[no-unverifiable-aggregates]]): the number DROPPED is
+ # named, here and in the run's counts. This used to `break` with a log line
+ # that said the cap was hit and never said how much was lost.
+ out["createCapped"] += len(new_rows) - i
+ log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — "
+ f"{len(new_rows) - i} row(s) not written")
+ break
+ cur[str(nxt)] = dict(vals)
+ nxt += 1
+ out["created"] += 1
+ ut_write_rows(rt, tkey, cur)
+ # Whatever the creates pass did NOT claim: tables this run only STAMPED. Written last and
+ # once each, so "one store write per table" holds whether a table was patched, created into,
+ # or both.
+ for tkey, cur in staged.items():
+ ut_write_rows(rt, tkey, cur)
+ return out
+
+
+def _lane_sentence(cond, top=True):
+ """One condition tree → the sentence a step's `detail` carries on the canvas.
+
+ C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a
+ label never claims a flat comparison the tree does not make. The SERVER composes it, for the
+ same reason it composes every other `detail` — a client paraphrase of a structure the engine
+ evaluates is a second implementation of the same sentence, free to drift from it.
+
+ ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is
+ live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a
+ rename would be churn on a working function to fix a word.
+ """
+ if cond is None:
+ return "Everything else" if top else ""
+ if isinstance(cond, dict):
+ for key, joiner in (("all", " and "), ("any", " or ")):
+ if key in cond:
+ parts = [_lane_sentence(c, False) for c in cond.get(key) or []]
+ parts = [p for p in parts if p]
+ if not parts:
+ return ""
+ inner = joiner.join(parts)
+ return inner if top or len(parts) == 1 else f"({inner})"
+ v = cond.get("value")
+ return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}")
+
+
+def _rid_num(rid):
+ return int(rid) if str(rid).isdigit() else 10 ** 9
+
+
+#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can
+#: grow a definition without limit is a serialisation cost wearing a compliance hat.
+MAX_REVIEWS = 100
+
+
+def review_audit(rt, auto_id, row_id, from_label, to_label, username,
+ by="user", note="", model=""):
+ """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to
+ the definition (newest first, bounded).
+
+ ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that
+ made it and the one-line reason it gave. One audit log, not two — a reader asking "who
+ decided this card" must not have to know there are two places to look, and the moment an
+ AI decision is invisible beside a human one the log stops being an audit.
+
+ ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone:
+ `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote
+ this shape off a stage field's `flowId` — a field the migration now drops. Kept because the
+ SHAPE is the contract a future decision action would write, and re-deriving an audit format
+ is how two of them end up existing.
+ """
+ entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id),
+ "from": _s(from_label, 60), "to": _s(to_label, 60),
+ "by": "ai" if by == "ai" else "user"}
+ if note:
+ entry["note"] = _s(note, 300)
+ if model:
+ entry["model"] = _s(model, 60)
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ d = cur.get(str(auto_id))
+ if d is not None:
+ d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS]
+ return cur
+
+ rt.update(STORE_KEY, _up, flush="sync")
+
+
+# ---------------------------------------------------------------------------------------------
+# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24)
+# ---------------------------------------------------------------------------------------------
+# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit —
+# the value reflects every pull the platform has, not just this tenant's). Computed by the
+# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files
+# no session owns this wave), human-write-refused at the grid door, and every number drills to
+# the exact snapshot rows behind it.
+#
+# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no
+# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero
+# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not
+# readable") — the `_bd_posts_count` law, one layer up.
+
+METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments")
+METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d")
+#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each
+#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer;
+#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned).
+PROFILE_MEASURES = ("followers", "avg_engagement")
+METRIC_AGGS = ("avg", "sum", "latest")
+
+
+def metric_value(series, measure, window, agg="", today=None):
+ """One metric over one handle's master series → `(value_string_or_None, drill_rows)`.
+
+ `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference
+ day; post-count windows count back from the newest post. `drill_rows` are the exact rows
+ the number came from, so the route can honour [[no-unverifiable-aggregates]] without
+ recomputing differently."""
+ series = series or {}
+ today = today or _now()
+ if measure in PROFILE_MEASURES:
+ rows = [r for r in series.get("snapshots") or []
+ if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""]
+ if window in ("last_7d", "last_30d"):
+ days = 7 if window == "last_7d" else 30
+ floor = today - _dt.timedelta(days=days)
+ rows = [r for r in rows
+ if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor]
+ if not rows:
+ return None, []
+ if window == "latest" or agg == "latest":
+ picked = [rows[-1]]
+ else:
+ picked = rows
+ vals = [_lane_num(r.get(measure)) for r in picked]
+ vals = [v for v in vals if v is not None]
+ if not vals:
+ return None, []
+ out = vals[-1] if (window == "latest" or agg == "latest") else \
+ (sum(vals) if agg == "sum" else sum(vals) / len(vals))
+ if measure == "avg_engagement":
+ # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs
+ # transform-pct scar) — scaled exactly once, here.
+ return f"{out * 100:.2f}", picked
+ return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked
+ # --- post measures: select POSTS by window, then each post's LATEST measured snapshot.
+ posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()]
+ posts.sort(key=lambda p: str(p.get("posted_at")))
+ if window == "latest":
+ picked_posts = posts[-1:]
+ elif window == "last_3_posts":
+ picked_posts = posts[-3:]
+ else:
+ days = 7 if window == "last_7d" else 30
+ floor = today - _dt.timedelta(days=days)
+ picked_posts = [p for p in posts
+ if (_parse_iso(str(p.get("posted_at")).replace(" ", "T"))
+ or _dt.datetime.min) >= floor]
+ vals, drill = [], []
+ for p in picked_posts:
+ snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or []
+ for snap in reversed(snaps):
+ raw = snap.get(measure)
+ v = _lane_num(raw)
+ if v is not None and str(raw).strip() != "":
+ vals.append(v)
+ drill.append(snap)
+ break
+ if not vals:
+ return None, []
+ out = sum(vals) if (agg or "sum") == "sum" else \
+ (vals[-1] if agg == "latest" else sum(vals) / len(vals))
+ return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill
+
+
+def metric_fields_of_table(t):
+ return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)]
+
+
+def _table_handle(row, url_field):
+ return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or ""))
+ or "").strip().lower()
+
+
+def compute_metric_cells(rt, table_key, today=None):
+ """Recompute every metric cell on ONE table from the master series. One coalesced write,
+ only when something actually changed (the flush-ceiling law); zero reads when the table
+ has no metric fields or the master is off. Returns the number of rows touched."""
+ t = ut_get(rt, table_key)
+ mfields = metric_fields_of_table(t)
+ if not mfields:
+ return 0
+ import ig_master
+ if not ig_master.configured():
+ return 0
+ url_field = next((f.get("key") for f in (t.get("fields") or [])
+ if f.get("type") == "url"), "")
+ rows = t.get("rows") or {}
+ handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()}
+ series = ig_master.series_for({h for h in handles.values() if h})
+ changes = {}
+ for rid, row in rows.items():
+ s = series.get(handles.get(rid) or "")
+ for f in mfields:
+ bag = f["metric"]
+ val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"),
+ bag.get("agg") or "", today=today)
+ if s else (None, []))
+ want = "" if val is None else str(val)
+ if str((row or {}).get(f["key"], "")) != want:
+ changes.setdefault(str(rid), {})[f["key"]] = want
+ if not changes:
+ return 0
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ tt = cur.get(table_key)
+ if tt is not None:
+ for rid, vals in changes.items():
+ tt.setdefault("rows", {}).setdefault(rid, {}).update(vals)
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ return len(changes)
+
+
+# ── ⭐⭐ THE RELATIONAL PASS (2026-08-07) — derived LINK cells and ROLLUP cells ────────────────
+#
+# Owner: *"adding relational database function with links and rollups… exactly like how Airtable
+# does it… and this rollup needs to have formula that we can use to calculate things like average
+# Views over last N posts."*
+#
+# ⛔ WHY THIS IS SERVER-SIDE AND MATERIALISED, when `formula` is client-side and is not. A formula
+# reads ONE ROW; a rollup reads ANOTHER TABLE'S ROWS, which the client has not loaded and must not
+# have to. So this rides `compute_metric_cells`' pattern exactly — recompute, diff, ONE coalesced
+# write only when something changed — and inherits its flush-ceiling discipline for free.
+#
+# ⭐ AND R1's "ONE STORE FOR ONE SERIES" SURVIVES, which is the thing to check before touching
+# this. The authoritative post record is `ut_ig_posts`; the authoritative engagement series is
+# `ut_ig_post_snapshots`. A rollup cell is a PROJECTION refreshed from them — the same standing as
+# a `metric` cell, and the same standing as the `posts` json window (which is even allowed to SHED
+# posts to fit, precisely because the store still holds them). ⛔ A rollup may only ever READ. The
+# moment one writes a number nothing else can re-derive, it has become a third copy.
+
+def _ut():
+ """`core.user_tables`, imported LAZILY — and the laziness is measured, not stylistic.
+
+ Pulling this module in at import time initialises the store layer earlier than
+ `automation_engine` used to, and the last time that happened it moved a store-commit COUNT
+ from three to four on an unrelated gate. The relational pass
+ needs three constants and two predicates from the field layer; it does not need to change
+ when this module is imported.
+ """
+ import core.user_tables as _m
+ return _m
+
+
+#: The fns by family — how a value is folded, and what a blank means in each.
+_ROLLUP_NUM_FNS = frozenset({"sum", "average", "stdev", "min", "max"})
+_ROLLUP_BOOL_FNS = frozenset({"and", "or", "xor"})
+#: The count/order family — folded by their own arms above the lanes.
+_ROLLUP_SEQ_FNS = frozenset({"countall", "counta", "count", "latest"})
+#: The text family. ⛔ IT IS A NAMED SET NOW AND IT USED TO BE THE FALL-THROUGH, which is how
+#: wave 28 nearly shipped a wrong number that looked like data: `stdev` validated and STORED
+#: (`core.user_tables.ROLLUP_FNS`) a commit before this function learned it, and an fn no arm
+#: claims fell past both lanes into the join below — so a "Std deviation" column rendered
+#: `"137684, 19561, 8123"`. Filled, plausible, and not a statistic. An unrecognised fn now
+#: returns `""` ([[gate-answers-the-wrong-question]]: blank is the honest answer to a question
+#: nothing can answer; a comma-joined list is a different question's answer wearing this label).
+_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
+#: ⭐⭐ WHAT THIS FOLD ACTUALLY IMPLEMENTS, DERIVED FROM THE ARMS RATHER THAN RESTATED.
+#: `verify_automation` asserts this is IDENTICAL to `core.user_tables.ROLLUP_FNS` name for name
+#: (contract C1's parity leg), so the validator can never again accept a function the fold cannot
+#: compute — in EITHER direction. A hand-listed copy in the gate would have gone green on the
+#: defect above, because the defect was that the two lists already disagreed.
+ROLLUP_FOLD_FNS = frozenset(_ROLLUP_NUM_FNS | _ROLLUP_BOOL_FNS | _ROLLUP_SEQ_FNS
+ | _ROLLUP_TEXT_FNS)
+_ROLLUP_TEXT_FNS = frozenset({"concatenate", "arrayjoin", "arraycompact", "arrayunique"})
+#: What `arrayjoin` puts between values. Airtable uses ", "; `concatenate` uses nothing.
+_ROLLUP_JOIN = ", "
+#: A rollup's own cell ceiling. The text fns can concatenate a whole column into one cell, and a
+#: cell nobody can read is not an aggregate.
+ROLLUP_MAX_CHARS = 4000
+
+
+def _sort_key(value, ftype):
+ """One cell → a sortable key, typed by the LINKED field's declared type. `None` when the cell
+ is blank or does not parse as its declared type.
+
+ ⛔ TYPE-AWARE ON PURPOSE. `posted_at` is a `date` and `views` is an `int`; sorting either as a
+ string puts `2026-9-1` before `2026-10-1` and `9` after `100`. Since `sortBy` is what decides
+ WHICH rows a `limit` keeps, getting this wrong does not mis-order a display — it silently
+ averages the wrong twelve posts.
+
+ ⛔⛔ **BLANKS ARE PARTITIONED OUT BY THE CALLER, NEVER RANKED BY A FLAG — AND THE FLAG VERSION
+ SHIPPED BROKEN FOR ONE DEPLOY.** This returned `(1, 0.0, "")` for a blank and `(0, key, "")`
+ otherwise, documented as "a blank sorts LAST under desc". It does the OPPOSITE: `reverse=True`
+ flips the whole tuple, so `(1, …)` sorted FIRST and an undated row displaced the most recent
+ real one out of the window. A sentinel inside the key cannot mean "last" in both directions,
+ because the direction is applied to the sentinel too.
+ ⚠ AND IT WAS NOT A CORNER CASE ON THE RUNG THAT MATTERS: `wave20-split` MEASURED `datetime` as
+ `None` on 24/24 posts from the Profiles dataset, and `_bd_post_identity` writes `posted_at`
+ only when the vendor sent one — so on a paid profile pull EVERY post row is blank here, every
+ key tied, and `limit 12` took whatever twelve came first in dict order. The e2e test passed
+ because its fixture posts all carried dates.
+ ⇒ `None` means "not comparable", the caller keeps those rows at the END whichever way it
+ sorts, and "an unknown date is not a recent one" is finally what the code does.
+ """
+ raw = "" if value is None else str(value).strip()
+ if raw == "":
+ return None
+ if ftype in ("int", "currency", "pct", "rating"):
+ n = _lane_num(raw)
+ return (n,) if n is not None else None
+ if ftype == "date":
+ d = _parse_iso(raw.replace(" ", "T"))
+ return (d.timestamp(),) if d is not None else None
+ return (raw.lower(),)
+
+
+def _sample_stdev(nums):
+ """SAMPLE standard deviation (n-1) of `nums`, or None under two values.
+
+ ⭐ ONE IMPLEMENTATION, TWO READERS, and that is the point of lifting four lines into a
+ function: `_rollup_fold` RENDERS this number into a "Std deviation" column and
+ `_rollup_ref_threshold` COMPARES rows against it inside a `sigmas` condition. A second copy
+ would let a column and the filter beside it disagree about the same word on the same set —
+ the exact drift `top_up_views` was extracted to prevent one module over
+ ([[one-evaluator-per-question]]).
+ ⛔ None means UNANSWERABLE and no caller may read it as 0.
+ """
+ if len(nums) < 2:
+ return None
+ mean = sum(nums) / len(nums)
+ return (sum((n - mean) ** 2 for n in nums) / (len(nums) - 1)) ** 0.5
+
+
+def _rollup_ref_threshold(rows, field, sigmas):
+ """`mean + sigmas*stdev` of `field` over `rows` — a statistic OF THE SCOPED SET (contract C1).
+
+ ⛔ `rows` MUST already be the scoped window: ranked by `sortBy`, deduped, and cut by `limit`,
+ and NOT yet filtered by the conditions this threshold feeds. That order is the contract
+ (`core.user_tables.ROLLUP_REF_OPS`'s note says so in as many words) and both other orders
+ produce a plausible number: computing it before `limit` answers "2 sigma of everything this
+ account ever posted" under a column that says "of the last 10", and computing it after the
+ filter makes the threshold depend on the rows it is choosing — a definition that chases
+ itself.
+
+ Returns None when the window cannot answer — fewer than two numeric values in `field`.
+ ⛔ THE CALLER MUST DROP THE ROW, NOT KEEP IT. "Beyond 2 sigma of one post" is not a question
+ with a permissive answer; letting an unanswerable leaf pass everything would silently turn
+ "the outliers" into "all of them", which is this module's worst failure mode wearing a filter.
+ """
+ nums = [n for n in (_lane_num((r or {}).get(field)) for r in rows) if n is not None]
+ sd = _sample_stdev(nums)
+ if sd is None:
+ return None
+ return (sum(nums) / len(nums)) + float(sigmas) * sd
+
+
+def _rollup_fold(fn, values, ftype="text"):
+ """`values` (raw cells, in the order the window kept them) → the aggregate, as a STRING.
+
+ Returns `""` for "nothing to aggregate", NEVER `0`. ⛔ That distinction is this module's
+ oldest law and it bites hardest here: `sum` over no linked records is not zero, it is a
+ question with no rows to answer it, and a 0 in an "Avg views" column reads as a measurement
+ that the creator gets no views.
+ ⚠ The ONE exception is the count family, where zero IS the answer — "how many linked records"
+ over an empty set is genuinely 0, not unknown.
+
+ `ftype` is the SOURCE column's DECLARED type on the linked table (2026-08-10, D-92). Only
+ `min`/`max` read it today; it defaults to `text` so every existing caller and every test that
+ folds a bare list keeps its exact previous answer.
+ """
+ if fn == "countall":
+ return str(len(values))
+ if fn == "latest":
+ # Ordering belongs to the rollup bag (`sortBy` is mandatory for this function). Preserve
+ # a blank on the newest row rather than reaching backwards and presenting an older value
+ # as current.
+ return "" if not values or values[0] is None else str(values[0])[:ROLLUP_MAX_CHARS]
+ if fn == "counta":
+ return str(len([v for v in values if str(v or "").strip() != ""]))
+ if fn == "count":
+ # Airtable's COUNT counts NUMERIC values; COUNTA counts non-empty ones. Keeping them
+ # distinct is the whole reason both exist.
+ return str(len([v for v in values if _lane_num(v) is not None]))
+ # ⭐⭐ 2026-08-10 — D-92 CLOSED: `min`/`max` OVER A DATE COLUMN.
+ #
+ # Both were numeric-only, so a rollup over `posted_at`, `due_date` or `order_date` rendered
+ # BLANK forever while looking completely configured — the failure this module refuses
+ # everywhere else, arriving through the one fold that had no type awareness. "Earliest order"
+ # and "latest invoice due" are the two most ordinary date rollups there are, and
+ # `odoo_relational` already ships `latest` over `due_date`, so the vocabulary claimed dates
+ # and the fold did not.
+ #
+ # ⛔ NOT FIXED BY SNIFFING THE VALUES. An ISO date sorts correctly as a string, so a
+ # string-compare fallback would have worked on well-formed data and silently mis-ordered a
+ # `Aug 5, 2026` or a `2026-9-1` — [[measure-the-real-call]]'s shape. The DECLARED type is
+ # already at this call site (`linked_types`), and `_sort_key` is already the one function that
+ # turns a typed cell into a comparable key, blanks partitioned out. This reuses both rather
+ # than growing a second idea of what a date is.
+ # ⚠ RETURNS THE CELL, NOT THE KEY. `_sort_key` yields a comparison tuple; the answer a person
+ # wants in the column is the stored date string exactly as the source row spells it.
+ # ⚠ `min`/`max` ONLY. `sum`/`average`/`stdev` over dates are not blank by oversight — the mean
+ # of two timestamps is a number this product has no column type for, and inventing one here
+ # would be a value with no author.
+ if fn in ("min", "max") and ftype == "date":
+ keyed = [(k, str(v)) for k, v in
+ ((_sort_key(v, "date"), v) for v in values) if k is not None]
+ if not keyed:
+ return ""
+ return (min(keyed) if fn == "min" else max(keyed))[1][:ROLLUP_MAX_CHARS]
+ if fn in _ROLLUP_NUM_FNS:
+ nums = [n for n in (_lane_num(v) for v in values) if n is not None]
+ if not nums:
+ return ""
+ if fn == "stdev":
+ # ⭐⭐ SAMPLE standard deviation (n-1), and the divisor is a ruling, not a preference
+ # (C1 / `core.user_tables.ROLLUP_FNS`'s note): a rollup folds the rows that happen to
+ # be LINKED, which is a sample of an account's posting history and not its entirety.
+ # ⚠ Fixture to check a refactor against: [2,4,4,4,5,5,7,9] -> 2.14. The POPULATION
+ # form gives 2.00 on the same input, so a test that ever reads 2.00 has silently
+ # switched divisors.
+ # ⛔ FEWER THAN TWO VALUES IS "" AND NEVER "0". n-1 = 0 would divide by zero, but the
+ # honest reason is upstream of the arithmetic: one measurement has no spread to
+ # report, and a 0 in a "Std deviation" column reads as PERFECT CONSISTENCY — the
+ # single most confident thing this column can say, asserted from a single row. Same
+ # law as the blank `sum`, and it bites harder here.
+ out = _sample_stdev(nums)
+ if out is None:
+ return ""
+ else:
+ out = (sum(nums) if fn == "sum" else min(nums) if fn == "min"
+ else max(nums) if fn == "max" else sum(nums) / len(nums))
+ return f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"
+ if fn in _ROLLUP_BOOL_FNS:
+ # A checkbox cell is '1'/'' in this product, so truth is "non-blank and not a zero".
+ flags = [str(v or "").strip() not in ("", "0", "false", "False") for v in values]
+ if not flags:
+ return ""
+ hit = (all(flags) if fn == "and" else any(flags) if fn == "or"
+ else sum(1 for f in flags if f) % 2 == 1)
+ return "1" if hit else ""
+ # --- the text family. ⛔ CLAIMED BY NAME, NEVER BY FALL-THROUGH — see `_ROLLUP_TEXT_FNS`.
+ # An fn no arm above recognises returns "" rather than a comma-joined dump of every value,
+ # which is the shape a not-yet-implemented aggregate wore for one commit of wave 28.
+ if fn not in _ROLLUP_TEXT_FNS:
+ return ""
+ vals = [str(v).strip() for v in values if str(v or "").strip() != ""]
+ if fn == "arrayunique":
+ seen, uniq = set(), []
+ for v in vals:
+ if v.lower() not in seen:
+ seen.add(v.lower())
+ uniq.append(v)
+ vals = uniq
+ if not vals:
+ return ""
+ text = ("".join(vals) if fn == "concatenate" else _ROLLUP_JOIN.join(vals))
+ return text[:ROLLUP_MAX_CHARS]
+
+
+def _link_from_key(fields, bag):
+ """Which column on THIS table supplies the join value.
+
+ Declared `from` wins; otherwise the PROFILE-flagged column, then the PINNED one. ⭐ That
+ fallback chain is what makes an Instagram database link up with no configuration at all —
+ the "automatically" in the owner's instruction — and it is the same chain the grid already
+ uses to decide a table's identity column, rather than a second opinion about it.
+ """
+ declared = str((bag or {}).get("from") or "").strip()
+ if declared:
+ return declared
+ prof = next((f for f in fields if isinstance(f.get("profile"), dict)), None)
+ if prof:
+ return str(prof.get("key") or "")
+ pin = next((f for f in fields if f.get("pinned") is True), None)
+ return str(pin.get("key") or "") if pin else ""
+
+
+def _linked_rows_by_join(linked, on_key):
+ """`{join value (lower-cased) -> [(row_id, row)]}` over one linked table, built ONCE.
+
+ ⚠ Lower-cased because the join values this exists for are Instagram handles, which the
+ profile flag already normalises to lower case on one side and which a hand-typed cell on the
+ other side may not. A join that misses on case is a relation that silently reports zero.
+ """
+ idx = {}
+ for rid, row in ((linked or {}).get("rows") or {}).items():
+ k = str((row or {}).get(on_key) or "").strip().lower()
+ if k:
+ idx.setdefault(k, []).append((str(rid), row or {}))
+ return idx
+
+
+def _rollup_condition_matches(row, condition, field_types, ref_value=None):
+ """Evaluate one Airtable-style linked-record condition against a candidate row.
+
+ `ref_value` is the threshold a `ref: {sigmas}` leaf compares against, already computed by the
+ caller over the SCOPED set (`_rollup_ref_threshold`). ⛔ It is passed IN rather than computed
+ here because this function sees one row and the statistic is a property of the whole window —
+ a version that reached for the set from inside would be recomputing the same mean once per
+ row, and would have to be handed the window anyway.
+ ⚠ `None` means the window could not answer, and the leaf then matches NOTHING. See the
+ threshold helper for why the permissive reading is the dangerous one.
+ """
+ field = str((condition or {}).get("field") or "")
+ op = str((condition or {}).get("op") or "")
+ raw = (row or {}).get(field)
+ text = str(raw or "").strip()
+ if op == "is_empty":
+ return text == ""
+ if op == "is_not_empty":
+ return text != ""
+ if (condition or {}).get("ref") is not None:
+ # ⛔ NUMERIC LANE ONLY, BOTH SIDES. The validator already restricts `ref` to the ordering
+ # ops, and a row whose cell is blank or unparseable has no position relative to a computed
+ # threshold — it is not "below" it. Dropping it is the same partition law `_sort_key`
+ # follows: unrankable is not a rank ([[sentinel-in-a-sort-key]]).
+ left_num = _lane_num(text)
+ if ref_value is None or left_num is None:
+ return False
+ return ((op == "gt" and left_num > ref_value)
+ or (op == "gte" and left_num >= ref_value)
+ or (op == "lt" and left_num < ref_value)
+ or (op == "lte" and left_num <= ref_value))
+ wanted = str((condition or {}).get("value") or "").strip()
+ if op == "contains":
+ return wanted.casefold() in text.casefold()
+ if op == "not_contains":
+ return wanted.casefold() not in text.casefold()
+ if op in ("eq", "neq"):
+ left_num, right_num = _lane_num(text), _lane_num(wanted)
+ equal = (left_num == right_num if left_num is not None and right_num is not None
+ else text.casefold() == wanted.casefold())
+ return equal if op == "eq" else not equal
+ ftype = (field_types or {}).get(field, "text")
+ left = _sort_key(text, ftype)
+ right = _sort_key(wanted, ftype)
+ if left is None or right is None:
+ return False
+ return ((op == "gt" and left > right) or (op == "gte" and left >= right)
+ or (op == "lt" and left < right) or (op == "lte" and left <= right))
+
+
+def compute_relation_cells(rt, table_key, tables=None):
+ """Recompute every DERIVED LINK cell and every ROLLUP cell on ONE table. Returns rows touched.
+
+ Zero store reads when the table declares neither kind — the same cheap-by-construction shape
+ `compute_metric_cells` has, so walking every table on a tick costs a dict scan per table.
+ """
+ store = tables if tables is not None else ut_all(rt)
+ t = (store or {}).get(table_key)
+ fields = list((t or {}).get("fields") or [])
+ links = [f for f in fields if _ut().is_derived_link(f)]
+ rollups = [f for f in fields if isinstance(f.get("rollup"), dict)]
+ if not links and not rollups:
+ return 0
+ rows = (t or {}).get("rows") or {}
+ by_key = {str(f.get("key")): f for f in fields}
+
+ # --- resolve every link field ONCE per table, not once per row.
+ # `resolved[link_key][row_id] = [(linked_row_id, linked_row), ...]`
+ resolved, linked_types = {}, {}
+ for f in links + [by_key.get(str((r.get("rollup") or {}).get("link"))) for r in rollups]:
+ lk_key = str((f or {}).get("key") or "")
+ if not lk_key or lk_key in resolved or not isinstance((f or {}).get("link"), dict):
+ continue
+ bag = f["link"]
+ linked = (store or {}).get(str(bag.get("table") or "")) or {}
+ linked_types[lk_key] = {str(lf.get("key")): str(lf.get("type") or "text")
+ for lf in (linked.get("fields") or [])}
+ if bag.get("inverse"):
+ # Airtable's reciprocal side: this row is linked to every SOURCE row whose ordinary
+ # link cell contains this row id. The source cell remains the one relationship truth.
+ source_rows = linked.get("rows") or {}
+ inverse_key = str(bag.get("inverse") or "")
+ inverse_index = {}
+ for source_id, source_row in source_rows.items():
+ for target_id in [part.strip() for part in
+ str((source_row or {}).get(inverse_key) or "").split(",")]:
+ if target_id:
+ inverse_index.setdefault(target_id, []).append(
+ (str(source_id), source_row or {}))
+ resolved[lk_key] = {str(rid): inverse_index.get(str(rid), []) for rid in rows}
+ elif bag.get("on"):
+ idx = _linked_rows_by_join(linked, str(bag["on"]))
+ from_key = _link_from_key(fields, bag)
+ resolved[lk_key] = {
+ str(rid): idx.get(str((row or {}).get(from_key) or "").strip().lower(), [])
+ for rid, row in rows.items()} if from_key else {}
+ else:
+ # An ORDINARY link: the cell IS the relation. Ids the linked table no longer holds
+ # are dropped rather than carried — a link to a deleted row is not a link.
+ lrows = (linked.get("rows") or {})
+ resolved[lk_key] = {
+ str(rid): [(i, lrows[i]) for i in
+ [s.strip() for s in str((row or {}).get(lk_key) or "").split(",")]
+ if i and i in lrows]
+ for rid, row in rows.items()}
+
+ changes = {}
+ for rid, row in rows.items():
+ rid = str(rid)
+ row = row or {}
+ for f in links:
+ fk = str(f["key"])
+ hits = (resolved.get(fk) or {}).get(rid) or []
+ if f["link"].get("single"):
+ hits = hits[:1]
+ # ⛔ THE CAP IS A DISPLAY CAP AND THE ROLLUPS DO NOT READ THROUGH IT. A derived cell
+ # is a projection of `resolved`, which is uncapped and is what every rollup below
+ # consumes — so a profile with 900 posts shows the first 500 ids and still averages
+ # over all 900. Same argument the `posts` window already makes: shedding is safe
+ # precisely because the authoritative store still holds everything.
+ want = ",".join(i for i, _r in hits[:_ut().LINK_MAX_IDS])
+ if str(row.get(fk, "")) != want:
+ changes.setdefault(rid, {})[fk] = want
+ for f in rollups:
+ fk, bag = str(f["key"]), f["rollup"]
+ lk_key = str(bag.get("link") or "")
+ hits = list((resolved.get(lk_key) or {}).get(rid) or [])
+ # ⚠ A rollup whose link field does not exist (renamed, deleted) resolves to NOTHING
+ # and therefore to a blank cell — never to a stale number. A column that keeps
+ # printing yesterday's answer after its input is gone is the worst of the options.
+ # ⭐⭐ WAVE 28 / CONTRACT C1 — SCOPE FIRST, FILTER SECOND, AND THE TWO USED TO BE THE
+ # OTHER WAY ROUND. The conditions block stood HERE, above the ranking, so
+ # "last 10 posts where views > X" meant *the 10 most recent of the posts over X*
+ # rather than *the ones over X among the last 10* — two different windows wearing one
+ # sentence. Harmless while a threshold was a literal; incoherent the moment a
+ # threshold is a statistic OF the window, because the set being described and the set
+ # doing the describing would be different sets.
+ # ⛔ THIS ORDER IS THE CONTRACT, not an implementation choice: `core.user_tables`'s
+ # `ROLLUP_REF_OPS` note states it ("the scope picks the window, THEN the threshold is
+ # computed over that window, THEN the conditions filter it") and the validator half
+ # was written against it.
+ # ⚠ IT IS A BEHAVIOUR CHANGE FOR EXACTLY ONE SHAPE: a stored rollup carrying BOTH
+ # `conditions` AND `limit`. No shipped preset does (measured across `odoo_relational`
+ # and the IG presets — the one preset with conditions, `_OPEN_ONLY`, is a `countall`
+ # with no limit), so the blast radius is user-built rollups only.
+ # ⭐⭐ 2026-08-09 (owner) — THE PRE-FILTER, ABOVE THE RANKING. Owner: *"instead of last
+ # 12 posts, we also want to make it so its last N record, where the record's Status is
+ # video."* `conditions` cannot answer that: C1 moved them BELOW the window on purpose,
+ # so they select among the rows the window already kept. `where` selects WHICH rows
+ # the window is spent on.
+ # ⛔ ABOVE `distinctBy` TOO, not merely above the sort. Dedup keeps the first row per
+ # identity; run it first and a carousel could claim the slot its reel sibling needed,
+ # so the window would come up short for a reason nothing on screen explains.
+ # ⚠ NO `ref` REACHES HERE — `_clean_rollup` refuses a set-statistic threshold in this
+ # list, because at this point there is no fixed set for a statistic to be about.
+ where = list(bag.get("where") or [])
+ if where:
+ where_matches = lambda pair: [
+ _rollup_condition_matches(pair[1], condition,
+ linked_types.get(lk_key) or {}, ref_value=None)
+ for condition in where]
+ if str(bag.get("whereConj") or "and") == "or":
+ hits = [pair for pair in hits if any(where_matches(pair))]
+ else:
+ hits = [pair for pair in hits if all(where_matches(pair))]
+ sort_by = str(bag.get("sortBy") or "")
+ if sort_by:
+ ftype = (linked_types.get(lk_key) or {}).get(sort_by, "text")
+ # ⛔ PARTITION, THEN SORT. A row whose sort cell is blank or unparseable is not
+ # rankable, and it must land at the END whichever direction is asked for — which
+ # a sentinel inside the sort key cannot do, because `reverse` flips the sentinel
+ # too (see `_sort_key`). Unrankable rows are appended, so a `limit` spends its
+ # window on rows that HAVE the value before it falls back to ones that do not.
+ keyed = [(pair, _sort_key(pair[1].get(sort_by), ftype)) for pair in hits]
+ rankable = [(p, k) for p, k in keyed if k is not None]
+ rankable.sort(key=lambda pk: pk[1],
+ reverse=str(bag.get("sortDir") or "desc") == "desc")
+ hits = [p for p, _k in rankable] + [p for p, k in keyed if k is None]
+ distinct_by = str(bag.get("distinctBy") or "")
+ if distinct_by:
+ seen, unique = set(), []
+ for pair in hits:
+ identity = str(pair[1].get(distinct_by) or "").strip().lower()
+ # A blank is not an identity. Keep it rather than collapsing every unknown
+ # record into one synthetic duplicate.
+ if identity and identity in seen:
+ continue
+ if identity:
+ seen.add(identity)
+ unique.append(pair)
+ hits = unique
+ limit = int(bag.get("limit") or 0)
+ if limit:
+ hits = hits[:limit]
+ # --- the window is now FIXED, so a set-statistic threshold has a set to be about.
+ conditions = list(bag.get("conditions") or [])
+ if conditions:
+ # ⚠ RESOLVED ONCE PER LEAF, NOT ONCE PER ROW. The threshold is a property of the
+ # window; computing it inside `matches` would recompute the same mean for every
+ # candidate and — worse — would invite computing it over a set that the filter is
+ # already shrinking underneath it.
+ # ⚠ `.get("sigmas", 0.0)`, never `... or 0.0` — a legitimate `sigmas: 0` ("beyond
+ # the mean") is falsy, and the `or` spelling would silently rewrite it to the same
+ # number by accident. It reads identically and is right for the wrong reason,
+ # which is how it survives a review.
+ refs = [_rollup_ref_threshold(
+ [r for _i, r in hits], str(c.get("field") or ""),
+ (c.get("ref") or {}).get("sigmas", 0.0))
+ if isinstance(c, dict) and c.get("ref") is not None else None
+ for c in conditions]
+ matches = lambda pair: [
+ _rollup_condition_matches(pair[1], condition,
+ linked_types.get(lk_key) or {}, ref_value=ref)
+ for condition, ref in zip(conditions, refs)]
+ if str(bag.get("conditionConj") or "and") == "or":
+ hits = [pair for pair in hits if any(matches(pair))]
+ else:
+ hits = [pair for pair in hits if all(matches(pair))]
+ src = str(bag.get("field") or "")
+ # D-92 — the SOURCE column's declared type, read from the same `linked_types` map the
+ # sort path above uses. `countall` folds a synthetic `[1]*n` with no source column at
+ # all, so the default stands for it.
+ want = _rollup_fold(str(bag.get("fn") or ""),
+ [r.get(src) for _i, r in hits] if src else [1] * len(hits),
+ (linked_types.get(lk_key) or {}).get(src, "text"))
+ if str(row.get(fk, "")) != want:
+ changes.setdefault(rid, {})[fk] = want
+ if not changes:
+ return 0
+
+ # A run that just wrote linked rows passes its in-flight user_tables bucket here so the
+ # relation refresh joins the SAME coalesced commit. The standalone/tick path below keeps
+ # the public helper's old persist-on-change behaviour.
+ if tables is not None:
+ tt = tables.get(table_key)
+ if tt is not None:
+ for r, vals in changes.items():
+ tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
+ return len(changes)
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ tt = cur.get(table_key)
+ if tt is not None:
+ for r, vals in changes.items():
+ tt.setdefault("rows", {}).setdefault(r, {}).update(vals)
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ return len(changes)
+
+
+def _refresh_relations_inplace(tables, log=print):
+ """Refresh every relation against one mutable user_tables bucket; perform no store write."""
+ touched = 0
+ for tk, table in list((tables or {}).items()):
+ fields = (table or {}).get("fields") or []
+ if not any(f.get("type") == "rollup" or _ut().is_derived_link(f) for f in fields):
+ continue
+ try:
+ touched += compute_relation_cells(None, tk, tables=tables)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] relation refresh {tk} failed: {type(e).__name__}: {e}")
+ return touched
+
+
+def refresh_relations(rt, log=print):
+ """The tick half of the relational pass — the twin of `refresh_metrics`.
+
+ ⚠ Runs for EVERY table, because a link can point anywhere: a rollup on table A goes stale
+ when table B gains a row, and A has no way to know that happened. Cheap by construction — a
+ table declaring neither kind costs one dict scan.
+ """
+ snapshot = {
+ str(key): {**(table or {}),
+ "rows": {str(rid): dict(row or {})
+ for rid, row in ((table or {}).get("rows") or {}).items()}}
+ for key, table in (ut_all(rt) or {}).items()
+ }
+ if not _refresh_relations_inplace(snapshot, log=log):
+ return 0
+ actual = [0]
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ actual[0] = _refresh_relations_inplace(cur, log=log)
+ return cur
+
+ # ⭐ ASYNC, and for TWO independent reasons (2026-08-09, the lost-record bug).
+ #
+ # 1. COST, which was wave 27 item 2's whole point: this pass runs after every row write, and
+ # `flush="sync"` made each one a blocking HF commit against the 256-commits/hr budget.
+ # A derived-cell recompute has no business forcing a commit on someone typing.
+ # 2. It was the deterministic TRIGGER of the bug: `add_row` writes `flush='async'`, so a new
+ # row lives only in the store cache for 2-20s, and this call — on the SAME key
+ # (`UT_STORE_KEY == user_tables`) — used to `_read_strict` that cache away and upload the
+ # result. `POST /rows` answered 201 and the row was gone.
+ #
+ # ⚠ THE ROOT FIX IS IN `core/store.py` (a sync RMW no longer discards a dirty cache) and it
+ # is what makes the other ~39 sync writers of this key safe. This line is not that fix and
+ # must not be mistaken for it — it removes the trigger and the cost, nothing more. Both
+ # landed together on purpose: one is correctness, one is the hot path.
+ rt.update(UT_STORE_KEY, _up, flush="async")
+ return actual[0]
+
+
+def refresh_metrics(rt, today=None, log=print):
+ """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window
+ metric can never go staler than one tick while a scheduler exists. Cheap by construction —
+ a table without metric fields costs a dict scan and nothing else."""
+ touched = 0
+ for tk, t in ut_all(rt).items():
+ if metric_fields_of_table(t):
+ try:
+ touched += compute_metric_cells(rt, tk, today=today)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}")
+ return touched
+
+
+def purge_subject(rt, handle):
+ """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the
+ tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half
+ non-optional: a purge that missed the pooled copy would not be erasure). Returns
+ `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is
+ now ABSENT, which is the one aggregate whose drill is emptiness."""
+ subject = str(handle or "").strip().lstrip("@").lower()
+ if not subject:
+ return {}
+ counts = {}
+ tables = ut_all(rt)
+ post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {}
+ codes = {str(r.get("shortcode") or "") for r in post_rows.values()
+ if str((r or {}).get("influencer_key") or "").strip().lower() == subject}
+
+ keeps = {
+ "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key")
+ or "").strip().lower() != subject,
+ "ut_ig_posts": lambda r: str((r or {}).get("influencer_key")
+ or "").strip().lower() != subject,
+ "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes,
+ DISCOVER_TABLE: lambda r: str((r or {}).get("handle")
+ or "").strip().lower() != subject,
+ }
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ for tk, keep in keeps.items():
+ t = cur.get(tk)
+ if t is None:
+ continue
+ rows = t.get("rows") or {}
+ nxt = {rid: r for rid, r in rows.items() if keep(r)}
+ counts[tk] = len(rows) - len(nxt)
+ t["rows"] = nxt
+ return cur
+
+ rt.update(UT_STORE_KEY, _up, flush="sync")
+ import ig_master
+ for bucket, n in (ig_master.purge_handle(subject) or {}).items():
+ counts[f"master:{bucket}"] = n
+ return counts
+
+
+# ---------------------------------------------------------------------------------------------
+# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33)
+# ---------------------------------------------------------------------------------------------
+# Six ways an automation starts, exactly: manual | schedule | event_field | record_created |
+# webhook | email. The first two are what always existed (Run now; cron via the tick). The four
+# new ones are EVENTS, and A2 makes their discipline LAW rather than taste:
+#
+# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state,
+# not for being in it. Implemented as per-record ARMED state over successive evaluations
+# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms
+# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing
+# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the
+# disarmed set with everything currently matching, so already-matching records do not fire
+# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar).
+# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only
+# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers
+# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never
+# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by
+# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as
+# a statusNote) catches whatever construction did not foresee.
+# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds
+# instead of running — a partial run entry names the count and the deliberate way through
+# (Run now). C4's discovery guard is this rule's special case.
+# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record
+# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire);
+# event_field fires every transition.
+
+# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ───────────────────────────────
+# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that
+# renames the same events is a second vocabulary to learn for no gain.
+#
+# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions".
+# Renaming the key would have orphaned every stored trigger in production for a caption; the key
+# is the contract with the store, the label is the contract with the reader, and they are allowed
+# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the
+# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4
+# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath.
+#
+# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show
+# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a
+# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop
+# class wearing a friendlier face; here the two lists are separate on purpose and the refusal
+# names the state rather than pretending the key is unknown.
+# ── WAVE 24 · C-TRIG (owner ruling R6). ───────────────────────────────────────────────────────
+# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the
+# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway —
+# it is the event this flow starts from. Picking it sets the definition's kind to
+# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now.
+#
+# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the
+# schedule (or on Run now), so it stays out of both lists below and its node switch flips the
+# CRON — see `TRIGGER_SCHEDULE_KEYS`.
+#
+# ⭐⭐ WAVE 29 (item 7 · D-9 · R1) — `tiktok_profile_match` MOVED HERE FROM `TRIGGER_PLANNED`, and
+# that move is the whole of "TikTok is a real trigger now". The label was written a wave early in
+# the owner's own words and has not changed; what changed is which tuple it sits in, because
+# `clean_trigger` refuses the planned list with a sentence and accepts this one.
+TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created",
+ "enters_view", "webhook", "email", "form_submitted", "ig_profile_match",
+ "tiktok_profile_match")
+#: ⭐ WAVE 25 · C2 / owner ruling R9 — `web_page_changed` JOINS THE PLANNED LIST, and joining THIS
+#: tuple rather than `TRIGGER_KEYS` is the whole of its implementation. `clean_trigger` refuses
+#: everything here with a sentence, so the faded row is a wall; the picker shows it so the Scraper
+#: section is not a section of one.
+#: ⚠ `web_page_changed` IS NOT THE WEB ACTION (D-51). A trigger that notices a page changed and an
+#: action that drives a browser are different builds; this row must not be read as progress on D-51.
+TRIGGER_PLANNED = ("button_clicked", "comment_added", "web_page_changed")
+TRIGGER_LABELS = {
+ "manual": "Manual",
+ "schedule": "At a scheduled time",
+ "event_field": "When a record matches conditions",
+ "record_updated": "When a record is updated",
+ "record_created": "When a record is created",
+ "enters_view": "When a record enters a view",
+ "webhook": "When a webhook is received",
+ "email": "When an email arrives",
+ "form_submitted": "When a form is submitted",
+ "ig_profile_match": "When an Instagram profile fits a criteria",
+ "button_clicked": "When a button is clicked",
+ "comment_added": "When a comment is added",
+ "web_page_changed": "When a website page changes",
+ "tiktok_profile_match": "When a TikTok profile fits a criteria",
+}
+
+# ── WAVE 25 · C2 — THE PICKER TAXONOMY, and it lives HERE beside the vocabulary it describes. ──
+# `group` already rode the wire as "Standard"/"Sources" (`routes_automation`), which is a
+# distinction about where a trigger came FROM rather than about what a person is choosing. The
+# question the picker actually asks is: does this fire on TIME, on your own DATA, or because
+# something OUTSIDE said so. Three answers, and every trigger has exactly one.
+#
+# ⛔ TWO CONTROLS, NOT ONE, AND THE FIRST DRAFT HAD ONLY THE WRONG HALF. Indexing this map
+# directly (`TRIGGER_GROUP_OF[k]`) makes an unclassified trigger a KeyError — which is exactly the
+# incident `_triggers_vocab`'s `per.get(k, ...)` comment records: a key added to `TRIGGER_KEYS`
+# without remembering a dict beside it 500'd `GET /automations`, the payload the whole automation
+# surface polls every 2.5 s, with every gate green. A mis-grouped row is a cosmetic bug; a 500 is
+# the surface. So:
+# * RUNTIME fails SOFT — an unclassified trigger falls into `other`, which sorts LAST (the rule
+# `ACTION_GROUP_ORDER` already uses: an unordered group sorts last, never first, because
+# appearing at the top looks deliberate) and is honestly captioned rather than smuggled into
+# Database.
+# * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`,
+# so the fallback is provably dead code in production and the classification is still
+# mandatory. The fallback catches the accident; the gate stops it shipping.
+TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1},
+ "database": {"label": "Database", "order": 2},
+ "connector": {"label": "Connector", "order": 3},
+ "other": {"label": "Other", "order": 99}}
+#: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to
+#: prevent — it is the soft landing, not a category anybody should be adding triggers to.
+TRIGGER_GROUP_FALLBACK = "other"
+TRIGGER_GROUP_OF = {
+ "manual": "time", "schedule": "time",
+ "event_field": "database", "record_updated": "database", "record_created": "database",
+ "enters_view": "database", "form_submitted": "database",
+ "button_clicked": "database", "comment_added": "database",
+ "email": "connector", "webhook": "connector", "ig_profile_match": "connector",
+ "web_page_changed": "connector", "tiktok_profile_match": "connector",
+}
+#: The SUB-group inside "Connector" — which connected thing this trigger comes through.
+#: ⚠ THESE KEYS ARE GROUPING HANDLES FOR THE PICKER, NOT connector-directory slugs, and the two
+#: genuinely differ: the directory's OAuth row for Gmail is `google` (the provider), while a
+#: person choosing a trigger is picking *Gmail* (the product). A client that joined this key
+#: against `/connectors/directory` would match `scraper` and `tiktok` and miss `gmail` — so it
+#: must group by it and render `label`, never look it up. Said here because the miss would be
+#: silent and partial, which is the worst shape.
+TRIGGER_CONNECTOR = {
+ "email": {"key": "gmail", "label": "Gmail"},
+ "webhook": {"key": "webhooks", "label": "Webhooks"},
+ # R9: the Scraper ships with ONE built trigger and one faded one.
+ "ig_profile_match": {"key": "scraper", "label": "Scraper"},
+ "web_page_changed": {"key": "scraper", "label": "Scraper"},
+ "tiktok_profile_match": {"key": "tiktok", "label": "TikTok"},
+}
+#: Triggers that watch a database and therefore need one named before they can fire.
+TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view",
+ "form_submitted")
+#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself.
+#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven,
+#: so flipping its node must flip the schedule.
+#:
+#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read
+#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`,
+#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For
+#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a
+#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns
+#: about eight lines into its own comment. Derived from one named set now, so a trigger added to
+#: `TRIGGER_KEYS` cannot silently join the wrong side of it.
+#: ⚠ WAVE 29 — `tiktok_profile_match` BELONGS HERE FOR THE SAME REASON `ig_profile_match` DOES,
+#: and forgetting it is precisely the failure this constant's own note describes: it watches no
+#: table and MAKES rows on the schedule, so its node switch has nothing to flip but the cron. Left
+#: out, a person clicking the TikTok trigger node's switch would toggle the trigger itself while
+#: the schedule kept firing — a switch that lies.
+TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match", "tiktok_profile_match")
+#: ⭐ WAVE 25 — DEBT D-55: "the cron drives this one", on the wire at last.
+#:
+#: ⛔ `TRIGGER_SCHEDULE_KEYS` MUST NOT SHIP VERBATIM, and the one-element difference is the entire
+#: reason this constant exists rather than the tuple above being sent. That set answers "which
+#: way does this trigger's NODE SWITCH flip" — and `manual` is in it only because a manual
+#: automation's switch has nothing else to flip. Shipping it as "the cron drives this" would draw
+#: a schedule face on the one trigger whose whole sentence is "It runs only when you press Run
+#: now": a control contradicting its own description.
+#:
+#: D-55's history is why it is DERIVED rather than listed: the client carried
+#: `CRON_DRIVEN_TRIGGERS = ["schedule", "ig_profile_match"]` — a hand-kept copy of a server fact
+#: that fails VISIBLY but silently (a new cron-driven trigger simply shows no schedule face).
+#: Subtracting from the engine's own set means a trigger added there cannot be forgotten here.
+TRIGGER_CRON_KEYS = frozenset(TRIGGER_SCHEDULE_KEYS) - {"manual"}
+#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so
+#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples.
+TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view")
+MAX_WATCH_FIELDS = 12
+#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there,
+#: and so would a deployment that prefers immediacy over coalescing.
+EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15)
+FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses
+FIRE_WINDOW_SECONDS = 300
+FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on
+EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation
+EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest
+CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota
+
+EMAIL_FIELDS = [
+ field_def("email_id", "Email id"), field_def("email_from", "From"),
+ field_def("email_subject", "Subject"), field_def("email_date", "Date"),
+ field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"),
+]
+
+
+def clean_trigger(raw, previous=None):
+ """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and
+ means what it always meant: manual + whatever `schedule` says.
+
+ ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry
+ with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT
+ rather than refused — the picker writes `{key}` first and the table/field after, the
+ wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the
+ wire so the surface says "finish setting this up" instead of snapping back to Manual. It
+ cannot fire while incomplete (the hooks match on the table it does not name), which is the
+ fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a
+ condition on a field the trigger does not watch) are still refused with the sentence —
+ incomplete is a state, wrong is not.
+ """
+ if raw in (None, "", {}):
+ return (dict(previous) if isinstance(previous, dict) and previous else None), None
+ if not isinstance(raw, dict):
+ return None, "the trigger must be an object"
+ prev = previous if isinstance(previous, dict) else {}
+ key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"),
+ 30).strip()
+ if key in TRIGGER_PLANNED:
+ # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence
+ # says WHY rather than "unknown trigger", because the picker legitimately showed it.
+ return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — "
+ f"it renders so you can see it is coming, and it cannot be saved")
+ if key not in TRIGGER_KEYS:
+ return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS))
+ if key == "schedule":
+ # ⛔ STILL NOT STORED, and the original reasoning holds for THIS key alone: `schedule`
+ # already owns the cron (`defn['schedule']` = `{cron, enabled}`), so a stored
+ # `{key:'schedule'}` would be a second copy of that fact, free to disagree with it.
+ return None, None
+ if key == "manual":
+ # ⭐⭐ 2026-08-07 (owner ruling) — **MANUAL IS A REAL, STORED CHOICE NOW.**
+ # Owner: *"Make it so that when you choose Manual, it IS a manual automation that the user
+ # can just press Run to make the full flow work."*
+ #
+ # ⛔ THIS SPLITS A PAIR THAT SHOULD NEVER HAVE BEEN ONE. The old line refused both keys
+ # together with one argument — *"storing a no-op trigger would be a second copy of that
+ # fact"* — and that argument is TRUE OF `schedule` AND FALSE OF `manual`. A schedule has
+ # another home; **manual has none.** Nothing anywhere recorded "this automation is
+ # manual", so storing it is not a duplicate: it is the only record there has ever been.
+ #
+ # ⚠ WHAT THE CONFLATION COST, measured live: picking Manual wrote nothing, so
+ # `chosen` (`!!trigger || schedule.enabled`) stayed false, the Builder kept showing the
+ # "nobody has decided yet" empty state, and Configuration — including the Database picker
+ # a plain automation cannot do without — never rendered. The owner reported it twice. The
+ # previous note reasoned that a manual option *"would bounce straight back to this state
+ # on the next reload"* and concluded the option should be HIDDEN; the honest conclusion
+ # was that it should be STORED.
+ #
+ # ⚠ DELIBERATELY BARE. No `enabled`, no `paused`: a manual trigger cannot be switched off
+ # (Run now always works, which is the whole of what it means) and a switch that governs
+ # nothing is worse than no switch. `graph()` keeps this node on the SCHEDULE panel so the
+ # cron stays reachable — picking Manual says how it fires today, never that it may not be
+ # scheduled tomorrow.
+ return {"key": "manual"}, None
+ out = {"key": key,
+ "enabled": bool(raw["enabled"]) if "enabled" in raw else
+ bool(prev.get("enabled", True)),
+ "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))}
+ if key in TRIGGER_TABLE_KEYS:
+ table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip()
+ if table and not table.startswith(UT_PREFIX):
+ return None, ("event triggers watch blank databases (ut_*) this wave — "
+ f"{table!r} is not one")
+ out["table"] = table
+ if key == "event_field":
+ # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record
+ # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a
+ # second, quieter way to express the same narrowing, and the owner asked for one.
+ # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is
+ # dropped on this definition's next clean — silently, and exactly once, because nothing
+ # writes the key back. A refusal here would have 400'd the live automations that carry it.
+ cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"),
+ where="the trigger")
+ if cerr:
+ return None, cerr
+ out["when"] = cond
+ if key == "record_updated":
+ # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty
+ # is the WIDER reading and it is the default there too, so it stays the default here.
+ watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields")
+ if watch_raw in (None, ""):
+ watch = []
+ elif not isinstance(watch_raw, list):
+ return None, "the watched-field list must be a list of field keys"
+ else:
+ watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()]
+ if len(watch) > MAX_WATCH_FIELDS:
+ return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} "
+ f"fields — leave the list empty to watch every field")
+ out["fields"] = watch
+ # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A
+ # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration:
+ # "a record was updated" is an event, and asking it to also be a filter was the overlap
+ # with `event_field` the owner asked to end. Stated loudly in the contract AND here so
+ # nobody restores it as a bug fix.
+ # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate`
+ # naturally returns "no gate" for it — the write itself becomes the event — rather than
+ # this needing a second removal anywhere.
+ if key == "enters_view":
+ out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"),
+ 80).strip()
+ if key == "form_submitted":
+ # Blank = any form on that database. Naming one narrows to it, which is what a table
+ # carrying an intake form AND a correction form needs.
+ out["formToken"] = _s(raw.get("formToken") if "formToken" in raw
+ else prev.get("formToken"), 64).strip()
+ if key == "webhook":
+ # The token is MINTED here, once, and survives every later patch — rotating it on
+ # every Save would silently break the external caller the URL was given to.
+ out["token"] = _s(prev.get("token"), 64) or _secrets_token()
+ # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config.
+ # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no
+ # map behaves exactly as it did — it fires the flow and reads nothing — so the live
+ # webhook automations are untouched. `webhook` deliberately stays OUT of
+ # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and
+ # every existing webhook trigger would go unconfigured and stop firing.
+ table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip()
+ if table and not table.startswith(UT_PREFIX):
+ return None, ("a webhook writes into a blank database (ut_*) — "
+ f"{table!r} is not one")
+ out["table"] = table
+ fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw
+ else prev.get("fieldMap"))
+ if ferr:
+ return None, ferr
+ out["fieldMap"] = fmap
+ if key == "email":
+ out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"),
+ 200).strip() or "in:inbox is:unread"
+ out["configured"] = _trigger_configured(out)
+ return out, None
+
+
+#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody
+#: else's schema and is legitimately wider than an action's hand-written value list.
+MAX_BODY_FIELDS = 40
+MAX_BODY_DEPTH = 5
+
+
+def clean_body_map(raw):
+ """D-41: `{"": ""}` for a webhook. Returns `(map, error)`.
+
+ Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated
+ rather than half-supported: `items.0.sku` would read as working for the first element and
+ silently write nothing the day a payload arrives with the list empty, which is the shape of
+ bug this module keeps paying for. A path that resolves to nothing writes nothing.
+ """
+ if raw in (None, ""):
+ return {}, None
+ if not isinstance(raw, dict):
+ return None, "the webhook field map must be an object of {body path: field key}"
+ if len(raw) > MAX_BODY_FIELDS:
+ return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record"
+ out = {}
+ for path, field in raw.items():
+ p = _s(path, 200).strip()
+ if not p:
+ return None, "a webhook mapping has an empty body path"
+ if len(p.split(".")) > MAX_BODY_DEPTH:
+ return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload — "
+ f"map a shallower value")
+ fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_")
+ if not fk:
+ return None, f"the value at {p!r} is not mapped to a field"
+ out[p] = fk[:60]
+ return out, None
+
+
+def body_value(body, path):
+ """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is
+ an object or a list answers None rather than a stringified `{...}` in a cell, because the
+ Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable."""
+ cur = body
+ for part in str(path or "").split("."):
+ if not isinstance(cur, dict):
+ return None
+ cur = cur.get(part)
+ return cur if isinstance(cur, (str, int, float, bool)) else None
+
+
+def webhook_row(rt, defn, body):
+ """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`.
+
+ ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same
+ `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable,
+ from the only side the caller is on, from a payload whose paths did not resolve. That is the
+ D-11 class (a table that quietly stops growing), and the caller here is a machine that will
+ keep posting. The note rides the 200: the flow still fires, and the answer says why no row
+ was written.
+
+ ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop
+ prevention law (A2(2)). A sibling automation watching this table does NOT fire on a
+ webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow
+ fires, because `hook_fire` fires it explicitly, which is the difference between "this
+ trigger fired" and "a write happened".
+ """
+ trg = (defn or {}).get("trigger") or {}
+ table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {})
+ if not table or not fmap or not isinstance(body, dict):
+ return "", 0, "" # no map configured — nothing to report
+ t = ut_get(rt, table)
+ if t is None:
+ return "", 0, f"{table} no longer exists, so nothing was written"
+ values = {}
+ for path, fkey in fmap.items():
+ v = body_value(body, path)
+ if v is not None:
+ values[fkey] = _s(v, 500) if isinstance(v, str) else v
+ if not values:
+ return "", 0, ("none of the mapped paths resolved to a value in this payload — "
+ "check the paths against what you are sending")
+ rows = dict((t.get("rows") or {}))
+ if len(rows) >= row_cap(table):
+ return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was "
+ f"written (the flow still ran)")
+ rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1)
+ rows[rid] = values
+ ut_write_rows(rt, table, rows)
+ return rid, len(values), ""
+
+
+def _trigger_configured(trg, config=None):
+ """Is this trigger complete enough to fire? One reader, because "configured" is asserted in
+ three places (the wire, the graph node, the hooks) and three copies of a boolean is how a
+ surface says "ready" about a trigger the engine skips.
+
+ ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is
+ the one trigger whose configuration lives in the DEFINITION's config (the discovery filters)
+ rather than on the trigger. `clean_trigger` calls this without it and so answers
+ conservatively (False); `clean_definition` calls it again with the validated config and
+ refines. Conservative-then-refined is the fail-closed order — the reverse would flash
+ "ready" on a trigger with nothing to search for.
+ """
+ key = str((trg or {}).get("key") or "")
+ # ⭐ WAVE 29 — BOTH discovery triggers, and they answer identically: a corpus search with no
+ # filter is not a search, it is a request for the whole index. Named as a pair rather than
+ # `or`-ed onto the Instagram line so a third network joins by adding a key, not by editing a
+ # boolean expression.
+ if key in ("ig_profile_match", "tiktok_profile_match"):
+ return bool((config or {}).get("predicates"))
+ if key in TRIGGER_TABLE_KEYS and not trg.get("table"):
+ return False
+ if key == "event_field":
+ # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything,
+ # ever" — which is not a trigger, it is a description of the table.
+ # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone
+ # and carried no condition becomes `configured: false` on its next clean. It stops
+ # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it
+ # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert.
+ return bool(trg.get("when"))
+ if key == "enters_view":
+ return bool(trg.get("viewId"))
+ return True
+
+
+def _secrets_token():
+ import secrets as _sec
+ return _sec.token_urlsafe(24)
+
+
+# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the
+# process would keep punishing an automation for a storm that ended with the restart.
+_FIRES = {}
+_FIRES_LOCK = threading.Lock()
+
+
+def _breaker_trips(tenant, auto_id, now=None):
+ now = now if now is not None else time.time()
+ key = (tenant, str(auto_id))
+ with _FIRES_LOCK:
+ log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS]
+ log.append(now)
+ _FIRES[key] = log
+ return len(log) > FIRE_LIMIT
+
+
+def _pause_trigger(rt, auto_id, note):
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ d = cur.get(str(auto_id))
+ if d is not None:
+ trg = d.get("trigger")
+ if isinstance(trg, dict):
+ trg["paused"] = True
+ d["statusNote"] = _s(note, 200)
+ return cur
+ rt.update(STORE_KEY, _up, flush="sync")
+
+
+def trigger_fire(rt, tenant, auto_id, log=print, rows=None):
+ """One trigger firing — breaker first, then the ordinary async run. False when it did not
+ start (breaker, or already running — both are answers, not errors)."""
+ if _breaker_trips(tenant, auto_id):
+ note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in "
+ f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's "
+ f"subject in a loop")
+ _pause_trigger(rt, auto_id, note)
+ log(f"[aios-auto] breaker: {auto_id} {note}")
+ return False
+ return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows)
+
+
+# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE
+# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates).
+_SETTLE = {}
+_SETTLE_LOCK = threading.Lock()
+
+
+def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print):
+ """Buffer ONE touched row for a coalesced evaluation, carrying the written cell.
+
+ ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave
+ dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the
+ reason is worth stating because it is invisible from this file: an ordinary ut cell typed at
+ the grid door lands in the editor's PER-USER OVERLAY stratum
+ (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables`
+ definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for
+ the common case the value that just changed exists ONLY in this event, and a definition-row
+ read sees the pre-write value — the trigger would evaluate stale and never fire.
+ `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree
+ read the record's other columns) with the written cells on top.
+ """
+ key = (tenant, str(auto_id))
+ cell = {str(field): after} if field else {}
+ if EVENT_SETTLE_SECONDS <= 0:
+ with _SETTLE_LOCK:
+ buf = _SETTLE.setdefault(key, {"rows": {}})
+ buf["rows"].setdefault(str(row_id), {}).update(cell)
+ _settle_eval(rt, tenant, auto_id, log=log)
+ return
+ with _SETTLE_LOCK:
+ buf = _SETTLE.setdefault(key, {"rows": {}})
+ buf["rows"].setdefault(str(row_id), {}).update(cell)
+ timer = buf.get("timer")
+ if timer is not None:
+ timer.cancel() # the burst continues — push the window out
+ timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval,
+ args=(rt, tenant, auto_id), kwargs={"log": log})
+ timer.daemon = True
+ buf["timer"] = timer
+ timer.start()
+
+
+def view_filter(rt, table_key, view_id):
+ """`(tree, fields, problem)` for one saved view on a user table — the substrate the
+ `enters_view` trigger tests membership against (C3-v2 / owner R2).
+
+ Personal strata first (`find_view`), then the shared bucket, because a view somebody shared
+ is exactly the kind an automation gets pointed at. A view that has been deleted answers a
+ PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one
+ would turn "when a record enters Overdue" into "on every write", which is the widening this
+ module refuses everywhere else.
+ """
+ key = str(table_key or "")
+ vid = str(view_id or "").strip()
+ if not key or not vid:
+ return None, [], "the trigger names no view"
+ try:
+ import core.table_store as table_store
+ tops = table_store.make(f"{key}_table_workspace", st=rt)
+ found = tops.find_view(vid)
+ view = (found[1] if found else None) or tops.shared_view(vid)
+ except Exception as e: # noqa: BLE001
+ return None, [], f"the view could not be read ({type(e).__name__})"
+ if not isinstance(view, dict):
+ return None, [], f"view {vid!r} no longer exists on {key}"
+ cfg = view.get("config") or {}
+ tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"}
+ return tree, list((ut_get(rt, key) or {}).get("fields") or []), ""
+
+
+def _row_gate(rt, defn, trg):
+ """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string.
+
+ None means the trigger has NO match gate — the write itself is the event (wave 22's "the
+ field changed", and `record_updated` over any field). A problem means the gate cannot be
+ built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that
+ passes.
+ """
+ key = trg.get("key")
+ if key == "enters_view":
+ tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId"))
+ if problem:
+ return None, problem
+ import harness.filter_eval as filter_eval
+ return (lambda row: filter_eval.matches(tree, row, fields)), ""
+ when = trg.get("when")
+ if when:
+ return (lambda row: lane_match(when, row)), ""
+ return None, ""
+
+
+def _settle_eval(rt, tenant, auto_id, log=print):
+ """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire."""
+ with _SETTLE_LOCK:
+ buf = _SETTLE.pop((tenant, str(auto_id)), None)
+ written = dict((buf or {}).get("rows") or {})
+ touched = list(written)
+ if not touched:
+ return
+ d = all_definitions(rt).get(str(auto_id))
+ trg = (d or {}).get("trigger") or {}
+ if trg.get("key") not in ("event_field", "record_updated", "enters_view") \
+ or trg.get("paused") or not trg.get("enabled", True) \
+ or not trg.get("configured", True):
+ return
+ gate, problem = _row_gate(rt, d, trg)
+ if problem:
+ # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken
+ # automation, not a quiet no-op — the note is what the surface shows instead of "On".
+ if (d.get("statusNote") or "") != problem:
+ _pause_trigger(rt, auto_id, problem)
+ log(f"[aios-auto] trigger gate: {auto_id} {problem}")
+ return
+ fired = []
+ if gate is not None:
+ rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
+ disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
+ nxt = set(disarmed)
+ for rid in touched:
+ # The merge (see `_settle_buffer`): the shared definition row underneath so a C4
+ # tree can read the record's other columns, the just-written cells on top because
+ # for an ordinary ut column this event is the ONLY place that value exists yet.
+ if gate({**(rows.get(rid) or rows.get(str(rid)) or {}),
+ **(written.get(rid) or {})}):
+ if rid not in disarmed:
+ fired.append(rid) # false→true THIS evaluation: the edge
+ nxt.add(rid)
+ else:
+ nxt.discard(rid) # left the state — re-armed (Airtable's rule)
+ if nxt != disarmed:
+ set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]})
+ if not fired:
+ return
+ else:
+ fired = list(touched) # "the field changed" — the burst is the edge
+ if len(fired) > FLOOD_LIMIT:
+ _commit_run(rt, auto_id, "partial",
+ f"the trigger matched {len(fired)} records in one evaluation — more than "
+ f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to "
+ f"process them deliberately", {"held": len(fired)}, True)
+ return
+ trigger_fire(rt, tenant, auto_id, log=log, rows=fired)
+
+
+def _seed_event_state(rt, defn):
+ """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so
+ turning the trigger on fires nothing — the first fire needs a real false→true transition.
+ Evaluated over the definition rows (the machine-written truth this trigger class watches).
+
+ Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation
+ reads. Two implementations of "does this row match" is how a seed disagrees with the edge it
+ is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it.
+ """
+ trg = (defn or {}).get("trigger") or {}
+ if trg.get("key") not in ("event_field", "record_updated", "enters_view") \
+ or not trg.get("configured"):
+ return
+ gate, problem = _row_gate(rt, defn, trg)
+ if gate is None or problem:
+ return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
+ rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
+ matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
+ set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
+
+
+def grid_hook(evt):
+ """THE listener the human write doors emit into (registered onto
+ `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may
+ import both sides). Never raises into a write path; a broken trigger listener must not
+ break typing into a cell."""
+ try:
+ st = evt.get("st")
+ if st is None:
+ return
+ tenant = str(getattr(st, "key", "") or "royal-imports")
+ table = str(evt.get("table") or "")
+ kind = str(evt.get("type") or "")
+ defs = all_definitions(st)
+ field = str(evt.get("field") or "")
+ rid = str(evt.get("rowId") or "")
+ for aid, d in defs.items():
+ trg = (d or {}).get("trigger") or {}
+ key = str(trg.get("key") or "")
+ if trg.get("paused") or not trg.get("enabled", True) \
+ or not trg.get("configured", True) \
+ or key not in TRIGGER_ROW_KEYS \
+ or str(trg.get("table") or "") != table:
+ continue
+ if kind == "record_created" and key == "record_created":
+ hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
+ ridn = int(rid) if rid.isdigit() else None
+ if ridn is None or ridn <= hw:
+ continue # once per record EVER (A2(4)) — undo-proof
+ set_state(st, aid, {"rcHighwater": ridn})
+ trigger_fire(st, tenant, aid, rows=[rid])
+ elif kind == "event_field" and key == "event_field":
+ # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every
+ # write on the watched table settles, and the CONDITION decides whether it fires
+ # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would
+ # now always take its first arm anyway; leaving a read of a key the validator no
+ # longer writes is the drift seat this migration exists to close.
+ _settle_buffer(st, tenant, aid, rid, field, evt.get("after"))
+ elif kind == "event_field" and key == "record_updated" and (
+ not (trg.get("fields") or []) or field in (trg.get("fields") or [])):
+ _settle_buffer(st, tenant, aid, rid, field, evt.get("after"))
+ elif key == "enters_view":
+ # BOTH kinds feed it: a row can enter a view by being edited into its filter or
+ # by being CREATED already inside it. Listening only to edits would silently miss
+ # every new record — the half of the definition a reader assumes is covered.
+ _settle_buffer(st, tenant, aid, rid, field, evt.get("after"))
+ except Exception as e: # noqa: BLE001
+ print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}")
+
+
+def form_fired(rt, table_key, row_id, values=None, form_token=""):
+ """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7).
+
+ One submitted form row → every `form_submitted` automation watching that database fires.
+ Returns the list of automation ids that started, so the door can log what it set off (and so
+ the gate can assert it, rather than asserting a side effect nobody can see).
+
+ ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form,
+ so it fires triggers exactly like typing into a cell does. The loop-prevention law is not
+ weakened by that — the engine's own writers still never reach here (only `routes_forms` calls
+ it), so an automation cannot create a form row and re-fire itself.
+
+ `values` is accepted and unused today: the row is already written when this is called, and
+ the flow reads it from the table. It stays in the signature because the caller HAS it and a
+ later refire policy ("only when field X was submitted") needs it — a parameter added later
+ would mean changing D's call site in a wave that does not own it.
+ """
+ started, table = [], str(table_key or "")
+ if not table:
+ return started
+ tenant = str(getattr(rt, "key", "") or "royal-imports")
+ for aid, d in all_definitions(rt).items():
+ trg = (d or {}).get("trigger") or {}
+ if trg.get("key") != "form_submitted" or trg.get("paused") \
+ or not trg.get("enabled", True) or not trg.get("configured", True):
+ continue
+ if str(trg.get("table") or "") != table:
+ continue
+ want = str(trg.get("formToken") or "")
+ if want and not hmac.compare_digest(want, str(form_token or "")):
+ continue # this automation watches a DIFFERENT form on that table
+ if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None):
+ started.append(aid)
+ return started
+
+
+def hook_fire(rt, tenant, auto_id, token, body=None):
+ """The webhook trigger's decision, separated from FastAPI so the gate can drive it.
+ Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind,
+ 409 already running, 200 started.
+
+ ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or
+ sent something that is not JSON. It is written onto a record BEFORE the flow fires — the
+ flow's actions walk the record the webhook just created, which is the whole point of mapping
+ it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected.
+ """
+ import hmac as _hmac
+ defn = all_definitions(rt).get(str(auto_id))
+ if defn is None:
+ return 404, {"error": "unknown_automation"}
+ trg = defn.get("trigger") or {}
+ want = str(trg.get("token") or "")
+ if trg.get("key") != "webhook" or not want:
+ return 403, {"error": "no_webhook", "message":
+ "this automation has no webhook trigger"}
+ if trg.get("paused") or not trg.get("enabled", True):
+ return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"}
+ if not _hmac.compare_digest(want, str(token or "")):
+ return 403, {"error": "bad_token", "message": "that token is not valid"}
+ # D-41: map the payload onto a record FIRST, so the flow that fires next walks it.
+ row_id, mapped, note = webhook_row(rt, defn, body)
+ started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None)
+ out = {"started": bool(started), "at": _iso(),
+ # Answered even when zero, so a caller wiring a map up can see whether their paths
+ # resolved. Silence here would make "my JSON is not landing" undebuggable from the
+ # outside, which is the only side the caller is on.
+ "rowId": row_id or None, "mapped": mapped}
+ if note:
+ out["note"] = note # a 200 that wrote no row SAYS which reason it was
+ return 200, out
+
+
+def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None):
+ """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a
+ row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the
+ gate drives this without a network; production leaves them None.
+
+ Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per
+ tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most
+ `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write.
+ """
+ trg = defn.get("trigger") or {}
+ if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True):
+ return None
+ import oauth_connect
+ creator = str(defn.get("createdBy") or "").strip() or "admin"
+ token, err = oauth_connect.google_creds(rt, creator)
+ if err:
+ if (defn.get("statusNote") or "") != err:
+ def _note(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ dd = cur.get(str(auto_id))
+ if dd is not None:
+ dd["statusNote"] = _s(err, 200)
+ return cur
+ rt.update(STORE_KEY, _note, flush="sync")
+ return None
+ lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n))
+ reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid))
+ ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT)
+ if lerr:
+ return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}",
+ {}, True)
+ seen = set((defn.get("state") or {}).get("emailSeen") or [])
+ fresh = [m for m in ids if m not in seen]
+ if not fresh:
+ return None # nothing new is not a run — no history spam
+ if len(fresh) > FLOOD_LIMIT:
+ return _commit_run(rt, auto_id, "partial",
+ f"{len(fresh)} new emails matched in one poll — more than the "
+ f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow "
+ f"the query, or press Run now after adjusting it",
+ {"held": len(fresh)}, True)
+ fresh = fresh[:EMAIL_MAX_PER_POLL]
+ rows_in, notes = [], []
+ for mid in fresh:
+ row, rerr = reader(mid)
+ if row:
+ rows_in.append(row)
+ elif rerr:
+ notes.append(rerr)
+ table_key = (defn.get("config") or {}).get("targetTable") or ""
+ if not table_key:
+ return _commit_run(rt, auto_id, "error",
+ "the email trigger has nowhere to write — the automation names no "
+ "target database", {}, False)
+ ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox",
+ EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key)
+ existing = dict((ut_get(rt, table_key) or {}).get("rows") or {})
+ merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key))
+ ut_write_rows(rt, table_key, merged)
+ new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:]
+ set_state(rt, auto_id, {"emailSeen": new_seen})
+ summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — "
+ f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written")
+ if notes:
+ summary += f" — {notes[0][:100]}"
+ entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts,
+ True, affected=list(merged)[:200])
+ trigger_fire(rt, tenant, auto_id, log=log)
+ return entry
+
+
+def compose_sentence(defn):
+ """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the
+ definition so it cannot lie about what runs."""
+ cfg = defn.get("config") or {}
+ trg = defn.get("trigger") or {}
+ sched = defn.get("schedule") or {}
+ kind = defn.get("kind")
+ if trg.get("key") == "event_field":
+ # Law 4: no watched field any more, so the sentence stops naming one. It said
+ # "When None changes on ut_x" the moment the key stopped being stored.
+ head = f"When a record in {trg.get('table')} matches conditions"
+ elif trg.get("key") == "record_updated":
+ head = f"When a record in {trg.get('table')} is updated"
+ elif trg.get("key") == "ig_profile_match":
+ head = "When an Instagram profile fits the criteria"
+ elif trg.get("key") == "record_created":
+ head = f"When a record is created in {trg.get('table')}"
+ elif trg.get("key") == "webhook":
+ head = "When the webhook is called"
+ elif trg.get("key") == "email":
+ head = f"When an email matches {trg.get('query')}"
+ elif sched.get("enabled"):
+ head = f"{_cron_label(sched.get('cron'))}"
+ else:
+ head = "When you press Run now"
+ if kind == "scrape_db":
+ host = urlparse(str(cfg.get("url") or "")).hostname or "the page"
+ body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}"
+ elif kind == "field_instagram":
+ # ⚠ NO RUNG CLAUSE (R5). There is one way to capture a profile now, so ", exact counts
+ # first" / ", anonymous only" described a choice that no longer exists.
+ body = f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}"
+ elif kind == "discover_instagram":
+ body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into "
+ f"{cfg.get('targetTable') or DISCOVER_TABLE}")
+ else:
+ # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has.
+ # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every
+ # plain automation would have introduced itself as "search Instagram for up to 0 profiles
+ # into ut_ig_candidates" — a sentence composed from a definition that says none of it,
+ # on the one surface whose whole promise is that it "cannot lie about what runs".
+ n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions")))
+ tbl = cfg.get("targetTable") or trg.get("table") or ""
+ body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else ""))
+ if n else "do nothing yet — this automation has no actions")
+ lanes = cfg.get("lanes") or []
+ tail = f", then route each record across {len(lanes)} lanes" if lanes else ""
+ return f"{head}, {body}{tail}."
+
+
+def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
+ """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it
+ directly. Returns the run entry, or None when it was already running (the 409)."""
+ defn = all_definitions(rt).get(str(auto_id))
+ if defn is None:
+ return None
+ if not _claim(tenant, auto_id):
+ return None
+ # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever
+ # happened to press Run — and above all not to the scheduler, which is not a person and
+ # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by
+ # whether a human or a cron got to the first run first.
+ owner = str(defn.get("createdBy") or "").strip()
+ if owner and username in MACHINE_OWNERS:
+ username = owner
+ try:
+ _step(tenant, auto_id, "running")
+ # A deferred metric snapshot is paid work already in Bright Data's queue. Collect it
+ # first and do not start another profile scrape while it is outstanding: re-running the
+ # action would buy duplicate engagement reads and reintroduce the timeout this handoff
+ # exists to remove. The normal scheduler calls this path too via `pending_collect_ids`.
+ # ⭐ 2026-08-09 — THE PROFILE HANDOFF IS COLLECTED FIRST, for the same reason and one
+ # rung earlier: a profile snapshot the vendor is still building is paid work, and
+ # starting a fresh scrape for the same handle would buy the identical row a second time.
+ # Ahead of the metric collector because the profile IS the thing the run was asked for;
+ # the engagement batches hang off it.
+ if _pending_profile_tasks(defn):
+ state, summary, counts, affected, steps = collect_pending_profile_snapshots(
+ rt, defn, username=username, log=log,
+ step=lambda text: _step(tenant, auto_id, text))
+ return _commit_run(rt, auto_id, state, summary,
+ {k: v for k, v in counts.items() if k != RUN_NOTES_KEY},
+ state != "error", affected, steps,
+ notes=counts.get(RUN_NOTES_KEY))
+ if _pending_metric_tasks(defn):
+ state, summary, counts, affected, steps = collect_pending_metric_snapshots(
+ rt, defn, username=username, log=log,
+ step=lambda text: _step(tenant, auto_id, text))
+ return _commit_run(rt, auto_id, state, summary,
+ {k: v for k, v in counts.items() if k != RUN_NOTES_KEY},
+ state != "error", affected, steps,
+ notes=counts.get(RUN_NOTES_KEY))
+ runner = RUNNERS.get(defn.get("kind"))
+ if runner is None:
+ return _commit_run(rt, auto_id, "error",
+ f"unknown automation kind {defn.get('kind')!r}", {}, False)
+ try:
+ # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run.
+ # `status.step` was already on the wire and D's half renders it; measured against the
+ # code, it was set exactly ONCE ("running") and never again, so the word would have
+ # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a
+ # constant as a progress indicator is worse than rendering nothing: it looks like an
+ # answer. The runners move it now, and the 120 s Bright Data wait counts out loud.
+ state, summary, counts, affected, steps = runner(
+ rt, defn, username=username, log=log,
+ step=lambda text: _step(tenant, auto_id, text), rows=rows)
+ except Refused as e:
+ return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}")
+ return _commit_run(rt, auto_id, "error",
+ f"{type(e).__name__}: {str(e)[:200]}", {}, False)
+ # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is
+ # committed, over the records this run actually touched. ONE call site rather than three
+ # inside the runners: every kind gets actions and endings for free, and a fourth runner
+ # cannot forget to opt in.
+ #
+ # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated,
+ # wired to the wire and covered by twelve gate checks that all called `apply_actions`
+ # DIRECTLY — so the whole feature was green and unreachable. A person would have built a
+ # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`.
+ #
+ # A failing action must not fail the RUN: the machine steps already wrote their rows and
+ # reporting that as an error would misdescribe what happened. It degrades to `partial`
+ # with the reason in the summary — the cap_note discipline.
+ # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`,
+ # which now reads this name — an assignment only on the success path would turn any
+ # action failure into a NameError inside the handler that exists to prevent exactly that.
+ # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked
+ # NOTHING — the branch where every candidate was a known-dead handle, i.e. exactly the
+ # run a person stares at wondering why the automation stopped doing anything.
+ run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or [])
+ try:
+ a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [],
+ username=username, log=log,
+ step=lambda text: _step(tenant, auto_id, text))
+ # ⭐⭐ D-103 — POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so
+ # the runner contract keeps its shape, and they must leave before the merge or they
+ # would be a "count" everywhere downstream.
+ run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or [])
+ counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}}
+ if a_counts.get("enrichMetricBatchesPending"):
+ batches = int(a_counts["enrichMetricBatchesPending"])
+ state = "partial" if state != "error" else state
+ summary += (f" — {batches} post-engagement batch"
+ f"{'' if batches == 1 else 'es'} still building; Views and other "
+ "metrics will be collected automatically without another paid read")
+ if a_counts.get("enrichUnbound"):
+ # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is
+ # `partial` because it genuinely did part of its job — it walked the records — and
+ # the sentence names the ONE thing that has to change, in the two places a person
+ # can change it. The old behaviour was `ok` with an empty table.
+ state = "partial" if state != "error" else state
+ summary += (" — the Instagram step did not run: this database has no profile "
+ "column. Name one on the step, or mark a text column as the "
+ "Instagram profile")
+ if a_counts.get("enrichProfileBatchesPending"):
+ # ⭐ The paid profile the vendor is still building. Said out loud so a run that
+ # looks like a failure is read as the handoff it is — the tick finishes it.
+ n = int(a_counts["enrichProfileBatchesPending"])
+ state = "partial" if state != "error" else state
+ summary += (f" — {n} profile read{'' if n == 1 else 's'} took longer than the "
+ "wait allows and will be collected automatically, at no extra cost")
+ if a_counts.get("enrichBlocked"):
+ state = "partial" if state != "error" else state
+ # ⭐⭐ D-103 — THE REASON IS IN THE SENTENCE, not only behind a click. "1 profile
+ # read(s) were blocked" is the exact string the owner read three mornings running
+ # before asking "wtf is going on"; it names a quantity and withholds the one
+ # thing that would let anybody act. The first note is the vendor's own words.
+ blocked_note = next((n for n in run_notes if ": " in n), "")
+ summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked"
+ + (f". {_s(blocked_note, 220)}" if blocked_note else ""))
+ # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the
+ # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it
+ # logged the cap and rolled up `ok`, so a flow that had silently stopped writing
+ # looked exactly like one that had nothing to write. Same rule, same sentence shape.
+ if a_counts.get("createCapped"):
+ state = "partial" if state != "error" else state
+ summary = (f"{summary} — {a_counts['createCapped']} row(s) NOT created: a target "
+ f"database is at its row cap")
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}")
+ state = "partial" if state != "error" else state
+ summary = f"{summary} — the actions did not finish ({type(e).__name__})"
+ return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
+ steps, notes=run_notes)
+ finally:
+ _release(tenant, auto_id)
+
+
+def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None):
+ """Start a run on a background thread. False when one is already in flight (→ 409)."""
+ if running(tenant, auto_id):
+ return False
+ th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows),
+ daemon=True, name=f"automation-{tenant}-{auto_id}")
+ th.start()
+ return True
+
+
+# ---------------------------------------------------------------------------------------------
+# THE TICK + the in-process scheduler
+# ---------------------------------------------------------------------------------------------
+
+def pending_collect_ids(rt):
+ """⭐ 2026-08-06 — automations holding a snapshot the vendor is still building.
+
+ ⛔ THE DEFECT THIS CLOSES, reported live: *"why is the SMALL 10 records test search taking
+ forever, its not populating"*. A corpus search takes ~20 minutes, so the run hands off and
+ stores `pendingSnapshot` with the summary *"The next run picks up the results"* — which is
+ true only if there IS a next run. The owner's automation is MANUAL (`schedule.enabled` false),
+ so `due_ids` never returns it, nothing ever collected it, and the results the account had
+ already been charged for sat ready at the vendor forever. The sentence was not wrong; it was
+ describing a run nobody had scheduled.
+
+ A pending snapshot is unfinished work the tenant has already paid for, so the tick finishes it
+ regardless of schedule. Independent of `due_ids` on purpose — a schedule says *start something
+ new*, this says *collect what is already running*, and folding the second into the first would
+ make an unscheduled automation's paid result depend on someone remembering to press a button.
+ """
+ out = []
+ for aid, d in all_definitions(rt).items():
+ if not isinstance(d, dict):
+ continue
+ if (d.get("trigger") or {}).get("paused"):
+ continue
+ pending_discovery = (d.get("kind") == "discover_instagram"
+ and str((d.get("state") or {}).get("pendingSnapshot") or "").strip())
+ # ⭐ 2026-08-09 — PROFILE handoffs join the other two. Without this line the profile
+ # deferral would be stored and never collected, which is the same defect it fixes wearing
+ # a queue: `due_ids` only returns SCHEDULED automations, and the automation this was
+ # measured on is `trigger: manual`. A capability written down but never walked is what
+ # this whole change is about, so it must not be reintroduced one function later.
+ if pending_discovery or _pending_metric_tasks(d) or _pending_profile_tasks(d):
+ out.append(aid)
+ return sorted(out)
+
+
+def due_ids(rt, now=None):
+ """Which of this tenant's automations a tick at `now` should start. Pure over the store."""
+ return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now))
+
+
+def tick(rt, tenant, now=None, log=print):
+ """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the
+ ids started. The email polls are bounded and fail-quiet per automation — one broken
+ mailbox connection must not stop the tenant's schedules."""
+ started = []
+ # ⭐ COLLECT-FIRST (2026-08-06). A snapshot the vendor has finished building is a result the
+ # tenant has already been charged for; it is collected whether or not this automation is on a
+ # schedule. `_claim` makes the union safe — an id in both lists starts once.
+ for aid in dict.fromkeys(list(pending_collect_ids(rt)) + list(due_ids(rt, now))):
+ if run_async(rt, tenant, aid, username="scheduler", log=log):
+ started.append(aid)
+ for aid, d in all_definitions(rt).items():
+ if (d.get("trigger") or {}).get("key") == "email":
+ try:
+ if email_poll(rt, tenant, aid, d, log=log) is not None:
+ started.append(aid)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}")
+ # C7's amendment: date-window metrics advance with the tick, so `today` is never staler
+ # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan.
+ try:
+ refresh_metrics(rt, log=log)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}")
+ # ⭐ 2026-08-07 — the relational pass, on the same tick and for the same reason. A rollup on
+ # table A goes stale when table B gains a row, and A cannot know that happened; the tick is
+ # the only place that sees both. ⚠ SEPARATE try/except from the metrics above deliberately —
+ # one pass failing must not silently cancel the other, which a shared block would do.
+ try:
+ refresh_relations(rt, log=log)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] relation refresh {tenant} failed: {type(e).__name__}: {e}")
+ # ⭐ THE READ-THROUGH ROLLUPS, on the same tick and for a stronger version of the same reason.
+ # A linked rollup goes stale when the LINKED TABLE gains a row; a source-backed one goes stale
+ # when ODOO does — and its window moves on its own besides (a `ytd` column is wrong on 1
+ # January without anybody writing anything). The tick is the only place that sees either.
+ #
+ # ⛔ ITS OWN try/except, like the two passes above, and the reason is the same one stated
+ # there: this pass REFUSES loudly on a truncated group set, and a shared block would let that
+ # honest refusal silently cancel the relational pass that had already succeeded.
+ #
+ # ⚠ IT WRITES NOTHING WHEN IT REFUSES — see `rollup_sql`'s header. A half-applied rollup mixes
+ # two vintages of one column and looks completely normal, which is why the refusal is total.
+ try:
+ import rollup_sql
+ for _tk in list((rt.get(UT_STORE_KEY) or {})):
+ if rollup_sql.source_fields((rt.get(UT_STORE_KEY) or {}).get(_tk) or {}):
+ # ⚠ `today` is DEFAULTED INSIDE `compute`, not passed: this scope has no
+ # such local, and `refresh_metrics` above resolves it the same way.
+ n = rollup_sql.compute(rt, _tk)
+ if n:
+ log(f"[aios-auto] source rollups {tenant}/{_tk}: {n}")
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] source rollup {tenant} failed: {type(e).__name__}: {e}")
+ if started:
+ log(f"[aios-auto] tick {tenant}: started {', '.join(started)}")
+ return started
+
+
+def tick_all(now=None, log=print):
+ """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop
+ the others' schedules."""
+ from harness import runtime as _rt
+ out = {}
+ for slug in _rt.known_tenants():
+ try:
+ out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}")
+ return out
+
+
+#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer
+#: would be a busy loop against a vocabulary that cannot express it.
+TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60)
+_SCHEDULER = [None]
+
+
+def scheduler_loop(log=print):
+ """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work.
+
+ Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and
+ `verify_seam` import this module through `main.py`, run against fake stores in seconds and
+ exit. A loop that ticked on entry would fire inside them.
+ """
+ while True:
+ time.sleep(TICK_SECONDS)
+ try:
+ tick_all(log=log)
+ except Exception as e: # noqa: BLE001
+ log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}")
+
+
+def start_scheduler(log=print):
+ """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.**
+
+ ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave
+ brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196
+ seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires
+ two or three times inside a gate that imports `main.py` — and while a tick over a fake store
+ finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and
+ mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread
+ that can move a gate's subject is a flaky suite waiting to happen.
+
+ So it takes the convention `main.py:353` already established for exactly this hazard —
+ `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup
+ event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the
+ external EventBridge tick (R5) does not depend on it either way, since that POSTs the
+ endpoint rather than riding this thread.
+ """
+ if os.environ.get("AIOS_AUTOMATIONS") != "1":
+ return False
+ if _SCHEDULER[0] is not None:
+ return False
+ th = threading.Thread(target=scheduler_loop, kwargs={"log": log},
+ daemon=True, name="automation-scheduler")
+ _SCHEDULER[0] = th
+ th.start()
+ return True
diff --git a/api/connectors_tt.py b/api/connectors_tt.py
new file mode 100644
index 0000000000000000000000000000000000000000..d73c0725ed7683d2b42741a06892d332611ba751
--- /dev/null
+++ b/api/connectors_tt.py
@@ -0,0 +1,293 @@
+"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
+
+Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
+automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
+exists at all rather than another thousand lines inside the engine.
+
+⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
+post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
+live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
+`.claude/wiki/research/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
+close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
+name below reads through a candidate list it is because the vendor has two names for one fact
+(`biography`/`signature`, `region`/`country`), never because the name is uncertain.
+
+⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
+public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
+vendor key is a key to a SUPPLIER.
+
+⚠ **THE TRANSPORT IS BORROWED FROM `connectors_ig`, ON PURPOSE.** `bd_call`, `bd_scrape` and
+`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
+Instagram's — so re-implementing them here would be a second copy of the deferral handling, the
+truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
+The name is wrong and the code is right; extracting a `connectors_bd` transport module is booked
+as a PENDING line in B's mailbox rather than done mid-wave, because moving those functions would
+land in the middle of three other tickets' diffs.
+
+⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
+ 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
+ envelope keys (`timestamp`, `input`) that no metadata call mentions;
+ 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
+ and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
+ TikTok claim that matters most (`play_count`) is exactly a declaration.
+"""
+from __future__ import annotations
+
+import automation_engine as engine
+from connectors_ig import ( # noqa: F401 (bd_call/bd_scrape are re-exported for the runners)
+ _bd_first_url,
+ _bd_flag,
+ _bd_list,
+ _bd_source_payload,
+ _first,
+ _ig_int,
+ bd_call,
+ bd_filter_start,
+ bd_key,
+ bd_ready,
+ bd_scrape,
+)
+
+# ---------------------------------------------------------------------------------------------
+# THE DATASETS
+# ---------------------------------------------------------------------------------------------
+# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
+# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
+# full field list, and the two DISCOVERY halves answer **404 for our key**:
+# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
+# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
+# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
+TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
+TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
+TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
+
+#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
+#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
+TT_POST_TYPE_VIDEO = "video"
+TT_POST_TYPE_CONTENT = "content"
+
+#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
+#: beside the dataset ids because it is the same class of vendor fact.
+#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
+#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
+#: batch calls, and the gate asserts the two agree rather than trusting that they do.
+TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
+
+
+def tt_profile_url(handle):
+ """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
+ h = str(handle or "").strip().lstrip("@")
+ return TT_PROFILE_URL.format(handle=h) if h else ""
+
+
+# ---------------------------------------------------------------------------------------------
+# THE FIELD MAPS
+# ---------------------------------------------------------------------------------------------
+# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
+# in `automation_engine`. The rules they all obey, stated once:
+#
+# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
+# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
+# returns `None` (never 0) when nothing matches, which is what makes that possible.
+# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
+# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
+# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
+# vendor's side is preserved rather than silently discarded while our column model catches up.
+# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
+# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
+# ride in `source_payload` where they make no claim.
+
+
+def _tt_str(node, *names):
+ """The first non-empty string among `names`, or None when the vendor sent nothing.
+
+ ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
+ an empty string written into a cell claims "we looked and it is empty".
+ """
+ v = _first(node, *names)
+ if v is None:
+ return None
+ s = str(v).strip()
+ return s or None
+
+
+def _tt_pct(node, *names):
+ """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
+
+ ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
+ the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
+ measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
+ """
+ v = _first(node, *names)
+ if v is None:
+ return None
+ out = engine._pct100(v)
+ return out or None
+
+
+def _tt_day(node, *names):
+ """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
+ v = _first(node, *names)
+ if v is None:
+ return None
+ return engine._day(v) or None
+
+
+def _drop_blanks(row):
+ """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
+ return {k: v for k, v in row.items() if v is not None and v != ""}
+
+
+def normalize_profile(node, handle=""):
+ """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
+
+ ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
+ * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
+ populated on 85% of rows and they carry the same text;
+ * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
+ two-letter-ISO description, `country` has no description at all.
+ ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
+ `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
+ only count of its kind the dataset offers.
+ """
+ node = node if isinstance(node, dict) else {}
+ account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
+ return _drop_blanks({
+ "platform": engine.PLATFORM_TIKTOK,
+ "handle": account,
+ "full_name": _tt_str(node, "nickname"),
+ "tt_id": _tt_str(node, "id"),
+ "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
+ "bio": _tt_str(node, "biography", "signature"),
+ "external_url": _bd_first_url(_first(node, "bio_link")),
+ "verified": _bd_flag(node, "is_verified"),
+ "is_private": _bd_flag(node, "is_private"),
+ # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
+ # on their own description. It is the closest thing TikTok has to Instagram's
+ # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
+ # the approximation only ever fills a cell the vendor actually answered.
+ "is_business": _bd_flag(node, "is_commerce_user"),
+ "followers": _ig_int(_first(node, "followers")),
+ "following": _ig_int(_first(node, "following")),
+ "posts_count": _ig_int(_first(node, "videos_count")),
+ # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
+ # nulls). It is not a post-level number and it is not our `likes` column, which is why it
+ # is stored under a different name.
+ "likes_received": _ig_int(_first(node, "likes")),
+ "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
+ "like_engagement": _tt_pct(node, "like_engagement_rate"),
+ "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
+ "country_code": _tt_str(node, "region", "country"),
+ "region": _tt_str(node, "region"),
+ "predicted_lang": _tt_str(node, "predicted_lang"),
+ # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
+ # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
+ # which is why the append law dates a snapshot by when WE read it.
+ "account_created_at": _tt_day(node, "create_time"),
+ "source_payload": _bd_source_payload(node),
+ })
+
+
+def tt_post_type(node):
+ """A TikTok post row → one of OUR three type options, or None.
+
+ ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
+ `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
+ vendor's token would fail `_clean_field`'s option check on the way in and would put an
+ untranslated API word in front of a user on the way out.
+
+ So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
+ row carries more than one `carousel_images` entry, which is what a carousel IS on either
+ network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
+ token cannot express it, so inferring `carousel` from the word would be inventing a fact.
+ ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
+ that is exactly what would make the wrong default invisible).
+ """
+ raw = str((node or {}).get("post_type") or "").strip().lower()
+ if raw == TT_POST_TYPE_VIDEO:
+ return "video"
+ if raw != TT_POST_TYPE_CONTENT:
+ return None
+ images = (node or {}).get("carousel_images")
+ return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
+
+
+def normalize_post(node):
+ """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
+
+ ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
+ Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
+ fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
+ `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
+ manufacture a second measurement that a rollup could average or double-count, which is a worse
+ outcome than the missing column it would paper over.
+
+ ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
+ the probe measured them as the same shape. That equality is what lets the comments→posts link
+ join with NO normaliser, which the Instagram side never had.
+ ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
+ ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
+ countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
+ no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
+ """
+ node = node if isinstance(node, dict) else {}
+ shortcode = _tt_str(node, "shortcode", "post_id")
+ if not shortcode:
+ return None
+ return _drop_blanks({
+ "platform": engine.PLATFORM_TIKTOK,
+ "shortcode": shortcode,
+ "influencer_key": _tt_str(node, "profile_username"),
+ "posted_at": _tt_day(node, "create_time"),
+ "type": tt_post_type(node),
+ "caption": _tt_str(node, "description"),
+ "url": _tt_str(node, "url"),
+ "hashtags": _bd_list(node, "hashtags"),
+ "tagged_location": _tt_str(node, "commerce_info"),
+ "views": _ig_int(_first(node, "play_count")),
+ "likes": _ig_int(_first(node, "digg_count")),
+ "comments": _ig_int(_first(node, "comment_count")),
+ "shares": _ig_int(_first(node, "num_share_count")),
+ "saves": _ig_int(_first(node, "collect_count")),
+ "video_duration": _ig_int(_first(node, "video_duration")),
+ "source_payload": _bd_source_payload(node),
+ })
+
+
+def normalize_comment(node):
+ """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
+
+ ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
+ `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
+ `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
+ a question the vendor already answers — and it would disagree, because a page of replies is not
+ all of them.
+ ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
+ text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
+ that changes its mind cannot change ours.
+ ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
+ PII) stay in `source_payload` and are promoted to no column, which is the same posture the
+ Instagram comment schema takes for the same D-24 reason.
+ """
+ node = node if isinstance(node, dict) else {}
+ comment_key = _tt_str(node, "comment_id")
+ if not comment_key:
+ return None
+ return _drop_blanks({
+ "platform": engine.PLATFORM_TIKTOK,
+ "comment_key": comment_key,
+ "shortcode": _tt_str(node, "post_id"),
+ "commented_at": _tt_day(node, "date_created"),
+ "likes": _ig_int(_first(node, "num_likes")),
+ "replies": _ig_int(_first(node, "num_replies")),
+ "source_payload": _bd_source_payload(node),
+ })
+
+
+#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
+#: rather than naming three functions, and so adding a fourth dataset is one entry.
+TT_NORMALIZERS = {
+ "tt_profile": normalize_profile,
+ "tt_post_metrics": normalize_post,
+ "tt_comments": normalize_comment,
+}
diff --git a/api/main.py b/api/main.py
index dd666f170eca42f46522f164d41ec27f60bbbf04..efe1afba1822a5fdcfff73f45242e532ee1594fb 100644
--- a/api/main.py
+++ b/api/main.py
@@ -1,532 +1,545 @@
-"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at.
-
-It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical,
-reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
-JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
-
-WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing:
-
- * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
- `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
- anonymous principal and no route could scope anything. Now a signed stateless cookie carries
- a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
- user RECORD. `/api/health` stays unauthenticated (liveness only).
- * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
- for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
- * **The SSL shim is env-gated.** It used to run at import, unconditionally.
-
-⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
-session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
-is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB
-commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
-"""
-import os
-import sys
-from pathlib import Path
-
-# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
-# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
-# unverified default context; the HF Space and the container are unaffected. This used to run
-# unconditionally at import — i.e. the shipped container disabled TLS verification for every
-# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
-# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
-# deliberately, and is never on by default.
-# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
-if os.environ.get("AIOS_INSECURE_SSL") == "1":
- import ssl
- ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
-
-# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
-# (where the layout differs from the local sibling-dir default). ---
-_HERE = Path(__file__).resolve()
-_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform"))
-for _p in (str(_RI), str(_HERE.parent)):
- if _p not in sys.path:
- sys.path.insert(0, _p)
-
-from dotenv import load_dotenv # noqa: E402
-load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
-
-from fastapi import Body, Depends, FastAPI, Request # noqa: E402
-from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
-from fastapi.responses import JSONResponse # noqa: E402
-from fastapi.staticfiles import StaticFiles # noqa: E402
-from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
-
-import aios_session # noqa: E402
-import routes_admin # noqa: E402
-import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox)
-import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery)
-import routes_auth # noqa: E402
-import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A)
-import routes_customers # noqa: E402
-import routes_grid # noqa: E402
-import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces)
-import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit)
-import routes_nav # noqa: E402
-import routes_pages # noqa: E402
-import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane)
-import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav)
-import routes_records # noqa: E402
-import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases)
-import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file)
-import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire)
-import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router)
-import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router)
-import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router)
-import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router)
-from core import grid_events # noqa: E402
-from deps import Session, module_gate # noqa: E402
-
-_WEB_DIST = _HERE.parents[1] / "web" / "dist"
-
-# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
-# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
-# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
-# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so
-# embed == standalone by construction.
-#
-# TWO REASONS THIS IS HERE and not folded into the routes:
-# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR
-# layout is broken, and discovering that from a 500 on the first customer request instead of
-# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
-# rewrite dropped it and this restores it.)
-# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and
-# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
-# API have not drifted. The rewrite removed them and the referee went red; retargeting the
-# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
-# not delete — but only when the subject genuinely moved. Here it should not have moved).
-#
-# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
-# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
-# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone
-# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
-# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
-import json as _json_contract # noqa: E402
-_FIELDS_PATH = _RI / "aios_grid_fields.json"
-if not _FIELDS_PATH.is_file():
- raise FileNotFoundError(
- f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
- "platform root (it also carries the data layer this API imports)."
- )
-_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
-FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
-# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded.
-# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be
-# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage.
-_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
- if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
- "date")}
-
-app = FastAPI(title="AIOS web API")
-
-# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with
-# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
-# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead.
-app.add_middleware(GZipMiddleware, minimum_size=1024)
-
-
-@app.exception_handler(StarletteHTTPException)
-def _error_shape(request: Request, exc: StarletteHTTPException):
- """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
-
- `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
- 422 from a malformed path param) is wrapped here so a client never has to branch on two
- different error bodies.
- """
- detail = exc.detail
- if isinstance(detail, dict) and "error" in detail:
- body = detail
- else:
- body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
- return JSONResponse(body, status_code=exc.status_code,
- headers=getattr(exc, "headers", None))
-
-
-@app.exception_handler(grid_events.StoreUnavailable)
-def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
- """THE SAFETY NET for "the store is down" → 503, from anywhere.
-
- ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
- `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own
- try/except — surfaced as a 500. A store outage is a normal operational state and every route
- here touches the store at least twice (the workspace read, then the write), so "remember to
- wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
- means a store outage can only ever be a 503, whichever call raised it.
-
- The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this
- adapter, which has no durable session dict to degrade into. A 200 over a write that
- evaporated is the failure the whole rule exists to prevent.
- """
- return JSONResponse(
- {"error": {"code": "store_unavailable",
- "message": "the tenant store is unavailable — no change was saved"}},
- status_code=503)
-
-
-@app.get("/api/health")
-def health():
- """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
- reveal anything about the deployment beyond "the process is up". No version, no tenant list,
- no config: a health endpoint is the one URL every scanner finds first.
-
- ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned
- release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated
- caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind
- a session — and the authoritative copy is the `VERSION` file in the Space repo, which
- `deploy_web.space_version()` reads without needing the app to be up at all."""
- return {"ok": True}
-
-
-app.include_router(routes_auth.router)
-app.include_router(routes_nav.router)
-app.include_router(routes_customers.router)
-app.include_router(routes_products.router)
-app.include_router(routes_assets.router)
-app.include_router(routes_tables.router)
-app.include_router(routes_grid.router)
-app.include_router(routes_records.router)
-# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
-# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so
-# a new page is a builder module plus one registry line, and nothing here changes.
-app.include_router(routes_pages.router)
-app.include_router(routes_admin.router)
-app.include_router(routes_automation.router)
-app.include_router(routes_keychain.router)
-app.include_router(routes_statements.router)
-# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view.
-# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path
-# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a
-# platform-operator route ever shadowing each other. Every path it declares is gated by
-# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it.
-app.include_router(routes_platform_admin.router)
-# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are
-# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is
-# something every user does with their own views/folders/databases.
-app.include_router(routes_alerts.router)
-app.include_router(routes_shares.router)
-app.include_router(routes_uploads.router)
-# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this
-# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by:
-# a router included AFTER it answers 404 forever while importing fine, type-checking fine and
-# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause.
-#
-# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished
-# features that would have shipped dead. The workers each posted a mount ask and said they would
-# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The
-# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the
-# control is that the enumeration must NAME every router in this file.**
-app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated
-app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory
-# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
-# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so
-# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
-# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot
-# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug.
-app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door
-# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed,
-# because the wave-23 scar is exactly this: three finished routers shipped with no include_router
-# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks
-# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED.
-app.include_router(routes_odoo_tables.router)
-
-
-# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
-# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic
-# behavior. An alias that kept the old auth would be a bypass of everything above it.
-_GATE = module_gate(routes_customers.MODULE)
-
-
-@app.get("/api/customers", deprecated=True)
-def _customers_alias(session: Session = Depends(_GATE)):
- return routes_customers._payload(session)
-
-
-@app.patch("/api/customers/{pid}", deprecated=True)
-def _patch_alias(pid: int, body: dict = Body(default=None),
- session: Session = Depends(_GATE)):
- return routes_customers.patch_customer(pid, body, session)
-
-
-def _assert_contract_floor():
- """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
-
- This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
- canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
- the embedded host are serving two different schemas and the grid's own contract has forked —
- the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
- on whatever machine is actually running.
- """
- import aios_grid
- base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
- if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
- raise RuntimeError(
- "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
- "disagree on the base field set — embed and standalone would serve different schemas. "
- "Run aios-web/verify_fields_contract.py.")
-
-
-def _startup_notes():
- """Say the two things an operator must know, once, at import — never in a response body."""
- if aios_session.EPHEMERAL_SECRET:
- print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process "
- "key. Sessions will not survive a restart and will not work across workers. "
- "Set it in production.")
- if os.environ.get("AIOS_INSECURE_SSL") == "1":
- print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound "
- "requests. Local development only; never in a deployed environment.")
-
-
-_assert_contract_floor()
-_startup_notes()
-
-
-# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
-# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
-# page cannot live behind a password prompt, so the shell's own assets must be public — which they
-# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
-#
-# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
-# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
-# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
-# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
-# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
-# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same
-# answer as any other path that is not part of the app.
-#
-# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox);
-# this guard is what makes the API safe regardless of what the bundle happens to contain.
-_DEV_FIXTURES = {"sample_customers.json"}
-
-
-class _AppStatic(StaticFiles):
- async def get_response(self, path, scope):
- if Path(path).name in _DEV_FIXTURES:
- raise StarletteHTTPException(status_code=404, detail="Not Found")
- resp = await super().get_response(path, scope)
- # Vite content-hashes everything under assets/ (a change is a NEW url), so those are
- # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle.
- # index.html must stay revalidated or a deploy would strand returning browsers on the old
- # bundle; ETag/304 makes that revalidation a header exchange, not a transfer.
- # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes,
- # and `startswith("assets/")` silently skipped every asset (measured on the local probe).
- if path.replace("\\", "/").lstrip("/").startswith("assets/"):
- resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
- else:
- resp.headers["Cache-Control"] = "no-cache"
- return resp
-
-
-# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
-if _WEB_DIST.is_dir():
- app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
-
-
-# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) -----------------------
-# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in
-# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's
-# default envelope; scoped users still pay their own scope's first build, once.
-# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
-# run against fakes) can never fire a live Odoo pull.
-def _seed_and_sync_store():
- """Bring the analytical store (harness.datastore) LIVE for this container — the app.py
- bootstrap, mirrored (2026-07-31, owner item 1).
-
- `harness.datastore` powers every measure column/condition, and ONLY app.py used to call
- `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store
- that never existed and every measure cell served blank. Seeding alone was NOT enough either,
- and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless
- `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands
- EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced
- forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the
- write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded
- passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still
- serve.
- """
- import time as _t
- t0 = _t.time()
- try:
- from harness import datastore as _ds
- if _ds.ensure_seed():
- print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s")
- # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's
- # search_reads must never interleave on the shared client's xmlrpc transport.
- import core.odoo as _odoo
- try:
- _odoo._tlocal.client = _odoo.OdooClient()
- except Exception:
- pass
- res = {}
- for i in range(12):
- res = _ds.sync_all(log=lambda *a, **k: None)
- print(f"[aios-api] datastore sync pass {i + 1}: "
- + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
- if all(v.get("phase") == "live" for v in res.values()):
- break
- # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see
- # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut —
- # one reconcile pass at boot removes both classes of phantom row before the first
- # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of
- # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran.
- try:
- _ds.reconcile_deletes(log=lambda *a, **k: None)
- except Exception:
- pass
- print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
- f"(ready={_ds.ready()})")
- # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE.
- #
- # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside
- # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a
- # freshly booted container could spawn the four locked databases was T+30 MINUTES. The
- # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s
- # across a ~20-minute window over two boots returned the pre-wave schema on all 20
- # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like
- # this — a thing that never ran, mistaken for a thing that ran and failed.
- #
- # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is
- # the exact line where the mirror has finished advancing — seed, up to twelve sync passes,
- # then the delete reconcile. Calling it from the other thread would race the seed and hit
- # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which
- # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild
- # refuses — correctly, but for a reason that reads like a data loss scare.
- # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it.
- _rebuild_odoo_relational("boot")
- except Exception as e: # noqa: BLE001
- print(f"[aios-api] datastore seed/sync skipped: {e}")
-
-
-def _rebuild_odoo_relational(why):
- """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
- 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
- could have asked" are different diagnoses.
-
- ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased
- for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran
- and was fine" from "the rebuild was never reached". Three failure markers and no success
- marker means silence is ambiguous; now it is not.
-
- ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The
- caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and
- royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks
- straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME
- process-global, and rebinding it in a daemon thread under live readers can serve one tenant's
- rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify.
- """
- try:
- import odoo_relational as _rel
- from harness import runtime as _runtime
- if not _rel.is_royal("royal-imports"):
- return
- _rt = _runtime.get_runtime("royal-imports")
- counts = _rel.refresh(_rt, "royal-imports")
- # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
- #
- # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field
- # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to
- # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a
- # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every
- # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the
- # tables simply look finished and answer nothing.
- #
- # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count
- # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes.
- # Calling the inner one here would return a plausible number and change no cell.
- #
- # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source
- # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest
- # refusal silently cancel a relational pass that had already succeeded.
- try:
- import automation_engine as _engine
- _engine.refresh_relations(_rt, log=lambda *_a: None)
- except Exception as e: # noqa: BLE001
- print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
- try:
- import rollup_sql as _rollup
- for _b, _key, _l, _f in _rel.TABLES:
- _rollup.compute(_rt, _key)
- except Exception as e: # noqa: BLE001
- print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
- print(f"[aios-api] odoo relational rebuild done ({why}): "
- + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
- except Exception as e: # noqa: BLE001
- # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
- # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
- # `BinderException` about a missing column is a different action from a timeout or an auth
- # failure, and the text alone often does not say which it was.
- print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}")
-
-
-def _store_resync_loop():
- """Keep the analytical mirror current — the API-process stand-in for the re-sync the
- Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
- refreshed pool re-reads the freshly synced store."""
- import time as _t
- passes = 0
- while True:
- _t.sleep(1800)
- passes += 1
- try:
- from harness import datastore as _ds
- import core.odoo as _odoo
- try:
- _odoo._tlocal.client = _odoo.OdooClient()
- except Exception:
- pass
- _ds.sync_all(log=lambda *a, **k: None)
- # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot
- # see (the boot pass's comment has the measured case). Cheap id-sweep per entity;
- # without it deleted Odoo lines inflate every sum on the mirror FOREVER.
- if passes % 4 == 0:
- _ds.reconcile_deletes(log=lambda *a, **k: None)
- except Exception as e: # noqa: BLE001
- print(f"[aios-api] store resync failed: {e}")
- # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt
- # AFTER the mirror they are derived from, in the same pass and in that order — deriving
- # from a mirror this loop is about to advance would publish a worklist one cycle stale
- # every single time.
- #
- # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must
- # not swallow the sync's error message, and — worse the other way — a sync failure must
- # not skip a rebuild that had nothing wrong with it. Two independent failures, two
- # independent logs. (`_rebuild_odoo_relational` carries its own handlers.)
- #
- # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed`
- # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so
- # "the wiring exists" and "the data is current" are different claims and only the stamp
- # can tell a user which one they are looking at.
- #
- # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which
- # is what made the boot path silent for 30 minutes after every restart; it is now the
- # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here
- # would be a second thing to keep in step, and the two would answer differently on the
- # next ruling — the exact shape the Views top-up was just fixed for on the other side of
- # this wave.
- _rebuild_odoo_relational("resync")
-
-
-def _prewarm():
- import time as _t
- t0 = _t.time()
- # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can
- # take many minutes of XML-RPC to close, and the first live probe of this arrangement
- # showed the pool warm (42s) silently queued behind it — the whole app cold for every
- # visitor while a background column channel caught up. Measure cells retry via the
- # transient rule until the sync lands; nothing else waits on it.
- import threading as _th
- _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
- try:
- from harness import runtime as _runtime
- rt = _runtime.get_runtime("royal-imports")
- routes_customers.warm_default(rt)
- import pages as _pages
- _pages.warm_default(rt)
- print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s")
- except Exception as e: # noqa: BLE001 — boot must not die on a warm-up
- print(f"[aios-api] prewarm skipped: {e}")
-
-
-if os.environ.get("AIOS_PREWARM") == "1":
- import threading as _threading
- _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
- _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
+"""AIOS web API — the ONE shared, stateless process the Streamlit exit is aimed at.
+
+It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical,
+reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
+JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
+
+WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bearing:
+
+ * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
+ `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
+ anonymous principal and no route could scope anything. Now a signed stateless cookie carries
+ a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
+ user RECORD. `/api/health` stays unauthenticated (liveness only).
+ * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
+ for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
+ * **The SSL shim is env-gated.** It used to run at import, unconditionally.
+
+⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
+session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
+is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB
+commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
+"""
+import os
+import sys
+from pathlib import Path
+
+# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
+# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
+# unverified default context; the HF Space and the container are unaffected. This used to run
+# unconditionally at import — i.e. the shipped container disabled TLS verification for every
+# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
+# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
+# deliberately, and is never on by default.
+# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
+if os.environ.get("AIOS_INSECURE_SSL") == "1":
+ import ssl
+ ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
+
+# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
+# (where the layout differs from the local sibling-dir default). ---
+_HERE = Path(__file__).resolve()
+_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform"))
+for _p in (str(_RI), str(_HERE.parent)):
+ if _p not in sys.path:
+ sys.path.insert(0, _p)
+
+from dotenv import load_dotenv # noqa: E402
+load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
+
+from fastapi import Body, Depends, FastAPI, Request # noqa: E402
+from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
+from fastapi.responses import JSONResponse # noqa: E402
+from fastapi.staticfiles import StaticFiles # noqa: E402
+from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
+
+import aios_session # noqa: E402
+import routes_admin # noqa: E402
+import routes_alerts # noqa: E402 (wave 20 item 25 — the Alerts inbox)
+import routes_assets # noqa: E402 (wave 18 C2-ASSET — catalog product imagery)
+import routes_auth # noqa: E402
+import routes_automation # noqa: E402 (wave 18 C4-AUTO — SESSION D's router, mounted by A)
+import routes_changes # noqa: E402 (wave 29 item 20 / C6 — F's change token, mounted by A)
+import routes_customers # noqa: E402
+import routes_grid # noqa: E402
+import routes_keychain # noqa: E402 (wave 18 C7 — keychain + connectors admin surfaces)
+import routes_statements # noqa: E402 (EXIT-6 — the statement sender, off Streamlit)
+import routes_nav # noqa: E402
+import routes_pages # noqa: E402
+import routes_platform_admin # noqa: E402 (wave 19 R3/R4 — the Loopable cross-tenant plane)
+import routes_products # noqa: E402 (wave 15 C-TOPIC — gated, not yet in the nav)
+import routes_records # noqa: E402
+import routes_shares # noqa: E402 (wave 20 R10 — grants for views, folders and databases)
+import routes_uploads # noqa: E402 (wave 21 C5 — tabular preview for Select-from-file)
+import routes_tables # noqa: E402 (wave 18 C3-UT — user-created databases over the wire)
+import routes_connectors # noqa: E402 (wave 23 C11 — the connectors directory; SESSION A's router)
+import routes_forms # noqa: E402 (wave 23 C9 — the PUBLIC form door; SESSION D's router)
+import routes_templates # noqa: E402 (wave 23 C12 — template apply doors; SESSION E's router)
+import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; SESSION E's router)
+from core import grid_events # noqa: E402
+from deps import Session, module_gate # noqa: E402
+
+_WEB_DIST = _HERE.parents[1] / "web" / "dist"
+
+# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
+# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
+# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
+# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so
+# embed == standalone by construction.
+#
+# TWO REASONS THIS IS HERE and not folded into the routes:
+# 1. It FAILS LOUDLY at import when the file is missing — which means the deploy or the RI_DIR
+# layout is broken, and discovering that from a 500 on the first customer request instead of
+# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
+# rewrite dropped it and this restores it.)
+# 2. `aios-web/verify_fields_contract.py` — the cross-side REFEREE — reads `FIELDS` and
+# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
+# API have not drifted. The rewrite removed them and the referee went red; retargeting the
+# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
+# not delete — but only when the subject genuinely moved. Here it should not have moved).
+#
+# ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
+# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
+# measure_ columns — the same list the Streamlit host renders. That is the point: the standalone
+# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
+# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
+import json as _json_contract # noqa: E402
+_FIELDS_PATH = _RI / "aios_grid_fields.json"
+if not _FIELDS_PATH.is_file():
+ raise FileNotFoundError(
+ f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
+ "platform root (it also carries the data layer this API imports)."
+ )
+_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
+FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
+# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded.
+# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be
+# wrongly rounded. `select` joined 2026-08-02 (dba) — a choice label rounded would be garbage.
+_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
+ if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
+ "date")}
+
+app = FastAPI(title="AIOS web API")
+
+# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED — with
+# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
+# payload to ~10–15% of that. minimum_size spares the tiny acks the overhead.
+app.add_middleware(GZipMiddleware, minimum_size=1024)
+
+
+@app.exception_handler(StarletteHTTPException)
+def _error_shape(request: Request, exc: StarletteHTTPException):
+ """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
+
+ `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
+ 422 from a malformed path param) is wrapped here so a client never has to branch on two
+ different error bodies.
+ """
+ detail = exc.detail
+ if isinstance(detail, dict) and "error" in detail:
+ body = detail
+ else:
+ body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
+ return JSONResponse(body, status_code=exc.status_code,
+ headers=getattr(exc, "headers", None))
+
+
+@app.exception_handler(grid_events.StoreUnavailable)
+def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
+ """THE SAFETY NET for "the store is down" → 503, from anywhere.
+
+ ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
+ `StoreUnavailable` raised while BUILDING THE PAYLOAD — before the route reached its own
+ try/except — surfaced as a 500. A store outage is a normal operational state and every route
+ here touches the store at least twice (the workspace read, then the write), so "remember to
+ wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
+ means a store outage can only ever be a 503, whichever call raised it.
+
+ The seam raises this only when the caller passed no `fallback_ws` — i.e. exactly on this
+ adapter, which has no durable session dict to degrade into. A 200 over a write that
+ evaporated is the failure the whole rule exists to prevent.
+ """
+ return JSONResponse(
+ {"error": {"code": "store_unavailable",
+ "message": "the tenant store is unavailable — no change was saved"}},
+ status_code=503)
+
+
+@app.get("/api/health")
+def health():
+ """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
+ reveal anything about the deployment beyond "the process is up". No version, no tenant list,
+ no config: a health endpoint is the one URL every scanner finds first.
+
+ ⛔ THE VERSION DOES NOT GO HERE, and it was asked to (2026-08-04, when LIVE became a pinned
+ release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated
+ caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind
+ a session — and the authoritative copy is the `VERSION` file in the Space repo, which
+ `deploy_web.space_version()` reads without needing the app to be up at all."""
+ return {"ok": True}
+
+
+app.include_router(routes_auth.router)
+app.include_router(routes_nav.router)
+app.include_router(routes_customers.router)
+app.include_router(routes_products.router)
+app.include_router(routes_assets.router)
+app.include_router(routes_tables.router)
+app.include_router(routes_grid.router)
+app.include_router(routes_records.router)
+# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
+# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder — so
+# a new page is a builder module plus one registry line, and nothing here changes.
+app.include_router(routes_pages.router)
+app.include_router(routes_admin.router)
+app.include_router(routes_automation.router)
+app.include_router(routes_keychain.router)
+app.include_router(routes_statements.router)
+# Wave 19 (owner item 13, R3): the LOOPABLE admin plane — the platform's own cross-tenant view.
+# Its own prefix (`/api/v1/platform-admin`), never nested under `/admin`, so there is no path
+# ambiguity with `routes_admin`'s `{username}` params and no chance of a tenant-admin route and a
+# platform-operator route ever shadowing each other. Every path it declares is gated by
+# `core.platform_admin.is_platform_admin`, and `verify_api` enumerates this router to prove it.
+app.include_router(routes_platform_admin.router)
+# Wave 20 (owner items 25 + 18/23/26): the Alerts inbox and the manage-access surface. Both are
+# session-gated rather than admin-gated — an alert is a person's own subscription, and sharing is
+# something every user does with their own views/folders/databases.
+app.include_router(routes_alerts.router)
+app.include_router(routes_shares.router)
+app.include_router(routes_uploads.router)
+# ⭐ WAVE 23 — THE THREE NEW ROUTERS. Mounted here, above the static catch-all at the bottom of this
+# file, because `app.mount("/", _AppStatic(...), html=True)` swallows everything it is reached by:
+# a router included AFTER it answers 404 forever while importing fine, type-checking fine and
+# passing its own gate. That is the wave-20 declared-but-unmounted shape with a different cause.
+#
+# ⛔ ALL THREE EXISTED, COMPLETE AND GATED, WITH NO MOUNT until the close-out audit — three finished
+# features that would have shipped dead. The workers each posted a mount ask and said they would
+# signal "ready" first; C waited for a signal that never came while the files landed anyway. **The
+# lesson is not "read the mailbox harder": `verify_api`'s enumeration is what caught it, so the
+# control is that the enumeration must NAME every router in this file.**
+app.include_router(routes_templates.router) # item 7 / C12 — template registry, session-gated
+app.include_router(routes_connectors.router) # item 10 / C11 — the connectors directory
+# ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
+# two paths are DELIBERATELY unauthenticated — a form is filled in by someone with no account — so
+# it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
+# by scanning tenants with a constant-time compare and answers a uniform 403, so a bad token cannot
+# distinguish "no such form" from "not yours", and it never echoes a tenant, table or slug.
+app.include_router(routes_forms.router) # item 8 / C9 — the PUBLIC form door
+# WAVE 27 item 17 — E's Odoo relational doors. Mounted in the SAME change that E's module landed,
+# because the wave-23 scar is exactly this: three finished routers shipped with no include_router
+# line — complete, gated, type-clean and 404 for every caller. `verify_api.section_mounts` walks
+# `main.app.routes` and pins these two paths by NAME, so an unmounted router now goes RED.
+app.include_router(routes_odoo_tables.router)
+# ⭐⭐ WAVE 29, item 20 / R11 / contract C6 — F's CHANGE TOKEN. `GET /api/v1/changes?scope=`
+# answers "did this bucket change" for ~zero cost (an in-memory counter; ZERO `store.get()` deep
+# copies), which is what lets a filtered view pick up a row created in another tab, by an automation
+# or by a connector sync without re-downloading the world.
+# ⛔ THIS LINE IS THE ARTIFACT THIS PROTOCOL LOSES MOST RELIABLY, AND IT WAS ALREADY LOST ONCE HERE:
+# F's router and the CLIENT half both shipped complete, so the poller was calling `/api/v1/changes`
+# six times a minute and taking a 404 while every one of F's own gates was green. `verify_api`'s D-48
+# leg caught it (`unmatched: [('/api/v1/changes', 'apiBridge.ts')]`) — a client fetch path with no
+# mounted route — which is precisely the control the wave-23 scar above was written to install.
+# ⚠ Mounted is NOT callable: `section_changes_callable` in `verify_api.py` SIGNS IN and CALLS this
+# route, because a route can be mounted and still raise before its own `try:` (D-107's plain-text 500).
+app.include_router(routes_changes.router) # item 20 / C6 — F's router, A's line
+
+
+# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
+# They are the v1 handlers with the v1 session requirement — NOT the old unauthenticated Basic
+# behavior. An alias that kept the old auth would be a bypass of everything above it.
+_GATE = module_gate(routes_customers.MODULE)
+
+
+@app.get("/api/customers", deprecated=True)
+def _customers_alias(session: Session = Depends(_GATE)):
+ return routes_customers._payload(session)
+
+
+@app.patch("/api/customers/{pid}", deprecated=True)
+def _patch_alias(pid: int, body: dict = Body(default=None),
+ session: Session = Depends(_GATE)):
+ return routes_customers.patch_customer(pid, body, session)
+
+
+def _assert_contract_floor():
+ """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
+
+ This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
+ canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
+ the embedded host are serving two different schemas and the grid's own contract has forked —
+ the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
+ on whatever machine is actually running.
+ """
+ import aios_grid
+ base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
+ if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
+ raise RuntimeError(
+ "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
+ "disagree on the base field set — embed and standalone would serve different schemas. "
+ "Run aios-web/verify_fields_contract.py.")
+
+
+def _startup_notes():
+ """Say the two things an operator must know, once, at import — never in a response body."""
+ if aios_session.EPHEMERAL_SECRET:
+ print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process "
+ "key. Sessions will not survive a restart and will not work across workers. "
+ "Set it in production.")
+ if os.environ.get("AIOS_INSECURE_SSL") == "1":
+ print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound "
+ "requests. Local development only; never in a deployed environment.")
+
+
+_assert_contract_floor()
+_startup_notes()
+
+
+# ⛔ THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
+# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
+# page cannot live behind a password prompt, so the shell's own assets must be public — which they
+# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
+#
+# ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
+# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
+# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
+# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
+# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
+# and it survives only because it sits in `web/public/`. So it is refused here — 404, the same
+# answer as any other path that is not part of the app.
+#
+# The right long-term fix is deleting it from `web/public/` (S2's lane — flagged in the mailbox);
+# this guard is what makes the API safe regardless of what the bundle happens to contain.
+_DEV_FIXTURES = {"sample_customers.json"}
+
+
+class _AppStatic(StaticFiles):
+ async def get_response(self, path, scope):
+ if Path(path).name in _DEV_FIXTURES:
+ raise StarletteHTTPException(status_code=404, detail="Not Found")
+ resp = await super().get_response(path, scope)
+ # Vite content-hashes everything under assets/ (a change is a NEW url), so those are
+ # immutable — a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle.
+ # index.html must stay revalidated or a deploy would strand returning browsers on the old
+ # bundle; ETag/304 makes that revalidation a header exchange, not a transfer.
+ # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes,
+ # and `startswith("assets/")` silently skipped every asset (measured on the local probe).
+ if path.replace("\\", "/").lstrip("/").startswith("assets/"):
+ resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
+ else:
+ resp.headers["Cache-Control"] = "no-cache"
+ return resp
+
+
+# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
+if _WEB_DIST.is_dir():
+ app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
+
+
+# --- boot prewarm (AIOS_PREWARM=1 — set by the Dockerfile, never by tests) -----------------------
+# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in
+# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's
+# default envelope; scoped users still pay their own scope's first build, once.
+# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
+# run against fakes) can never fire a live Odoo pull.
+def _seed_and_sync_store():
+ """Bring the analytical store (harness.datastore) LIVE for this container — the app.py
+ bootstrap, mirrored (2026-07-31, owner item 1).
+
+ `harness.datastore` powers every measure column/condition, and ONLY app.py used to call
+ `ensure_seed()` — so on a fresh Space disk this container resolved measures against a store
+ that never existed and every measure cell served blank. Seeding alone was NOT enough either,
+ and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless
+ `api:measure-column: ModelError the data cache is still warming up` — `ready()` demands
+ EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced
+ forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the
+ write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded
+ passes). Fail-quiet throughout: no seed/token/Odoo → the columns stay blank, the rows still
+ serve.
+ """
+ import time as _t
+ t0 = _t.time()
+ try:
+ from harness import datastore as _ds
+ if _ds.ensure_seed():
+ print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s")
+ # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's
+ # search_reads must never interleave on the shared client's xmlrpc transport.
+ import core.odoo as _odoo
+ try:
+ _odoo._tlocal.client = _odoo.OdooClient()
+ except Exception:
+ pass
+ res = {}
+ for i in range(12):
+ res = _ds.sync_all(log=lambda *a, **k: None)
+ print(f"[aios-api] datastore sync pass {i + 1}: "
+ + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
+ if all(v.get("phase") == "live" for v in res.values()):
+ break
+ # ⭐ Wave 21 (item 2, "make sure the metrics are correct"): a cursor sync can never see
+ # a HARD DELETE, and the downloaded seed carries whatever was deleted since it was cut —
+ # one reconcile pass at boot removes both classes of phantom row before the first
+ # measure is served. MEASURED 2026-08-05: five deleted sale_order_line rows = $284.25 of
+ # phantom YTD revenue, stable across re-syncs, zero the moment reconcile ran.
+ try:
+ _ds.reconcile_deletes(log=lambda *a, **k: None)
+ except Exception:
+ pass
+ print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
+ f"(ready={_ds.ready()})")
+ # ⭐⭐ 2026-08-09 (wave 28, D-107) — THE RELATIONAL REBUILD RUNS AT BOOT, HERE.
+ #
+ # ⛔ IT DID NOT BEFORE, AND NOTHING SAID SO. The rebuild lived only inside
+ # `_store_resync_loop`, whose very first statement is `sleep(1800)` — so the earliest a
+ # freshly booted container could spawn the four locked databases was T+30 MINUTES. The
+ # symptom was read as "the boot path is silent": `/odoo-tables/status` polled every 30 s
+ # across a ~20-minute window over two boots returned the pre-wave schema on all 20
+ # samples. It was not silent, it had not been asked yet. Both halves of D-107 were like
+ # this — a thing that never ran, mistaken for a thing that ran and failed.
+ #
+ # ⚠ WHY HERE AND NOT IN `_prewarm`: the tables are DERIVED FROM THE MIRROR, and this is
+ # the exact line where the mirror has finished advancing — seed, up to twelve sync passes,
+ # then the delete reconcile. Calling it from the other thread would race the seed and hit
+ # either `ro_con()`'s "still warming" RuntimeError or, worse, a HALF-SYNCED mirror, which
+ # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild
+ # refuses — correctly, but for a reason that reads like a data loss scare.
+ # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it.
+ _rebuild_odoo_relational("boot")
+ except Exception as e: # noqa: BLE001
+ print(f"[aios-api] datastore seed/sync skipped: {e}")
+
+
+def _rebuild_odoo_relational(why):
+ """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
+ 'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
+ could have asked" are different diagnoses.
+
+ ⭐ THE SUCCESS LINE IS NOT DECORATION — it is the control this path lacked. D-107 was chased
+ for a day on the strength of *absent* log markers, which cannot distinguish "the rebuild ran
+ and was fine" from "the rebuild was never reached". Three failure markers and no success
+ marker means silence is ambiguous; now it is not.
+
+ ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The
+ caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and
+ royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks
+ straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME
+ process-global, and rebinding it in a daemon thread under live readers can serve one tenant's
+ rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify.
+ """
+ try:
+ import odoo_relational as _rel
+ from harness import runtime as _runtime
+ if not _rel.is_royal("royal-imports"):
+ return
+ _rt = _runtime.get_runtime("royal-imports")
+ counts = _rel.refresh(_rt, "royal-imports")
+ # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
+ #
+ # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field
+ # DEFINITIONS; every Link and Rollup cell comes from a separate pass. Those passes used to
+ # live only on the automation `tick`, which fires from an EXTERNAL EventBridge cron — so a
+ # fresh boot landed 71,954 rows with eleven fully-configured relational columns and every
+ # one of them BLANK until an unrelated scheduler happened to run. Nothing errors; the
+ # tables simply look finished and answer nothing.
+ #
+ # ⚠ `refresh_relations`, NOT `compute_relation_cells`. The latter computes a change count
+ # over a blob it was handed and PERSISTS NOTHING; the former walks the tenant and writes.
+ # Calling the inner one here would return a plausible number and change no cell.
+ #
+ # ⚠ ONE try/except PER PASS, for the reason `tick` states at its own copies: a source
+ # rollup REFUSES loudly on a truncated group set, and a shared block would let that honest
+ # refusal silently cancel a relational pass that had already succeeded.
+ try:
+ import automation_engine as _engine
+ _engine.refresh_relations(_rt, log=lambda *_a: None)
+ except Exception as e: # noqa: BLE001
+ print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
+ try:
+ import rollup_sql as _rollup
+ for _b, _key, _l, _f in _rel.TABLES:
+ _rollup.compute(_rt, _key)
+ except Exception as e: # noqa: BLE001
+ print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
+ print(f"[aios-api] odoo relational rebuild done ({why}): "
+ + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
+ except Exception as e: # noqa: BLE001
+ # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
+ # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
+ # `BinderException` about a missing column is a different action from a timeout or an auth
+ # failure, and the text alone often does not say which it was.
+ print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}")
+
+
+def _store_resync_loop():
+ """Keep the analytical mirror current — the API-process stand-in for the re-sync the
+ Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
+ refreshed pool re-reads the freshly synced store."""
+ import time as _t
+ passes = 0
+ while True:
+ _t.sleep(1800)
+ passes += 1
+ try:
+ from harness import datastore as _ds
+ import core.odoo as _odoo
+ try:
+ _odoo._tlocal.client = _odoo.OdooClient()
+ except Exception:
+ pass
+ _ds.sync_all(log=lambda *a, **k: None)
+ # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot
+ # see (the boot pass's comment has the measured case). Cheap id-sweep per entity;
+ # without it deleted Odoo lines inflate every sum on the mirror FOREVER.
+ if passes % 4 == 0:
+ _ds.reconcile_deletes(log=lambda *a, **k: None)
+ except Exception as e: # noqa: BLE001
+ print(f"[aios-api] store resync failed: {e}")
+ # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt
+ # AFTER the mirror they are derived from, in the same pass and in that order — deriving
+ # from a mirror this loop is about to advance would publish a worklist one cycle stale
+ # every single time.
+ #
+ # ⛔ OUTSIDE the try/except above, NOT folded into it. A failing relational rebuild must
+ # not swallow the sync's error message, and — worse the other way — a sync failure must
+ # not skip a rebuild that had nothing wrong with it. Two independent failures, two
+ # independent logs. (`_rebuild_odoo_relational` carries its own handlers.)
+ #
+ # ⚠ THE REBUILD IS NOT THE FRESHNESS GUARANTEE — the rows carry a visible `_refreshed`
+ # stamp for that. This loop is a daemon thread whose failure path is a `print` (D-29), so
+ # "the wiring exists" and "the data is current" are different claims and only the stamp
+ # can tell a user which one they are looking at.
+ #
+ # ⭐ ONE IMPLEMENTATION, TWO CALLERS (wave 28). This block used to be the only copy, which
+ # is what made the boot path silent for 30 minutes after every restart; it is now the
+ # SECOND caller of the same function `_seed_and_sync_store` calls at boot. A copy here
+ # would be a second thing to keep in step, and the two would answer differently on the
+ # next ruling — the exact shape the Views top-up was just fixed for on the other side of
+ # this wave.
+ _rebuild_odoo_relational("resync")
+
+
+def _prewarm():
+ import time as _t
+ t0 = _t.time()
+ # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can
+ # take many minutes of XML-RPC to close, and the first live probe of this arrangement
+ # showed the pool warm (42s) silently queued behind it — the whole app cold for every
+ # visitor while a background column channel caught up. Measure cells retry via the
+ # transient rule until the sync lands; nothing else waits on it.
+ import threading as _th
+ _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
+ try:
+ from harness import runtime as _runtime
+ rt = _runtime.get_runtime("royal-imports")
+ routes_customers.warm_default(rt)
+ import pages as _pages
+ _pages.warm_default(rt)
+ print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s")
+ except Exception as e: # noqa: BLE001 — boot must not die on a warm-up
+ print(f"[aios-api] prewarm skipped: {e}")
+
+
+if os.environ.get("AIOS_PREWARM") == "1":
+ import threading as _threading
+ _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
+ _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
diff --git a/api/odoo_relational.py b/api/odoo_relational.py
index 64c8895fc05650b53978df0b9067f5d1c9a6ba1b..ca2f5090764d4ddd6c19471c0b8df08078a59795 100644
--- a/api/odoo_relational.py
+++ b/api/odoo_relational.py
@@ -713,13 +713,40 @@ def read_products(cur):
def read_customers(cur, excluded=None):
- """[(row dict)] — every partner Odoo has TRANSACTED with, keyed on the `res.partner` id.
-
- ⭐ THE POPULATION IS THE UNION OF THE TWO DOCUMENT UNIVERSES, and both legs are load-bearing:
- Amazon books as direct invoices with no sale order (the `odoo-api` gotcha), so a sale-order
- leg alone would silently drop a real customer. Partners with no document at all are left out
- on purpose — a row that can never appear in any topic has nothing to roll up and would render
- as an all-blank line.
+ """[(row dict)] — every CUSTOMER partner, keyed on the `res.partner` id.
+
+ ⭐ THE POPULATION IS A UNION OF THREE LEGS, and every one of them is load-bearing.
+
+ The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale
+ order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer.
+
+ ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 —
+ the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old
+ docstring said partners with no document are *"left out on purpose — a row that can never
+ appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same
+ join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records**
+ (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer
+ a salesperson has not sold to yet is exactly the row a prospecting view needs.
+
+ ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the
+ document legs for the rank leg would drop **16 partners that hold posted documents** (7
+ archived, 9 active with rank <= 0), and `ut_odoo_invoices` / `ut_odoo_orders` rows carry
+ `partner_id` LINKS straight back here — so those links would dangle with nothing reporting
+ it. When other tables point AT a population, a widening must be a SUPERSET.
+
+ ⚠ THE RANK LEG IS SKIPPED WHEN THE MIRROR HAS NO `customer_rank` COLUMN, which is the same
+ `columns()`/`_col` discipline every other optional column here uses — but note the difference
+ honestly: an absent `agent_id` blanks a CELL, while an absent `customer_rank` narrows the
+ POPULATION back to the document union. It degrades to today's behaviour rather than to an
+ empty or a wrong table, and `verify_odoo_relational` carries a check that goes RED while the
+ column is missing so the narrowing can never pass for done.
+
+ ⛔ NOT FROM LIVE ODOO, THOUGH `customer_rank` IS TRIVIAL TO ASK IT. `excluded_ids` above
+ states the rule for this module and it applies with more force to a POPULATION than to a name
+ list: a live call makes the spawn fail whenever Odoo is unreachable, and a Space hydrates its
+ mirror from a SNAPSHOT at boot. The population would then be "whichever source answered this
+ time" — swinging ~45% against `MAX_SHRINK`'s 50% refusal, deleting and re-adding rows on the
+ weather. One source, always present at spawn time: the mirror.
⛔ NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR
partners; sourcing a customer registry from its own receivables is what kept most Odoo ids
@@ -732,6 +759,11 @@ def read_customers(cur, excluded=None):
agent = ("ag.name" if "agent_id" in have else "NULL")
agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
+ # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an
+ # archived prospect out, and a rank test without it would re-admit the 47 archived partners
+ # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above.
+ rank_leg = (" OR (p.customer_rank > 0 AND p.active) "
+ if {"customer_rank", "active"} <= have else "")
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} "
"FROM res_partner p "
@@ -740,7 +772,8 @@ def read_customers(cur, excluded=None):
f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL "
" UNION "
" SELECT partner_id FROM account_move "
- f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)")
+ f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)"
+ f"{rank_leg}")
out = []
for r in cur.execute(sql).fetchall():
(pid, name, city, state, country, agent, agent_id) = r
diff --git a/api/providers.py b/api/providers.py
index 0eecba5f274bee8868a4086274519abaa07fa186..5e1a78fdcf84345be650175a85ea17986f9d9d9b 100644
--- a/api/providers.py
+++ b/api/providers.py
@@ -100,6 +100,26 @@ PROVIDERS: dict[str, Provider] = {
"creator (identical for two shortcodes in one call, ticking upward between "
"calls) and never populates video_play_count. Their own documented example "
"account no longer reproduces it."),
+ # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our
+ # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own
+ # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`.
+ "tt_profile": Capability(True, _COST_BRIGHTDATA,
+ "40 fields incl. bio/engagement rates/region"),
+ # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM.
+ # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says
+ # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post
+ # record and a separate view rung would be a second bill for a number we already have.
+ # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e:
+ # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an
+ # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it
+ # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a
+ # quiet fallback bolted onto this one.
+ "tt_post_metrics": Capability(
+ True, _COST_BRIGHTDATA,
+ "likes + comments + shares + saves, and play_count inline (DECLARED "
+ "no-empties-or-zeros, UNPROVEN until one live pull)"),
+ "tt_comments": Capability(True, _COST_BRIGHTDATA,
+ "separate paid dataset, opt-in — 17 fields"),
}),
"apify": Provider(
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
@@ -121,6 +141,17 @@ DEFAULT_CHAINS: dict[str, tuple] = {
# would be a lie that costs a wasted call on every single post.
"ig_post_views": ("apify",),
"ig_comments": ("brightdata",),
+ # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER.
+ # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a
+ # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data
+ # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap
+ # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here
+ # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result,
+ # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable;
+ # it is not declared until a runner walks it.
+ "tt_profile": ("brightdata",),
+ "tt_post_metrics": ("brightdata",),
+ "tt_comments": ("brightdata",),
}
diff --git a/api/routes_automation.py b/api/routes_automation.py
index 2695ff6760f55dffce3f60192c4a31105f6f6982..e002faa05be91844a0df3baa69ad31de2e42c15b 100644
--- a/api/routes_automation.py
+++ b/api/routes_automation.py
@@ -127,6 +127,12 @@ def _triggers_vocab(session):
# still ACCEPTS it either way — readiness is a deployment fact, not a validity one, which
# is the same split `email` already makes.
"ig_profile_match": (engine.bd_ready(), "" if engine.bd_ready() else "configure_brightdata"),
+ # ⭐ WAVE 29 (D-9 / R1) — TikTok, live. ⛔ IT NEEDS ITS OWN ROW EVEN THOUGH THE ANSWER IS
+ # IDENTICAL, and the reason is the `.get(k, (True, ""))` default below: a trigger this dict
+ # forgets is reported READY, so a deployment with no vendor key would offer TikTok search
+ # as configured and the search would find nothing. Same key, same readiness bit, stated.
+ "tiktok_profile_match": (engine.bd_ready(),
+ "" if engine.bd_ready() else "configure_brightdata"),
}
#: ⭐ WAVE 24 — the server's own one-line description per trigger. C-TYPES: the picker renders
#: THIS under the option, because a CLIENT paraphrase of a server vocabulary is a second copy
@@ -212,6 +218,14 @@ def _tick_state():
"source": "in-process" if inproc else ("external" if ext else "")}
+#: W29-T01 — the Board retirement ran, per tenant, this process. Same shape and same reasoning as
+#: `_C8_MIGRATED` below: a one-way cleanup of data no living code path creates any more, whose
+#: cost is a full walk of every row-cell in the tenant and whose result on pass 2..N is always
+#: "nothing to do". ⚠ A module-global keyed by tenant is deliberately NOT reset by a table create
+#: or delete — a new database cannot contain the legacy stage cells this retires.
+_BOARD_RETIRED = set()
+
+
@router.get("/automations")
def list_automations(session: Session = Depends(_GATE)):
"""`{automations: [...], kinds: [...], cronPresets: [...]}` — the rail's whole payload.
@@ -219,10 +233,29 @@ def list_automations(session: Session = Depends(_GATE)):
The vocabularies ride WITH the list rather than sitting in a client constant: the cron
presets and the kind list are the server's, and a client copy of either is a thing that goes
stale silently (the editor would offer a preset the parser rejects)."""
- # Idempotent Board retirement removes only engine-marked stage fields and legacy Board state.
- # User-created Status/Stage columns remain intact.
- if session.runtime.available():
- engine.retire_automation_board_state(session.runtime)
+ # ⭐⭐ WAVE 29 (W29-T01, owner item 5: "Automation takes a while to appear"). THE COMPLAINT WAS
+ # THIS ROUTE, and the cost was never the automations: it read the whole `user_tables` bucket
+ # THREE TIMES per call — once for the stage-field retirement scan, twice more inside
+ # `discover_default_table`. That bucket's documented ceiling is 35.8 MB / ~1.4 s to serialize
+ # (`automation_engine.py` header) and `core/store.py` re-serializes on EVERY `.get()`, hit or
+ # miss, UNDER THE STORE LOCK — so the reads also serialize behind each other. ~4 s of deep
+ # copying to render a rail that shows a name and a toggle.
+ #
+ # Now: ONE read, lent to both callers. And the retirement SCAN — O(all row-cells in the
+ # tenant), which hits its own `if not stage_keys: return` only AFTER walking every row of
+ # every table — runs once per tenant per process, mirroring the `_C8_MIGRATED` guard below.
+ #
+ # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent":
+ # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s
+ # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can
+ # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either —
+ # wave 26's R6 deleted the board that made them.
+ tables = engine.ut_all(session.runtime) if session.runtime.available() else None
+ if tables is not None and session.tenant not in _BOARD_RETIRED:
+ _BOARD_RETIRED.add(session.tenant)
+ # Idempotent Board retirement removes only engine-marked stage fields and legacy Board
+ # state. User-created Status/Stage columns remain intact.
+ engine.retire_automation_board_state(session.runtime, tables=tables)
defs = engine.all_definitions(session.runtime)
items = [_wire(d, session.tenant) for _, d in
sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())]
@@ -272,6 +305,13 @@ def list_automations(session: Session = Depends(_GATE)):
# validator refuses.
"seedSources": list(engine.SEED_SOURCES),
"seedMaxRows": engine.SEED_MAX_ROWS,
+ # W29-T01: `tables` is the snapshot read once at the top of this handler.
+ # ⚠ IT WAS TAKEN BEFORE THE RETIREMENT WROTE, and that is deliberate and
+ # harmless: retirement only removes `stage_auto_*` FIELDS and pops those
+ # same keys off rows, and the election reads the preset-profile vocabulary
+ # (`handle` + N preset keys) and whether a table has ANY rows. A stage key
+ # is in neither set, and no row is ever deleted — so the pre-write
+ # snapshot and the post-write bucket cannot elect different tables.
# 2026-08-10 — the OFFER must name the table the SAVE will actually use.
# This was the bare `DISCOVER_TABLE` constant while `create`/`patch` now
# resolve a targetless discovery flow to the profile database the tenant
@@ -279,7 +319,7 @@ def list_automations(session: Session = Depends(_GATE)):
# save would have written somewhere else — a default that disagrees with
# itself across two panels, which is the shape wave 25's R2 fixed for
# `targetTable` vs the action's `table`.
- "table": (engine.discover_default_table(session.runtime)
+ "table": (engine.discover_default_table(session.runtime, tables=tables)
or engine.DISCOVER_TABLE)},
"storeAvailable": bool(session.runtime.available()),
# Wave 22 C3 — the trigger vocabulary, session-scoped because email readiness is a
@@ -299,7 +339,19 @@ def list_automations(session: Session = Depends(_GATE)):
"maxCondDepth": engine.MAX_COND_DEPTH,
"maxCondChildren": engine.MAX_COND_CHILDREN,
"maxGroupDepth": engine.MAX_GROUP_DEPTH,
- "maxActions": engine.MAX_ACTIONS},
+ "maxActions": engine.MAX_ACTIONS,
+ # ⭐ W29-T09 (owner item 11) — THE POST-GROUP VOCABULARY, on the wire for the
+ # same reason `cronPresets` is: `clean_post_groups` REFUSES a type it does
+ # not know, so a client that invented one would build a config the save door
+ # rejects with a sentence about a word the person never typed. Both halves
+ # ride — the stored key and the label a person reads — because a client-side
+ # translation of `video` into "Reels" is a second copy of this list.
+ "postTypes": [{"key": t, "label": engine.POST_TYPE_LABELS.get(t, t)}
+ for t in engine.POST_TYPES],
+ # The ceiling a group's limit is judged against. `clean_post_groups` bounds a
+ # group by the action's OWN `maxPosts`, not by a constant, so the control can
+ # only warn honestly if it reads the same number the validator uses.
+ "maxPostsPerPull": engine.MAX_POSTS_PER_PULL},
# Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see
# `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie.
"tick": _tick_state()}
diff --git a/api/routes_changes.py b/api/routes_changes.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ec0a9d785429f9a4d52c06b6547e0a3d57c1ed4
--- /dev/null
+++ b/api/routes_changes.py
@@ -0,0 +1,159 @@
+"""routes_changes.py — THE CHANGE TOKEN (wave 29, item 20 / ruling R11, contract C6).
+
+ GET /api/v1/changes?scope=
+ -> {"scope": "ut_leads",
+ "tokens": {"rows": "b3f1c9:14", "overlay": "b3f1c9:3"},
+ "backend": "hf"}
+
+The owner's report: *a record created in another tab, by an automation, or by a connector sync
+does not appear in a filtered view until you reload.* The client had no way to ask whether
+anything had changed except to re-download everything, so it never asked.
+
+⛔ WHY THIS ROUTE EXISTS INSTEAD OF THE OBVIOUS "JUST POLL THE ROWS ENDPOINT", and the arithmetic
+is the whole argument. `GET /tables/{key}/rows` costs THREE full-tenant deep copies under ONE
+lock, on ONE uvicorn process (the container runs no `--workers`), against a documented ceiling of
+35.8 MB / ~1.4 s per bucket ⇒ roughly 4.2 s of lock-held CPU per poll, so TWO TABS SATURATE THE
+SERVER and every unrelated write queues behind them. This handler performs ZERO `store.get()`
+calls: `revision_of` below reads an in-memory counter that `Store.put`/`Store.update` bump at the
+moment a write is accepted. `verify_live_workspace.py`'s server half counts
+`get`/`_read_strict`/`_download` on this path and goes RED at one — because a token endpoint that
+deep-copies pays the identical 1.4 s and wins nothing.
+
+⭐ IT REACHES ALL THREE WRITERS THE OWNER NAMED, and that is a property of the QUESTION, not of
+the plumbing: it asks *"did this bucket change"*, never *"did somebody emit an event"*. The event
+seam (`user_tables.ROW_HOOKS`) fires only from HUMAN write doors by construction — the automation
+engine's own writer deliberately stays silent, and `verify_automation.py` has a negative control
+enforcing that silence. A push channel could therefore only ever carry 1 of the 3 writers. A
+counter on the store sees the automation and the connector sync for free, and the loop-prevention
+law is untouched.
+
+WHAT WALLS THIS, and why it is not `may_open`:
+
+ * The TENANT is the wall, and it is structural. Every bucket name is namespaced through the
+ session's OWN runtime (`TenantRuntime.store_key`), so a scope naming another tenant's table
+ resolves to a bucket this tenant does not have and reads 0. There is no cross-tenant answer
+ to give.
+ * A per-table `may_open` check would cost the very deep copy this route exists to avoid —
+ `user_tables.get()` reads the whole `user_tables` bucket to answer it.
+ * ⭐ And it would protect nothing, because `user_tables.STORE_KEY` is ONE bucket holding EVERY
+ `ut_*` table in the tenant: the rows token for `ut_a` and `ut_b` is literally the same
+ number. There is no per-table fact here to withhold. (Which also means a write to any one of
+ them invites every open tab to re-read its own table — over-signalling, i.e. the safe
+ direction: the cost is one extra fetch, never a missed change.)
+
+So: a valid session, a scope sanitised to a bucket-safe shape, and nothing else.
+"""
+from fastapi import APIRouter, Depends
+
+import core.shared_overlay as shared_overlay
+import core.store as store
+import core.user_tables as user_tables
+import modules.customer_data as customer_data
+import modules.product_data as product_data
+
+from deps import Session, err, require_session
+
+router = APIRouter(prefix="/api/v1")
+
+#: A scope reaches a STORE KEY, so it is sanitised as strictly as one: lowercase, digits and
+#: underscores only. Not paranoia about SQL — `Store` addresses files and `PgStore` binds
+#: parameters — but a scope is user-supplied text that becomes part of a path in the HF backend,
+#: and "it cannot escape today" is a property of the current backend rather than of this route.
+_SCOPE_MAX = 64
+
+
+def revision_of(handle, name):
+ """`{'rev','updated_at','token'}` for bucket `name` as THIS caller addresses it — or None.
+
+ ⭐ ONE resolver, and the gate calls the HANDLER rather than reproducing this dance, so the
+ route and its control cannot disagree about which physical key was read. A control that
+ re-derives the address is how a gate ends up certifying a key nobody serves.
+
+ `handle` is whatever the caller holds — a tenant runtime, a `Store`, or `core.store` itself.
+ The runtime is recognised DUCK-TYPED (`store_key` + `store_handle`) rather than imported:
+ `store_key()` is the ONE place a tenant's namespace prefix is applied, and applying it here a
+ second time is how one tenant reads another's counter.
+
+ ⛔ None — never a fabricated token — when the active backend publishes no revision. `PgStore`
+ maintains `rev` in `store_kv` and does not yet expose it, so under `STORE_BACKEND=pg` this
+ answers None, the client stops polling, and the app behaves exactly as it does today. A token
+ frozen at 0 would instead promise liveness the backend is not delivering.
+ """
+ target, key = handle, str(name)
+ namespaced = getattr(handle, "store_key", None)
+ if callable(namespaced): # a tenant runtime owns the namespace prefix
+ key = namespaced(name)
+ bound = getattr(handle, "store_handle", None)
+ target = bound if bound is not None else store._d()
+ fn = getattr(target, "revision", None)
+ if not callable(fn):
+ return None
+ try:
+ return fn(key)
+ except Exception:
+ # A backend that raises here must not take the poll down with it: the caller degrades to
+ # "no token for this bucket", which is the same honest null the pg path returns.
+ return None
+
+
+def _clean_scope(raw):
+ s = str(raw or "").strip().lower()
+ if not s or len(s) > _SCOPE_MAX or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789_"
+ for c in s):
+ return ""
+ return s
+
+
+def buckets_for(scope):
+ """`{label: store bucket}` for one topic scope — the map the CLIENT never gets to write.
+
+ ⛔ Every value is imported from the module that WRITES it, never re-spelled here.
+ `user_tables.STORE_KEY`, `customer_data.TABLE_KEY` and `product_data.TABLE_KEY` are the
+ literal keys those modules read and update, so a rename moves this map with it. A hand-copied
+ `'customer_table_workspace'` would keep answering 200 with a token for a bucket nobody writes
+ — a poll that costs nothing, changes never, and looks exactly like "there were no changes".
+
+ An absent label means "no store residency": `customer` and `product` rows come from Odoo
+ through a 15-minute server cache, which R11 deliberately leaves alone. The client reads a
+ missing/null token as "do not poll this", never as "unchanged".
+ """
+ s = _clean_scope(scope)
+ if not s:
+ return None
+ #: The shared stratum's bucket is asked FOR rather than spelled, so C5's `BUCKET_SUFFIX` has
+ #: exactly one definition and a rename cannot leave the poller watching a dead key.
+ if s.startswith(user_tables.KEY_PREFIX):
+ workspace = f"{s}_table_workspace"
+ elif s == "customer":
+ workspace = customer_data.TABLE_KEY
+ elif s == "product":
+ workspace = product_data.TABLE_KEY
+ else:
+ return {}
+ names = {"overlay": workspace, "shared": shared_overlay.bucket(workspace)}
+ if s.startswith(user_tables.KEY_PREFIX):
+ names["rows"] = user_tables.STORE_KEY
+ return names
+
+
+@router.get("/changes")
+def changes(scope: str = "", session: Session = Depends(require_session)):
+ """One tiny read per tab per interval. The response is small, uncacheable and boring."""
+ names = buckets_for(scope)
+ if names is None:
+ raise err(400, "bad_scope", "name the table you are watching, e.g. ?scope=customer")
+ if not names:
+ # ⛔ A well-formed scope this map does not know is a 404, NOT an empty `tokens` object.
+ # An empty map is indistinguishable from "nothing here ever changes", so a topic added
+ # later would poll forever against silence and read as a working live view that never
+ # updates. Loud beats plausible.
+ raise err(404, "unknown_scope", f"nothing here watches {_clean_scope(scope)!r}")
+ tokens = {}
+ for label, bucket in names.items():
+ rev = revision_of(session.runtime, bucket)
+ # ⛔ A NULL IS AN ANSWER, not an omission. The pg backend maintains `rev` in `store_kv`
+ # and does not yet expose it, so after D-4 flips this reads None until `PgStore` grows a
+ # `revision()`. Sending null and letting the client stop polling is the honest failure;
+ # a token frozen at 0 would promise liveness the backend is not delivering.
+ tokens[label] = rev.get("token") if isinstance(rev, dict) else None
+ return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()}
diff --git a/api/routes_forms.py b/api/routes_forms.py
index 3fc769ea39091243657794f4cefb37775bfb33e1..545819fa7d97216f1136881ba4633413abc7aa44 100644
--- a/api/routes_forms.py
+++ b/api/routes_forms.py
@@ -32,15 +32,32 @@ bypassed. Booked as DEBT rather than described as a guarantee — a rate limit t
being one is worse than none.
"""
import hmac
+import os
import secrets
import time
-from fastapi import APIRouter, Request
+from fastapi import APIRouter, Depends, Request
-from deps import err
+from deps import Session, err, require_session
router = APIRouter(prefix="/api/v1")
+#: ⭐⭐ WAVE-29 R7 — THE TOKEN INDEX, one bucket per tenant: `{token: {table, view}}`.
+#:
+#: ⛔ WHY THE TOKEN IS NOT IN THE FORM SPEC, which is where wave 23 assumed it would live. The spec
+#: rides `view.config.display.form`, which a BROWSER writes — so a token stored beside it is a
+#: token a browser can choose. `_resolve` walks tenants and answers with the FIRST match, so a
+#: tenant that set its token to a value it had seen elsewhere would quietly receive another
+#: tenant's submissions. Nothing about that is exotic: it needs one `PATCH` and a token somebody
+#: pasted into a chat. Minting server-side into a bucket the client cannot write closes it by
+#: construction rather than by validation, and `aios_grid._clean_form` drops a wire `token`.
+#:
+#: ⭐ It is also FASTER than what it replaces. The old resolution walked every table's workspace,
+#: every user's stratum and every view of every tenant looking for a matching token — booked in
+#: this file's own header as "a FULL SCAN, deliberately, and its cost is BOOKED". Now it is one
+#: small dict read per tenant, and the view is loaded ONCE, after the token has already matched.
+TOKENS_KEY = "form_tokens"
+
#: The mint. 24 bytes url-safe = 32 characters, 192 bits — the same shape the automation hook's
#: per-automation token uses, so there is ONE token strength in the product rather than two.
TOKEN_BYTES = 24
@@ -126,35 +143,40 @@ def _same(a: str, b: str) -> bool:
# index becomes worth its risk when either number grows.
-def _iter_form_views(rt):
- """Yield `(table_key, view_id, form_spec)` for every form-enabled view in one tenant.
+def _find_view(rt, table_key: str, view_id: str):
+ """`(view, owner_username)` for one view in one table's workspace, or `(None, None)`.
- A form spec lives at `view.config.display.form` and carries the token — see W23-C9. Reads
- only; nothing here writes, so a malformed stratum is skipped rather than repaired.
+ The bucket is `{username: {views: {id: view}}}` — one stratum per user plus the shared one —
+ and a view id is unique across them, so the first match is the view.
"""
- import core.user_tables as user_tables
-
try:
- tables = user_tables.all_tables(st=rt) or {}
+ bucket = rt.get(f"{table_key}_table_workspace") or {}
except Exception: # noqa: BLE001
- return
- for key in list(tables):
- try:
- bucket = rt.get(f"{key}_table_workspace") or {}
- except Exception: # noqa: BLE001
- continue
- if not isinstance(bucket, dict):
+ return None, None
+ if not isinstance(bucket, dict):
+ return None, None
+ for owner, ws in bucket.items():
+ if not isinstance(ws, dict):
continue
- for ws in bucket.values():
- if not isinstance(ws, dict):
- continue
- for view_id, view in (ws.get("views") or {}).items():
- if not isinstance(view, dict):
- continue
- display = ((view.get("config") or {}).get("display") or {})
- spec = display.get("form")
- if isinstance(spec, dict) and spec.get("token"):
- yield str(key), str(view_id), spec
+ view = (ws.get("views") or {}).get(view_id)
+ if isinstance(view, dict):
+ return view, str(owner)
+ return None, None
+
+
+def _spec_of(view) -> dict:
+ """The form spec on a view, or `{}`. One place knows the key path."""
+ spec = (((view or {}).get("config") or {}).get("display") or {}).get("form")
+ return spec if isinstance(spec, dict) else {}
+
+
+def _tokens(rt) -> dict:
+ """This tenant's `{token: {table, view}}` index, always a dict."""
+ try:
+ found = rt.get(TOKENS_KEY) or {}
+ except Exception: # noqa: BLE001
+ return {}
+ return found if isinstance(found, dict) else {}
def _resolve(token: str):
@@ -164,6 +186,12 @@ def _resolve(token: str):
tenant whose store answered would make "the third tenant's form" depend on the first two
being healthy, and a form that stops working because somebody else's store blipped is the
kind of failure nobody can reproduce.
+
+ ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. A token that resolves to a view which has since
+ been DELETED, or whose form spec has been removed, answers the same 403 as a token that never
+ existed — the index is written by the mint route and nothing deletes the tenant's views on its
+ behalf, so "the row is in the index" cannot be allowed to mean "the form is live". A stale
+ pointer outliving its view is exactly how a door that was closed stays open.
"""
from harness import runtime as _rt
@@ -174,9 +202,17 @@ def _resolve(token: str):
rt = _rt.get_runtime(slug)
except Exception: # noqa: BLE001
continue
- for table_key, _view_id, spec in _iter_form_views(rt):
- if _same(str(spec.get("token") or ""), token):
- return rt, slug, table_key, spec
+ for stored, where in _tokens(rt).items():
+ if not _same(str(stored), token) or not isinstance(where, dict):
+ continue
+ table_key = str(where.get("table") or "")
+ view, _owner = _find_view(rt, table_key, str(where.get("view") or ""))
+ spec = _spec_of(view)
+ # No view, no questions, no form. Checked HERE rather than by the caller so every
+ # public route inherits it from one place.
+ if not table_key or not spec.get("fields"):
+ return None
+ return rt, slug, table_key, spec
return None
@@ -192,6 +228,7 @@ def _public_form(rt, table_key: str, spec: dict) -> dict:
somebody arrange these questions, and re-deriving the order from the schema would silently
rearrange a form every time a column was added.
"""
+ import aios_grid
import core.user_tables as user_tables
defn = user_tables.get(table_key, st=rt) or {}
@@ -204,6 +241,24 @@ def _public_form(rt, table_key: str, spec: dict) -> dict:
# and never an error: losing one column must not take a live form offline.
if not f or f.get("source") not in (None, "overlay"):
continue
+ # ⭐ WAVE-29 R7 — TWO WALLS, AND NEITHER IS REDUNDANT.
+ #
+ # (1) The TYPE allow-list: a form may only ask questions this door can answer honestly.
+ # `_clean_values` below coerces twelve types and passes everything else through as
+ # text, so an `image`, a `json` document or a `link` reached through a form would be
+ # a string typed into a column that means something else.
+ # (2) `is_computed_cell`: the SERVER owns that number. And it catches what the type list
+ # structurally cannot — a METRIC bag rides an ordinary `int`, so a machine-computed
+ # column passes the allow-list by type and must still be refused.
+ #
+ # ⚠ Enforced HERE rather than only in the builder, because the builder is a courtesy and
+ # this is the wall: the spec is a stored bag, and a field can become computed (a formula,
+ # a rollup, a metric binding) long after it was added to a live form
+ # ([[schema-role-is-not-a-value-wall]]).
+ if str(f.get("type") or "text") not in aios_grid.FORM_FIELD_TYPES:
+ continue
+ if user_tables.is_computed_cell(f):
+ continue
row = {"key": str(f.get("key")), "label": str(f.get("label") or f.get("key")),
"type": str(f.get("type") or "text"), "required": str(key) in required}
options = f.get("options")
@@ -218,9 +273,32 @@ def _public_form(rt, table_key: str, spec: dict) -> dict:
"submitLabel": str(spec.get("submitLabel") or "")[:60] or "Submit",
"fields": out,
"honeypot": HONEYPOT_FIELD,
+ # ⛔ THE FLAG, NEVER THE LIST. `requireEmail` tells the page to ask for an address; the
+ # ADDRESSES stay server-side. Returning them would hand an unauthenticated caller the
+ # guest list — every colleague's work email, to anyone who has the link — which is a
+ # worse leak than the door this restriction exists to close. It is also the only reason
+ # this key can be computed here rather than shipped as `spec['emails']`.
+ "requireEmail": spec.get("access") == "emails",
}
+def _invited(spec: dict, email: str) -> bool:
+ """May this address submit? Case- and space-insensitive against the stored list.
+
+ ⚠ **IDENTIFICATION, NOT AUTHENTICATION, and the honest boundary is stated rather than
+ implied** (this file's header does the same for the token). Nothing here proves the submitter
+ owns the address they typed: anyone holding the link who knows an invited address can pass.
+ What it buys is real but bounded — a link forwarded outside the invited group stops working
+ for whoever cannot name a member of it, and every submission carries the address it claimed.
+ A per-recipient link, which WOULD prove possession, is booked rather than half-built.
+ """
+ if spec.get("access") != "emails":
+ return True
+ wanted = str(email or "").strip().lower()
+ return bool(wanted) and wanted in {str(e).strip().lower()
+ for e in (spec.get("emails") or [])}
+
+
# --- validation -------------------------------------------------------------------------------
@@ -290,6 +368,135 @@ def _clean_values(fields: list, raw: dict):
# --- the routes -------------------------------------------------------------------------------
+# --- the AUTHENTICATED half: minting, reading and revoking a form's share link ----------------
+#
+# ⚠ PATH: `/form-link`, NOT `/forms/link`. The public routes are `/forms/{token}`, and a literal
+# segment beside a path parameter is decided by declaration ORDER — a reader (or a later
+# refactor that moves a decorator) would have to know that to know which handler answers. A
+# different noun cannot be shadowed by a token somebody chooses.
+
+
+def _public_base() -> str:
+ """The origin a share link is built on. Empty = the client builds it from its own location.
+
+ `AIOS_PUBLIC_BASE` is the same variable the OAuth callback resolves against, so a domain move
+ takes the form links with it. Absent, we return a PATH rather than guessing a hostname:
+ a link with the wrong origin looks right, gets pasted into an email and 404s for the reader.
+ """
+ return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/")
+
+
+def _link_of(token: str) -> str:
+ return f"{_public_base()}/#/form/{token}"
+
+
+def _may_administer_view(session: Session, table_key: str, view_id: str):
+ """`(view, spec)` when this caller may publish THIS view, else raises.
+
+ ⭐ `_may_administer`, deliberately, not `_may_edit`. Minting a public link is a PERMISSIONS
+ act — it opens a write door onto the tenant's table for anybody holding the URL — and
+ `table_store` already draws that line for views: a collaborator may edit a shared view, only
+ its creator or an admin may change who reaches it. Publishing is the strongest form of that
+ question, so it takes the narrower answer.
+ ⚠ A view sitting in the caller's OWN stratum is theirs even with no `createdBy` stamp (a
+ personal view predating the stamp has none), which `_may_administer` alone would refuse.
+ """
+ import core.table_store as table_store
+
+ if not table_key.startswith("ut_"):
+ # `add_row` is the only write a form can perform and it is a `ut_*` door; an Odoo-backed
+ # grid has no such door, so a link over one could never be anything but a 500 later.
+ raise err(400, "not_a_database", "forms collect into your own databases only")
+ view, owner = _find_view(session.runtime, table_key, view_id)
+ if not isinstance(view, dict):
+ raise err(404, "no_such_view", "that view no longer exists")
+ # ⛔ `uname` / `admin`, NOT `username` / `is_admin`. `Session` has no such attributes, and
+ # reading one raises OUTSIDE any handler's own try: — which is exactly D-107, where four
+ # finished Odoo routes were mounted, uncallable, and answered a bare plain-text 500 for a day
+ # ([[mounted-is-not-callable]]). Written wrong here first; caught by this file's own gate.
+ if not (owner == session.uname
+ or table_store._may_administer(view, session.uname, session.admin)):
+ raise err(403, "not_yours", "only this view's creator or an admin can publish it")
+ return view, _spec_of(view)
+
+
+@router.get("/form-link")
+def read_form_link(topic: str, view: str, session: Session = Depends(require_session)):
+ """This form's current share link, or `{"token": null}` — never a mint.
+
+ ⛔ A GET MUST NOT MINT. The builder reads this every time the panel opens, and a route that
+ created a token on read would publish a form the moment somebody looked at the tab.
+ """
+ _v, spec = _may_administer_view(session, topic, view)
+ token = ""
+ for stored, where in _tokens(session.runtime).items():
+ if isinstance(where, dict) and where.get("table") == topic and where.get("view") == view:
+ token = str(stored)
+ break
+ return {
+ "token": token or None,
+ "url": _link_of(token) if token else None,
+ # ⭐ THE SERVER'S OWN VIEW OF THE SPEC, and it exists to make ONE failure visible rather
+ # than silent: the builder's `spec` prop comes from the client's `cleanDisplay`, so if the
+ # client ever stops carrying `display.form`, the panel would render an empty form over a
+ # stored one and the next save would erase real questions. Reporting what the SERVER holds
+ # lets the panel say so instead ([[read-path-cannot-witness-write-path]]).
+ "stored": ({"fields": len(spec.get("fields") or []),
+ "access": spec.get("access") or "public",
+ "title": str(spec.get("title") or "")[:120]} if spec else None),
+ }
+
+
+@router.post("/form-link")
+def mint_form_link(body: dict, session: Session = Depends(require_session)):
+ """Mint (or regenerate) this view's share link. `{topic, view, regenerate?}`.
+
+ Idempotent without `regenerate`: opening the panel twice must not invalidate the link
+ somebody already emailed. With it, the previous token is DROPPED in the same write — a
+ "regenerate" that leaves the old link alive has revoked nothing.
+ """
+ topic, view = str((body or {}).get("topic") or ""), str((body or {}).get("view") or "")
+ _v, spec = _may_administer_view(session, topic, view)
+ if not spec.get("fields"):
+ raise err(400, "empty_form", "add at least one question before sharing this form")
+ fresh = secrets.token_urlsafe(TOKEN_BYTES)
+ keep = not (body or {}).get("regenerate")
+
+ minted = {"token": fresh}
+
+ def _set(cur):
+ cur = dict(cur or {})
+ for stored, where in list(cur.items()):
+ if isinstance(where, dict) and where.get("table") == topic and where.get("view") == view:
+ if keep:
+ minted["token"] = str(stored)
+ return cur
+ cur.pop(stored, None) # regenerate: the old link dies in this same write
+ cur[fresh] = {"table": topic, "view": view}
+ return cur
+
+ # `sync`: a share link the user is about to paste into an email must be durable before the
+ # response says it exists. This is not a hot path.
+ session.runtime.update(TOKENS_KEY, _set, flush="sync")
+ return {"token": minted["token"], "url": _link_of(minted["token"])}
+
+
+@router.delete("/form-link")
+def revoke_form_link(topic: str, view: str, session: Session = Depends(require_session)):
+ """Turn the link off. The form spec stays; only the door closes."""
+ _may_administer_view(session, topic, view)
+
+ def _drop(cur):
+ cur = dict(cur or {})
+ for stored, where in list(cur.items()):
+ if isinstance(where, dict) and where.get("table") == topic and where.get("view") == view:
+ cur.pop(stored, None)
+ return cur
+
+ session.runtime.update(TOKENS_KEY, _drop, flush="sync")
+ return {"ok": True}
+
+
@router.get("/forms/{token}")
def get_form(token: str, request: Request):
"""Render-time payload for a public form. Rate-limited like the POST: an unauthenticated GET
@@ -363,6 +570,19 @@ async def submit_form(token: str, request: Request):
form = _public_form(rt, table_key, spec)
if not form["fields"]:
raise _refuse()
+
+ # ⭐ WAVE-29 R7 — the authorized-by-email door. Checked BEFORE `_clean_values` so an
+ # uninvited caller learns nothing about the form's questions from the shape of its refusal,
+ # and before the daily cap so strangers cannot spend a live form's allowance.
+ # ⚠ NOT `_refuse()`: this one is deliberately a DIFFERENT answer. `_refuse` exists so a wrong
+ # token cannot be told apart from a missing form; here the caller already holds a valid link
+ # and the only useful thing we can say is "not with that address" — telling them nothing
+ # would leave a colleague staring at a form that silently does nothing.
+ submitter = str((body or {}).get("email") or "").strip().lower()[:254]
+ if not _invited(spec, submitter):
+ raise err(403, "not_invited",
+ "this form only accepts submissions from invited email addresses")
+
clean, problem = _clean_values(form["fields"], values)
if problem:
raise err(400, "invalid_submission", problem)
@@ -384,7 +604,10 @@ async def submit_form(token: str, request: Request):
# `record_created` row event, so every trigger and audit downstream sees `form:`
# rather than a blank actor. Booked as a dated amendment in the split doc; a persistent
# per-row stamp needs `add_row` to grow one, which is SESSION A's file.
- actor = f"form:{token[:8]}"
+ # ⭐ WAVE-29 — an invited submitter is stamped BY ADDRESS rather than by token prefix: on an
+ # email-restricted form the address is the most identifying thing we honestly have, and it is
+ # what makes `record_created` legible in a run log ("form:ana@…" beats "form:Kx2p9Lm4").
+ actor = f"form:{submitter or token[:8]}"[:120]
row_id = user_tables.add_row(table_key, clean, username=actor, st=rt)
if not row_id:
raise err(409, "not_accepted", "that submission could not be saved — try again later")
diff --git a/api/routes_oauth.py b/api/routes_oauth.py
index 658f16e60fdaab292a168070fd4fb21fa8cabfcc..78b419c01cb4ee102bb4ef4b44d0f915c2252969 100644
--- a/api/routes_oauth.py
+++ b/api/routes_oauth.py
@@ -32,7 +32,18 @@ def _redirect_uri(request: Request, provider: str) -> str:
"""The redirect URI this deployment registers at the provider — env-pinned when the
container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
the request. MUST match a console-registered URI verbatim, so it is computed in exactly
- one place."""
+ one place.
+
+ ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
+ the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
+ the request-derived fallback below is effectively dev-only.
+ ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
+ this returns is where the provider sends the user BACK, and the session cookie is host-only
+ (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
+ the user actually browsed plants the session on the wrong hostname and they return logged
+ out. Moving the app to a new hostname means moving this value AND re-registering the
+ resulting URI in the provider console; one without the other fails closed.
+ Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
if not base:
base = f"{request.url.scheme}://{request.url.netloc}"
diff --git a/api/routes_products.py b/api/routes_products.py
index cb83bff460bd787fdae4ee6afb4a408c8d7f5e80..d198f28dbb2d4da95f2c80af35004ccb22a85705 100644
--- a/api/routes_products.py
+++ b/api/routes_products.py
@@ -63,6 +63,17 @@ def _pool_for(rt, team_id):
# naming the pause when there is none. Minting a product snapshot here would be a new
# persistence format at a wave tail; refusing with a sentence is the answer the user can act
# on, and it is the same 503 the customer path already gives for an unsnapshotted scope.
+ # ⭐ WAVE 29 / R12 — THE POOL IS THE CATALOGUE NOW (5,875 active products, not the 2,717 that
+ # happened to sell in a revenue window). Two consequences that land HERE rather than in
+ # `product_data`:
+ # · the 503 below now also covers a TRUNCATED catalogue read. `modules.product_data.
+ # _catalogue_by_code` RAISES instead of degrading, precisely so a short pull arrives as a
+ # refusal a user can act on rather than as a plausible smaller grid nobody questions;
+ # `pool_unavailable` names the cause in its message.
+ # · the first build for a scope got slower (MEASURED 2026-08-11: ~44 s consolidated, of
+ # which the catalogue read is ~3 s; a BU-scoped build is ~9 s). Only a scope's FIRST-ever
+ # build blocks — the stale-while-refresh discipline above is what keeps that off the
+ # request path afterwards.
import routes_keychain
if routes_keychain.odoo_paused(rt):
hit = rt.pool_cache.get(key)
diff --git a/api/routes_tables.py b/api/routes_tables.py
index b4b6cb6fc581d734417212728d794c3c894b0407..e35c9accd9cb2f44e306e21c1abfaf5bf2d58d0f 100644
--- a/api/routes_tables.py
+++ b/api/routes_tables.py
@@ -684,6 +684,56 @@ def add_row(table_key: str, body: dict = Body(default=None),
return {"rid": rid, "pid": int(rid)}
+@router.post("/tables/{table_key}/rows/import", status_code=201)
+def import_rows(table_key: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """⭐⭐ WAVE-29 T25 (owner item 6) — the IMPORT door: N mapped rows, ONE store write.
+
+ Body: `{"rows": [{fieldKey: value, ...}, ...]}` — already MAPPED by the client's dialog, so a
+ spreadsheet column name never reaches the store. APPEND-ONLY in v1: every row is a new record,
+ nothing is matched or overwritten, and the dialog says so before the button is pressed.
+
+ ⛔ COMPUTED COLUMNS ARE REFUSED HERE, NOT FILTERED. `is_computed_cell` is the same predicate
+ the cell wall uses (one evaluator for one question), and a rollup or formula key arriving in
+ an import is not a stray to be tidied away — it means the client offered a target it should
+ not have, and silently dropping it would leave the user looking for a column of values that
+ never arrived. The refusal names the column.
+
+ ⛔ ATOMIC. `add_rows` writes nothing unless the whole batch fits under `MAX_ROWS` and every
+ profile cell validates, because a half-imported file is the worst outcome available: the user
+ cannot tell which rows landed without reconciling the spreadsheet by hand.
+ """
+ _records_or_refuse(session, table_key)
+ ut = _ut()
+ rows_in = (body or {}).get("rows")
+ if not isinstance(rows_in, list) or not rows_in:
+ raise err(400, "bad_rows", "rows must be a non-empty array of {fieldKey: value} objects")
+ if any(not isinstance(r, dict) for r in rows_in):
+ raise err(400, "bad_rows", "every row must be an object of {fieldKey: value}")
+ defn = _defn_or_refuse(session, table_key)
+ by_key = {f["key"]: f for f in (defn.get("fields") or [])}
+ asked = {k for r in rows_in for k in r}
+ unknown = sorted(k for k in asked if k not in by_key)
+ if unknown:
+ raise err(400, "unknown_field",
+ f"this database has no column {unknown[0]!r}")
+ computed = sorted(k for k in asked if ut.is_computed_cell(by_key[k]))
+ if computed:
+ label = by_key[computed[0]].get("label") or computed[0]
+ raise err(400, "computed_field",
+ f"{label!r} is worked out from other columns, so it cannot be imported into")
+ try:
+ made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime)
+ except Exception:
+ raise err(503, "store_unavailable", "nothing was imported — the store refused")
+ if made is None:
+ raise err(400, "refused",
+ f"nothing was imported — {len(rows_in)} rows would take this database past "
+ f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value")
+ _refresh_relations(session)
+ return {"imported": len(made), "pids": [int(r) for r in made]}
+
+
# ---------------------------------------------------------------------------------------------
# THE SHARED FIELD SCHEMA (contract C-FIELD, owner ruling R2)
# ---------------------------------------------------------------------------------------------
@@ -897,11 +947,18 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None),
except grid_events.StoreUnavailable:
raise err(503, "store_unavailable",
"the tenant store is unavailable — your change was not saved")
- # ⚠ THE READ-BACK SPANS BOTH STRATA, and it has to (wave 25, C3-A1). An ordinary cell lands
- # in this user's OVERLAY; a PROFILE cell writes THROUGH to the shared DEFINITION row. Reading
- # only the overlay — which is what this did — reports a perfectly accepted profile write as
- # `refused`, because the key is simply not in the bucket it looked in. Same merge order the
- # rows envelope uses (`table_rows`): definition under, overlay over.
+ # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT.
+ #
+ # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH
+ # STRATA, and it has to (wave 25, C3-A1)" while the two lines under it read exactly one bucket.
+ # It was true of the wave-25 world it was written in — an ordinary cell landed in the caller's
+ # OVERLAY and only a PROFILE cell wrote through — and it stayed after `grid_events` began
+ # routing EVERY accepted cell on a `ut_` scope to `user_tables.patch_cells`
+ # (`grid_events.py:1979-1984`). There is no second stratum this read is missing; a docstring
+ # claiming otherwise is what makes the next reader look for a merge bug that is not here.
+ # ⚠ Legacy overlay values, written before that routing existed, are still merged for DISPLAY
+ # by `table_rows` (:587-595) — display only, and deliberately not re-asserted here: `_took`
+ # asks whether THIS write landed, and this write goes to the definition.
stored = dict(((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {})
.get(str(pid)) or {})
accepted = {k: stored.get(k) for k in updates if k in stored}
diff --git a/platform/aios_grid.py b/platform/aios_grid.py
index 8dfd737177be5d9eeed5fb271be61bc03dda72fa..01c569b2ea282c41e974cf12dd0217757ea3be06 100644
--- a/platform/aios_grid.py
+++ b/platform/aios_grid.py
@@ -673,7 +673,10 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_
"source": "overlay",
"default": bool(field.get("default", True)),
"custom": True,
- "agg": field.get("agg") if field.get("agg") in {"sum"} else None,
+ # WAVE-29 C7: the whole vocabulary, not the `{"sum"}` literal that was here — a
+ # picker offering Average against a projection that only passes Sum through is the
+ # silent half of this feature.
+ "agg": field.get("agg") if field.get("agg") in FIELD_AGGS else None,
"note": str(field.get("note") or "")[:2000],
**({"options": options} if ftype in ("select", "multiselect") else {}),
**(_choice_appearance(field, options)
@@ -880,8 +883,109 @@ def _clean_rhs(raw, valid_keys):
#: the whole reason the two modes above it needed a staged hold: `_clean_display` DROPS an
#: unknown mode, so a client that offers a mode this set does not carry lets a user build a view
#: that silently reverts to a grid on the next read.
+#: ⭐⭐ WAVE-29 R6/R7 (owner item 10, contract C3) — 'form', which CLOSES D-90. The client has
+#: carried `form` in its union, with an icon, a label and a tone, since wave 23; this set never
+#: did, so `_clean_display` DROPPED both the mode and the `display.form` spec on every write —
+#: while `routes_forms.py` reads exactly that key to serve the public submit door. The public door
+#: has therefore been live and UNREACHABLE for two waves: not broken, just impossible to point at
+#: anything. The mirror is one name, and it is the half nobody could see was missing because
+#: BOTH sides were individually consistent.
+#: ⚠ Being a legal stored mode is NOT the same as being offered: `form` is deliberately held out
+#: of the client's `CREATABLE_MODES` until `CustomerGrid` mounts a renderer for it (the hold law
+#: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by
+#: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered
+#: one is red too, so the hold cannot outlive its reason the way wave 27's did).
DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart',
- 'timeseries', 'catalog', 'swipe'}
+ 'timeseries', 'catalog', 'swipe', 'form'}
+
+#: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at
+#: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read
+#: exactly this key since wave 23 and NOTHING HAS EVER BEEN ABLE TO WRITE IT: `form` was not a
+#: legal mode and this function had no branch for the key, so every spec a client sent was dropped
+#: on the way in. That is D-90 stated precisely — not a broken feature, an unreachable one.
+#:
+#: ⛔ THE TOKEN IS NOT HERE, AND IT NEVER WILL BE. A share token that rides the wire is a token a
+#: browser can CHOOSE, and `routes_forms._resolve` walks tenants and answers with the FIRST match —
+#: so one tenant setting its token to another tenant's value would silently receive that tenant's
+#: submissions. The token is minted server-side and lives in a bucket no client write can reach;
+#: `_clean_form` drops any `token` key that arrives here, rather than validating its shape.
+FORM_ACCESS = ('public', 'emails')
+MAX_FORM_FIELDS = 60
+MAX_FORM_EMAILS = 200
+MAX_FORM_TITLE, MAX_FORM_DESC, MAX_FORM_SUBMIT = 120, 1000, 60
+MAX_FORM_EMAIL = 254
+
+#: The field types a form may COLLECT, as an ALLOW-LIST rather than a list of exclusions — the
+#: fail-closed direction, because the cost of the two mistakes is not symmetric. A type missing
+#: from here is a question the builder cannot ask yet; a type wrongly present is a public door
+#: writing values the column cannot mean (an `image` with no upload channel, a `json` document
+#: typed into a text box, a `link` naming a record id a stranger guessed).
+#: ⚠ NOT sufficient on its own, and the reason is a shape this codebase has been bitten by before:
+#: a METRIC bag rides ANY field type (`core/user_tables.py` — `metric` is not a field kind), so an
+#: `int` column can be machine-computed while passing this list. `routes_forms` therefore asks
+#: `user_tables.is_computed_cell` as well — one evaluator for "is this computed", reused rather
+#: than re-derived ([[one-evaluator-per-question]]).
+#: Client mirror: `customer-grid/FormInterface.tsx` FORM_FIELD_TYPES; `verify_forms.py` compares
+#: the two files name-for-name.
+FORM_FIELD_TYPES = ('text', 'select', 'multiselect', 'int', 'currency', 'pct', 'date',
+ 'checkbox', 'phone', 'email', 'url', 'rating')
+
+#: Deliberately looser than a full RFC parse and stricter than `routes_forms._clean_values`' "@ in
+#: it": this list decides who MAY SUBMIT, so a typo that silently locks a colleague out is the
+#: expensive failure, not an odd address that gets in.
+_FORM_EMAIL = re.compile(r'^[^@\s,;]+@[^@\s,;]+\.[^@\s,;]+$')
+
+
+def _clean_form(raw, valid_keys):
+ """One form spec, fail-closed. Returns None when nothing is configured.
+
+ ⚠ FIELD ORDER IS THE FORM'S OWN and is preserved here, not re-derived from the schema: the
+ builder let somebody arrange these questions, and sorting them by column order would silently
+ rearrange a live form every time a column was added (`_public_form` states the same rule from
+ the serving end).
+
+ ⚠ PARTIAL-DROP, not whole-key drop, and the asymmetry against `swipe` above is deliberate. A
+ swipe binding is one three-part machine: two of its parts is not a degraded deck, it is a deck
+ that can never write. A form is a LIST of questions — losing the column behind question three
+ costs the asker question three, and taking the whole form away because one field was deleted
+ would be a far larger loss than the one that happened.
+ """
+ if not isinstance(raw, dict):
+ return None
+ fields, seen = [], set()
+ for k in (raw.get('fields') or [])[:MAX_FORM_FIELDS]:
+ if k in valid_keys and k not in seen:
+ seen.add(k)
+ fields.append(k)
+ out = {}
+ if fields:
+ out['fields'] = fields
+ # A required flag on a question the form no longer asks is not a rule, it is a trap: the
+ # submitter can never satisfy it and the sentence names a field they cannot see.
+ req = [k for k in dict.fromkeys(raw.get('required') or []) if k in seen]
+ if req:
+ out['required'] = req
+ for key, cap in (('title', MAX_FORM_TITLE), ('desc', MAX_FORM_DESC),
+ ('submitLabel', MAX_FORM_SUBMIT)):
+ text = str(raw.get(key) or '').strip()[:cap]
+ if text:
+ out[key] = text
+ # `public` is the ABSENT default (the `kanbanClamp` law: one way to say one thing), so only
+ # the restrictive value is ever stored. ⇒ a spec that loses its `access` key fails OPEN, which
+ # is why `emails` is what gets written rather than a `public: false`.
+ if raw.get('access') == 'emails':
+ out['access'] = 'emails'
+ emails = []
+ for e in (raw.get('emails') or [])[:MAX_FORM_EMAILS]:
+ e = str(e or '').strip().lower()[:MAX_FORM_EMAIL]
+ if _FORM_EMAIL.match(e) and e not in emails:
+ emails.append(e)
+ # Kept even while `access` is public: a person toggling the door open to test it and back
+ # again must not lose the list of people they typed. It is never served publicly.
+ if emails:
+ out['emails'] = emails
+ return out or None
+
#: C3 — the swipe binding's option cap. `leftOption`/`rightOption` are stored VALUES of a
#: single-select, and `_clean_options` trims every choice to 120 chars, so this is that same
@@ -937,6 +1041,35 @@ _LEGACY_MODES = {'dashboard': 'chart'}
#: GroupTableView over salesParity.tableFromSpec.
CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'}
CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'}
+
+#: ⭐ WAVE-29 C7 (item 17) — THE COLUMN-SUMMARY vocabulary: what a FIELD's `agg` may be, which is
+#: what the grid's totals row and its per-group subtotals compute. ORDERED, because the order is
+#: the picker's order; membership tests read it as a tuple perfectly well.
+#:
+#: ⛔ IT IS NOT `CHART_AGGS` AND THE TWO MUST NOT BE MERGED, however alike they look. `CHART_AGGS`
+#: gatekeeps a STORED value with live data behind it (`charts[].agg`, `calendarMetrics[].agg`):
+#: `_clean_chart` falls back to 'sum' on an unknown agg and `_clean_display` DROPS a whole
+#: calendarMetrics entry, so renaming its 'avg' would silently turn every saved chart into a sum
+#: and delete calendar cards, with nothing red. A chart's aggregation and a column's summary are
+#: also different questions — one reduces a SERIES, the other a COLUMN — and one list serving both
+#: would have to be the intersection of what each can express.
+#:
+#: ⭐ `average`, NOT `avg`, and the tie is broken by the vocabulary we cannot rename: `ROLLUP_FNS`
+#: (`core/user_tables.py`, 16 names, 47 rollups live in production) already spells it `average`,
+#: and it is the aggregate vocabulary a user actually reads today. Spelling it `avg` here would
+#: give the product two words for one operation on two menus a click apart.
+#:
+#: ⚠ `median` is net-new: it is in NEITHER `CHART_AGGS` nor `ROLLUP_FNS`, so a Median column
+#: summary has no rollup equivalent and this list is NOT a subset of either of its neighbours.
+#:
+#: ⚠ `count` counts ROWS in the scope (the group, or every matched row) — not non-blank cells.
+#: `ROLLUP_FNS` splits that hair three ways (count / counta / countall); a column summary does not,
+#: and must not grow a second spelling of it.
+#:
+#: Client mirror: `customer-grid/iconShapes.ts` FIELD_AGGS — ONE client list, imported by
+#: `aggregations.ts` and the field editor rather than re-declared, so the only boundary left to
+#: police is this one. `verify_icons.py::agg_parity` reads BOTH FILES and compares them.
+FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count')
MAX_CHARTS = 12 #: per view — a dashboard, not an unbounded render loop
MAX_CHART_TITLE = 60
@@ -1379,6 +1512,10 @@ def _clean_display(raw, valid_keys):
# `kanbanClamp` law) wearing a control that promises a choice.
if left and right and left.casefold() != right.casefold():
out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right}
+ # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here.
+ form = _clean_form(raw.get('form'), valid_keys)
+ if form:
+ out['form'] = form
return out
diff --git a/platform/core/shared_overlay.py b/platform/core/shared_overlay.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f99d3b6d97cbeac8c610daa309747304aa111cc
--- /dev/null
+++ b/platform/core/shared_overlay.py
@@ -0,0 +1,264 @@
+"""The TENANT-WIDE overlay stratum — one value per (row, column) for the whole workspace.
+
+Wave 29, item 20 / owner ruling R11, contract C5. The sibling of `core/table_store.py`, which
+holds the PER-USER strata (`store.get(key)[username]`) and always has.
+
+⛔ THE DEFECT THIS EXISTS FOR IS NOT "SHARING WOULD BE NICE" — IT IS THAT A SHARED VIEW SILENTLY
+WIDENS. `modules/product_data.py` already states it, in the comment above the supplier master:
+
+ "A user-created field and its values live in the PER-USER strata; only VIEWS are shared. So a
+ shared 'Buy list' view that filtered on a user-created column would, for every OTHER account,
+ name a column that does not exist — and an unknown column is an INACTIVE condition in the
+ tri-state engine, which IGNORES it and therefore WIDENS. The buy list would silently show the
+ whole catalogue to everyone but its author."
+
+That is why the four supplier columns were frozen as read-only CONTRACT columns rather than made
+editable, and why the owner's ask ("turn this Excel sheet into a User created Field that we can
+edit") has been parked for two waves. A column whose value is the same for every reader makes the
+shared view mean ONE thing, which is the precondition for editing it at all.
+
+────────────────────────────────────────────────────────────────────────────────────────────────
+⛔⛔ THIS MODULE IS NOT A PERMISSION WALL, AND MUST NEVER BECOME ONE BY ACCIDENT.
+
+It refuses no reader and no writer. The caller has already answered "may this session open this
+surface" (`user_tables.may_open`, `Session.require`, the BU pool) and this module answers only
+"what is stored". Two different questions; one of them belongs upstream, where the session is.
+
+What it DOES enforce is the one thing a caller can get wrong silently:
+
+ ⭐ `cells(table_key, pids)` — `pids` IS REQUIRED, POSITIONALLY, AND THERE IS NO "EVERYTHING"
+ CALL. A default of "all rows" is the widening hazard above wearing a friendly face: the
+ caller passes the row set it has ALREADY scoped, so a cell for a row this reader may not see
+ cannot come back to be merged. There is deliberately no `all_cells()` to reach for; a caller
+ that genuinely needs the lot passes the lot, in writing, where a reviewer can see it.
+
+The row wall then holds twice over, because `aios_grid.rows_from_pool` iterates the POOL and looks
+each pid UP in the overlay — never the reverse. A cell for a row outside the pool has nothing to
+attach to. This module is built to keep that true rather than to re-implement it.
+────────────────────────────────────────────────────────────────────────────────────────────────
+
+RESIDENCY. Its own bucket, `__shared`, beside the per-user one — never a `__shared__`
+member inside it. Two reasons, and the first is `product_data.py`'s own argument for separate
+topic keys ("separate store keys make that structurally impossible, which is the whole reason the
+table-page factory exists"): a per-user reader iterating usernames cannot encounter shared data
+when there is no shared data in that bucket to encounter. The second is cost — the per-user bucket
+carries every user's views, and a cell edit should not read-modify-write all of it.
+
+SHAPE, chosen to be drop-in:
+
+ {"fields": {field_key: Field}, # the tenant-wide column DEFINITIONS
+ "cells": {"": {field_key: value}}} # exactly `rows_from_pool(overlays=...)`'s shape
+
+`cells()` returns STRING pid keys for that reason — `rows_from_pool` does `overlays.get(str(pid))`,
+so a caller merges the shared stratum over the per-user one with one `dict.update` and cannot get
+the key type wrong. (`table_store` stores `{str(pid): {...}}` too; one shape, three readers.)
+"""
+import core.store as store
+
+#: The suffix that turns a topic's per-user workspace key into its shared one. Callers pass the
+#: key they ALREADY hold (`product_data.TABLE_KEY`, `f'{ut_key}_table_workspace'`) — one
+#: identifier for both strata, so there is no second naming convention to get wrong.
+BUCKET_SUFFIX = '__shared'
+
+#: One cell value's ceiling, matching the `json` field kind's. A shared cell is read by every user
+#: in the tenant, so an unbounded one is an unbounded cost for all of them.
+MAX_VALUE_CHARS = 32_768
+
+
+def bucket(table_key):
+ """The store key this topic's shared stratum lives under."""
+ key = str(table_key or '').strip()
+ if not key:
+ raise ValueError('shared_overlay: a table_key is required — it names the bucket')
+ return f'{key}{BUCKET_SUFFIX}'
+
+
+def _st(st):
+ return st if st is not None else store
+
+
+def _read(table_key, st=None):
+ """The whole stratum, always in its full two-key shape. Lenient like every other display
+ read: an unreachable store degrades to "nothing shared yet", never to an exception on a page
+ that would otherwise render."""
+ try:
+ data = _st(st).get(bucket(table_key)) or {}
+ except Exception:
+ data = {}
+ return {'fields': dict(data.get('fields') or {}),
+ 'cells': dict(data.get('cells') or {})}
+
+
+def _write(table_key, change, st=None, flush='async'):
+ """Read-modify-write one shared stratum.
+
+ `flush='async'` by default for the same reason `table_store._update` uses it: a typed cell
+ lands here inside the request round-trip, and the historical synchronous hub commit cost
+ seconds per keystroke. Structural writes (a field definition) pass `flush='sync'` — they are
+ rare, and a lost column definition is a worse failure than a lost keystroke.
+ """
+ def _up(data):
+ data = data if isinstance(data, dict) else {}
+ data.setdefault('fields', {})
+ data.setdefault('cells', {})
+ change(data)
+ return data
+ return _st(st).update(bucket(table_key), _up, flush=flush)
+
+
+def _pid(pid):
+ """Pids are ints everywhere in the grid and strings in JSON. ONE coercion, here, so a caller
+ passing either cannot write a row that a reader keyed the other way never finds."""
+ return str(int(pid))
+
+
+def _value(value):
+ """A cell holds a SCALAR. The Row contract is scalar on every surface — the grid, the filter
+ engine, formulas, export — and the `json` kind is a validated STRING rather than an object.
+
+ ⛔ A dict or a list RAISES rather than being dropped. This is called by our own code, so a
+ non-scalar is a programming error, and silently storing nothing would surface later as "the
+ shared column is blank for everyone" with no failure anywhere near the cause.
+ """
+ if isinstance(value, (dict, list, tuple, set)):
+ raise ValueError(f'shared_overlay: a cell holds a scalar, not {type(value).__name__}')
+ if value is None or isinstance(value, bool) or isinstance(value, (int, float)):
+ return value
+ text = str(value)
+ return text[:MAX_VALUE_CHARS]
+
+
+# ----------------------------------------------------------------- the column DEFINITIONS
+def fields(table_key, st=None):
+ """`{field_key: Field}` — the columns this topic shares tenant-wide.
+
+ Unscoped on purpose, and it is the one thing here that is: a shared column's EXISTENCE is
+ tenant-wide by definition — that is the whole feature, and it is what stops a shared view
+ naming a column half the workspace lacks. Its VALUES are scoped by `cells()`.
+ """
+ return _read(table_key, st)['fields']
+
+
+def is_shared(table_key, field_key, st=None):
+ """Is this column's value tenant-wide? The predicate a caller uses to decide WHICH stratum to
+ read and write — one evaluator, so the read path and the write path cannot disagree about
+ where a given column lives ([[one-evaluator-per-question]])."""
+ return str(field_key or '') in _read(table_key, st)['fields']
+
+
+def put_field(table_key, field_key, defn, st=None):
+ """Declare (or redefine) a shared column. Returns what was stored."""
+ key = str(field_key or '').strip()
+ if not key:
+ raise ValueError('shared_overlay: a field needs a key')
+ entry = dict(defn or {})
+ entry['key'] = key
+
+ def _add(data):
+ data['fields'][key] = entry
+
+ _write(table_key, _add, st, flush='sync')
+ return entry
+
+
+def drop_field(table_key, field_key, st=None):
+ """Remove a shared column AND every stored value for it.
+
+ ⛔ The cells go with the definition, exactly as `table_store.delete_field` scrubs its own:
+ orphaned values would silently resurface if the key were ever reused — and a resurrected
+ value in a TENANT-WIDE stratum reappears for everybody at once.
+ """
+ key = str(field_key or '').strip()
+ if not key:
+ return False
+ hit = [False]
+
+ def _drop(data):
+ hit[0] = data['fields'].pop(key, None) is not None
+ for row in data['cells'].values():
+ if isinstance(row, dict):
+ row.pop(key, None)
+ # ⚠ AND DROP THE ROWS THAT ARE NOW EMPTY. `cells()` already skips a blank row, so this
+ # changes nothing a reader sees — but this bucket is read by every user in the tenant,
+ # and a `{}` per pid that once held a since-deleted column is dead weight that only ever
+ # grows. Caught by `prune()` reporting rows nobody could see as dropped.
+ for pid in [p for p, row in data['cells'].items() if not row]:
+ data['cells'].pop(pid, None)
+
+ _write(table_key, _drop, st, flush='sync')
+ return hit[0]
+
+
+# ----------------------------------------------------------------------------- the VALUES
+def cells(table_key, pids, st=None):
+ """`{"": {field_key: value}}` for the rows named by `pids` — and ONLY those.
+
+ ⛔ `pids` IS REQUIRED. See the module header: this is the row wall expressed as a signature,
+ so a caller cannot get "every shared cell in the tenant" by forgetting an argument. Pass the
+ pool you have already scoped for this session.
+
+ An empty `pids` legitimately returns `{}` — a reader with no rows sees no cells, which is the
+ correct answer rather than a special case.
+ """
+ if pids is None:
+ raise TypeError('shared_overlay.cells: pids is required — pass the row set you have '
+ 'already scoped for this reader (there is deliberately no "all" call)')
+ wanted = {_pid(p) for p in pids}
+ if not wanted:
+ return {}
+ stored = _read(table_key, st)['cells']
+ return {pid: dict(row) for pid, row in stored.items()
+ if pid in wanted and isinstance(row, dict) and row}
+
+
+def put_cell(table_key, pid, field_key, value, st=None):
+ """Write ONE shared cell. Returns the value as stored (which may be truncated)."""
+ return put_cells(table_key, pid, {field_key: value}, st=st).get(str(field_key or ''))
+
+
+def put_cells(table_key, pid, values, st=None):
+ """Write several shared cells on one row, in one store update. Returns what was stored."""
+ row_id = _pid(pid)
+ clean = {str(k): _value(v) for k, v in dict(values or {}).items() if str(k)}
+ if not clean:
+ return {}
+
+ def _patch(data):
+ data['cells'].setdefault(row_id, {}).update(clean)
+
+ _write(table_key, _patch, st)
+ return clean
+
+
+def clear_row(table_key, pid, st=None):
+ """Forget every shared cell on one row. True when there was something to forget."""
+ row_id = _pid(pid)
+ hit = [False]
+
+ def _clear(data):
+ hit[0] = data['cells'].pop(row_id, None) is not None
+
+ _write(table_key, _clear, st)
+ return hit[0]
+
+
+def prune(table_key, live_pids, st=None):
+ """Drop shared cells for rows that no longer exist. Returns how many rows were dropped.
+
+ ⛔ NOT a scoped read wearing another name, and the difference is the whole reason `cells()`
+ refuses to serve everything: `live_pids` here means *every row this TABLE has*, which is a
+ fact about the data, whereas `pids` in `cells()` means *every row this READER may see*, which
+ is a fact about the session. Calling this with one session's pool would delete the shared
+ values of every row that session cannot see. The name says which is which; so does this note.
+ """
+ keep = {_pid(p) for p in live_pids}
+ dropped = [0]
+
+ def _prune(data):
+ gone = [pid for pid in data['cells'] if pid not in keep]
+ for pid in gone:
+ data['cells'].pop(pid, None)
+ dropped[0] = len(gone)
+
+ _write(table_key, _prune, st, flush='sync')
+ return dropped[0]
diff --git a/platform/core/store.py b/platform/core/store.py
index 332610d7227fca6ddb624ec8ee029cc22803fc4e..a062cc3169a225d3d853ef5122d25b29a5f6c510 100644
--- a/platform/core/store.py
+++ b/platform/core/store.py
@@ -33,6 +33,16 @@ REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
+#: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP.
+#:
+#: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
+#: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude
+#: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
+#: change, which is the truthful answer: the client's copy predates this process entirely.
+#:
+#: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality.
+REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
+
def _token():
return os.environ.get('HF_TOKEN') or None
@@ -63,6 +73,12 @@ class Store:
self._flushers = {} # name -> live worker thread
self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
self._last_flush = {} # name -> monotonic time of the last completed upload
+ # --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------
+ # name -> how many accepted writes this process has applied to that bucket, and when.
+ # Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for
+ # why that constraint is the whole feature rather than an optimisation.
+ self._revs = {}
+ self._rev_at = {}
def available(self):
"""True when a token is configured — callers degrade gracefully without one."""
@@ -131,6 +147,49 @@ class Store:
except Exception:
return True
+ # ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6)
+ def _bump(self, name):
+ """Advance one bucket's write counter. THE CALLER HOLDS THE LOCK.
+
+ Called from the two places an accepted write becomes visible to readers of this process
+ — `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT
+ from `_flush_worker`: that uploads data a bump already accounted for, and counting it
+ again would report a change to every browser each time the coalescing worker woke.
+ """
+ n = str(name)
+ self._revs[n] = self._revs.get(n, 0) + 1
+ self._rev_at[n] = _time.time()
+
+ def revision(self, name):
+ """`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else.
+
+ ⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why
+ this method exists at all. A browser asking "has anything changed" by re-fetching rows
+ costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way
+ in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s
+ per bucket — so two tabs polling saturate the server and every unrelated write queues
+ behind them. This performs no download, no `get()`, and no copy of the value.
+ ⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace`
+ counts `get`/`_read_strict`/`_download` on the serving path and goes red on one).
+
+ `rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend
+ ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops
+ being inert the moment D-4 flips, with the wire shape unchanged.
+
+ ⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write
+ by a SECOND container against the same HF dataset is invisible here. That is not a limit
+ this introduces — `get()` caches a bucket for the life of the process and never re-reads
+ it, so the other container's write is invisible to every reader on this backend already.
+ The token is exactly as fresh as the store it reports on. Postgres has no such gap.
+ ⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted:
+ an attachment is fetched on explicit user action, never polled.
+ """
+ n = str(name)
+ with self._lock:
+ rev = int(self._revs.get(n, 0))
+ at = self._rev_at.get(n)
+ return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'}
+
def _upload(self, name, data):
"""The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra
API round-trip on every put(), pure latency after the first success."""
@@ -193,6 +252,7 @@ class Store:
self._cache[name] = json.loads(json.dumps(data))
self._owned[name] = True
self._last_flush[name] = _time.monotonic()
+ self._bump(name) # C6 — AFTER the upload: a raise here is not a write
return self._cache[name]
def _flush_worker(self, name):
@@ -322,6 +382,7 @@ class Store:
raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
self._cache[name] = json.loads(json.dumps(data))
self._owned[name] = True
+ self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time
self._schedule_flush(name)
return data
self.put(name, data)
@@ -475,6 +536,20 @@ def update(name, fn, flush='sync'):
return _d().update(name, fn, flush=flush)
+def revision(name):
+ """The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing.
+
+ ⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS
+ HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely
+ so a capability one backend lacks cannot be reached through the shared door and die at the
+ cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change
+ and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half
+ had no owner: a method is a capability the HF store has, a module function is a promise both
+ backends keep.
+ """
+ return _d().revision(name)
+
+
def _flush_all_at_exit():
with _INSTANCES_LOCK:
instances = list(_INSTANCES.values())
diff --git a/platform/core/store_backend.py b/platform/core/store_backend.py
index d8bda2c3dae751315225835737d92fcf37178968..24185f19b9875fd7312ac2c0c62a738644741ddd 100644
--- a/platform/core/store_backend.py
+++ b/platform/core/store_backend.py
@@ -55,9 +55,13 @@ def active():
return _hf
-# --- the nine delegates. Written out rather than generated with setattr so that reading this file
+# --- the TEN delegates. Written out rather than generated with setattr so that reading this file
# tells you the whole interface, and so a new store operation cannot be added to core/store.py
# without something here failing to match (verify_store_pg.py compares the signatures).
+#
+# ⭐ That guard did its job in wave 29: `revision()` landed in `core/store.py` and `core/store_pg.py`
+# and the gate went red HERE, on the one file of the four that the change had missed. A generated
+# delegate list would have covered it silently and left this file unable to describe itself.
def available():
return active().available()
@@ -92,3 +96,13 @@ def delete_path(path_in_repo):
def flush(name_=None, timeout=30.0):
return active().flush(name_, timeout=timeout)
+
+
+def revision(name_):
+ """The bucket's change token (wave 29, C6) — `{'rev', 'updated_at', 'token'}`.
+
+ Both backends answer, and they answer with the same keys and the same units (`updated_at` is
+ epoch seconds on each). That parity is the reason the change-token endpoint needs no branch on
+ the backend, and it is asserted rather than assumed by `verify_store_pg`'s interface section.
+ """
+ return active().revision(name_)
diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py
index ddc303c909d30661c3be16415bcad2c4db3c5361..50edd2df6e0f8b0fa270a9ad9a8896466c29e73d 100644
--- a/platform/core/store_pg.py
+++ b/platform/core/store_pg.py
@@ -233,6 +233,37 @@ def delete_path(path_in_repo, tenant_slug=None):
return True
+def revision(name, tenant_slug=None):
+ """`{'rev', 'updated_at', 'token'}` for ONE bucket — wave 29, item 20 / contract C6.
+
+ ⭐ THIS BACKEND HAS MAINTAINED THE ANSWER SINCE X4 AND NEVER PUBLISHED IT. Every `put` and
+ `update` above already ends `updated_at = now(), rev = {bare}.rev + 1`; all this adds is the
+ read door. Which is why C6 insisted the HF side use these two column NAMES for its in-memory
+ counter: the wire shape does not change at the cutover, so the client keeps polling the same
+ endpoint and the token simply stops being per-process.
+
+ ⭐ AND HERE THE TOKEN IS DURABLE, which is the real upgrade. `core.store`'s counter lives in
+ one process, so it cannot see a write by a second container and it restarts at zero — hence
+ the process stamp in its token. `rev` is a column: it survives a restart, and it is shared by
+ every process pointed at this schema. The prefix below is therefore a constant rather than a
+ per-process value, and a backend flip changes the token exactly once (one refetch, correct).
+
+ An absent key is `rev 0`, not an error: a bucket nobody has written yet has changed zero times,
+ which is a fact a client can hold a baseline against.
+ """
+ with _pool().connection() as con:
+ row = con.execute(
+ _q('SELECT rev, updated_at FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
+ (str(name),)).fetchone()
+ rev = int(row[0]) if row and row[0] is not None else 0
+ stamp = row[1] if row else None
+ # ⚠ EPOCH SECONDS ON BOTH BACKENDS. `core.store` records `time.time()`; this column is a
+ # `timestamptz`. Same key, two types, is how a consumer that works on one backend breaks on
+ # the other — the exact class this module's interface parity exists to prevent.
+ at = stamp.timestamp() if hasattr(stamp, 'timestamp') else None
+ return {'rev': rev, 'updated_at': at, 'token': f'pg:{rev}'}
+
+
def flush(name=None, timeout=30.0):
"""A no-op that returns True: every write above is already committed when it returns. Kept in
the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
@@ -306,6 +337,9 @@ class PgStore:
def update(self, name, fn, flush='sync'):
return update(name, fn, flush=flush, tenant_slug=self.tenant_slug)
+ def revision(self, name):
+ return revision(name, tenant_slug=self.tenant_slug)
+
def upload_bytes(self, path_in_repo, data, message=None):
return upload_bytes(path_in_repo, data, message=message, tenant_slug=self.tenant_slug)
diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py
index c54f123c8fbff466b9feab7ce7a27071fed90320..34a3821dad7bc41b85809b0e884f7e781b083f02 100644
--- a/platform/core/user_tables.py
+++ b/platform/core/user_tables.py
@@ -165,7 +165,12 @@ MAX_JSON_CELL = 32 * 1024 * 1024
#: Where a profile handle points. A CLOSED vocabulary, for the reason every other closed enum
#: here is closed: a source nobody wrote a reader for renders as a column that promises a link.
-PROFILE_SOURCES = ('instagram',)
+#:
+#: ⭐ WAVE-29 CONTRACT C2 (owner R1/R2, 2026-08-11): `tiktok` joins it, so a `ut_tt_*` row can
+#: carry a profile flag the enrich action resolves. **A source is not a NAME, it is three rules**
+#: (below) — the vocabulary and the rules move together, because adding the word alone would
+#: accept a TikTok handle and then hand it an `instagram.com` link from `profile_url`.
+PROFILE_SOURCES = ('instagram', 'tiktok')
#: Hosts whose `//` path IS an Instagram profile. `www.` is stripped before the lookup.
_IG_HOSTS = ('instagram.com', 'instagr.am')
@@ -179,6 +184,55 @@ _IG_RESERVED = frozenset({'p', 'reel', 'reels', 'tv', 'stories', 'explore', 'acc
#: Instagram's own handle rule: 1–30 of letters, digits, period, underscore.
_IG_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{1,30}$')
+#: TikTok's profile hosts. ⛔ `vm.tiktok.com` is deliberately ABSENT: those are short links that
+#: usually resolve to a VIDEO, and accepting one would store a "handle" (the opaque code) that can
+#: never enrich — the same failure `_IG_RESERVED` exists to prevent, one layer earlier.
+_TT_HOSTS = ('tiktok.com', 'm.tiktok.com')
+
+#: TikTok's non-handle path segments. `tiktok.com/@user/video/123` is already refused by the
+#: one-segment rule; these are the ONE-segment paths that are still not people.
+_TT_RESERVED = frozenset({'video', 'tag', 'music', 'discover', 'search', 'foryou', 'explore',
+ 'live', 'upload', 'about', 'legal', 'privacy', 'terms', 'effect',
+ 'business', 'ads', 'node', 'login', 'signup'})
+
+#: TikTok's own handle rule: 2–24 of letters, digits, period, underscore. ⚠ NOT Instagram's —
+#: the charset matches but the bounds do not, and a shared regex would accept a 30-character
+#: TikTok "handle" that the vendor answers nothing for.
+_TT_HANDLE_RE = re.compile(r'^[A-Za-z0-9._]{2,24}$')
+
+#: ⭐ C2 — the per-source rules, ONE table, so `PROFILE_SOURCES` cannot grow a member that no
+#: normaliser knows. Every function below reads THIS rather than branching on the word: a source
+#: with no entry refuses at the door instead of silently taking Instagram's rules and Instagram's
+#: URL, which is what "add the word and ship" would have produced.
+#:
+#: `at_handle` = the profile path wears a leading `@` (TikTok's `/@name`; Instagram's `/name`).
+#: It is stripped before validation and re-added by `profile_url`, so the STORED truth is the bare
+#: handle in both families and a filter on the column cannot miss half of it.
+_PROFILE_RULES = {
+ 'instagram': {
+ 'hosts': _IG_HOSTS,
+ 'reserved': _IG_RESERVED,
+ 'handle': _IG_HANDLE_RE,
+ 'at_handle': False,
+ 'url': 'https://www.instagram.com/{handle}/',
+ },
+ 'tiktok': {
+ 'hosts': _TT_HOSTS,
+ 'reserved': _TT_RESERVED,
+ 'handle': _TT_HANDLE_RE,
+ 'at_handle': True,
+ 'url': 'https://www.tiktok.com/@{handle}',
+ },
+}
+
+
+def _profile_rules(source):
+ """The rule bag for a source, or None — the ONE resolution of "is this source real"."""
+ key = str(source or '').strip().lower()
+ if key not in PROFILE_SOURCES:
+ return None
+ return _PROFILE_RULES.get(key)
+
#: The C1 preset keys a profile field OWNS — the cells R6 clears when its handle is blanked.
#:
#: MIRRORED from the engine's C1 set as local literals, the same discipline (and for the same
@@ -227,17 +281,24 @@ def _clean_profile(raw):
def normalize_profile(value, source='instagram'):
"""A typed profile cell → `(handle, ok)`. THE cell validator the flag promises (C3).
- Accepts a bare handle (`name`), an at-handle (`@name`) or an Instagram profile URL in any of
- the forms a person actually pastes (`https://www.instagram.com/name/?hl=en`,
- `instagram.com/name`). Returns the BARE HANDLE — the URL form is derived for display, so the
- stored truth has exactly one spelling and a filter on it cannot miss half the column.
+ Accepts a bare handle (`name`), an at-handle (`@name`) or a profile URL of THIS SOURCE in any
+ of the forms a person actually pastes (`https://www.instagram.com/name/?hl=en`,
+ `instagram.com/name`, `https://www.tiktok.com/@name`). Returns the BARE HANDLE — the URL form
+ is derived for display, so the stored truth has exactly one spelling and a filter on it cannot
+ miss half the column.
+
+ ⛔ C2: the host, the reserved segments and the handle rule all come from `_PROFILE_RULES[source]`.
+ A TikTok URL pasted into an Instagram-flagged column is REFUSED, and vice versa — a cell that
+ names the wrong network is not a handle this row's automation can enrich, and quietly accepting
+ it would defer the failure to a run that then answers nothing.
`('', True)` for a blank: clearing a profile cell is legal and is the whole of R6.
`(None, False)` for anything else, and the caller refuses the write with a sentence — a
value that is neither is not a handle we can enrich, and storing it would put the failure
off until the automation runs and answers nothing.
"""
- if str(source or '').strip().lower() not in PROFILE_SOURCES:
+ rules = _profile_rules(source)
+ if rules is None:
return (None, False)
raw = str(value if value is not None else '').strip()
if not raw:
@@ -259,16 +320,19 @@ def normalize_profile(value, source='instagram'):
host = (parts.netloc or '').split('@')[-1].split(':')[0].strip().lower()
if host.startswith('www.'):
host = host[4:]
- if host in _IG_HOSTS:
+ if host in rules['hosts']:
segs = [s for s in (parts.path or '').split('/') if s]
- if len(segs) != 1 or segs[0].lower() in _IG_RESERVED:
- # Two segments is a post/reel/highlight, zero is the site itself. Refused rather
+ # C2: TikTok's profile path is `/@name`; Instagram's is `/name`. The `@` is stripped
+ # here and re-added by `profile_url`, so both families STORE a bare handle.
+ first = segs[0].lstrip('@') if segs else ''
+ if len(segs) != 1 or not first or first.lower() in rules['reserved']:
+ # Two segments is a post/reel/video, zero is the site itself. Refused rather
# than taking the first segment, which is what turns `/p/Cxyz` into handle `p`.
return (None, False)
- cand = segs[0]
+ cand = first
elif host:
- return (None, False) # a URL, but not an Instagram one
- if not _IG_HANDLE_RE.match(cand) or cand.lower() in _IG_HOSTS:
+ return (None, False) # a URL, but not one of THIS source's
+ if not rules['handle'].match(cand) or cand.lower() in rules['hosts']:
# The host check catches a bare `instagram.com` — it matches the handle charset, so
# without this it would normalise to a "handle" that can only ever resolve to nothing.
return (None, False)
@@ -277,11 +341,18 @@ def normalize_profile(value, source='instagram'):
def profile_url(handle, source='instagram'):
"""The DERIVED display form of a stored handle (C3: *"the URL form is derived for display"*).
- Derived and never stored, so the two spellings cannot drift apart."""
+ Derived and never stored, so the two spellings cannot drift apart.
+
+ ⛔ C2 — the template comes from the SOURCE's own rules. This function used to hardcode
+ `instagram.com` while its guard merely checked membership of `PROFILE_SOURCES`, so the moment
+ that tuple grew a second member it would have handed every TikTok handle an Instagram link —
+ a guard that passes and a body that lies ([[hardcoded-fallback-not-hardcoded-key]]).
+ """
h = str(handle or '').strip().lstrip('@')
- if not h or str(source or '').strip().lower() not in PROFILE_SOURCES:
+ rules = _profile_rules(source)
+ if not h or rules is None:
return ''
- return f'https://www.instagram.com/{h}/'
+ return rules['url'].format(handle=h)
def _profile_of(fields, exclude=None):
@@ -355,6 +426,54 @@ FORMAT_TZ = ('local', 'utc')
FORMAT_MAX_DECIMALS = 4
+#: ⭐ WAVE-29 CONTRACT C7 — the COLUMN-SUMMARY vocabulary a field's `agg` may take.
+#:
+#: MIRRORED from `aios_grid.FIELD_AGGS` (session C publishes it) and from the client's
+#: `aggregations.FIELD_AGGS`, as a local literal for the reason `METRIC_MEASURES` and
+#: `PROFILE_PRESET_KEYS` are local literals here: `core/` must stay importable without the layers
+#: above it. Three copies, one law, held in step by C's parity gate rather than by an import.
+#:
+#: ⛔ NOT `CHART_AGGS` (which spells it `avg` and gatekeeps STORED chart values with live data)
+#: and NOT `ROLLUP_FNS` (16 fold names, overlapping but not the same list). Merging any two of the
+#: three silently turns live stored charts into sums.
+FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count')
+
+#: A choice colour, as the client's editor spells one. `#RRGGBB`, upper-cased on the way in so a
+#: stored value has exactly ONE spelling and two fields cannot disagree about `#ebf6ef`.
+_OPTION_COLOR_RE = re.compile(r'#[0-9A-F]{6}')
+
+
+def _clean_option_colors(raw, options):
+ """`{option label → #RRGGBB}`, capped to the field's own option vocabulary (wave-29 C1).
+
+ ⛔ MIRRORED from `aios_grid._clean_option_colors` as a LOCAL function rather than imported, for
+ the reason `METRIC_MEASURES` and `PROFILE_PRESET_KEYS` are local literals here: `core/` must
+ stay importable without the layers above it, and `aios_grid` sits above this module. The pair
+ is held in step by a gate leg, never by an import.
+
+ ⚠ THE CAP IS THE POINT, not tidiness: colours are keyed by LABEL, so a renamed or deleted
+ option would otherwise leave a colour behind that nothing renders and no editor can reach —
+ and on the next rename that orphan could re-attach to a different option with the same name.
+ Case-insensitive on the way in, canonical on the way out, so `Done` and `done` cannot both
+ claim the same option.
+ """
+ if not isinstance(raw, dict):
+ return {}
+ supplied = {}
+ for label, color in raw.items():
+ if not isinstance(label, str) or not isinstance(color, str):
+ continue
+ clean = color.strip().upper()
+ if _OPTION_COLOR_RE.fullmatch(clean):
+ supplied[label.strip().lower()] = clean
+ out = {}
+ for option in options or []:
+ color = supplied.get(str(option).strip().lower())
+ if color:
+ out[str(option)] = color
+ return out
+
+
def _clean_format(raw):
"""One `format` bag → the stored shape, or None.
@@ -489,10 +608,12 @@ def clean_fields(raw):
if entry is None:
seen.discard(key)
continue
- # `agg` decides whether the grid's totals row SUMS the column. A rollup that declares it
- # through one door and not the other totals differently depending on who made it.
- if f.get('agg') in {'sum'}:
- entry['agg'] = 'sum'
+ # `agg` decides HOW the grid's totals row summarises the column. A rollup that declares it
+ # through one door and not the other summarises differently depending on who made it.
+ # ⭐ WAVE-29 C7: the vocabulary is `FIELD_AGGS`, not the single word `sum` — see its note.
+ agg_in = str(f.get('agg') or '').strip()
+ if agg_in in FIELD_AGGS:
+ entry['agg'] = agg_in
out.append(entry)
return out or None
@@ -702,6 +823,72 @@ def add_row(table_key, values=None, username=None, st=None, rid=None):
return rid
+def add_rows(table_key, rows_in, username=None, st=None):
+ """⭐⭐ WAVE-29 T25 (owner item 6) — APPEND MANY ROWS IN ONE STORE WRITE. Returns the list of
+ new row ids, or None if the table refuses records at all.
+
+ ⛔ WHY THIS EXISTS RATHER THAN A LOOP OVER `add_row`. Every `add_row` is a read-modify-write of
+ the WHOLE table document plus a row event; importing a 2,000-row spreadsheet that way is 2,000
+ full-document copies under one lock, which is the same shape as the 1.4 s-per-bucket problem
+ item 20 exists to avoid — on the one uvicorn process this product runs. One `update()` writes
+ them all.
+
+ ⛔ AND WHY NOT `automation_engine.upsert_rows`, which this module's own note (below) says to
+ reuse: that function is UPSERT-BY-KEY shaped, and v1 of the import door is deliberately
+ APPEND-ONLY (no key to merge on, no silent overwrite of a row somebody edited). It also lives
+ in the API layer, and `core/` may not import upward. When the import door grows an "update
+ matching rows" mode, the engine's is the one to lift — the note stands, it just is not this.
+
+ ⚠ THE CAP IS CHECKED AGAINST THE WHOLE BATCH, not per row: a partial import that stops at
+ `MAX_ROWS` leaves the user reconciling a spreadsheet against a table, which is exactly what
+ the refusal-with-a-sentence exists to prevent. Nothing is written if the batch does not fit.
+
+ ⚠ Ids continue the table's own sequence, and a PROFILE column validates every cell exactly as
+ `add_row` does — a row born with an unusable handle is a row the enrich action can never
+ answer, and the failure would otherwise surface as an automation quietly returning nothing.
+ """
+ if not is_user_table(table_key, st) or not records_mutable(table_key, st):
+ return None
+ batch = [dict(r) for r in (rows_in or []) if isinstance(r, dict)]
+ if not batch:
+ return []
+ defn = get(table_key, st) or {}
+ rows = defn.get('rows') or {}
+ if len(rows) + len(batch) > MAX_ROWS:
+ return None
+ valid = {f['key'] for f in (defn.get('fields') or [])}
+ pf = _profile_of(defn.get('fields') or [])
+ next_id = max((int(r) for r in rows if str(r).isdigit()), default=0)
+ made = []
+ for values in batch:
+ clean = {k: str(v) for k, v in values.items() if k in valid}
+ if pf and pf['key'] in clean:
+ handle, ok = normalize_profile(clean[pf['key']], pf['profile'].get('source'))
+ if not ok:
+ return None # atomic: one bad handle refuses the FILE
+ clean[pf['key']] = handle
+ next_id += 1
+ made.append((str(next_id), clean))
+
+ def _add(cur):
+ t = cur.get(str(table_key))
+ if t is not None:
+ bucket = t.setdefault('rows', {})
+ for rid, clean in made:
+ bucket[rid] = clean
+ return cur
+
+ _st(st).update(STORE_KEY, _add, flush='sync')
+ # ⚠ ONE EVENT PER ROW, deliberately: `record_created` is a trigger substrate, and an import of
+ # 500 rows genuinely IS 500 created records. The engine's own flood hold is what decides
+ # whether that runs an automation 500 times — a decision that belongs there, not to a writer
+ # that quietly emits less than it did.
+ for rid, _clean in made:
+ emit_row_event({'type': 'record_created', 'table': str(table_key), 'rowId': rid,
+ 'st': _st(st), 'user': str(username or '')})
+ return [rid for rid, _ in made]
+
+
# ⛔ `upsert_rows` USED TO LIVE HERE, AND DELETING IT IS THE FIX (DEBT D-6, wave 20).
#
# There were TWO bulk-upsert implementations — this one and `automation_engine.upsert_rows` —
@@ -1150,6 +1337,57 @@ def _clean_field(raw, previous=None):
opts = [' '.join(str(o).split())[:60] for o in (opts_raw or []) if str(o).strip()][:50]
if opts:
out['options'] = opts
+ # ⭐⭐ WAVE-29 CONTRACT C1 (owner item 13) — **THE OPTION COLOURS, WHICH THIS DOOR DROPPED.**
+ #
+ # The client has carried `optionColors`/`colorCodeOptions` since wave 14 (`types.ts`, the
+ # `BRAND_SWATCHES` editor, `cells.optionTint`, the kanban card) and `aios_grid._clean_field`
+ # keeps both — but THIS validator, the one every `ut_*` create and patch goes through, had
+ # zero occurrences of either key. So the owner picked a colour on a user database, the pane
+ # saved, and the colour was gone on the next read: the same crossed-validator shape as item 2a,
+ # in a different pair of doors. `grid_events._FIELD_ECHO_PROPS` already carried both, so the
+ # echo wire was plumbed for a value nothing stored.
+ #
+ # ⛔ SIBLING KEYS, and `options` IS NOT RESHAPED. Every consumer treats options as `string[]`
+ # (this function flattens each to a bare `str` twelve lines up); folding colour into the option
+ # entries would be a wire change across the grid, the kanban, the editor and the filter panel.
+ #
+ # ⛔ OMIT-WHEN-ABSENT, exactly like `format`/`metric`/`link`, because `verify_api.py` and
+ # `verify_automation.py` assert `_clean_field(f) == f` over every engine-seeded field list —
+ # a branch that emitted `optionColors: {}` would redden both gates for every preset column in
+ # the product. ([[default-must-pass-its-own-guard]])
+ #
+ # ⚠ INHERITED ACROSS A PATCH, for `agg`'s reason: a PATCH that sends only a label must not
+ # silently strip the colours somebody chose, which is precisely how a money column stopped
+ # totalling.
+ #
+ # ⚠ A HEX IS VALIDATED; BRAND MEMBERSHIP IS NOT, and that is deliberate. `_clean_option_colors`
+ # (`aios_grid.py:200`) — the sibling this mirrors — accepts any `#RRGGBB`, and the client
+ # renders a stored colour through `choiceColors.brandedTint`, which maps it to its NEAREST
+ # brand swatch at DISPLAY time while leaving what is stored byte-stable. Refusing a non-brand
+ # hex here would refuse values the other validator accepts on the same UI, and would break a
+ # column that moved between the two.
+ if ftype in ('select', 'multiselect'):
+ colors_raw = (raw.get('optionColors') if 'optionColors' in raw
+ else prev.get('optionColors'))
+ colors = _clean_option_colors(colors_raw, out.get('options') or [])
+ if colors:
+ out['optionColors'] = colors
+ color_code = (raw.get('colorCodeOptions') if 'colorCodeOptions' in raw
+ else prev.get('colorCodeOptions'))
+ if isinstance(color_code, bool):
+ out['colorCodeOptions'] = color_code
+ # ⭐ WAVE-29 T22's own repair: a `rating` column's star count. Grid-created columns on a `ut_*`
+ # database now come through this door rather than the per-user overlay, and this validator had
+ # no `max` — so a 3-star or 10-star column silently became 5-star on the way in. Bounded rather
+ # than refused, mirroring `aios_grid._clean_rating_max`: the field still holds its 1..max
+ # integers whatever the number was, so an unusable value is a default and never a lost column.
+ if ftype == 'rating':
+ max_raw = raw.get('max') if 'max' in raw else prev.get('max')
+ if max_raw is not None:
+ try:
+ out['max'] = max(2, min(int(max_raw), 10))
+ except (TypeError, ValueError):
+ pass
role = str(raw.get('editRole') or prev.get('editRole') or 'admins').strip().lower()
out['editRole'] = role if role in FIELD_EDIT_ROLES else 'admins'
if prev.get('default') is True or raw.get('default') is True:
@@ -1182,8 +1420,19 @@ def _clean_field(raw, previous=None):
# which found `agg` instead.
# ⚠ Sticky across a patch for the same reason as `default` and `pinned`: a client PATCH that
# omits the key must not un-total a column as a side effect of renaming its header.
- if str(raw.get('agg') or prev.get('agg') or '').strip() == 'sum':
- out['agg'] = 'sum'
+ # ⭐ WAVE-29 C7 — the vocabulary widened from the single word `sum` to `FIELD_AGGS`. The
+ # STICKINESS is unchanged and now applies to every name: a PATCH that omits `agg` keeps
+ # whatever the column had, because dropping it is how a money column silently stopped
+ # totalling. ⛔ An unknown name is DROPPED rather than stored — a summary nothing can compute
+ # would render blank under a picker that says otherwise.
+ # ⚠ `'agg' in raw`, not `raw.get('agg') or …` — the same idiom `format` and `description` use
+ # two screens up, and it is what makes the difference between OMITTING the key (keep what is
+ # stored: the wave-28 stickiness that stops a rename un-totalling a money column) and sending
+ # it EMPTY (clear the summary, which the picker's "None" row has to be able to do). Written
+ # the loose way, "None" would have been a no-op with the menu cheerfully showing the change.
+ agg_name = str((raw.get('agg') if 'agg' in raw else prev.get('agg')) or '').strip()
+ if agg_name in FIELD_AGGS:
+ out['agg'] = agg_name
# An automation column carries the binding the engine reads; it is preserved across a patch
# rather than re-declared, because the automation editor owns it and this route does not.
auto = raw.get('automation') if 'automation' in raw else prev.get('automation')
diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py
index 21a1d00a1e088d387cc4afc263ebff309aaa26b8..390bd20f9c366621c901b9109ceda4633918d414 100644
--- a/platform/harness/datastore.py
+++ b/platform/harness/datastore.py
@@ -170,9 +170,20 @@ ENTITIES = {
# table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent")
# marks agents who came from staff; the owner ruling on whether those count as agents is
# OUTSTANDING, so both flags are synced and neither reading is baked in.
+ # ⭐ WAVE 29 (W29-T54, amendment A4). `customer_rank` is the ONLY predicate that
+ # distinguishes "a partner Odoo has TRANSACTED with" (today's population, 2,465) from
+ # "a customer" (3,614 with `rank>0 active`) — a 1,149-partner drop, the same join-drop
+ # class as item 22's SKU pool. It cannot be derived from the document tables, because
+ # the missing partners are precisely the ones with no document.
+ # ⛔ It is synced HERE rather than read from live Odoo because `read_customers` also runs
+ # at BOOT off a hydrated snapshot, where Odoo may be unreachable — sourcing it live would
+ # make the population "whatever source answered this time", changing size with the network.
"fields": ["name", "team_id", "city", "state_id", "country_id", "active",
- "agent", "salesman_as_agent", "write_date"],
- "backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28
+ "agent", "salesman_as_agent", "customer_rank", "write_date"],
+ # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next
+ # sync_all() instead of needing a reseed.
+ "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28
+ "customer_rank"], # added 2026-08-11 (W29-T54)
# m2m fields land as _id (the FIRST linked id — the Customers-module convention:
# a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a
# JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
@@ -256,6 +267,16 @@ BATCH = 2000
# from "unknown".
BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free")
+# ⭐ WAVE 29 — fields that are genuinely INTEGER COUNTS, and this list exists because its absence
+# shipped a silent defect within minutes of `customer_rank` being added to the res_partner spec.
+# `_cols` types anything it does not recognise as VARCHAR, so `customer_rank` landed as text, the
+# mirror populated correctly, every row looked right — and `odoo_relational`'s own predicate
+# `customer_rank > 0` died with `Binder Error: Cannot compare values of type VARCHAR and type
+# INTEGER_LITERAL`. ⛔ THE COLUMN WAS PRESENT AND THE DATA WAS CORRECT; only its TYPE was wrong,
+# which is why "is the field in the mirror?" answered yes and the feature still could not run.
+# ⇒ A new numeric field belongs HERE, exactly as a new boolean belongs in BOOL_FIELDS above.
+INT_FIELDS = ("customer_rank", "supplier_rank")
+
import threading as _thr
@@ -444,6 +465,8 @@ def _cols(spec):
cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"]
elif f in BOOL_FIELDS:
cols.append(f"{f} BOOLEAN")
+ elif f in INT_FIELDS:
+ cols.append(f"{f} BIGINT")
elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed",
"price_subtotal", "margin", "purchase_price", "product_uom_qty",
"standard_price", "debit", "credit", "balance", "amount"):
diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py
index ada17a006d90572237c1205aa310c1eb4d3e8d21..f84b4ccadef38b0b4ad40b5c25c30ae77cb67892 100644
--- a/platform/modules/product_data.py
+++ b/platform/modules/product_data.py
@@ -30,11 +30,32 @@ template and this deliberately mirrors its signature and its row shape.
Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
does not pretend to cover columns that are absent.
+
+4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
+ 2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the
+ union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
+ showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
+
+ · ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
+ window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
+ that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
+ ~2,550 active SKUs have never sold in wholesale scope at all.
+ · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
+ revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and
+ not this one, and that zero is a measurement. A row that appears in NO revenue read
+ carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
+ client renders as an empty cell. Blank is an admission; zero is a measurement.
+ · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
+ that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
+ UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
+ carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two
+ sources are JOINED rather than merged.
"""
import json
import zlib
from pathlib import Path
+import core.odoo as O
import core.periods as P
import core.table_store as table_store
import modules.products as products
@@ -155,31 +176,66 @@ def _inventory_by_code(t):
return {}
+def _catalogue_by_code():
+ """`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from.
+
+ ⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
+ Inventory degrades to `{}` because a product table with blank stock columns beats one that
+ will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that
+ failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
+ green and nothing on screen saying so, which is the defect this seam exists to end
+ ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
+ into a 503 that names the cause, so the loud failure has somewhere honest to land.
+
+ It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
+ same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
+ to build a pool without Odoo.
+ """
+ return products.catalogue()
+
+
def pool(team_id=None, t=None):
- """One row per SKU across the scoped book — the PRODUCT analogue of `customer_data.pool`.
+ """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
+
+ CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
+ is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
+ that sold in its window and contributes NO rows of its own.
`team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
- driving it rather than a post-filter deciding the BU.
+ driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
+ catalogue has no team.
"""
t = t or P.today()
consolidated = team_id is None
inv = _inventory_by_code(t) if consolidated else {}
sup = supplier_master()
+ cat = _catalogue_by_code()
+ # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
+ # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those
+ # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
+ # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
+ rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
rows = []
- for r in products.directory(t=t, team_id=team_id):
- code = r["code"]
+ for code, meta in cat.items():
+ r = rev.get(code)
row = {
"pid": sku_pid(code),
"code": code,
- "product": r.get("product") or code,
- "category": r.get("category") or "(uncategorized)",
- "rev_ytd": r.get("rev_ytd", 0.0),
- "rev_ly": r.get("rev_ly", 0.0),
- "yoy_pct": r.get("yoy_pct"),
- "qty_ytd": r.get("qty_ytd", 0.0),
- "orders_ytd": r.get("orders_ytd", 0),
+ # ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's
+ # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
+ # record's `display_name` is the current truth, and it is the same string for every
+ # SKU that is not re-SKUed. `directory()` derives the category identically.
+ "product": meta.get("product") or code,
+ "category": meta.get("category") or "(uncategorized)",
+ # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
+ # keeps its measured numbers including a real 0.0 (it sold last year, not this one).
+ "rev_ytd": r.get("rev_ytd", 0.0) if r else None,
+ "rev_ly": r.get("rev_ly", 0.0) if r else None,
+ "yoy_pct": r.get("yoy_pct") if r else None,
+ "qty_ytd": r.get("qty_ytd", 0.0) if r else None,
+ "orders_ytd": r.get("orders_ytd", 0) if r else None,
}
# Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock).
s = sup.get(code) or {}
@@ -227,6 +283,10 @@ def pool(team_id=None, t=None):
row["buy_now"] = None
rows.append(row)
+ # `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU
+ # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
+ # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
+ # nothing sold".
rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
_assert_no_pid_collision(rows)
return rows
@@ -277,20 +337,68 @@ def validate(team_id=None, t=None):
"""Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number
which does not tie to Odoo does not ship.
- The check is deliberately narrow and says so: Σ per-SKU YTD revenue must equal the SKU
- module's own YTD total over the same scope. It does NOT cover the inventory columns (they are
- a different module's reconciliation, and they are absent on a scoped pull anyway).
+ ⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
+ (wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the
+ same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
+ not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
+ No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
+ That is the leg below, and it is the control that would have caught it
+ ([[no-unverifiable-aggregates]]).
+
+ ⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join
+ lose money?") and is still the right shape for that one. What changed is that it now has to
+ account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
+ products out of the grid; the total is decomposed rather than compared loosely, so a growing
+ "outside the grid" figure shows up as a number rather than as slack in a tolerance.
+
+ Still deliberately NOT covered, and said rather than implied: the inventory columns are a
+ different module's reconciliation, and they are absent on a scoped pull anyway.
"""
t = t or P.today()
rows = pool(team_id=team_id, t=t)
- ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
- yf, yt = P.ytd(t)
- theirs = round(sum(v.get("rev", 0.0)
- for v in (products._sku_rev(yf, yt, team_id) or {}).values()), 2)
+
+ # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
+ # products, asked fresh — not a `len()` over anything this module or `products.catalogue()`
+ # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
+ n_active = products.catalogue_count()
checks = [{
- "check": "Σ per-SKU YTD revenue == modules.products._sku_rev over the same window",
- "ours": ours, "theirs": theirs, "ok": abs(ours - theirs) < 0.01,
+ "check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT "
+ "Odoo search_count('product.product', active=True)",
+ "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
}]
+
+ yf, yt = P.ytd(t)
+ sku_rev = products._sku_rev(yf, yt, team_id) or {}
+ in_pool = {r["code"] for r in rows}
+ outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
+ ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
+ theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
+ dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
+ checks.append({
+ "check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product "
+ "carries == modules.products._sku_rev over the same window",
+ "ours": round(ours + dropped, 2), "theirs": theirs,
+ "ok": abs((ours + dropped) - theirs) < 0.01,
+ "detail": {"in_the_grid": ours, "outside_the_grid": dropped,
+ "codes_outside": len(outside)},
+ })
+ # ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
+ # trivially true when checked against the dict that did the dropping — that is the
+ # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
+ # product carries means `products.catalogue()` missed a row that has revenue, which is the
+ # 2,717 defect returning in a smaller costume.
+ # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
+ # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
+ coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
+ still_active = (O.get_odoo().search_count(
+ 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
+ if coded_outside else 0)
+ checks.append({
+ "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
+ "stay out) — asked of Odoo, never of the catalogue that did the dropping",
+ "ours": still_active, "theirs": 0, "ok": still_active == 0,
+ "detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
+ })
# WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every
# member of it must be re-derivable from the two columns beside it. A signal somebody buys
# stock on cannot be "mostly right": the failure that matters is a row that says OK because
diff --git a/platform/modules/products.py b/platform/modules/products.py
index b5b6f1435abd29d24aeea8498422793ee840a087..e996a9895c5eecddff30c3a8febf5a07cf8e36f8 100644
--- a/platform/modules/products.py
+++ b/platform/modules/products.py
@@ -166,6 +166,62 @@ def directory(t=None, team_id=None):
return rows
+def catalogue():
+ """`{sku_code: {'product', 'category'}}` — EVERY ACTIVE product, sold or not.
+
+ ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
+ row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
+ that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`,
+ `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe —
+ and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
+ 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
+ the consumer, never a window removed from `directory()`: removing the window alone lands at
+ 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
+
+ ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
+ catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
+ merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
+
+ Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active
+ records that carry none — so the join against `directory()` is code-for-code with no
+ normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
+ off a sale line, so a never-sold row wears the same format as a sold one.
+
+ ⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull
+ would put the grid back at a plausible wrong number with every gate green — the exact failure
+ this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
+ `modules/backorders.py:161` already uses).
+ """
+ dom = [('active', '=', True)]
+ prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
+ limit=50000)
+ n = O.get_odoo().search_count('product.product', dom)
+ if len(prods) != n:
+ raise ValueError(
+ f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against "
+ f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
+ f"nothing reporting it; raise the limit before shipping this.")
+ code_cat = _code_category()
+ out = {}
+ for p in prods:
+ code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
+ # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
+ # active products share a code, so this branch is a guard, not a merge policy.
+ out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
+ 'category': code_cat.get(code, '(uncategorized)')})
+ return out
+
+
+def catalogue_count():
+ """The INDEPENDENT population oracle: Odoo's own count of active products.
+
+ Deliberately a bare `search_count` and not a `len()` over anything this module built — the
+ 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
+ of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
+ """
+ return O.get_odoo().search_count('product.product', [('active', '=', True)])
+
+
def categories(t=None, team_id=None):
"""Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
return sorted({r['category'] for r in directory(t, team_id)})
diff --git a/web/src/automation/AutomationBuilder.tsx b/web/src/automation/AutomationBuilder.tsx
index 9ace4a3e0bf722b16460656f5b63ba667899be12..aab76973023167a6ad9d10beb4fc5df936636117 100644
--- a/web/src/automation/AutomationBuilder.tsx
+++ b/web/src/automation/AutomationBuilder.tsx
@@ -2544,6 +2544,95 @@ export function ActionProps({
At most 12 — the provider returns a profile’s top 12 posts and no more.
+ {/*
+ ⭐⭐ W29-T09 (owner item 11: *"the last 12 reels"*) — THE ONLY DOOR TO `config.postGroups`.
+ The server side has been whole since wave 28 — `clean_post_groups` validates it, the
+ enrich runner applies it — and `postGroups` had **zero occurrences** anywhere in
+ `web/src`, so NO USER COULD ASK FOR REELS-ONLY AT ALL. That is why this is a blocker for
+ the reels routing work and not a nicety: those tickets' negative controls need a config
+ a person can actually produce.
+
+ ⛔ THE KEYS ARE THE SERVER'S AND SO ARE THE WORDS. `vocab.postTypes` carries both; a
+ client that translated `video` into "Reels" locally would be a second copy of a
+ vocabulary `clean_post_groups` refuses deviations from. Absent vocabulary renders
+ NOTHING rather than three guessed names.
+
+ ⛔ ABSENT IS OFF, AND OFF IS THE DEFAULT FOREVER. An enrich action stored before this
+ control carries no `postGroups`, and the runner returns the post list unchanged when the
+ key is missing — so unchecking every box must DELETE the key, never store `[]`-with-
+ meaning or a group of zero. Deleting it is what keeps an old automation capturing
+ exactly what it always captured.
+
+ ⚠ ONE WRITER FOR ONE KEY [W-12]. This is the only control in `web/src` that writes
+ `postGroups`; the `maxPosts` box above writes `maxPosts` and nothing else. The two are
+ related only in that a group cannot keep more posts than the pull captures — which is
+ the server's rule, printed here rather than re-implemented.
+ */}
+ {(vocab?.postTypes || []).length ? (
+
+
Keep only certain posts
+ {(vocab?.postTypes || []).map((pt) => {
+ const groups =
+ ((cfg as { postGroups?: { type: string; limit: number }[] }).postGroups) || [];
+ const mine = groups.find((g) => g && g.type === pt.key);
+ const cap = Math.max(
+ 1,
+ Number((cfg as { maxPosts?: number }).maxPosts || 10) || 10
+ );
+ /* ⛔ WRITE THROUGH ONE FUNCTION, so the "no groups left ⇒ remove the key" rule
+ exists once. Two call sites each deciding it is how `[]` starts meaning
+ "capture nothing". */
+ const write = (next: { type: string; limit: number }[]) =>
+ setCfg({ postGroups: next.length ? next : undefined });
+ return (
+
+
+ write(
+ e.target.checked
+ ? [...groups.filter((g) => g.type !== pt.key),
+ { type: pt.key, limit: Math.min(cap, 12) }]
+ : groups.filter((g) => g.type !== pt.key)
+ )
+ }
+ />
+ {pt.label}
+ {mine ? (
+
+ write([
+ ...groups.filter((g) => g.type !== pt.key),
+ {
+ type: pt.key,
+ limit: Math.max(1, Number(e.target.value) || 1),
+ },
+ ])
+ }
+ />
+ ) : null}
+
+ );
+ })}
+
+ Leave these unticked to keep every post. Ticked, only the kinds you name are
+ kept — and a group cannot keep more than the {" "}
+ {Number((cfg as { maxPosts?: number }).maxPosts || 10) || 10} posts this
+ enrichment captures.
+
+
+ ) : null}
{/*
⭐ WAVE 28 · R6/R7 — THE TWO CHECKBOXES THAT STOOD HERE MOVED UP INTO THE INCLUDE
GROUP, KEYS AND ALL (`postMetrics`, `commentMetrics`). They are not deleted features:
diff --git a/web/src/automation/AutomationSurface.tsx b/web/src/automation/AutomationSurface.tsx
index 5254c2469a730c59b326ec061aa9b06715c44c67..a4f925f5b42428bc4312a0999a33545f0c69a173 100644
--- a/web/src/automation/AutomationSurface.tsx
+++ b/web/src/automation/AutomationSurface.tsx
@@ -216,7 +216,7 @@ export default function AutomationSurface() {
* asked for" and "what is on screen" are written in one statement and cannot drift apart.
* A resolution that wants to steer the rail compares itself against `wantedId` instead.
*/
- const select = useCallback((id: string, _stage = "") => {
+ const select = useCallback((id: string, _stage = "") => {
wantedId.current = id;
setActiveId(id);
}, []);
@@ -527,7 +527,21 @@ export default function AutomationSurface() {
time a third kind shipped, and printed inside a `--lp-rail-w` column. What the
kinds are belongs to the create form, which is one click away and lists all of
them from the server. */}
- {!items.length && !error ? (
+ {/* ⭐ WAVE 29 (W29-T01) — A LOADING RAIL IS NOT AN EMPTY ONE. `data` is null until the
+ first `listAutomations()` resolves, and this branch read only `items.length`, so
+ every visit to Automation asserted "No automations yet." to a person who has
+ several — then replaced it with their list. Half of owner item 5's "takes a while
+ to appear" is that sentence: the wait is real, but being told you own nothing is
+ what makes it read as broken rather than slow. `data === null` is the one state
+ that means NOBODY HAS LOOKED, and it gets skeleton rows, not a claim. */}
+ {data === null && !error ? (
+
+
+
+
+
+ ) : null}
+ {data !== null && !items.length && !error ? (
No automations yet.
) : null}
diff --git a/web/src/automation/TriggerPicker.tsx b/web/src/automation/TriggerPicker.tsx
index 7a05510a501a71bf3606f554f672ff25744aa08f..ba792d6aa2b5e53261d1b5a359d38e1daa17109c 100644
--- a/web/src/automation/TriggerPicker.tsx
+++ b/web/src/automation/TriggerPicker.tsx
@@ -154,8 +154,13 @@ export function TriggerMark({ kind = "" }: { kind?: string }) {
);
// A MUSICAL NOTE inside the same search frame as `ig_profile_match`: same act (search a corpus
- // of profiles), different corpus. Drawing it as a second person-in-a-magnifier would say the
- // two are interchangeable, which is exactly the claim D-9 is still open about.
+ // of profiles), different corpus. Drawing it as a second person-in-a-magnifier would say the two
+ // are interchangeable, and they are not — the two networks have different schemas and, per wave
+ // 29's R2, different `ut_*` families behind them.
+ // ⚠ WAVE 29: this row is LIVE now (D-9 closed the trigger half). Nothing in this file changed to
+ // make it so — `planned` rides the wire, which is the property that makes a server vocabulary
+ // worth having. The sentence that used to end this note ("the claim D-9 is still open about")
+ // was true when it was written and would have quietly aged into a lie.
if (kind === "tiktok_profile_match")
return (
diff --git a/web/src/automation/automationApi.ts b/web/src/automation/automationApi.ts
index ccfafafcffe25857a81b315246dd0327edc06f71..52fe1dba17b0c2713d32ac6194423c53823e3f69 100644
--- a/web/src/automation/automationApi.ts
+++ b/web/src/automation/automationApi.ts
@@ -472,6 +472,18 @@ export interface FlowVocab {
maxCondChildren: number;
maxGroupDepth: number;
maxActions: number;
+ /**
+ * ⭐ W29-T09 (owner item 11: *"the last 12 reels"*) — the post kinds a capture group may name,
+ * with the words a person reads. OPTIONAL, like every other vocabulary here: absent means a
+ * server older than this client, and the control renders nothing rather than inventing three
+ * names the save door would refuse.
+ * ⛔ `key` is what `config.postGroups` STORES; `label` is only ever painted. Writing the label
+ * would build a filter `clean_post_groups` rejects with a sentence naming a word nobody typed.
+ */
+ postTypes?: { key: string; label: string }[];
+ /** The ceiling `clean_post_groups` judges a group's limit against (the action's own maxPosts
+ * is the real bound; this is the absolute one, so a hint can be honest about both). */
+ maxPostsPerPull?: number;
/** Legacy read compatibility only; the server no longer sends Board review vocabulary. */
reviewDeciders?: string[];
/** Legacy read compatibility only; the server no longer sends Board review readiness. */
diff --git a/web/src/customer-grid/ColumnMenu.tsx b/web/src/customer-grid/ColumnMenu.tsx
index 819309eb539f29f8b525cf39ce850306530d853f..704dbfbd499edc9348d486309d3796c9c8dd2afe 100644
--- a/web/src/customer-grid/ColumnMenu.tsx
+++ b/web/src/customer-grid/ColumnMenu.tsx
@@ -7,13 +7,14 @@ import { CODE_LANGUAGE_LABELS, CODE_LANGUAGES, CREATABLE_TYPES, choiceOptions, c
codeLanguageOf, directionLabel, isDerivedLink, isMachineOwned,
isProfileField, parseOptions, ratingMax, ROLLUP_FN_LABELS, ROLLUP_FNS,
ROLLUP_REF_OPS } from "./types";
-import type { Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
+import type { AggName, Field, FieldFormat, FieldScope, FieldType, Measure, RollupCondition, RollupFn,
RollupRefOp, RollupSource, Viewer } from "./types";
import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
import type { WindowSpec } from "./windows";
import { normalizeWindow, windowLabel } from "./windows";
import { WindowPicker } from "../filter-kit";
import { TYPE_LABELS } from "./iconShapes";
+import { AGG_LABELS, aggOptions } from "./aggregations";
import { validateFormula } from "./formulaEngine";
import { BRAND_SWATCHES, defaultOptionColor, nearestSwatch, normalizeOptionColor,
OPTION_PALETTE } from "./choiceColors";
@@ -238,6 +239,10 @@ interface ColumnMenuProps {
/** Wave-5 item 10 — save a display format. Supplied for number/currency/formula and
* date/created_time fields. */
onFormat?: (format: FieldFormat) => void;
+ /** ⭐ WAVE-29 T33 (owner item 17) — set (or clear, with `undefined`) this column's SUMMARY: the
+ * value its group footers and the table's totals row show. Supplied wherever a field write is
+ * possible; the pane offers only what `aggregations.aggOptions` allows for the type. */
+ onAggregate?: (agg: AggName | undefined) => void;
/** Wave-5 item 1 — Sort by this field (replaces the view's sort), and the conditional
* clears (item 3). All of these mutate the EXISTING view config — no new host surface. */
onSort: (dir: "asc" | "desc") => void;
@@ -273,7 +278,7 @@ type CreatePosition = "left" | "right" | "end";
* choices/stars/formula config), and the Change-field control that swaps the column to show
* another field.
*/
-type MenuPane = "menu" | "note" | "edit" | "permissions" | "format";
+type MenuPane = "menu" | "note" | "edit" | "permissions" | "format" | "summary";
/**
* How each creatable type reads in the menu. DERIVED from CREATABLE_TYPES rather than listed
@@ -1738,6 +1743,7 @@ export default function ColumnMenu({
onDuplicate,
onPermissions,
onFormat,
+ onAggregate,
onSort,
onClearSort,
onFilterBy,
@@ -3146,6 +3152,63 @@ export default function ColumnMenu({
}
// ------------------------------------------------------------------ FORMAT pane (item 10)
+ /**
+ * ⭐⭐ WAVE-29 T33 (owner item 17) — THE COLUMN SUMMARY PANE.
+ *
+ * A flat list of the six aggregates plus "None", because that is the whole decision — there is
+ * no draft to reconcile and no Save button to press, so the click IS the answer and the pane
+ * closes on it (the same shape as Sort, one group up, and unlike Format, which composes four
+ * settings into one bag).
+ *
+ * ⛔ THE OPTIONS COME FROM `aggOptions`, never from a list written here. `sum · average ·
+ * median · min · max` are numeric-only and `count` counts ROWS, so a text column offers exactly
+ * one row — and it offers it, rather than hiding the pane, because "how many records is this
+ * group" is the summary a text column CAN answer.
+ */
+ if (pane === "summary" && onAggregate) {
+ const options = aggOptions(field);
+ return (
+
+
+
+ {
+ onAggregate(undefined);
+ onClose();
+ }}
+ >
+
+
+ {options.map((name) => (
+ {
+ onAggregate(name);
+ onClose();
+ }}
+ >
+
+
+ ))}
+
+
+ );
+ }
+
if (pane === "format" && formatKind) {
return (
)}
+ {/* ⭐ WAVE-29 T33 (owner item 17) — the COLUMN SUMMARY. Offered on every column, because
+ `count` applies to any type; the numeric folds are filtered per type inside the pane
+ by `aggOptions`. ⚠ NOT gated on `schemaLocked`: a summary is a reading of the column,
+ not a change to its schema, so it belongs on a pre-set column too — the same reason
+ Sort and Filter sit below without that gate. */}
+ {onAggregate && (
+ setPane("summary")}>
+
+
+ )}
diff --git a/web/src/customer-grid/CustomerGrid.tsx b/web/src/customer-grid/CustomerGrid.tsx
index 22d697a89667b76c352c97eef5ab73c4ed1726ee..dff2701b26b4911b1c84f332edf495322847148a 100644
--- a/web/src/customer-grid/CustomerGrid.tsx
+++ b/web/src/customer-grid/CustomerGrid.tsx
@@ -1,6270 +1,6570 @@
-import {
- memo,
- useCallback,
- useEffect,
- useLayoutEffect,
- useMemo,
- useRef,
- useState,
-} from "react";
-import { DataEditor, GridCellKind } from "@glideapps/glide-data-grid";
-import type {
- CellClickedEventArgs,
- DataEditorRef,
- EditableGridCell,
- GridKeyEventArgs,
- GridMouseEventArgs,
- GridSelection,
- DrawHeaderCallback,
- HeaderClickedEventArgs,
- Item,
- Rectangle,
- Theme,
-} from "@glideapps/glide-data-grid";
-import "@glideapps/glide-data-grid/dist/index.css";
-
-import { useCustomerData } from "./useCustomerData";
-import type { SurfaceScope } from "./apiBridge";
-import { defaultViewConfig, useGridColumns } from "./useGridColumns";
-import { activeMeasureRuleIds, pendingMeasures, runPipeline, unresolvedConditions, useVisibleRows }
- from "./useVisibleRows";
-import type { CohortSets } from "./useVisibleRows";
-import type { MeasureSets } from "./useVisibleRows";
-import { useGetCellContent } from "./useGetCellContent";
-import { useGridSelection } from "./useGridSelection";
-import Toolbar from "./Toolbar";
-import JsonViewer from "./JsonViewer";
-import RecordDetail from "./RecordDetail";
-import ViewSidebar from "./ViewSidebar";
-import ColumnMenu from "./ColumnMenu";
-import type { ColumnMenuState } from "./ColumnMenu";
-import { HEADER_ICONS } from "./iconShapes";
-import { emitHostEvent, eventId } from "./hostBridge";
-// ⭐ WAVE 27 item 21 (R14) — which of this table's views carry an alert. The SERVER's
-// list is the only answer; the rail's alert door is a one-way create.
-import { fetchAlerts } from "../alerts/alertsApi";
-import { NAV_MINIMIZE_EVENT, ROWS_STALE_EVENT, TOAST_EVENT, VIEW_OPEN_EVENT,
- WORKSPACE_STALE_EVENT, signal }
- from "../apiContract";
-import type { ViewOpenDetail } from "../apiContract";
-import { addTableField, addTableRow, deleteTableField, deleteTableRow, fetchLinkTargets,
- fetchRollupSources, patchTableField } from "./apiBridge";
-import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
-import SelectFromFile from "./SelectFromFile";
-// Owner item 16 / R4 / C-UNDO — the stack and every inverse. Pure, so a node gate can run it.
-import { describe, directed, popRedo, popUndo, pushUndo, stackFor } from "./undoStack";
-import type { CellChange, UndoBook, UndoEntry, UndoRow, UndoValue } from "./undoStack";
-import { exportFilename, runExport, triggerDownload } from "./export";
-import type { ExportFormat } from "./export";
-// owner item 3 (2026-08-03) — a time-series view exports its SHEET, not the rows under it.
-import { buildTsCsv, tsSheetToTable } from "./timeSeriesData";
-import type { TsSheet } from "./timeSeriesData";
-import { echoReemit, reconcileEchoView } from "./viewEcho";
-import { pruneStamps, reconcileFields } from "./optimism";
-import { applyViewOrder, newFolderId, pruneFolderStamps, reconcileFolders,
- resolveFolderId } from "./folders";
-import type { FolderStamps } from "./folders";
-// ⭐ WAVE 27 item 22 — the cells the active view's filter forces on a record added under it.
-import { filterSeedValues } from "./filterSeed";
-import { adoptNewFields, adoptNewViews, pruneTombstones, seedLocalViews, stampTombstone }
- from "./liveWorkspace";
-import type { Tombstones } from "./liveWorkspace";
-import type { GridFolder, ViewPermissions } from "./types";
-import type { FieldStamps } from "./optimism";
-import { evalFormula, orderFormulas, parseFormula } from "./formulaEngine";
-import type { FormulaAst } from "./formulaEngine";
-import { CalendarView, KanbanView, ListView, ModeSwitch } from "./viewModes";
-import { SwipeView } from "./SwipeView";
-import type { SwipeSpec } from "./SwipeView";
-// The chart engine lives in `viz/` since EXIT wave 2 (W2-5/Y3) — the grid is now
-// one of its two callers, the Y1 page envelope being the other.
-import { DashboardView } from "../viz/DashboardView";
-import { cleanCharts } from "../viz/chartData";
-import type { ChartSpec } from "../viz/chartData";
-import { MapView } from "./MapView";
-/* ═══ W18-C CATALOG ═══ (owner item 4, contract C6) */
-import { CatalogView } from "./CatalogView";
-import { CATALOG_CODE_FIELD } from "./catalogData";
-import type { CatalogSpec } from "./types";
-/* ═══ end W18-C CATALOG ═══ */
-import { lightTheme, STATUS_ROW_THEME, HOVER_ROW_THEME, HOVER_NEUTRAL,
- ACTIVE_ROW_NEUTRAL, CUSTOM_FIELD_MARK } from "./theme";
-import { avatarInitials, formatDisplay, imageCellRenderer, ratingCellRenderer, setAvatarRepaint,
- userCellRenderer } from "./cells";
-import { optionTint, pickTint } from "./choiceColors";
-import {
- EMPTY_GRID_COPY_PROVENANCE,
- markGridCopy,
- observeCopyEvent,
- planFieldPaste,
- pasteRowCount,
-} from "./clipboard";
-import { cellTipText, expandButtonRect, GROUP_HEADER_FONT, GROUP_LABEL_PAD, headerMarkLayout,
- headerMarkSizes, tipLeft } from "./overlayPlacement";
-import { AnchoredOverlay, BodyPortal, useOverlayLayer } from "./OverlaySurface";
-import type { AnchorRect } from "./OverlaySurface";
-import { StarIcon } from "./Stars";
-import { ALL_VIEW_ID, MAX_CALENDAR_METRICS, allViewName,
- MAX_FROZEN, choiceOptions, choiceVocabulary, clampFrozenCount, cleanDisplay, formulaOf,
- topicForScope,
- isDateFamilyType, isFilterGroup, isGroupableField, isMachineOwned, isMachineWritten,
- isDerivedLink,
- isNumericFieldType,
- isPickType, mayEditField,
- isModeFrozen, isUndeletableView, mayEditView, mayToggleViewLock,
- measureColumnIndex, ratingMax, ruleColumnKeys,
- tableMode, uniqueDisplayName } from "./types";
-import type {
- DisplayMode,
- DisplaySpec,
- Field,
- HostEvent,
- FieldScope,
- FieldType,
- FilterNode,
- FilterRule,
- Row,
- RowHeightMode,
- SavedView,
- ViewConfig,
-} from "./types";
-// Item 7 (C-TS). Default-exported because it is a leaf VIEW like MapView, and because RECORD
-// imports the same default for the record-detail Insights tab (C-EMBED).
-import TimeSeriesPanel from "./TimeSeriesPanel";
-import type { WindowSpec } from "./windows";
-import { windowLabel } from "./windows";
-
-const ROW_PX: Record = { short: 28, medium: 34, tall: 48 };
-
-/**
- * Wave-14 R7 — **a button label is ONE LINE.** Every button row in this file is a flex row with
- * no width reservation, so on a narrow grid box the items shrink and their labels wrap:
- * "Remove from cohort" becomes two lines, the row grows, and the bar stops being a bar. It only
- * happens below a width no fixed-size screenshot is taken at, which is why it survived.
- *
- * ⚠ Applied INLINE rather than on `.cg-btn`, where it belongs: `index.css` is PANEL's fence this
- * wave. A mailbox line asks them to hoist it onto `.cg-btn` globally — after which this is
- * redundant, and harmless, because it says exactly the same thing.
- */
-const ONE_LINE = { whiteSpace: "nowrap" } as const;
-
-/**
- * How many VISIBLE columns glide freezes. Module-level and used twice on purpose (item 15): the
- * `freezeColumns` prop and the group bar's label fit must read ONE answer, because the label is
- * clipped to exactly this strip and a second copy of the clamp would let the two disagree by a
- * column — which shows up as a label cut early for no visible reason.
- */
-function frozenCountOf(config: ViewConfig): number {
- return Math.min(MAX_FROZEN, Math.max(1, config.frozenCount ?? 1));
-}
-/* WAVE 21 item 3 (R6): the id and the NAME both moved to `types.ts` — the id because a second
- copy of a pinned literal is the drift class this repo gates against, the name because it is
- now topic-derived and the host mints the same string. */
-
-/**
- * owner item 2 (2026-08-03) — the measure-cell skeleton's pulse, and its ceiling.
- *
- * 140ms against the 4-step colour ramp is a ~0.6s cycle: a wait, not a strobe. The ceiling is
- * ~60s, after which the cells fall back to ordinary blanks — a measure that has not resolved in
- * a minute is not "still loading", and a shimmer that never ends promises a number that is not
- * coming. Shared constant so the two are read in one place rather than tuned apart.
- */
-const PULSE_MS = 140;
-const PULSE_MAX_TICKS = Math.round(60_000 / PULSE_MS);
-/** Referentially stable, so the fallback does not change `getCellContent`'s identity per render
- * and repaint the canvas forever. */
-const NO_PENDING_KEYS: ReadonlySet = new Set();
-
-/**
- * I2 — measure a cell's text in GLIDE'S OWN font, so "is this cut off?" is a fact rather than
- * a character-count guess (a guess is wrong in both directions: "IIIII" is narrow, "WWWWW" is
- * wide, and being wrong means either a missing tip or a tip over text you can already read).
- * One lazily-built offscreen context for the whole module — `measureText` is cheap, but
- * creating a canvas per mouse-move would not be.
- */
-let _tipCtx: CanvasRenderingContext2D | null | undefined;
-function measureCellText(text: string): number {
- if (_tipCtx === undefined) {
- _tipCtx = document.createElement("canvas").getContext("2d");
- // glide's default `baseFontStyle` is 13px; the family comes from our own theme.
- if (_tipCtx) _tipCtx.font = `13px ${lightTheme.fontFamily ?? "Inter, sans-serif"}`;
- }
- return _tipCtx ? _tipCtx.measureText(text).width : 0;
-}
-/**
- * Item 15 — the GROUP BAR's measurer, and it is a SECOND context on purpose.
- *
- * `measureCellText` above is set to plain "13px", which is right for ordinary cells and WRONG for
- * a group bar: that row carries `baseFontStyle: GROUP_HEADER_FONT` (semibold), and glide paints a
- * cell with the merged theme's font. Measuring the fit with the lighter weight under-truncates
- * and the label overflows into the frozen strip's hard clip — the exact failure `fitGroupLabel`
- * exists to prevent. Font string built from the SAME constant the cell's themeOverride uses, so
- * the two cannot drift.
- */
-let _groupCtx: CanvasRenderingContext2D | null | undefined;
-function measureGroupText(text: string): number {
- if (_groupCtx === undefined) {
- _groupCtx = document.createElement("canvas").getContext("2d");
- if (_groupCtx)
- _groupCtx.font = `${GROUP_HEADER_FONT} ${lightTheme.fontFamily ?? "Inter, sans-serif"}`;
- }
- return _groupCtx ? _groupCtx.measureText(text).width : 0;
-}
-/**
- * Owner item 8 (2026-07-27): the first render shows this many rows; "See more" reveals the rest.
- * A DISPLAY cap only — the pipeline still runs over the whole book, the toolbar count is still
- * the full matched count, and the cap is stated beside the control that lifts it, which is what
- * keeps [[no-unverifiable-aggregates]] satisfied: nothing is silently truncated.
- */
-const DISPLAY_PAGE = 50;
-const DISPLAY_STEP = 250;
-
-/* ═══════════════════════════════════════════════════════════════════════════════════════════
- ═══ W18-B VOID ═══ (wave 18, owner item 1b) — the two constants the void geometry needs
- that belong to the BROWSER and to GLIDE rather than to us. Both are mirrors, and both say
- here what they are mirroring, because a mirror that does not name its original is how the
- two copies stop agreeing.
- ═══════════════════════════════════════════════════════════════════════════════════════════ */
-
-/** The header band. `headerHeight={36}` at the DataEditor mount is the original.
- * Not `+ groupHeaderHeight`: glide adds that only when `enableGroups` is on, which it turns on
- * when a COLUMN carries a `group` — `useGridColumns` never sets one (scrolling-data-grid.js:13). */
-const HEADER_PX = 36;
-
-/** glide's own row-marker width ladder, `data-editor.js:103`, verbatim.
- *
- * ⚠ It is passed BACK to glide as `rowMarkerWidth` rather than merely predicted here. The
- * marker column is part of the content width the void's left edge is measured from, so a
- * four-pixel disagreement between what glide draws and what we compute is a four-pixel seam of
- * white against the tint — visible, and invisible to every gate. Pinning makes the two the same
- * number by construction; mirroring the ladder (rather than pinning one constant) is what keeps
- * the pin from changing the marker column's width on tables of more than 100 rows. */
-function rowMarkerPx(rows: number): number {
- return rows > 10_000 ? 48 : rows > 1000 ? 44 : rows > 100 ? 36 : 32;
-}
-
-/** The scrollbar gutter this browser steals from a scrollable box, measured once.
- *
- * ⛔ WHY THE VOID HAS TO KNOW. The void rectangles sit ON TOP of glide's scroller, so a
- * rectangle drawn to the box's own edge paints over the scrollbar and hides it. The two cases
- * are CROSSED, which is the part that is easy to get backwards: the BOTTOM void has to stop
- * short when the columns overflow (a horizontal scrollbar), and the RIGHT void has to stop
- * short when the rows overflow (a vertical one). Both happen in ordinary use — twelve columns
- * filtered to five rows is exactly the owner's case.
- *
- * 0 on overlay-scrollbar platforms (macOS, touch), which is correct: nothing is stolen there. */
-let scrollbarGutterPx: number | null = null;
-function scrollbarGutter(): number {
- if (scrollbarGutterPx !== null) return scrollbarGutterPx;
- if (typeof document === "undefined") return 0;
- const probe = document.createElement("div");
- probe.style.cssText =
- "position:absolute;top:-9999px;width:100px;height:100px;overflow:scroll";
- document.body.append(probe);
- scrollbarGutterPx = probe.offsetWidth - probe.clientWidth;
- probe.remove();
- return scrollbarGutterPx;
-}
-/* ═══ end W18-B VOID (module scope) ═══ */
-
-interface LocalWorkspace {
- fields: Field[];
- views: SavedView[];
- activeViewId: string;
- /** When this copy was written (browser clock — same-machine freshness only,
- * never date semantics). Absent on copies from before the echo reconcile. */
- savedAt?: number;
- /** Wave-6 item 3c — per-key freshness for this browser's own field-def
- * writes (edits + tombstones), consumed by reconcileFields at init. */
- fieldStamps?: FieldStamps;
- /** BUG-1 (wave 11) — viewId → the filter tree already pushed back to the host, so the
- * echo re-emit happens once per user edit rather than once per remount. NOT a timestamp:
- * `savedAt` beside it is rewritten on every init, so any time-based bound is refreshed by
- * the remount it is meant to bound. See viewEcho.echoReemit. */
- reemitted?: Record;
- /** 2026-08-04 — viewId → when THIS browser deleted it. The views half of the tombstone
- * rule fields and folders already had, needed the moment the rail adopts host views
- * after mount (liveWorkspace.ts). Persisted, not a ref, because a remount inside the
- * echo window would otherwise re-adopt a view the user deleted a second ago. */
- viewTombstones?: Tombstones;
- /** D-19 (wave 20) — viewId → when THIS browser last WROTE it. The counterpart of the
- * tombstones above: they stop a deleted view coming back, this stops a just-created one
- * being dropped by the host-list rule at init, before its upsert has round-tripped.
- * Same window, same prune, same browser-clock caveat. */
- viewWrites?: Tombstones;
-}
-
-function sameConfig(a: ViewConfig, b: ViewConfig): boolean {
- return JSON.stringify(a) === JSON.stringify(b);
-}
-
-function normalizeConfig(config: Partial | undefined, fields: Field[]): ViewConfig {
- const base = defaultViewConfig(fields);
- return {
- ...base,
- ...config,
- filters: Array.isArray(config?.filters) ? config.filters : [],
- // Legacy views persisted a flat rule array with an implicit AND — anything
- // other than an explicit "or" normalizes to "and", so old views are unchanged.
- filterConj: config?.filterConj === "or" ? "or" : "and",
- sorts: Array.isArray(config?.sorts) ? config.sorts : [],
- order: Array.isArray(config?.order) && config.order.length ? config.order : base.order,
- visible:
- Array.isArray(config?.visible) && config.visible.length
- ? config.visible
- : base.visible,
- widths: config?.widths && typeof config.widths === "object" ? config.widths : {},
- memberPids: Array.isArray(config?.memberPids) ? config.memberPids : [],
- // Wave-6 items 10/11. Both normalize junk to ABSENT (grid / 1 frozen), so
- // a legacy view is byte-identical to itself after a round trip. W13 (C4 as
- // amended): cleanDisplay keeps grid-carried field picks.
- display: cleanDisplay(config?.display),
- frozenCount: clampFrozenCount(config?.frozenCount),
- };
-}
-
-/**
- * The pinned system view, as this browser mints it when no host copy has arrived.
- *
- * WAVE 21 item 3 (R6) — the NAME is the scope's now (`allViewName`), not the literal "All
- * customers" this minted on every topic including `ut_*` databases of anything at all. The
- * ID is unchanged and deliberately so (see `ALL_VIEW_ID`).
- */
-function allRecordsView(fields: Field[], scope: string): SavedView {
- return {
- id: ALL_VIEW_ID,
- name: allViewName(scope),
- kind: "system",
- locked: true,
- config: defaultViewConfig(fields),
- };
-}
-
-function readLocal(storageKey: string): LocalWorkspace | null {
- try {
- const raw = localStorage.getItem(`aios-grid:${storageKey}`);
- return raw ? (JSON.parse(raw) as LocalWorkspace) : null;
- } catch {
- return null;
- }
-}
-
-function writeLocal(storageKey: string, value: LocalWorkspace): void {
- try {
- localStorage.setItem(`aios-grid:${storageKey}`, JSON.stringify(value));
- } catch {
- // Storage can be blocked in hardened/opaque iframes. The host bridge remains durable.
- }
-}
-
-function nextViewId(): string {
- return `view_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
-}
-
-/** A multiselect cell's SET, out of its comma-joined string (the `multi` contract). */
-function splitMulti(v: string): string[] {
- return v
- .split(",")
- .map((s) => s.trim())
- .filter((s) => s !== "");
-}
-
-/** Wave-5 item 3 — does any leaf of the filter tree name this column?
- *
- * ⚠ Wave-20 item 2 RETARGETED the note that stood here ("measure leaves carry measure keys,
- * which are never field keys, so they cannot false-positive"). True about false POSITIVES,
- * and precisely why it was a false NEGATIVE: a measure column's own condition names the
- * MEASURE, so this answered "not filtered" for a column the user had visibly filtered, and
- * the menu never offered "Don't filter by this field" on it. Both doors now go through
- * `ruleColumnKeys` — the same resolution the tint uses, so the menu and the colour can never
- * disagree about which columns a filter is about. */
-function treeNamesField(
- nodes: FilterNode[],
- key: string,
- measureCols: Map
-): boolean {
- for (const n of nodes) {
- if (isFilterGroup(n)) {
- if (treeNamesField(n.children, key, measureCols)) return true;
- } else if (ruleColumnKeys(n as FilterRule, measureCols).includes(key)) return true;
- }
- return false;
-}
-
-/** Wave-5 item 3 — "Don't filter by this field": drop every leaf naming the column, prune
- * groups that end up empty. Returns new arrays throughout (the config is state). */
-function dropFieldFromTree(
- nodes: FilterNode[],
- key: string,
- measureCols: Map
-): FilterNode[] {
- const out: FilterNode[] = [];
- for (const n of nodes) {
- if (isFilterGroup(n)) {
- const children = dropFieldFromTree(n.children, key, measureCols);
- if (children.length) out.push({ ...n, children });
- } else if (!ruleColumnKeys(n as FilterRule, measureCols).includes(key)) {
- out.push(n);
- }
- }
- return out;
-}
-
-// The header sprite map moved to iconShapes.ts (wave-8 I20/I21) — it is data +
-// SVG-source builders with no React in it, which is what lets verify_icons.py
-// import it under node and assert the (i) actually contrasts.
-
-function slugify(label: string): string {
- return (
- label
- .toLowerCase()
- .replace(/[^a-z0-9]+/g, "_")
- .replace(/^_|_$/g, "")
- .slice(0, 28) || "field"
- );
-}
-
-interface FieldBuildExtra {
- formula?: string;
- max?: number;
- scope?: FieldScope;
- label?: string;
- colorCodeOptions?: boolean;
- optionColors?: Record;
- /**
- * ⭐⭐ 2026-08-09 — THE RELATIONAL BAGS, WHICH THIS INTERFACE USED TO DROP ON THE FLOOR.
- *
- * `ColumnMenu.FieldConfigExtra` has carried them since 2026-08-07 and `extraFor` builds a
- * complete bag; this type — the receiving end of the same `extra` argument — never declared
- * either key, so `buildOverlayField` could not copy what it could not see. The measured result
- * on nurilab: a column stored as
- * `{key:'custom_video_views_90j26', label:'Video Views', type:'rollup', source:'overlay'}`
- * with NO bag, in one user's workspace stratum, which `compute_relation_cells` never reads.
- * Named, configured, rendering, and permanently blank — the owner's *"the Rollup doesn't
- * work"*. ⛔ Both halves were needed: this type, and the `createField` route below.
- */
- link?: Record;
- rollup?: Record;
- /** Item 15 (C-RENAME) — the option renames this save carries, by row identity. Consumed by
- * `retypeField` (which emits `choice_rename`) and ignored by every create path: a field
- * being CREATED has no values to migrate. */
- renames?: { from: string; to: string }[];
-}
-
-/**
- * One constructor for a user-created field, shared by "insert field" and Change-field's
- * "New field" half (wave-2 item 5) so the two doors cannot drift. A `multiselect` carries its
- * declared options AND `multi: true` — the cell is a comma-joined SET, and the flag is what the
- * grouping contract keys on (belt to the type-derived brace in groupRows).
- *
- * Wave-5: `formula` and `created_time` are built in the HOST's OWN emission shape (source
- * 'odoo' + derived — aios_grid.READONLY_CUSTOM_TYPES), so the optimistic local def and the
- * next payload's echo are byte-identical and nothing restyles on the round trip. `rating`
- * carries top-level `max`, `formula` carries top-level `formula` (the ~20:20 contract
- * amendment: options stays the select-family list). Wave-6 item 6: both types are now
- * `filterable: true` — the host flipped its emission, and conditions on them evaluate in the
- * client engine over computedRows (this table's counts are client-mode; a windowed table's
- * columns come from the semantic model, so the SQL engine never sees these types).
- */
-/**
- * ⭐⭐ 2026-08-09 (owner ruling) — DOES THE PRE-SET LOCK CLOSE THIS COLUMN'S SCHEMA?
- *
- * Owner: *"No rollup field should be uneditable, everything is custom and changeable always."*
- * A ROLLUP holds no data of its own — it is a question asked of other rows, re-askable at any
- * time, and re-asking it costs nothing because the answer is recomputed from the authoritative
- * store on the next pass. `preset` still closes every column that HOLDS something: retyping a
- * pre-set `followers` column would strand real measurements in a column that can no longer read
- * them.
- *
- * ⛔ THE CLIENT MIRROR OF `core.user_tables.preset_editable`, and `verify_rollup_editor.py`
- * holds the two in step. This check has THREE enforcement points (here, `may_edit_field`, and
- * `routes_tables._field_or_refuse`'s sentence); a client that kept hiding the editor while the
- * route allowed the PATCH would read as "you fixed nothing", which is the failure mode a fix in
- * one of three places always wears.
- */
-function isSchemaLocked(field: Field): boolean {
- return field.automation?.preset === true && field.type !== "rollup";
-}
-
-function buildOverlayField(
- label: string,
- type: FieldType,
- options?: string[],
- extra?: FieldBuildExtra
-): Field {
- const key = `custom_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`;
- if (type === "formula" || type === "created_time") {
- return {
- key,
- label,
- type,
- source: "odoo",
- derived: true,
- filterable: true,
- default: true,
- custom: true,
- ...(type === "formula"
- ? { agg: "sum", ...(extra?.formula ? { formula: extra.formula } : {}) }
- : {}),
- };
- }
- return {
- key,
- label,
- type,
- source: "overlay",
- default: true,
- custom: true,
- agg: ["currency", "int"].includes(type) ? "sum" : undefined,
- // The relational bags ride the local definition too, so the optimistic field and the
- // server's echo describe the same column. Without them the grid held a bagless twin of a
- // field the store had configured, and `mayEditField`/`isDerivedLink` read the twin.
- ...(extra?.link ? { link: extra.link as Field["link"] } : {}),
- ...(extra?.rollup ? { rollup: extra.rollup as Field["rollup"] } : {}),
- ...((type === "select" || type === "multiselect") && options?.length ? { options } : {}),
- ...((type === "select" || type === "multiselect")
- ? {
- colorCodeOptions: extra?.colorCodeOptions !== false,
- ...(extra?.optionColors && Object.keys(extra.optionColors).length
- ? { optionColors: extra.optionColors }
- : {}),
- }
- : {}),
- ...(type === "multiselect" ? { multi: true } : {}),
- ...(type === "rating" ? { max: extra?.max ?? 5 } : {}),
- };
-}
-
-/** The surface this grid draws. `cohort` is the same table over hand-curated SETS — the server
- * confirms it by stamping `workspace.cohortMode`, which is the only thing the body below reads.
- *
- * MEMOIZED (owner item 1, 2026-07-31): the shell re-renders on every chrome state flip — nav
- * collapse, toast, settings — and an unmemoized grid re-rendered its whole 3,400-line tree
- * each time. Measured cost: 200ms-5s of main-thread block, which ate the 240ms rail-fold
- * transition whole (the fold froze, then SNAPPED — the exact "static" the owner named). The
- * props surface is one stable string, so memo makes chrome state changes free; the grid still
- * re-renders for its own state (edits, resize) and remounts on route change via `key`. */
-interface CustomerGridProps {
- scope?: SurfaceScope;
- /** Read-only Grid view embedded in a linked-record modal. It keeps the standard
- * filter/sort/search toolbar while withdrawing schema and row mutations. */
- embedded?: boolean;
- /** Exact linked pids to project from the target database. */
- embeddedRecordIds?: readonly number[];
- /** Selection mode used by an editable ordinary-Link modal. */
- embeddedSelectable?: boolean;
- embeddedSelectedIds?: readonly number[];
- onEmbeddedSelectionChange?: (recordIds: number[]) => void;
-}
-
-interface LinkGridModalProps {
- label: string;
- table: SurfaceScope;
- recordIds: readonly number[];
- editable?: boolean;
- single?: boolean;
- onSave?: (recordIds: number[]) => void;
- onClose: () => void;
-}
-
-function LinkGridModal({ label, table, recordIds, editable = false, single = false,
- onSave, onClose }: LinkGridModalProps) {
- const panelRef = useRef(null);
- const [selectedIds, setSelectedIds] = useState(() => [...recordIds]);
- const changeSelectedIds = useCallback((ids: number[]) => {
- const next = single ? ids.slice(-1) : ids;
- setSelectedIds((current) =>
- current.join(",") === next.join(",") ? current : next
- );
- }, [single]);
- useOverlayLayer({
- panelRef,
- onDismiss: onClose,
- dismissOnOutside: true,
- initialFocus: "[data-overlay-autofocus]",
- trapFocus: true,
- });
- return (
-
-
-
-
-
-
-
- {editable ? (
-
- Cancel
- { onSave?.(selectedIds); onClose(); }}
- >
- Save links
-
-
- ) : null}
-
-
-
- );
-}
-
-function CustomerGrid({
- scope = "customer",
- embedded = false,
- embeddedRecordIds,
- embeddedSelectable = false,
- embeddedSelectedIds = [],
- onEmbeddedSelectionChange,
-}: CustomerGridProps = {}) {
- // Wave 16 C-TOPIC: which TABLE this tree is drawing, derived from the one scope prop.
- const topic = topicForScope(scope);
- const {
- fields: payloadFields,
- rawRows: fetchedRows,
- payload,
- loading,
- overlayEdits,
- setOverlayEdits,
- patchOverlay,
- } = useCustomerData(scope, {
- bindSurface: !embedded,
- includeWorkspace: !embedded,
- writable: !embedded,
- });
- const embeddedIdsKey = embeddedRecordIds?.join(",") ?? "";
- /**
- * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — **ROWS THIS BROWSER JUST CREATED**, held locally
- * until the server's own copy comes back.
- *
- * ⛔ THE OWNER'S WORDS: *"adding a new record visually takes too long, I need to be able to
- * spam it."* The old path awaited the POST, then fired `ROWS_STALE_EVENT` and re-read the
- * WHOLE table — so every "+" cost a round trip plus a full refetch before anything moved, and
- * pressing it five times queued five full refetches of a table that grew by five rows.
- *
- * ⚠ ADD-ONLY AND SELF-PRUNING, which is `liveWorkspace.ts`'s discipline applied to rows: a
- * pending row is merged in only while the fetched payload does NOT carry its pid, and the
- * effect below drops it the moment the server's copy arrives. There is no third state and no
- * merge of values — the server's row wins outright, because by then it IS this row.
- */
- const [pendingRows, setPendingRows] = useState([]);
- const rawRows = useMemo(() => {
- const base = embeddedRecordIds
- ? fetchedRows.filter((row) => new Set(embeddedRecordIds).has(row.pid))
- : fetchedRows;
- if (!pendingRows.length) return base;
- const known = new Set(base.map((r) => r.pid));
- const extra = pendingRows.filter((r) => !known.has(r.pid));
- return extra.length ? [...base, ...extra] : base;
- // The scalar key keeps this stable when a caller reconstructs the same id list.
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [fetchedRows, embeddedIdsKey, pendingRows]);
- useEffect(() => {
- // ⚠ PRUNED AGAINST THE FETCHED PAYLOAD, not against `rawRows` — `rawRows` contains the
- // pending rows, so it can never report that one has been absorbed. Pruning against the
- // memo's own output is the loop that keeps a placeholder alive forever.
- if (!pendingRows.length) return;
- const known = new Set(fetchedRows.map((r) => r.pid));
- if (!pendingRows.some((r) => known.has(r.pid))) return;
- setPendingRows((cur) => cur.filter((r) => !known.has(r.pid)));
- }, [fetchedRows, pendingRows]);
- /**
- * The rows as of the last render, for callbacks that must not be rebuilt per payload.
- * `appendRow` reads it to mint the next row id the way the server does (`max + 1`); putting
- * `rawRows` in that callback's dependency list would give the trailing "+" a new identity on
- * every refetch, which is a re-render of the grid for a value only used at click time.
- */
- const rowsRef = useRef(rawRows);
- rowsRef.current = rawRows;
- /**
- * ⭐ WAVE 27 · ITEM 2 — THE NEXT ROW ID, ADVANCED SYNCHRONOUSLY AT MINT TIME.
- *
- * ⛔ A RENDER-TIME SNAPSHOT IS NOT ENOUGH FOR THE GESTURE THIS ITEM IS ABOUT. `rowsRef.current`
- * is assigned during RENDER, so two "+" presses that land before React commits the first
- * `setPendingRows` both read the same list and both mint the same id. The owner's words were
- * *"I need to be able to spam it"* — that IS the failing case, and the damage is not a
- * duplicate row: `minted` is the identity key the failure and re-anchor branches filter and
- * map on, so ONE refused write would withdraw BOTH rows and one taken id would rewrite both.
- *
- * ⚠ SEEDED FROM THE ROWS ON EVERY MINT, then advanced: `Math.max(seen, storeMax) + 1`. Reading
- * the store's max each time is what keeps the client in step after a refetch brings rows in
- * from elsewhere; the ref is what stops a burst from colliding with itself in between.
- */
- const nextRidRef = useRef(0);
- const mintRid = useCallback(() => {
- const storeMax = rowsRef.current.reduce((m, r) => Math.max(m, r.pid), 0);
- nextRidRef.current = Math.max(nextRidRef.current, storeMax) + 1;
- return nextRidRef.current;
- }, []);
- /**
- * Withdraw rows this browser is still holding locally, because they have been DELETED.
- *
- * ⛔ THE PRUNE EFFECT CANNOT DO THIS AND THAT IS THE WHOLE BUG IT FIXES. Pruning is "the
- * server's payload now CONTAINS this row, so drop my copy" — a deleted row will never appear
- * in a payload again, so a pending row that is deleted before its first refetch stays merged
- * into `rawRows` for the life of the mount. Add a record, press Ctrl+Z, and the row sits there
- * looking undeleted while the store has already lost it.
- * ⚠ CALLED AT EVERY `deleteTableRow` SITE (the selection delete and undo's `rowDelete`
- * inverse), because those are the only two doors that remove a row — and the undo one is
- * exactly the path that produces a still-pending victim.
- */
- const withdrawPending = useCallback((rids: (string | number)[]) => {
- if (!rids.length) return;
- const gone = new Set(rids.map((r) => Number(r)));
- setPendingRows((cur) => (cur.some((r) => gone.has(r.pid))
- ? cur.filter((r) => !gone.has(r.pid))
- : cur));
- }, []);
- /**
- * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — WHICH VIEWS ON THIS TABLE CARRY AN ALERT.
- *
- * ⛔ THE SERVER'S LIST IS THE ONLY ANSWER. The rail's "Alert me about new records…" is a
- * one-way CREATE (it emits an event the shell turns into a POST) and nothing in the grid
- * hears the outcome — a `no_filter` refusal is a real answer, so an optimistic local set
- * would paint a badge on a view that has no alert. `GET /alerts` is the state.
- *
- * ⚠ RE-READ ON FOCUS, the same idiom the shell's own inbox uses. An alert created or deleted
- * in another surface (the Alerts pane owns the delete door) reaches this rail on the next
- * focus rather than instantly; the alternative was for the grid to guess at the outcome of a
- * request another component made.
- * ⚠ EMBEDDED GRIDS DO NOT ASK. A linked-record grid inside a modal is not a table anyone
- * alerts on, and a second fetch per relation cell would be a workspace read per click.
- */
- const [alerted, setAlerted] = useState([]);
- useEffect(() => {
- if (embedded) return;
- let live = true;
- const pull = () => {
- void fetchAlerts().then((r) => {
- if (!live || !r.ok) return; // a failed read leaves the badges as they were
- setAlerted(r.value.filter((a) => a.topic === scope).map((a) => a.viewId));
- });
- };
- pull();
- window.addEventListener("focus", pull);
- return () => {
- live = false;
- window.removeEventListener("focus", pull);
- };
- }, [embedded, scope]);
-
- const [fields, setFields] = useState([]);
- const [views, setViews] = useState([]);
- const [activeViewId, setActiveViewId] = useState(ALL_VIEW_ID);
- const [config, setConfig] = useState(() => defaultViewConfig([]));
- const [workspaceReady, setWorkspaceReady] = useState(false);
- const [saveState, setSaveState] = useState<"saved" | "saving">("saved");
- const [search, setSearch] = useState("");
- const [collapsed, setCollapsed] = useState>(new Set());
- const [hoverRow, setHoverRow] = useState(undefined);
- /**
- * Owner item 19 — the hover-only Expand button: which record it opens and where it sits, in
- * VIEWPORT coordinates (glide's `getBounds` space, the same one `.cg-header-tip` uses).
- *
- * A real DOM button rather than a mark painted into the canvas, and that is the whole design:
- * a drawn affordance needs a second hit-test that drifts out of step with the drawing, and
- * neither `tsc` nor a screenshot can see the drift — the button is painted, and clicking it
- * does nothing ([[ui-invisible-to-assertions]]). An element IS its own hit test.
- */
- const [expandAt, setExpandAt] = useState<
- { pid: number; x: number; y: number; size: number } | null
- >(null);
- const [detailPid, setDetailPid] = useState(null);
- const [columnMenu, setColumnMenu] = useState(null);
- /** Open choice list for a `select` / `user` cell — see onCellClicked. */
- const [picker, setPicker] = useState<{
- pid: number;
- fieldKey: string;
- anchor: AnchorRect;
- } | null>(null);
- /** Owner item 5 (2026-07-31) — closing a picker RETURNS FOCUS TO THE GRID, so Enter keeps
- * navigating (pick → Enter → next record) instead of stranding focus on a dead overlay. */
- const closePicker = useCallback(() => {
- setPicker(null);
- requestAnimationFrame(() => gridRef.current?.focus());
- }, []);
- /** ⭐ Wave-23 C7 — which `json` cell the big viewer is open on. NOT anchored like the picker
- * above: a document is not a choice list, so it opens as a centred modal (the record drawer's
- * surface) rather than a popover the size of the cell it came from. */
- const [jsonAt, setJsonAt] = useState<{ pid: number; fieldKey: string } | null>(null);
- const closeJson = useCallback(() => {
- setJsonAt(null);
- requestAnimationFrame(() => gridRef.current?.focus());
- }, []);
- /** A relation opens as the target database's real Grid view, projected to the linked ids.
- * This is intentionally distinct from JsonViewer: filters, sorts, search, column visibility,
- * and the standard cell renderers all remain available inside the large modal. */
- const [linkAt, setLinkAt] = useState<{ pid: number; fieldKey: string } | null>(null);
- const closeLink = useCallback(() => {
- setLinkAt(null);
- requestAnimationFrame(() => gridRef.current?.focus());
- }, []);
- /** WAVE 21 item 11 (R10) — is "Select records from a list" open? Opened from the view
- * rail's "…" and closed by the dialog; the SELECTION it produces outlives it. */
- const [selectFromFile, setSelectFromFile] = useState(false);
- /** The selection bar's "Add to cohort" popover (owner item 2's purpose for the checkboxes). */
- const [selAddOpen, setSelAddOpen] = useState(false);
- const [selListName, setSelListName] = useState("");
- const selAddRef = useRef(null);
- /** Owner item 9 (wave 20) — its counterpart: "Remove from cohort…", in ORDINARY views. */
- const [selRemoveOpen, setSelRemoveOpen] = useState(false);
- const selRemoveRef = useRef(null);
- /** Owner item 4 / C-ADDROW — the pid the trailing "+" just created, so the cursor can land on
- * it once the re-read actually brings it back (the row does not exist locally before that). */
- const [newRowPid, setNewRowPid] = useState(null);
- /**
- * ⭐ WARN-THEN-ALLOW on delete (owner, 2026-08-06) — the SIGNATURE of the selection currently
- * armed, or `""`.
- *
- * A signature and not a boolean, deliberately: it is the pid list itself, so changing the
- * selection DISARMS automatically. A boolean would stay true while you re-selected, and the
- * second Delete would then destroy a set nobody had been warned about — which is worse than no
- * warning at all, because the warning is what made it feel checked.
- */
- const [delArmed, setDelArmed] = useState("");
- /** Wave-2 item 2c — cohort mode: the active cohort and the "+ Add customers" picker. Picks
- * ACCUMULATE across searches (search, tick, search again, confirm once). */
- const [activeCohortId, setActiveCohortId] = useState(null);
- const [addCustOpen, setAddCustOpen] = useState(false);
- const [addCustQuery, setAddCustQuery] = useState("");
- const [addCustPicked, setAddCustPicked] = useState>(new Set());
- const addCustRef = useRef(null);
- /** Wave-5 item 1 — "Filter by this field": a monotonic signal the Toolbar consumes (it
- * appends the seeded condition and opens the builder). Wave-6 item 6c: a MEASURE-carrying
- * column seeds the equivalent measure condition instead (same measure key + window). */
- const [filterSeed, setFilterSeed] = useState<
- | { key?: string; measure?: { key: string; window: WindowSpec }; n: number }
- | null
- >(null);
- /** Wave-6 item 4 — the header-row "+" (insert at end): anchors the create-only menu. */
- const [plusMenu, setPlusMenu] = useState(null);
- /**
- * Wave-6 item 3c — per-key freshness of this browser's own field-def writes, persisted with
- * the workspace copy and consumed by reconcileFields at init. A ref, not state: stamps ride
- * along with the setFields that caused them; nothing renders from a stamp.
- */
- const fieldStampsRef = useRef<{ edited: Record; deleted: Record }>(
- { edited: {}, deleted: {} }
- );
- /** Wave-5 item 6 — the header description tip. Floated, pointer-events:none — a tooltip
- * that can receive the pointer swallows the NEXT click ([[ui-invisible-to-assertions]]). */
- const [headerTip, setHeaderTip] = useState<{ text: string; x: number; y: number } | null>(
- null
- );
- /** Wave-9 I2 — the CELL tip: hovering a cell whose text is cut off reveals it in full.
- * Same contract as the header tip above (floated, pointer-events:none, aria-hidden) for
- * the same reason — a tooltip that can receive the pointer swallows the NEXT click, which
- * this surface has already paid for once ([[ui-invisible-to-assertions]]). */
- const [cellTip, setCellTip] = useState<{ text: string; x: number; y: number } | null>(null);
-
- /** BUG-1 — viewId → the filter tree we have already pushed back to the host. Persisted with
- * the local workspace because the remount it guards against destroys any ref. */
- const reemittedRef = useRef>({});
- /** 2026-08-04 — views THIS browser deleted, by id. Read by the live adopt below AND by
- * init, so the mount path and the after-mount path answer "was this deleted here?" the
- * same way. Persisted with the local workspace for the same reason `reemitted` is. */
- const viewTombstonesRef = useRef({});
- /** D-19 — viewId → when this browser last wrote it (see LocalWorkspace.viewWrites). */
- const viewWritesRef = useRef({});
- const initializedKey = useRef(null);
- const saveTimer = useRef(null);
- const gridRef = useRef(null);
- const gridBoxRef = useRef(null);
- /** Internal clipboard provenance. The browser exposes pasted text, not its source column;
- * recording the grid's own copy keeps cross-field paste fail-closed without blocking
- * ordinary external text pasted into an editable field. */
- const copyProvenanceRef = useRef(EMPTY_GRID_COPY_PROVENANCE);
- useEffect(() => {
- const onCopy = () => {
- // The native copy event follows our grid key event. Preserve that provenance once;
- // every later copy (address bar, another app surface, another input) invalidates it.
- copyProvenanceRef.current = observeCopyEvent(copyProvenanceRef.current, Date.now());
- };
- const onWindowBlur = () => {
- copyProvenanceRef.current = EMPTY_GRID_COPY_PROVENANCE;
- };
- document.addEventListener("copy", onCopy, true);
- window.addEventListener("blur", onWindowBlur);
- return () => {
- document.removeEventListener("copy", onCopy, true);
- window.removeEventListener("blur", onWindowBlur);
- };
- }, []);
- const [gridSize, setGridSize] = useState({ width: 800, height: 600 });
-
- /**
- * ⛔ WAVE 21 item 3 (3c) — THE FALLBACK KEY IS THE SCOPE'S, and the literal it replaces is
- * the cross-table bleed the owner reported as "RI fields on a new database".
- *
- * MEASURED, not theorised. `fetchWorkspace` collapses 403 / 5xx / network / unreadable into
- * one silent `null` — deliberately, because the workspace is an ENHANCEMENT of the table and
- * an older host simply does not serve one. But `null` also meant "no `storageKey`", and the
- * fallback was the LITERAL `"customer-grid-standalone"` — ONE localStorage bucket for every
- * scope this component can be mounted at. So on a tenant whose `/workspace` 403s for `ut_*`
- * (routes_grid gated the route on `module_gate("customer_data")` — C4 fixes that half), the
- * FIRST user table to be opened writes its views and custom fields into that bucket and the
- * SECOND one reads them back as its own. Two databases with nothing to do with each other,
- * sharing a schema, with no error anywhere.
- *
- * Scoping the key makes that structurally impossible: a bucket belongs to one surface, so
- * the worst a failed workspace read can now do is show THIS table its own last-known local
- * copy. The prefix marks it as the local-only stratum — a key the server never issues, so it
- * can never collide with a real `storageKey` (those are `_table_workspace`-shaped).
- *
- * ⚠ Existing browsers lose the contents of the old shared bucket. That is the point: every
- * byte in it is a workspace some other surface persisted, and there is no way to tell whose.
- */
- const storageKey = payload?.workspace?.storageKey ?? `local:${scope}`;
-
- // Initialize once per permission/data scope. Host state wins; local state
- // fills only missing objects and keeps the standalone path useful.
- useEffect(() => {
- if (!payload || payloadFields.length === 0 || initializedKey.current === storageKey) return;
- const local = embedded ? null : readLocal(storageKey);
- // Item 3c: the def half of the no-blip layer. A RECENT local stamp beats a
- // lagged host echo (rename survives, retype holds, a delete stays deleted);
- // a caught-up echo returns host objects byte-identical (see optimism.ts).
- const stamps = pruneStamps(local?.fieldStamps, Date.now());
- fieldStampsRef.current = {
- edited: { ...(stamps.edited ?? {}) },
- deleted: { ...(stamps.deleted ?? {}) },
- };
- const initialFields = reconcileFields(
- payloadFields,
- local?.fields ?? [],
- stamps,
- Date.now(),
- payload.workspace != null
- );
- const hostViews = embedded ? [] : payload.workspace?.views ?? [];
- const byId = new Map();
- byId.set(ALL_VIEW_ID, allRecordsView(initialFields, scope));
- // D-19 — the HOST'S LIST DECIDES WHICH VIEWS EXIST. A local copy the host no longer
- // names is a ghost (deleted elsewhere, share revoked, store moved) and is dropped here,
- // guarded by this browser's own recent writes so an optimistic create survives.
- viewWritesRef.current = pruneTombstones(local?.viewWrites, Date.now());
- for (const view of seedLocalViews(local?.views ?? [], hostViews, viewWritesRef.current,
- Date.now(), payload.workspace != null))
- byId.set(view.id, view);
- // Host state wins — EXCEPT when it is a lagged echo of this browser's own
- // in-flight edit, where taking it would turn a just-completed measure rule
- // valueless: inactive, no pending marker, whole-book count. A rerun replaces
- // the iframe, so this init runs after every host round trip; the reconcile
- // is what keeps a rule with a typed rhs PENDING rather than inactive across
- // that remount (see viewEcho.ts).
- //
- // BUG-1 (wave 11) — which views carry a FILTER TREE the host copy does not have, and have
- // not already been told about. `echoReemit` owns both guards (see viewEcho.ts): identity
- // alone would also fire on a display-only reconcile, and a time-based bound cannot work
- // because the local-workspace effect below restamps `savedAt` on every init.
- const reemit = new Map();
- // 2026-08-04 — the views tombstone map, seeded from the persisted copy so a remount
- // INSIDE the echo window cannot walk back a view this browser just deleted. Init had
- // no such guard before (fields and folders both did), which was survivable only while
- // every host round trip was itself a remount.
- const tombstones = pruneTombstones(local?.viewTombstones, Date.now());
- viewTombstonesRef.current = tombstones;
- for (const view of hostViews) {
- // Both doors, the `reconcileFields` rule: a tombstone drops the host copy AND any
- // local copy, so a delete that raced localStorage cannot re-enter through either.
- if (tombstones[view.id] !== undefined) {
- byId.delete(view.id);
- continue;
- }
- const merged = reconcileEchoView(
- view, byId.get(view.id), local?.savedAt, Date.now()
- );
- const key = echoReemit(merged, view, local?.reemitted?.[view.id]);
- if (key !== null) reemit.set(view.id, key);
- byId.set(view.id, merged);
- }
- const initialViews = [...byId.values()].map((view) => ({
- ...view,
- config: normalizeConfig(view.config, initialFields),
- }));
- const requested =
- local?.activeViewId ?? payload.workspace?.activeViewId ?? ALL_VIEW_ID;
- const active = initialViews.find((view) => view.id === requested) ?? initialViews[0];
- setFields(initialFields);
- setViews(initialViews);
- setActiveViewId(active.id);
- setConfig(active.config);
- setWorkspaceReady(true);
- initializedKey.current = storageKey;
-
- /**
- * BUG-1 (wave 11) — TELL THE HOST. The reconcile above was right about the state and silent
- * about the consequence: it restored a rule this browser completed and the host copy does
- * not have, and then nothing sent it. The autosave effect cannot: init seeds `views` AND
- * `config` from the same reconciled object, so `sameConfig` reports "saved" and no upsert
- * is ever emitted. The rule stays ACTIVE client-side, PENDING forever, and unknown to
- * `_cl_resolve_measures` — which is exactly W11-HOST's live trace, a pending view carrying
- * no measure rule at all while the saved-view path resolves correctly.
- *
- * ⚠ THIS TERMINATES, and NOT because the host will store what we send. It might not: a
- * measure key outside this user's BU offer is stripped by `clean_filter_tree`, so the host
- * copy never gains the rule and `lagged` stays true forever. Termination comes from the
- * IDEMPOTENCE KEY — the filter tree itself, persisted per view — so we tell the host about
- * a given tree exactly once and only speak again when the USER edits the conditions.
- * ⚠ A ref cannot hold that: the remount this exists to survive destroys it. It rides in
- * localStorage beside the views, and `writeLocal` below carries it forward.
- *
- * Emitted from the NORMALIZED copy (`initialViews`), never from the raw host view: that is
- * the config the client is actually filtering with, and sending anything else would ask the
- * host to resolve a question nobody on screen is asking.
- */
- if (!embedded && reemit.size > 0) {
- const stamped = { ...(local?.reemitted ?? {}) };
- for (const view of initialViews) {
- const key = reemit.get(view.id);
- if (key === undefined) continue;
- viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id,
- Date.now());
- viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id, Date.now());
- emitHostEvent({ id: eventId("view"), type: "view_upsert", view });
- stamped[view.id] = key;
- }
- reemittedRef.current = stamped;
- } else {
- reemittedRef.current = { ...(local?.reemitted ?? {}) };
- }
- }, [payload, payloadFields, storageKey, embedded, scope]);
-
- useEffect(() => {
- if (!workspaceReady || embedded) return;
- writeLocal(storageKey, {
- fields,
- views,
- activeViewId,
- savedAt: Date.now(),
- fieldStamps: pruneStamps(fieldStampsRef.current, Date.now()),
- // BUG-1 — carried forward, never recomputed here. Dropping it would restore the loop:
- // the next init would find no record of what it already told the host and say it again.
- reemitted: reemittedRef.current,
- // Pruned on the way out, like every other stamp map here: the persisted blob is a
- // recent window, never an archive of everything this tab ever deleted.
- viewTombstones: pruneTombstones(viewTombstonesRef.current, Date.now()),
- // D-19's other half — without persisting these, a reload inside the echo window would
- // drop a view this browser created seconds ago.
- viewWrites: pruneTombstones(viewWritesRef.current, Date.now()),
- });
- }, [workspaceReady, storageKey, fields, views, activeViewId, embedded]);
-
- /**
- * ⭐ THE LIVE WORKSPACE (owner report, 2026-08-04) — what has APPEARED since we mounted.
- *
- * The init effect above runs ONCE per mount (`initializedKey`), which under Streamlit was
- * indistinguishable from "once per host round trip" because a rerun replaces the iframe.
- * The standalone shell has no rerun: this component stays mounted until `key={active.key}`
- * changes in Shell.tsx — switching modules. So a cohort created here reached the store,
- * came back in the very next `/workspace` re-read, sat in `payload.workspace.views`, and
- * never reached the rail. That is the owner's report, exactly: it appears after you visit
- * another module and come back, because that is the only thing that remounts this tree.
- *
- * ⚠ FIELDS FIRST, AND THE ORDER IS LOAD-BEARING. `fields_from_workspace(ws,
- * cohorts=bool(cohort_lists))` emits the derived "Locked views" column only once a cohort
- * EXISTS — so a user's FIRST cohort changes the field contract in the same payload that
- * carries the new view. The adopted views are normalized against `nextFields`, not against
- * the `fields` state (which React has not committed yet), or a projected `config.order`
- * naming the new column would be normalized against a list that does not have it.
- *
- * ⚠ Add-only, tombstoned, and identity-stable — see liveWorkspace.ts for why each of those
- * is a correctness property rather than a nicety.
- */
- const hostWorkspaceViews = payload?.workspace?.views;
- useEffect(() => {
- if (!workspaceReady || embedded) return;
- const now = Date.now();
- const nextFields = adoptNewFields(
- fields, payloadFields, fieldStampsRef.current.deleted, now
- );
- if (nextFields !== fields) setFields(nextFields);
- setViews((current) =>
- adoptNewViews(current, hostWorkspaceViews, viewTombstonesRef.current, now,
- (config) => normalizeConfig(config, nextFields))
- );
- }, [workspaceReady, hostWorkspaceViews, payloadFields, fields, embedded]);
-
- /** Item 3c — stamp a def write / a delete. Pruned at every touch so the persisted blob
- * stays a recent window, never an archive. */
- const stampFieldEdit = useCallback((key: string) => {
- const s = pruneStamps(fieldStampsRef.current, Date.now());
- fieldStampsRef.current = {
- edited: { ...(s.edited ?? {}), [key]: Date.now() },
- deleted: { ...(s.deleted ?? {}) },
- };
- }, []);
- const stampFieldDelete = useCallback((key: string) => {
- const s = pruneStamps(fieldStampsRef.current, Date.now());
- const edited = { ...(s.edited ?? {}) };
- delete edited[key];
- fieldStampsRef.current = {
- edited,
- deleted: { ...(s.deleted ?? {}), [key]: Date.now() },
- };
- }, []);
-
- const updateConfig = useCallback(
- (next: ViewConfig) => {
- if (next.groupBy !== config.groupBy) setCollapsed(new Set());
- setConfig(next);
- },
- [config.groupBy]
- );
-
- const {
- visibleCols,
- fieldByKey,
- order,
- visible,
- lockedKey,
- onColumnResize,
- onColumnMoved,
- onColumnProposeMove,
- setColumnVisible,
- insertColumn,
- } = useGridColumns(fields, config, updateConfig);
-
- /**
- * ⭐ Wave-20 owner item 4 (ruling R8, contract C-ADDROW) — **THE GHOST ROW.**
- *
- * A trailing "+" row at the bottom of the grid, the way Airtable grows a table, replacing the
- * "Add record" button that used to sit in the shell's header bar (S4 deleted it this wave).
- *
- * ⛔ USER DATABASES ONLY. A connector's rows are read-synced from Odoo; a "+" there could only
- * refuse, and R8 names that a fake affordance. The test is the SCOPE (`ut_*`), which is also
- * the only scope with a rows endpoint to POST to — so the affordance and the capability come
- * from the same fact rather than from two lists that can drift.
- *
- * ⚠ THE ROW IS NOT ADDED LOCALLY. `rawRows` is the server's answer, and a client-invented row
- * would have no rid, no defaults and no place in anyone else's copy. The POST clears the rows
- * cache and fires `ROWS_STALE_EVENT`, `useCustomerData` re-reads, and the row arrives with the
- * id the store gave it. The cursor then follows it (`newRowPid` + the effect below) — which is
- * why the pid is remembered instead of glide's `"bottom"` being returned here: at the moment
- * this resolves, the appended row does not exist yet, so "bottom" would land on the last OLD
- * row.
- */
- const isUserTable = !embedded && scope.startsWith("ut_");
- const recordsMutable = payload?.recordsMutable !== false;
- const canMutateRecords = isUserTable && recordsMutable;
- /**
- * ⭐ 2026-08-07 — the databases a `link` column may point at, fetched when the column menu
- * OPENS rather than on every render of the grid.
- *
- * ⚠ ON DEMAND IS THE WHOLE DESIGN. `GET /tables` walks every table in the tenant and returns
- * their full field lists; hanging that off the grid's mount would put a workspace-wide read
- * behind every page view for a picker almost nobody opens. The column menu is the only surface
- * that needs it, so it is the surface that asks.
- * ⚠ The list is left standing once fetched — a database created in another tab mid-session is
- * a staleness a menu re-open corrects, and re-fetching per open would spend the same read
- * repeatedly for a list that changes about once a week.
- */
- const [linkTargets, setLinkTargets] = useState([]);
- useEffect(() => {
- // ⛔⛔ 2026-08-09 — `plusMenu` TOO, AND THAT OMISSION WAS THE WHOLE BUG. Owner: *"I am not
- // able to see Choose Column for Post rows link"*. There are TWO doors into this editor —
- // a column's own menu (`columnMenu`) and the "+" Add-field button (`plusMenu`) — and this
- // effect knew about one. Via "+", `linkTargets` stayed `[]`, so `target` never resolved and
- // the rollup Column picker held nothing but its placeholder, permanently. Editing an
- // EXISTING column worked, which is what made it look like a rollup bug rather than a fetch
- // that never fired. MEASURED on the live app: 5 selects rendered, "Column to roll up" with
- // exactly 1 option while `GET /tables` returns `ut_ig_posts` with 22 fields.
- // ⚠ A picker that is EMPTY and a picker whose data never loaded look identical, which is
- // why this survived: both render "Choose a column…" over nothing.
- if ((!columnMenu && !plusMenu) || linkTargets.length) return;
- let live = true;
- void fetchLinkTargets().then((t) => {
- if (live) setLinkTargets(t);
- });
- return () => {
- live = false;
- };
- }, [columnMenu, plusMenu, linkTargets.length]);
- /**
- * ⭐⭐ 2026-08-09 — the READ-THROUGH rollup's offer, fetched on the same terms as the link
- * targets above: only when the column menu opens, and once per session.
- *
- * ⚠ THE GUARD IS A SEPARATE `asked` FLAG, NOT `topics.length`. An empty offer is the CORRECT
- * and common answer (a tenant with nothing connected), so guarding on the length would re-ask
- * on every single menu open for exactly the workspaces where the answer can never change —
- * the one case the "fetch once" rule exists for.
- */
- const [rollupSourceOffer, setRollupSourceOffer] =
- useState({ topics: [], windows: [] });
- const rollupOfferAsked = useRef(false);
- useEffect(() => {
- // ⛔ SAME TWO DOORS as the link targets above. This one is less visible because an empty
- // offer is the correct answer for a tenant with nothing connected — so a source offer that
- // never loaded is indistinguishable from one that is legitimately empty, and the mode switch
- // simply never appears. Fixing only the link half would have left that asymmetry in place.
- if ((!columnMenu && !plusMenu) || rollupOfferAsked.current) return;
- rollupOfferAsked.current = true;
- let live = true;
- void fetchRollupSources().then((o) => {
- if (live) setRollupSourceOffer(o);
- });
- return () => {
- live = false;
- };
- }, [columnMenu, plusMenu]);
- /**
- * ⭐ WAVE 27 · OWNER ITEM 22 — the values the ACTIVE VIEW forces on a new record.
- *
- * ⛔ THE DEFECT: "+" posted `{values: {}}` on every view, so a record added while a filter was
- * on was created, was real, and was invisible — the button read as broken and the row the user
- * then typed into did not exist as far as they could see. `filterSeed.ts` owns the derivation
- * and its whole design is how much it REFUSES to derive (negations, ranges, `or` branches,
- * contradictions, machine columns — each with its reason at the clause).
- *
- * ⚠ ONE FUNCTION, TWO CALLERS, deliberately: item 2's optimistic add must insert the row
- * carrying these same cells, or the row it paints locally fails the filter it was added under
- * and disappears on the next read — the exact blip C2 exists to prevent.
- */
- const seedValues = useCallback(
- () => filterSeedValues(config.filters, config.filterConj, fields, payload?.viewer),
- [config.filters, config.filterConj, fields, payload?.viewer]
- );
- /**
- * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — the "+" paints its row NOW.
- *
- * ⛔ THE ID IS MINTED THE WAY THE SERVER MINTS IT — `max(numeric row id) + 1`, which is
- * `core.user_tables.add_row`'s own rule — and then POSTed as `{rid, values}` through the door
- * the undo path already opened (C-ADDROW). That is what makes the optimism honest rather than
- * hopeful: on the overwhelmingly common single-writer path the server stores exactly the id
- * this browser drew, so the row on screen and the row in the store are the same record from
- * the first frame.
- *
- * ⚠ AND WHEN THEY ARE NOT, IT RE-ANCHORS. Two people adding at once means the second POST
- * finds the id taken; `add_row` falls back to `max+1` and ANSWERS with what it actually wrote
- * (its own note says so), so the local row is re-keyed to the returned pid. Assuming the
- * requested id came back is the one way this could leave a row on screen that no longer
- * matches anything in the store.
- *
- * ⛔ NO `ROWS_STALE_EVENT` ON THE HAPPY PATH. That event re-reads the whole table, which is
- * the cost item 2 exists to remove — and re-reading would also un-paint the row for the
- * duration of the fetch, which is the NO-BLIP law's exact subject. The pending row is pruned
- * by the effect above when a payload that already contains it arrives, from whatever cause.
- *
- * ⚠ THE NARROW RACE, STATED: a cell PATCHed within the POST's round trip could reach the
- * server before the row exists. The window is now ~20ms (A moved the relation refresh off
- * this path), it needs a keystroke inside it, and the PATCH's failure is a rolled-back
- * overlay edit rather than lost data — the alternative, blocking the paint until the POST
- * resolves, is the defect being fixed.
- */
- const appendRow = useCallback(async (): Promise => {
- if (!canMutateRecords) return undefined;
- const seeded = seedValues();
- const minted = mintRid();
- const optimistic: Row = { ...seeded, pid: minted };
- setPendingRows((cur) => [...cur, optimistic]);
- setNewRowPid(minted);
- const made = await addTableRow(scope, seeded, minted);
- if (!made) {
- // `addTableRow` already said WHY, in the server's own words (a row cap, a refused profile
- // cell, an unreachable host). The row is withdrawn rather than left standing: a row that
- // survives its own failed write is the lie this whole path has to avoid.
- setPendingRows((cur) => cur.filter((r) => r.pid !== minted));
- setNewRowPid(null);
- return undefined;
- }
- if (made.pid !== minted) {
- setPendingRows((cur) =>
- cur.map((r) => (r.pid === minted ? { ...r, pid: made.pid } : r))
- );
- setNewRowPid(made.pid);
- }
- // R4 — the append is undoable: Ctrl+Z deletes the row it just created, and a redo restores
- // it under the SAME rid (the server's `{rid}` passthrough, C-ADDROW).
- // ⛔ ITEM 22 MOVED THIS LINE. It read `values: {}` with a comment saying "an empty new record
- // carries no values" — true until this wave, and now false: a row added on a filtered view is
- // born holding the cells that filter forces. `redo` replays `r.values` verbatim
- // (`:1569`), so leaving the literal would have made Ctrl+Z → Ctrl+Y restore the row under the
- // right id with its seeded cells silently dropped — [[undo-capture-before-the-write]] in its
- // quietest form, since the id and the row count would both be right.
- undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
- kind: "rowAdd", table: scope, rows: [{ rid: made.rid, values: seeded }],
- });
- // ⛔ NO `signal(ROWS_STALE_EVENT)` HERE ANY MORE (item 2 / C2). It re-read the whole table
- // on every "+", which is the whole cost the owner reported — and every OTHER caller of it
- // still fires, so a row created by an automation, another tab or an undo still arrives.
- // The one thing that used to depend on this refetch was the cursor landing on the new row,
- // and the row is now on screen before the POST resolves, so it lands immediately.
- // `undefined` is returned rather than glide's "bottom": "bottom" is resolved against the
- // rows glide knew about when the gesture started, which is the row BEFORE this one.
- return undefined;
- }, [canMutateRecords, scope, seedValues, mintRid]);
-
- /* wave20 item 2 — measure key + window -> the columns that display it. Built ONCE per field
- list and handed to every consumer of "which column is this rule about", so the tint
- (`columnTones`, inside the hook above) and the column menu's filter doors resolve a
- measure condition the same way. See `types.ruleColumnKeys`. */
- const measureCols = useMemo(() => measureColumnIndex(fields), [fields]);
-
- // Airtable behavior: configuration changes to the active view autosave.
- useEffect(() => {
- if (!workspaceReady || embedded) return;
- const active = views.find((view) => view.id === activeViewId);
- if (!active || sameConfig(active.config, config)) {
- setSaveState("saved");
- return;
- }
- setSaveState("saving");
- if (saveTimer.current !== null) window.clearTimeout(saveTimer.current);
- saveTimer.current = window.setTimeout(() => {
- const updated = { ...active, config };
- setViews((current) =>
- current.map((view) => (view.id === updated.id ? updated : view))
- );
- viewWritesRef.current = stampTombstone(viewWritesRef.current, updated.id,
- Date.now());
- emitHostEvent({ id: eventId("view"), type: "view_upsert", view: updated });
- setSaveState("saved");
- saveTimer.current = null;
- }, 420);
- return () => {
- if (saveTimer.current !== null) window.clearTimeout(saveTimer.current);
- };
- }, [workspaceReady, activeViewId, config, views, embedded]);
-
- // Numeric dimensions force a glide relayout when either the component frame
- // or Streamlit's main column changes width (notably sidebar collapse).
- useLayoutEffect(() => {
- const element = gridBoxRef.current;
- if (!element) return;
- const measure = () => {
- const rect = element.getBoundingClientRect();
- if (rect.width > 0 && rect.height > 0) {
- setGridSize({
- width: Math.floor(rect.width),
- height: Math.floor(rect.height),
- });
- }
- };
- measure();
- const observer = new ResizeObserver(measure);
- observer.observe(element);
- window.addEventListener("resize", measure);
- return () => {
- observer.disconnect();
- window.removeEventListener("resize", measure);
- };
- }, [workspaceReady]);
-
- // CG-3: one mode per table, decided by whether the payload is a window. See types.ts
- // `TableMode` for why both engines running at once is the failure this prevents.
- const mode = tableMode(payload?.counts);
- const serverWindowed = mode === "server-windowed";
- // Owner items 4+6 — the Cohort page's grid renders without the Views sidebar.
- const hideViews = embedded || payload?.workspace?.hideViews === true;
- // Wave-6 item 10 — how this view displays. A WINDOWED table is always the grid: list/
- // calendar/kanban compute over the whole matched set, and one page is not it (CG-3's rule,
- // the same reason grouping is off there).
- const displaySpec = cleanDisplay(config.display);
- const displayMode: DisplayMode = serverWindowed ? "grid" : displaySpec?.mode ?? "grid";
-
- // CG-8 — measure conditions, answered by the host as `{ruleId: pid[]}`. Converted to Sets
- // once here rather than per row: the pipeline runs this over every row of the book.
- //
- // ⚠ REPLACED, not merged. Merging looks like the safer choice — "hold the previous result"
- // is the design — but it holds it FOREVER: the host omits a rule's id when it could not
- // resolve it (a warming store, a scope change mid-flight), and a merged map keeps yesterday's
- // answer under today's question with `pendingMeasures` reporting zero, so the marker never
- // shows. That is the widening sin wearing a confident count.
- //
- // Holding the previous result across a LOCAL edit does not need the merge and never did: the
- // memo is keyed on `payload.measureSets` identity, which does not change while the user is
- // typing, so the ref already carries the last answers through to the next payload. Merge and
- // replace therefore differ only in the dangerous case.
- const measures = useMemo(() => payload?.measures ?? [], [payload?.measures]);
- const measureSetsRef = useRef({});
- const measureSets = useMemo(() => {
- const incoming = payload?.measureSets;
- if (incoming) {
- const next: MeasureSets = {};
- for (const [ruleId, pids] of Object.entries(incoming)) next[ruleId] = new Set(pids);
- measureSetsRef.current = next;
- }
- return measureSetsRef.current;
- }, [payload?.measureSets]);
- const pendingMeasureCount = useMemo(
- () => pendingMeasures(config.filters, measureSets),
- [config.filters, measureSets]
- );
- // BUG-1 (wave 11) — the two key sets, published for the handshake. See the `.cg-shell`
- // attributes below for why they are DOM attributes and not console logs.
- const measureRuleKeys = useMemo(
- () => activeMeasureRuleIds(config.filters).join(","),
- [config.filters]
- );
- const measureSetKeys = useMemo(() => Object.keys(measureSets).join(","), [measureSets]);
-
- // Owner item 5. Cohort membership rides `workspace.lists`, the same array the "Add to list"
- // picker already reads, so the names offered and the sets tested come from one place. A
- // cohort with no `pids` is a target you can add to but cannot filter on; the engine treats it
- // as unanswerable (matches nothing) rather than absent (matches everything).
- const lists = useMemo(() => payload?.workspace?.lists ?? [], [payload?.workspace?.lists]);
- /* wave17 R1 / C-LOCKV — `railCohortId` is gone. It held the TRANSIENT lock the retired
- Cohorts section applied on click: a lock that lived in component state and was never
- persisted. A locked view now carries its own `config.cohortLock`, so SELECTING the view
- is the lock and the saved config is the only source. One mechanism, and it survives a
- reload — which the transient one never did. */
- const cohortSets = useMemo(() => {
- const out: CohortSets = {};
- for (const l of lists) if (l.pids) out[l.id] = new Set(l.pids);
- return out;
- }, [lists]);
- // The TENANT'S today. Never `new Date()`: a browser a day ahead of the server would resolve
- // "the past month" to a different month than the host summed, and the count would deny the
- // rows with nothing erroring.
- const today = payload?.today;
- // Wave-5 item 1 — who is looking (host-computed). ONE editability verdict for every edit
- // door (cells, pickers, drawer): types.mayEditField — stratum + read-only-by-nature +
- // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown.
- const viewer = payload?.viewer;
- const canEditField = useCallback(
- (f: Field): boolean => !embedded && recordsMutable && mayEditField(f, viewer),
- [embedded, recordsMutable, viewer]
- );
- const unresolvedCount = useMemo(
- () => unresolvedConditions(config.filters, { cohortSets, today }),
- [config.filters, cohortSets, today]
- );
-
- // Wave-5 items 9/11 — CLIENT-COMPUTED cells, injected AT their field keys so the whole
- // pipeline (filter/search/sort/group), the cells and the drawer read them like any other
- // value. Formulas parse ONCE per definition change (never per row); evaluation reads the
- // row WITH this session's overlay edits layered, so editing a referenced field recomputes
- // live. A formula that does not parse, or errors on a row, yields BLANK — never a wrong
- // number (formulaEngine.ts). `created_time` copies the row's `_created`.
- // 2026-07-31 (owner item 2): formulas may reference OTHER formulas now, so parse order is
- // TOPOLOGICAL (orderFormulas) — a formula runs after the formulas it reads, cycle members
- // never run (blank, never a stale number), and each row's results feed the next formula
- // through a per-row scope.
- const formulaAsts = useMemo(() => {
- const sources = new Map();
- for (const f of fields) {
- if (f.type !== "formula") continue;
- const src = formulaOf(f);
- if (src) sources.set(f.key, src);
- }
- const { order, cyclic } = orderFormulas(sources);
- const out: { key: string; ast: FormulaAst }[] = [];
- for (const key of order) {
- if (cyclic.has(key)) continue;
- const p = parseFormula(sources.get(key)!);
- if (p.ok) out.push({ key, ast: p.ast });
- }
- return out;
- }, [fields]);
- const createdTimeKeys = useMemo(
- () => fields.filter((f) => f.type === "created_time").map((f) => f.key),
- [fields]
- );
- const computedRows = useMemo(() => {
- if (formulaAsts.length === 0 && createdTimeKeys.length === 0) return rawRows;
- const env = { today: payload?.today };
- return rawRows.map((r) => {
- const edits = overlayEdits[r.pid];
- const scope: Row = edits ? { ...r, ...edits } : { ...r };
- const out: Row = { ...r };
- for (const k of createdTimeKeys) {
- out[k] = (r._created as string | undefined) ?? null;
- scope[k] = out[k];
- }
- for (const { key, ast } of formulaAsts) {
- const v = evalFormula(ast, (k) => scope[k], env);
- out[key] = v;
- scope[key] = v; // later formulas read this one's result — the topo order above
- }
- return out;
- });
- }, [rawRows, overlayEdits, formulaAsts, createdTimeKeys, payload?.today]);
-
- // Wave-2 item 2c — COHORT MODE (the Cohort page). The host serves the WHOLE pool (rows +
- // derived values over the pool); the ACTIVE cohort scopes the table to its pids CLIENT-side.
- const cohortMode = payload?.workspace?.cohortMode === true;
- useEffect(() => {
- if (!cohortMode) return;
- if (activeCohortId && lists.some((l) => l.id === activeCohortId)) return;
- setActiveCohortId(lists[0]?.id ?? null);
- }, [cohortMode, lists, activeCohortId]);
- const activeCohort = cohortMode
- ? lists.find((l) => l.id === activeCohortId) ?? null
- : null;
- const cohortMemberSet = useMemo(
- () => new Set(activeCohort?.pids ?? []),
- [activeCohort]
- );
- // The FIXED subset is the scope. A cohort with no membership shows nothing — falling back to
- // the whole pool would put 1,500 rows under a rail entry that says 12, which is the widening
- // sin wearing a sidebar. Everything downstream (pipeline, counts, selection, detail) runs
- // over this scope, so the toolbar count is the cohort's matched count by construction.
- const scopedRows = useMemo(
- () =>
- cohortMode
- ? computedRows.filter((r) => cohortMemberSet.has(r.pid))
- : computedRows,
- [cohortMode, computedRows, cohortMemberSet]
- );
-
- const { visibleRows, pidToIndex } = useVisibleRows(
- scopedRows,
- fields,
- config.filters,
- search,
- config.sorts,
- // grouping a WINDOW would headline a page's count/subtotals as the group's
- serverWindowed ? null : config.groupBy,
- collapsed,
- config.memberPids,
- config.filterConj,
- serverWindowed,
- measureSets,
- cohortSets,
- today,
- // Item 12 (C-LOCK) — the 14th positional, RECORD's engine input. The lock intersects
- // FIRST, so everything downstream (conditions, memberPids, the ranking domain) operates
- // inside the cohort. Absent = today's behaviour exactly.
- // wave17 R1 / C-LOCKV: the view's SAVED lock is now the only source. A locked view is a
- // saved view whose `config.cohortLock` names its own id, so opening it IS applying the
- // lock — the wave-15 transient rail pick that used to outrank this is gone with its rail.
- config.cohortLock
- );
-
- // Owner item 8 — the DISPLAY cap. The pipeline above still ran over the whole book (counts,
- // "Add to list" and selection-by-pid all depend on that); only what glide PAINTS is sliced.
- // pidToIndex is rebuilt over the slice because every index consumer (selection, detail
- // navigation, hover) is positional against what is actually on screen.
- const [displayCap, setDisplayCap] = useState(DISPLAY_PAGE);
- const capped = visibleRows.length > displayCap;
- const displayRows = useMemo(
- () => (capped ? visibleRows.slice(0, displayCap) : visibleRows),
- [capped, visibleRows, displayCap]
- );
- const displayPidToIndex = useMemo(() => {
- if (!capped) return pidToIndex;
- const m = new Map();
- displayRows.forEach((vr, i) => {
- if (vr.kind === "data" && !m.has(vr.record.pid)) m.set(vr.record.pid, i);
- });
- return m;
- }, [capped, displayRows, pidToIndex]);
-
- // C-AVATAR (wave-14 item 11) — profile photos, absent until HOST serves them. The tick is the
- // async half: `cells.setAvatarRepaint` fires it when an Image finishes decoding, and it is a
- // dep of `getCellContent`, which is the only thing glide watches. See useGetCellContent.
- const [avatarTick, setAvatarTick] = useState(0);
- useEffect(() => {
- setAvatarRepaint(() => setAvatarTick((t) => t + 1));
- return () => setAvatarRepaint(undefined);
- }, []);
- const userAvatars = payload?.workspace?.userAvatars;
- /**
- * Item 15 — the frozen strip's usable width, which is what a GROUP BAR's label is clipped to
- * now that the first column is pinned in every mode. `null` when nothing is grouped (no bar to
- * clip) — see `fitGroupLabel` for why the clip exists and why the count outranks the name.
- */
- const groupLabelSpace = useMemo(() => {
- if (!config.groupBy) return null;
- const n = Math.min(frozenCountOf(config), visibleCols.length);
- let px = 0;
- // `GridColumn` is a union — only its SIZED member declares `width`, and `useGridColumns`
- // always builds that one. Narrowed rather than asserted so an auto-sized column (which we
- // do not create) would read 0 and simply not be counted, instead of throwing in a memo.
- for (let i = 0; i < n; i += 1)
- px += (visibleCols[i] as { width?: number } | undefined)?.width ?? 0;
- return Math.max(0, px - GROUP_LABEL_PAD);
- }, [config, visibleCols]);
- /**
- * ⭐ owner item 2 (2026-08-03) — WHICH MEASURE COLUMNS ARE STILL BEING CALCULATED.
- *
- * A measure column's numbers are resolved by the server (one aggregate over the whole book)
- * and reach the browser on the workspace re-read that follows the create — seconds later.
- * In between, the column is on screen with nothing in it, which the owner correctly read as
- * an error rather than as a wait.
- *
- * ⛔ `editRequestId` IS THE TEST, and it is the honest one. It is set optimistically at the
- * moment the create is emitted and cleared by `reconcileFields` when the host's own copy of
- * that field comes back — and the host's copy travels in the SAME `/workspace` response as
- * `derived`, which is where the values are. So the flag is true across exactly the window
- * where the column exists and its numbers do not, and false the instant they land.
- *
- * The alternative — "no value anywhere in the column" — cannot tell a pending column from
- * one that permanently failed to resolve (a BU-scoped caller on a company-level measure,
- * say), and would spin forever on the second. This one resolves either way: when the echo
- * arrives with no values, the flag clears and the cells go honestly blank.
- */
- const pendingMeasureKeys = useMemo(() => {
- const out = new Set();
- for (const f of fields)
- if (f.editRequestId && f.key.startsWith("measure_")) out.add(f.key);
- return out;
- }, [fields]);
- /**
- * The skeleton's pulse. A canvas cell cannot hold a CSS animation, so the motion is repaints:
- * this counter is a dependency of `getCellContent`, which is the only thing glide watches.
- *
- * ⚠ IT RUNS ONLY WHILE SOMETHING IS PENDING, and the effect's own guard is what stops it —
- * an interval left running would repaint the whole canvas ~7×/s forever, on every grid, to
- * animate nothing. 140ms × the 4-step ramp is a ~0.6s cycle: a wait, not a strobe.
- */
- const [pulse, setPulse] = useState(0);
- /**
- * ⛔ AND IT GIVES UP. A skeleton that never resolves is worse than the blank it replaced: a
- * blank cell is at least honest about having no number, while a permanent shimmer promises
- * one that is never coming.
- *
- * The window it guards is narrow but real. `editRequestId` clears when `reconcileFields` takes
- * the host's copy — and that function has a branch (`hostAuthoritative: false`, i.e. a payload
- * with no workspace) whose `{...local, ...host}` spread would PRESERVE the flag forever. Today
- * that branch cannot be reached with a measure column on screen (measures are offered through
- * the workspace, so a payload without one cannot have produced this field), which is an
- * argument about the current call graph and not a property of the code. This bound holds
- * whether or not the argument stays true, and it costs one boolean.
- */
- const [pendingGaveUp, setPendingGaveUp] = useState(false);
- useEffect(() => {
- setPendingGaveUp(false);
- if (pendingMeasureKeys.size === 0) return;
- let n = 0;
- const id = window.setInterval(() => {
- n += 1;
- if (n > PULSE_MAX_TICKS) {
- window.clearInterval(id);
- setPendingGaveUp(true); // fall back to ordinary blank cells
- return;
- }
- setPulse((p) => p + 1);
- }, PULSE_MS);
- return () => window.clearInterval(id);
- }, [pendingMeasureKeys]);
- const activePendingKeys = pendingGaveUp ? NO_PENDING_KEYS : pendingMeasureKeys;
- const getCellContent = useGetCellContent(
- displayRows,
- visibleCols,
- fieldByKey,
- overlayEdits,
- canEditField,
- userAvatars,
- avatarTick,
- groupLabelSpace,
- measureGroupText,
- activePendingKeys,
- pulse
- );
- const { gridSelection, selectedPids, onGridSelectionChange, selectPids, togglePid,
- setActiveCell, clearSelection } =
- useGridSelection(
- displayRows,
- displayPidToIndex,
- visibleCols.length,
- embeddedSelectable ? embeddedSelectedIds : []
- );
- const embeddedSelectedKey = embeddedSelectedIds.join(",");
- const selectedPidKey = [...selectedPids].join(",");
- useEffect(() => {
- if (!embeddedSelectable) return;
- if (selectedPidKey !== embeddedSelectedKey)
- selectPids(embeddedSelectedIds as number[], "replace");
- }, [embeddedSelectable, embeddedSelectedKey, embeddedSelectedIds, selectedPidKey, selectPids]);
- useEffect(() => {
- if (!embeddedSelectable) return;
- onEmbeddedSelectionChange?.([...selectedPids]);
- }, [embeddedSelectable, onEmbeddedSelectionChange, selectedPids]);
- /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════
- THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one
- thing it cannot: reading the value a cell held BEFORE the write, which only exists at the
- call site. Everything below funnels through `patchAndRecord` / `patchManyAndRecord`, so a
- write path that forgets to record is a write path that does not reach the store either.
-
- ⚠ A REF, NOT STATE. Nothing on screen depends on the stack in v1 (no undo button), so
- keeping it in state would repaint the grid on every keystroke of a paste for no pixels.
- Per SCOPE (a Ctrl+Z on the Customer grid must never rewrite a user table's cell) and per
- tab (nothing persists it — a stack restored into a session that did not make those edits
- would undo somebody else's work). */
- const undoBook = useRef({});
- const applyingUndo = useRef(false);
-
- /** The value a cell holds RIGHT NOW: the overlay stratum wins over the payload row, exactly
- * as `useGetCellContent` renders it — so what undo restores is what was on screen. */
- const rowByPid = useMemo(() => {
- const map = new Map();
- for (const r of rawRows) map.set(r.pid, r);
- return map;
- }, [rawRows]);
- const currentValue = useCallback(
- (pid: number, key: string): UndoValue => {
- const edited = overlayEdits[pid]?.[key];
- const raw = edited !== undefined ? edited : rowByPid.get(pid)?.[key];
- return raw == null ? null : (raw as UndoValue);
- },
- [overlayEdits, rowByPid]
- );
-
- const recordCells = useCallback(
- (changes: CellChange[], label: string) => {
- if (applyingUndo.current || changes.length === 0) return;
- undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
- kind: "cells", label, changes,
- });
- },
- [scope]
- );
-
- /** ONE cell write, recorded. Every editor, picker and drag goes through this. */
- const patchAndRecord = useCallback(
- (pid: number, updates: Partial, label = "an edit") => {
- const changes: CellChange[] = Object.entries(updates).map(([key, value]) => ({
- pid, key,
- before: currentValue(pid, key),
- after: (value ?? null) as UndoValue,
- }));
- patchOverlay(pid, updates);
- recordCells(changes, label);
- },
- [currentValue, patchOverlay, recordCells]
- );
-
- /** MANY cells, ONE stack entry — a paste and a bulk clear are each one user action (R4). */
- const patchManyAndRecord = useCallback(
- (writes: { pid: number; updates: Partial }[], label: string) => {
- const changes: CellChange[] = [];
- for (const w of writes)
- for (const [key, value] of Object.entries(w.updates))
- changes.push({
- pid: w.pid, key,
- before: currentValue(w.pid, key),
- after: (value ?? null) as UndoValue,
- });
- for (const w of writes) patchOverlay(w.pid, w.updates);
- recordCells(changes, label);
- },
- [currentValue, patchOverlay, recordCells]
- );
-
- /**
- * Apply one stack entry in one direction. The INVERSE lives in `undoStack.directed` — this
- * only knows how to WRITE each op, and it writes through the same doors the user does
- * (`patchOverlay`, the rows endpoint), so an undone edit is persisted exactly like the edit
- * was. Nothing here is optimistic-only: a Ctrl+Z that reverted the screen and not the store
- * would come back on the next reload.
- */
- const applyEntry = useCallback(
- async (entry: UndoEntry, dir: "back" | "forward") => {
- const op = directed(entry, dir);
- if (op.kind === "cells") {
- const byPid = new Map>();
- for (const c of op.changes) {
- const at = byPid.get(c.pid) ?? {};
- at[c.key] = (c.after ?? "") as Row[string];
- byPid.set(c.pid, at);
- }
- applyingUndo.current = true;
- try {
- for (const [pid, updates] of byPid) patchOverlay(pid, updates);
- } finally {
- applyingUndo.current = false;
- }
- } else if (op.kind === "rowAdd") {
- // Restore under the OLD id where the store still has it free; the server answers with
- // what it actually wrote and the re-read is what puts the rows back on screen.
- //
- // ⚠ SEQUENTIAL, not `Promise.all`. `add_row` picks `max(id)+1` when the requested id is
- // taken, and it reads the store to do it — firing ten restores concurrently is ten
- // readers racing one counter, which is how two rows end up sharing an id.
- let any = false;
- let moved = 0;
- for (const r of op.rows) {
- const made = await addTableRow(op.table, r.values as Record, r.rid);
- any = any || !!made; // a refusal already said why, in the server's words
- // ⚠ THE ID IS PART OF WHAT IS BEING UNDONE, and `add_row` falls back to `max(id)+1`
- // when the one it was asked for is no longer free — which happens if anything created
- // a row in the gap. The restore is still the right thing to do, but it is no longer
- // the SAME record to anything that named the old id (a cohort, a comment, a filter),
- // and a redo would then look for an id that is not there. Said out loud rather than
- // discovered later.
- if (made && String(made.rid) !== String(r.rid)) moved += 1;
- }
- if (!any) return;
- if (moved)
- signal(
- TOAST_EVENT,
- `${moved} restored record${moved === 1 ? "" : "s"} came back under a new id — ` +
- `something had taken the original while it was gone.`
- );
- signal(ROWS_STALE_EVENT);
- } else if (op.kind === "rowDelete") {
- let any = false;
- for (const r of op.rows) any = (await deleteTableRow(op.table, r.rid)) || any;
- if (!any) return;
- // ⭐ ITEM 2 — the row may still be one of THIS browser's pending copies (add, then
- // Ctrl+Z). A deleted row never reappears in a payload, so the prune effect can never
- // absorb it and it would sit on screen looking undeleted. See `withdrawPending`.
- withdrawPending(op.rows.map((r) => r.rid));
- signal(ROWS_STALE_EVENT);
- } else if (op.kind === "choiceRename") {
- // Item 15 — the inverse mapping. The host rewrites the values and the saved views that
- // name them, exactly as it did on the way out; `directed` already turned the pairs
- // around, so this emits what it is given.
- emitHostEvent({
- id: eventId("choicerename"),
- type: "choice_rename",
- key: op.fieldKey,
- renames: op.renames,
- });
- }
- signal(TOAST_EVENT, describe(entry, dir));
- },
- [patchOverlay, withdrawPending]
- );
-
- /**
- * Ctrl+Z / Ctrl+Shift+Z (and Ctrl+Y, which is the same request on Windows).
- *
- * ⚠ ON `window`, IN CAPTURE, and it steps aside for real text fields. The grid is a canvas —
- * glide's key handling only fires while the canvas has focus, so a Ctrl+Z after clicking the
- * toolbar would do nothing, which is exactly the "sometimes it works" the owner would report
- * next. But an with a cursor in it has its OWN undo that belongs to the browser, and
- * stealing that would be worse than not having ours: the search box, the rename field and
- * glide's own cell editor are all inputs, so `activeElement` decides.
- *
- * Nothing to undo says so out loud rather than silently ignoring the key — an undo that
- * appears to do nothing is indistinguishable from one that is broken.
- */
- useEffect(() => {
- const onKey = (event: KeyboardEvent) => {
- if (!(event.ctrlKey || event.metaKey) || event.altKey) return;
- const key = event.key.toLowerCase();
- if (key !== "z" && key !== "y") return;
- const el = document.activeElement as HTMLElement | null;
- if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable))
- return;
- const forward = key === "y" || event.shiftKey;
- event.preventDefault();
- const book = undoBook.current;
- const state = stackFor(book, scope);
- const { state: next, entry } = forward ? popRedo(state) : popUndo(state);
- if (!entry) {
- signal(TOAST_EVENT, forward ? "Nothing to redo." : "Nothing to undo.");
- return;
- }
- book[scope] = next;
- void applyEntry(entry, forward ? "forward" : "back");
- };
- window.addEventListener("keydown", onKey, true);
- return () => window.removeEventListener("keydown", onKey, true);
- }, [scope, applyEntry]);
-
-
- /**
- * R4 — **BULK BACKSPACE/DELETE IS ONE ACTION.** glide's own delete walks the selection and
- * calls `onCellEdited` per cell, which would put forty entries on the stack for one keypress;
- * returning `false` takes the whole operation over so it lands as one.
- *
- * ⚠ PRESETS ARE NEVER ATTEMPTED (the contract says so, and it is also the only honest
- * behaviour): `canEditField` is the same verdict the editor and the paste path use, so a
- * selection spanning read-only columns clears the editable ones and leaves the rest exactly
- * as they were — rather than firing writes the server will refuse one by one.
- */
- /**
- * ⭐ 2026-08-06 (owner) — DELETE RECORDS, with Ctrl+Z.
- *
- * Owner, verbatim: *"I should always be able to delete records that I created myself manually
- * (not from automation), because right now it only shows 'Add to cohort'… make sure I can use
- * 'Delete/Backspace' to delete the record and use Ctrl Z if i want to undo."*
- *
- * Everything under this was already built and had no door: `DELETE /tables/{key}/rows/{rid}`,
- * `deleteTableRow`, and `undoStack`'s `rowDelete` — whose inverse (`rowAdd` under the SAME rid)
- * is exactly "put it back". The only missing piece was something that calls them.
- *
- * ⛔ THE VALUES ARE CAPTURED BEFORE THE DELETE, and this is the line the whole feature rests on.
- * `deleteTableRow` answers a boolean; it does not hand the row back. Push the undo entry after
- * the round trip and there is nothing left to read, so Ctrl+Z would faithfully restore an EMPTY
- * row under the right id — a silent data loss wearing the costume of a working undo.
- *
- * ⚠ ONE ENTRY FOR THE WHOLE GESTURE (R4's "bulk Backspace = ONE grouped stack entry"), which is
- * why `UndoEntry` carries a row LIST now. Ten rows deleted must be one Ctrl+Z, not ten.
- *
- * ⚠ NO CONFIRM DIALOG, deliberately. The owner named Ctrl+Z as the safety net in the same
- * sentence as the delete; a modal on top of a working undo is friction that teaches people to
- * dismiss modals.
- */
- const deleteRecords = useCallback(
- async (pids: number[]): Promise => {
- if (!canMutateRecords || !pids.length) return false;
- // The row as it stands NOW, straight off the rendered records — the same values the
- // reader can see, so a restore puts back what they watched disappear.
- const byPid = new Map();
- for (const vr of displayRows)
- if (vr.kind === "data") byPid.set(vr.record.pid, vr.record);
- const captured: UndoRow[] = [];
- /*
- * ⛔ A SKIPPED ROW IS COUNTED AND NAMED, never dropped quietly ([[no-unverifiable-aggregates]]).
- * `displayRows` is FILTERED and CAPPED, and the selection is not: Select-from-file matches
- * on a value and can tick pids the current view does not render — `fileSelect` has a whole
- * bucket for exactly that ("not in this view"). Capturing only what is on screen and
- * `continue`ing past the rest would delete two rows and report three, which is the silent-cap
- * defect in the one place it is least forgivable.
- *
- * REFUSED WHOLE rather than partially applied: a delete that half-happened leaves the reader
- * reconciling a count against a table, and the fix (scroll or clear the filter, then select
- * again) is one sentence away.
- */
- const offscreen: number[] = [];
- for (const pid of pids) {
- const rec = byPid.get(pid);
- if (!rec) {
- offscreen.push(pid);
- continue;
- }
- const values: Record = {};
- for (const f of fields) {
- const v = rec[f.key];
- if (v !== undefined && v !== null && v !== "")
- values[f.key] = v as UndoValue;
- }
- captured.push({ rid: pid, values });
- }
- if (offscreen.length) {
- signal(
- TOAST_EVENT,
- `Nothing was deleted — ${offscreen.length} of the ${pids.length} selected ` +
- `record${pids.length === 1 ? " is" : "s are"} not shown in this view. Clear the ` +
- `filter (or scroll them into view) and select again.`
- );
- return false;
- }
- if (!captured.length) return false;
-
- /*
- * ⭐ WARN THEN ALLOW (owner ruling, 2026-08-06) — the arm fires ONLY where something is
- * actually at stake, so deleting your own scratch rows stays one keypress.
- *
- * ⛔ THE PREDICATE IS "WHAT WOULD BE LOST", NOT "WHO CREATED THIS". There is no stored
- * creator on a row, and deriving one from a stage column would flip meaning as the
- * automation runs. But the question the warning answers is a different and answerable one:
- * *has an automation written to this row* — because that is exactly what a re-find cannot
- * give back. MEASURED in `run_discover_instagram`: the seen-before test reads the rows
- * CURRENTLY IN THE TABLE, so a deleted candidate returns as NEW — `found_count` back to 1,
- * `first_found` re-stamped, and its stage reset to Review. Deleting a candidate somebody
- * had already judged throws that judgement away.
- *
- * ⚠ This paragraph used to name a "Declined" stage. WAVE 26 / R6 DELETED THE BOARD'S
- * BUILT-IN TERMINALS — the lanes are whatever the user defined now — so the warning names
- * the mechanism (the stage resets to Review) and no longer a stage that does not ship.
- */
- /* ⭐ WAVE 27 · ITEM 12 / R3 — `f.automation?.stageField` SURVIVES THE STAGE DELETION, and
- that is SESSION B's recommendation taken rather than an oversight (their ASK ->C, in the
- wave doc): the engine stops WRITING the flag, but the guarded migration that removes an
- existing stage column rides `ut_ensure`, so a tenant's table is only cleaned the next
- time something writes to it. Until then a legacy stage column still exists on real
- data, and this reader is exactly what keeps it read-only. Deleting it would hand a
- machine-written column back to the user for the length of that window — which is the
- one thing R3's deletion must not do on its way out. */
- const machineKeys = fields
- .filter((f) => isMachineWritten(f) || f.automation?.stageField)
- .map((f) => f.key);
- // `captured` holds only NON-EMPTY values, so a key being present IS "a machine wrote here".
- const risky = captured.filter((r) => machineKeys.some((k) => r.values[k] !== undefined));
- const sig = captured.map((r) => r.rid).join(",");
- if (risky.length && delArmed !== sig) {
- setDelArmed(sig);
- signal(
- TOAST_EVENT,
- `${risky.length} of these ${risky.length === 1 ? "records was" : "records were"} ` +
- `written by an automation. Deleting ${risky.length === 1 ? "it" : "them"} resets ` +
- `first-found and times-found, and puts the stage back to Review — so a card ` +
- `somebody had already moved on comes back undecided. Press Delete again ` +
- `(or click again) to confirm.`
- );
- return false;
- }
- setDelArmed("");
- // SEQUENTIAL, matching the restore path: the store is one document and ten concurrent
- // read-modify-writes against it is how a delete silently misses a row.
- const gone: UndoRow[] = [];
- for (const r of captured)
- if (await deleteTableRow(scope, r.rid)) gone.push(r);
- if (!gone.length) {
- signal(TOAST_EVENT, "Nothing was deleted — the server refused.");
- return false;
- }
- undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
- kind: "rowDelete", table: scope, rows: gone,
- });
- // ⭐ ITEM 2 — same reason as the undo path: a row added and then deleted in one session
- // is still a PENDING copy here, and no future payload will ever prune it.
- withdrawPending(gone.map((r) => r.rid));
- clearSelection();
- signal(ROWS_STALE_EVENT);
- // ⚠ THE CONSEQUENCE IS NAMED IN FULL, and the first version of this sentence undersold it.
- // It said only that a matching record "will return", which is true and reassuring and
- // leaves out the part that actually costs something.
- //
- // MEASURED in `run_discover_instagram`: the seen-before test is built from the rows
- // CURRENTLY IN THE TABLE (`seen` comes from `existing2`). A deleted profile is therefore
- // not "seen again" on the next run — it is NEW. `found_count` resets to 1, `first_found`
- // is re-stamped, and the stage goes back to Review, because a new candidate's card starts
- // at the human gate. So deleting a candidate somebody had already moved out of Review puts
- // it back there undecided — the human judgement is the thing the re-find cannot give back.
- // (W26/R6 deleted the board's built-in terminals, so there is no named stage to cite here.)
- //
- // (The follower SNAPSHOT history is safe either way — it lives in the platform master keyed
- // by handle, a different store, which only `purge_handle` touches.)
- // Same survival, same reason as `machineKeys` above (item 12 / R3, B's ASK ->C): the flag
- // is no longer WRITTEN, and a table that has not been migrated yet still carries it.
- const fedByAutomation = fields.some((f) => f.automation?.stageField);
- signal(
- TOAST_EVENT,
- `Deleted ${gone.length} record${gone.length === 1 ? "" : "s"}. Ctrl+Z to undo.` +
- (fedByAutomation
- ? " If this automation finds one again it comes back as a NEW candidate — first-found" +
- " and times-found reset, and its stage back to Review. Undo keeps all of that."
- : "")
- );
- return true;
- },
- [canMutateRecords, displayRows, fields, scope, clearSelection, delArmed,
- withdrawPending]
- );
-
- const onGridDelete = useCallback(
- (sel: GridSelection): boolean => {
- /*
- * ⭐ 2026-08-06 (owner) — A ROW SELECTION + Delete/Backspace DELETES THE RECORDS.
- *
- * Checking rows and pressing Delete used to CLEAR every editable cell in them, which on the
- * owner's Instagram table now clears nothing at all: every column but one is machine-written
- * and refuses the write. So the key appeared to do nothing, which is what they reported.
- *
- * ⚠ ROWS ONLY, AND ONLY WHEN NO **MULTI-CELL** RANGE IS DRAWN. Checking rows is an explicit
- * gesture about RECORDS; DRAGGING a range is a gesture about CELLS, and a Delete that
- * destroyed records because some rows happened to be checked would feel unrecoverable even
- * with an undo behind it. When both are present, the narrower reading (clear the cells) wins.
- *
- * ⛔ A SINGLE ACTIVE CELL IS NOT A RANGE, and the first version of this got it wrong.
- * `sel.current.range` is ALWAYS set when a cell is merely focused — a 1×1 rect — so
- * `!sel.current?.range` meant the feature worked only if you had never clicked a cell.
- * Caught by driving it live rather than by the gate: click a cell, then check a row, then
- * press Delete, and the key silently went back to clearing cells. The gesture the rule is
- * actually about is a DRAG, which is width or height greater than one.
- */
- const dragged = !!sel.current
- && (sel.current.range.width > 1 || sel.current.range.height > 1);
- if (canMutateRecords && sel.rows.length > 0 && !dragged) {
- const pids: number[] = [];
- for (const rowIndex of sel.rows) {
- const vr = displayRows[rowIndex];
- if (vr && vr.kind === "data") pids.push(vr.record.pid);
- }
- // ⛔ `true`, NOT `false`, WHEN THIS BRANCH DOES NOT DO THE WORK — the scar the comment at
- // the bottom of this handler already records, pointed at a second case. `false` cancels
- // glide's own delete, so bailing with it would leave Delete doing nothing at all on a
- // selection this branch declined. `deleteRecords` is async and the keypress cannot wait
- // for it, so the branch commits here and reports the outcome through its own toast.
- if (!pids.length) return true;
- void deleteRecords(pids);
- return false;
- }
- const byPid = new Map>();
- const clear = (rowIndex: number, colIndex: number) => {
- const vr = displayRows[rowIndex];
- if (!vr || vr.kind !== "data") return;
- const column = visibleCols[colIndex];
- const field = column ? fieldByKey.get(column.id!) : undefined;
- if (!field || !canEditField(field)) return;
- const at = byPid.get(vr.record.pid) ?? {};
- at[field.key] = "";
- byPid.set(vr.record.pid, at);
- };
- for (const rowIndex of sel.rows)
- for (let c = 0; c < visibleCols.length; c++) clear(rowIndex, c);
- const range = sel.current?.range;
- if (range)
- for (let y = range.y; y < range.y + range.height; y++)
- for (let x = range.x; x < range.x + range.width; x++) clear(y, x);
- const writes = [...byPid.entries()].map(([pid, updates]) => ({ pid, updates }));
- if (!writes.length) return true;
- patchManyAndRecord(writes, "clearing cells");
- // ⚠ `false` ONLY WHEN THIS ACTUALLY DID THE WORK. Returning it unconditionally cancels
- // glide's own delete for cases this handler does not cover — a COLUMN selection, which
- // glide deletes from `toDelete.columns` and the loops above never look at — so Delete
- // would silently clear nothing. Handing the keypress back when there is nothing to
- // group is strictly safer than swallowing it: the per-cell path still refuses read-only
- // fields (`onCellEdited`'s own `canEditField`), it just does not arrive as one entry.
- return false;
- },
- [canMutateRecords, deleteRecords, displayRows, visibleCols, fieldByKey, canEditField,
- patchManyAndRecord]
- );
-
- /**
- * Owner item 4 / C-ADDROW — the cursor FOLLOWS the appended row, once it exists.
- *
- * The append is a server round trip, so at click time there is nothing to focus; this waits
- * for the re-read to bring the pid back and then lands the active cell on its first column,
- * scrolled into view. `newRowPid` is cleared either way — a row the re-read never produced
- * (a refused write, a filter that excludes it) must not leave a cursor waiting forever.
- */
- useEffect(() => {
- if (newRowPid === null) return;
- const index = displayPidToIndex.get(newRowPid);
- if (index === undefined) return;
- setActiveCell(0, index);
- gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" });
- setNewRowPid(null);
- }, [newRowPid, displayPidToIndex, setActiveCell]);
-
- /** Owner item 23 — the fields in the view's column order, for the Hide-fields panel. Built
- * from `order` (already reconciled by useGridColumns) rather than from `config.order` so the
- * panel and the grid can never disagree about which fields exist or where they sit. */
- const orderedFields = useMemo(
- () => order.map((key) => fieldByKey.get(key)).filter((f): f is Field => !!f),
- [order, fieldByKey]
- );
- /** Owner item 17 — the row of the ACTIVE cell, i.e. what the last click highlighted. Read off
- * `gridSelection` rather than tracked separately so keyboard navigation moves the wash too;
- * `useGridSelection` already guards this index against a grid that shrank this render. */
- const activeRow = gridSelection.current?.cell[1];
-
- const statusValues = useMemo(() => {
- const map: Record = {};
- for (const field of fields) {
- // A `user`'s people come from the HOST, never from the rows — an assignee nobody has been
- // given yet is still assignable.
- if (field.type === "user") {
- map[field.key] = payload?.userOptions ?? [];
- continue;
- }
- // ⭐ Owner item 24 — every other choice column goes through ONE rule (`choiceVocabulary`):
- // the DECLARED list when the field has one, the values seen in the data when it does not.
- // This used to branch on the type, which sent `stock_bucket` — a `select` whose vocabulary
- // is computed server-side and declares no `options` — down the declared path to an empty
- // list, and an empty supplied list WINS in the filter panel. See the function's own note;
- // it is pure so the gate can run it, which nothing inside this file can be.
- if (field.type === "select" || field.type === "multiselect" || field.type === "status")
- map[field.key] = choiceVocabulary(field, rawRows);
- }
- return map;
- }, [fields, rawRows, payload?.userOptions]);
-
- const rowPx = ROW_PX[config.rowHeightMode];
- // DISTINCT customers, not painted data rows. Grouping a MULTI field (Cohorts) puts the same
- // customer under every group it belongs to, so counting rows would report more records than
- // there are customers — a number nobody could reconcile against the book. `pidToIndex` is
- // first-wins per pid, so its size IS the distinct count, and it equals the row count for every
- // non-multi grouping and for no grouping at all.
- //
- // ⚠ The FULL pipeline's map, never the display slice's: the toolbar count must state what the
- // view MATCHES. What is painted is the "Showing first N" bar's job (owner item 8).
- const recordCount = pidToIndex.size;
- const shownRecords = displayPidToIndex.size;
- /* wave17 GRID — item 2 / owner R5. The whole-table row band is GONE, and with it the
- `bandTone` / `bandMask` pair that used to sit here: the winning control's tone, and the
- alternating-row mask built from the DATA-ROW ORDINAL so group headers could not flip the
- stripe at a boundary. R5 keeps the involved-COLUMN wash and the header tints — those are
- `useGridColumns.columnTones` + `COLUMN_TONE_THEME` and are untouched. What follows is now
- three row states, not four. */
- const getRowThemeOverride = useCallback(
- (row: number): Partial | undefined => {
- const visibleRow = displayRows[row];
- if (!visibleRow || visibleRow.kind !== "data") return undefined;
- // Owner item 17 — the row holding the ACTIVE cell stays washed after the pointer moves
- // on. Ordered AFTER hover on purpose: the pointer is the more immediate signal, and a
- // hovered-and-active row reading as merely active would make hover look broken.
- if (row === hoverRow) {
- if (!config.colorBy) return HOVER_NEUTRAL;
- return (
- HOVER_ROW_THEME[String(visibleRow.record[config.colorBy] ?? "").toLowerCase()] ??
- HOVER_NEUTRAL
- );
- }
- // A colour-by wash already marks this row with meaning the user chose; overlaying the
- // active tint on top would blend two hues into a third that means neither. There, glide's
- // accent ring is the active marker and this stays out of the way.
- if (!config.colorBy) {
- if (row === activeRow) return ACTIVE_ROW_NEUTRAL;
- return undefined;
- }
- return STATUS_ROW_THEME[
- String(visibleRow.record[config.colorBy] ?? "").toLowerCase()
- ];
- },
- [displayRows, config.colorBy, hoverRow, activeRow]
- );
- const onItemHovered = useCallback(
- (args: GridMouseEventArgs) => {
- const row = args.kind === "cell" ? args.location[1] : undefined;
- setHoverRow((previous) => (previous === row ? previous : row));
- // Owner item 19 — the hover-only Expand. Placed from glide's OWN bounds for the primary
- // cell of this row, so freeze, horizontal scroll and row-height mode are handled by the
- // component that owns them rather than re-derived here.
- //
- // ⚠ Only ever CLEARED from here for a different row, never for "the pointer left the
- // canvas". Moving the pointer ONTO the button leaves the canvas, so clearing on
- // out-of-bounds would unmount the control between the mouse arriving and the click
- // landing — an affordance that vanishes exactly when you reach for it, and one that
- // every assertion still sees because it exists in every state except the one that
- // matters. `.cg-grid-box`'s own onMouseLeave is what dismisses it.
- const vrow = row !== undefined ? displayRows[row] : undefined;
- if (args.kind === "cell" && vrow?.kind === "data" && row !== undefined) {
- const pid = vrow.record.pid;
- const b = gridRef.current?.getBounds(0, row);
- // Clamped to the grid's own box. ⚠ RETARGETED wave-14 item 15: this used to say "with a
- // grouping active `freezeColumns` is 0, so the primary column scrolls away". It no
- // longer does — the first column is now frozen in EVERY mode. The clamp stays because
- // its other half is still live: glide's bounds are VIEWPORT coordinates, so a row
- // scrolled under the header or past the bottom still reports a rect outside the grid,
- // and a `position: fixed` button with no clamp would paint over the chrome on a row
- // nobody can see. The horizontal legs are now defence in depth (a column drag, a box
- // narrower than the frozen strip) rather than the everyday case.
- const at = b
- ? expandButtonRect(b, gridBoxRef.current?.getBoundingClientRect())
- : null;
- // Recomputed EVERY move, not memoised on the pid: a scroll can leave the pointer over
- // the same record at a new y, and a button that keeps the pid but not the position
- // floats over the wrong row. Referential stability is preserved by comparing values,
- // which is the cheap half — `getBounds` is arithmetic over glide's own layout.
- setExpandAt((prev) =>
- !at
- ? null
- : prev && prev.pid === pid && prev.x === at.x && prev.y === at.y
- ? prev
- : { pid, x: at.x, y: at.y, size: at.size }
- );
- }
- // Wave-5 item 6 — hovering a header whose field carries a description floats the text
- // under the header. The tip element is pointer-events:none and aria-hidden: it can
- // NEVER become the click target, which is the tooltip trap this page has already paid
- // for once ([[ui-invisible-to-assertions]]).
- if (args.kind === "header") {
- const colDef = visibleCols[args.location[0]];
- const field = colDef ? fieldByKey.get(colDef.id!) : undefined;
- const note = field ? field.note || field.description : undefined;
- // Owner item 16 — a title the fit SHORTENED must still be readable somewhere, and the
- // tip is where. `colDef.title !== field.label` IS the truncation test: `fitHeaderTitle`
- // returns the label by identity when it fits, so this asks the renderer what it did
- // rather than re-running the measurement and hoping the two agree.
- const cut = !!field && !!colDef && colDef.title !== field.label;
- const text = cut && field ? (note ? `${field.label} — ${note}` : field.label) : note;
- const bounds = args.bounds;
- if (text && bounds) {
- setHeaderTip((prev) =>
- prev && prev.text === text && prev.x === bounds.x
- ? prev
- : { text, x: bounds.x, y: bounds.y + bounds.height + 4 }
- );
- return;
- }
- }
- setHeaderTip((prev) => (prev === null ? prev : null));
-
- // I2 — the cell half. Only text that is genuinely CUT OFF gets a tip: a tooltip on
- // every cell just repeats what is already legible and covers the row under it.
- // Truncation is MEASURED with glide's own font rather than guessed from a character
- // count, which is what makes "e.g. a Note" work and a short currency cell stay quiet.
- if (args.kind === "cell" && row !== undefined) {
- const vr = displayRows[row];
- const cellCol = visibleCols[args.location[0]];
- // Group headers and footers carry no cell text of their own — a tip over the group
- // bar would repeat the label glide has already drawn across it.
- if (vr?.kind === "data" && cellCol) {
- const field = fieldByKey.get(cellCol.id!);
- const text = field ? formatDisplay(field, vr.record[field.key]) : "";
- const bounds = args.bounds;
- const tip = bounds ? cellTipText(text, measureCellText(text), bounds.width) : null;
- if (tip && bounds) {
- const x = tipLeft(bounds.x, window.innerWidth, measureCellText(tip) + 20);
- const y = bounds.y + bounds.height + 4;
- setCellTip((prev) =>
- prev && prev.text === tip && prev.x === x && prev.y === y
- ? prev
- : { text: tip, x, y }
- );
- return;
- }
- }
- }
- setCellTip((prev) => (prev === null ? prev : null));
- },
- [visibleCols, fieldByKey, displayRows]
- );
- const rowHeight = useMemo(
- () =>
- config.groupBy
- ? (row: number) => (displayRows[row]?.kind === "group-header" ? 32 : rowPx)
- : rowPx,
- [config.groupBy, rowPx, displayRows]
- );
-
- /* ═══ W18-B VOID ═══ (wave 18, owner item 1b) — WHERE THE TABLE ENDS, in canvas pixels.
- Consumed by the two `.cg-grid-void` rectangles at the DataEditor mount; see the CSS region
- of the same name for why the void is painted in the DOM rather than in the glide theme.
-
- ⭐ THE PROPERTY THAT MAKES THIS ARITHMETIC AND NOT SCROLL-TRACKING, and the reason there is
- no `onVisibleRegionChanged` handler anywhere near it: **a void can only exist in an axis the
- content does not overflow.** glide's scroll extent is EXACTLY the content — `scrollWidth =
- nonGrowWidth (+ overscrollX)`, `scrollHeight = header + rows (+ overscrollY)`, and we pass
- neither overscroll prop (scrolling-data-grid.js:12-24) — so at maximum scroll the last
- row/column lands flush on the client edge. Wherever there is something to paint, the scroll
- offset in that axis is 0 and cannot become anything else. The rectangles therefore depend on
- the columns, the rows and the box, and on nothing that moves while the user drags.
-
- Every column carries an explicit `width` (`useGridColumns` sets `config.widths[key] ??
- DEFAULT_WIDTH`) and none carries `grow`, so glide's column sizer passes them through
- untouched and this sum is the width it actually lays out. */
- const gridVoid = useMemo(() => {
- const gutter = scrollbarGutter();
- let colsPx = rowMarkerPx(displayRows.length);
- for (const c of visibleCols) colsPx += (c as { width?: number }).width ?? 0;
- let rowsPx = HEADER_PX;
- if (typeof rowHeight === "number") rowsPx += displayRows.length * rowHeight;
- else for (let i = 0; i < displayRows.length; i++) rowsPx += rowHeight(i);
- // ⚠ Owner item 4 — THE GHOST ROW IS PART OF THE TABLE, and this line is what makes it
- // visible. glide draws its trailing row AFTER the last data row, but `displayRows` (our
- // rows) does not contain it, so the void started exactly where the ghost row does and
- // painted flat #F6F8FC straight over it. The "+" was still clickable the whole time
- // (`.cg-grid-void` is `pointer-events: none`), which is the worst version of this bug:
- // the gate went green on an affordance nobody could see. Found by READING THE SCREENSHOT
- // ([[ui-invisible-to-assertions]], [[finalize-visual-review-sop]]).
- if (canMutateRecords)
- rowsPx += typeof rowHeight === "number" ? rowHeight : rowHeight(displayRows.length);
-
- /* Fit is tested against the client box the OTHER axis's scrollbar leaves behind — the same
- `clientWidth`/`clientHeight` glide's own scroll handler reads (infinite-scroller.js:
- 110-116), so this branch and glide's cannot disagree about whether a bar is there. The
- two `if`s resolve the circularity in the only direction it can run: a bar in one axis can
- CREATE one in the other, but two bars can never un-create each other. */
- let vBar = rowsPx > gridSize.height;
- let hBar = colsPx > gridSize.width;
- if (vBar && !hBar) hBar = colsPx > gridSize.width - gutter;
- if (hBar && !vBar) vBar = rowsPx > gridSize.height - gutter;
- const clientW = gridSize.width - (vBar ? gutter : 0);
- const clientH = gridSize.height - (hBar ? gutter : 0);
-
- /* `null` = the content reaches that edge, so there is no void and no rectangle. The
- comparison is strict: a table ending exactly on the edge has nothing past it. */
- return {
- clientW,
- clientH,
- below: rowsPx < clientH ? rowsPx : null,
- right: colsPx < clientW ? colsPx : null,
- };
- }, [visibleCols, displayRows, rowHeight, gridSize, canMutateRecords]);
- /* ═══ end W18-B VOID (geometry) ═══ */
-
- // The record drawer resolves positions against what the MODE paints: the display slice for
- // grid/list (the cap is real there), the FULL pipeline for calendar/kanban/map (a month, a
- // stack or a pin reaches past the cap by design — a drawer that refused those pids would
- // close itself on a card the user can plainly see).
- // ⭐ WAVE-27 item 8 (C3) — `swipe` belongs here for calendar/kanban/map's own reason: the deck
- // is derived from `modeDataRows` (the FULL pipeline), so a card past the display cap is one
- // the user can plainly see, and a drawer that refused its pid would close itself on open.
- const fullSetMode =
- displayMode === "calendar" || displayMode === "kanban" || displayMode === "map" ||
- displayMode === "swipe";
- const detailRows = fullSetMode ? visibleRows : displayRows;
- const detailIdxMap = fullSetMode ? pidToIndex : displayPidToIndex;
- const detailIndex = detailPid !== null ? detailIdxMap.get(detailPid) : undefined;
- useEffect(() => {
- if (detailPid !== null && detailIndex === undefined) setDetailPid(null);
- }, [detailPid, detailIndex]);
- const detailRecord = useMemo(() => {
- if (detailIndex === undefined) return null;
- const row = detailRows[detailIndex];
- if (!row || row.kind !== "data") return null;
- return overlayEdits[row.record.pid]
- ? { ...row.record, ...overlayEdits[row.record.pid] }
- : row.record;
- }, [detailIndex, detailRows, overlayEdits]);
- const dataPosition = useMemo(() => {
- if (detailIndex === undefined) return 0;
- return detailRows
- .slice(0, detailIndex + 1)
- .filter((row) => row.kind === "data").length;
- }, [detailIndex, detailRows]);
- const neighborExists = useCallback(
- (delta: -1 | 1) => {
- if (detailIndex === undefined) return false;
- for (
- let index = detailIndex + delta;
- index >= 0 && index < detailRows.length;
- index += delta
- )
- if (detailRows[index]?.kind === "data") return true;
- return false;
- },
- [detailIndex, detailRows]
- );
- const go = useCallback(
- (delta: -1 | 1) => {
- if (detailIndex === undefined) return;
- for (
- let index = detailIndex + delta;
- index >= 0 && index < detailRows.length;
- index += delta
- ) {
- const row = detailRows[index];
- if (row?.kind !== "data") continue;
- setDetailPid(row.record.pid);
- if (displayMode === "grid")
- gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" });
- return;
- }
- },
- [detailIndex, detailRows, displayMode]
- );
-
- /**
- * Wave-9 I3 — the description "(i)", drawn RIGHT-ALIGNED and vertically centred with the
- * field name.
- *
- * ⚠ This is the header-draw alternative, taken because glide's `overlayIcon` PROVABLY
- * cannot do it: `drawHeaderInner` paints an overlay at a hard-coded `drawX + 9` /
- * `(height - 18) / 2 + 6` — a badge on the bottom-right corner of the TYPE mark at the far
- * LEFT of the header — and no prop moves it. `drawHeader` is glide's supported escape
- * hatch: it hands over the ctx, the rect, the menu bounds and the sprite manager, plus a
- * `drawContent()` that runs its own rendering first. So glide still draws the header it
- * always drew (type mark, title, menu); only the (i) is ours, and `overlayIcon` is no
- * longer set on any column.
- *
- * The geometry is `infoMarkRect()` in overlayPlacement.ts, so "right-aligned and centred"
- * is asserted numerically by a gate rather than judged from one screenshot at one width.
- */
- const drawGridHeader = useCallback(
- (args, drawContent) => {
- const field = args.column.id ? fieldByKey.get(args.column.id) : undefined;
- // ⚠⚠ WAVE-14 ITEM 2 — THE ONE LINE THAT MAKES AN INVOLVED HEADER BOLD, and it is not a
- // font setting anywhere near where you would look for one.
- //
- // glide's `drawGridHeaders` does, verbatim:
- // if (theme !== outerTheme) ctx.font = theme.baseFontFull; // :38-40
- // i.e. the moment a column carries ANY `themeOverride` its header title is painted in the
- // CELL font ("13px") instead of the header font ("600 13px"). That is why the owner's
- // filtered/sorted/grouped headers had already lost their weight — nothing in our code
- // asked for it, and setting `headerFontStyle` on the involved theme alone would have
- // changed precisely nothing, because glide never reads it on that path. Re-asserting the
- // MERGED theme's header font on the ctx here is what puts it back, for every column,
- // uniformly: on a column with no override this is a no-op (glide set the same value at
- // :21), and glide's own save/restore around each column stops it leaking to the next.
- //
- // Built from `headerFontStyle` + `fontFamily` rather than reading `headerFontFull`,
- // which is real at runtime but is declared on `FullTheme`, not the public `Theme` the
- // callback is typed with — same string, no cast, and it is exactly how
- // `mergeAndRealizeTheme` composes it.
- //
- // Bonus fix, worth knowing: `drawHeaderInner` passes `theme.headerFontFull` to
- // `getMiddleCenterBias`, which measures with the CURRENT ctx font but CACHES under the
- // string it was handed. Before this line, an overridden column measured its baseline bias
- // in the 13px font and filed it under the "600 13px" key, so whichever column drew first
- // decided the vertical centring for all of them.
- args.ctx.font = `${args.theme.headerFontStyle} ${args.theme.fontFamily}`;
- drawContent();
- if (!field) return;
- const hasInfo = !!(field.note || field.description);
- // ⭐ OWNER ITEM 4 (2026-08-06): *"why is the Field still have the Dot at the header to
- // mark its a custom editable field?"* Because the predicate was `source === "overlay"`
- // alone — true of columns a person added BESIDE the Odoo ones, and true of literally
- // every column in a `ut_*` database, including the ones an automation spawns and fills.
- // So on the owner's Instagram table the dot marked all 23 columns as "yours", which is
- // both meaningless (it never varies) and wrong (they are not yours to edit).
- const isCustom = field.source === "overlay" && !isMachineOwned(field);
- // ONE layout for both marks, from the same `headerMarkSizes` order `useGridColumns` used
- // to reserve the label's room — so the number of marks drawn and the number reserved for
- // cannot disagree. null = the column is too narrow to carry them without covering its own
- // label, and then NOTHING is drawn (all-or-nothing; see headerMarkLayout).
- const marks = headerMarkLayout(
- args.rect,
- args.menuBounds.width,
- headerMarkSizes(hasInfo, isCustom)
- );
- if (!marks) return;
- let slot = 0;
- if (hasInfo) {
- const at = marks[slot++];
- args.spriteManager.drawSprite(
- "aiosInfo",
- "normal",
- args.ctx,
- at.x,
- at.y,
- at.size,
- args.theme
- );
- }
- if (isCustom) {
- // Wave-14 item 1 / R11 — the user-created-field marker that REPLACES the yellow header
- // wash. A muted dot: quiet enough to ignore while reading, present enough to answer
- // "which of these columns are mine?" at a glance. Never a background — the owner killed
- // the wash, and a paler wash would have been the same answer in a lower voice.
- const at = marks[slot++];
- const ctx = args.ctx;
- ctx.save();
- ctx.beginPath();
- ctx.arc(at.x + at.size / 2, at.y + at.size / 2, at.size / 2, 0, Math.PI * 2);
- ctx.fillStyle = CUSTOM_FIELD_MARK;
- ctx.fill();
- ctx.restore();
- }
- },
- [fieldByKey]
- );
-
- const onCellClicked = useCallback(
- (cell: Item, event: CellClickedEventArgs) => {
- const row = displayRows[cell[1]];
- if (!row) return;
- // Owner item 10: clicking into the cells is "I am working now" — the frame folds its
- // navigation rail to the slim strip (a no-op in the embed; the shell listens).
- signal(NAV_MINIMIZE_EVENT);
- if (row.kind === "group-header") {
- event.preventDefault();
- setCollapsed((current) => {
- const next = new Set(current);
- if (next.has(row.groupKey)) next.delete(row.groupKey);
- else next.add(row.groupKey);
- return next;
- });
- return;
- }
- const column = visibleCols[cell[0]];
- const field = column ? fieldByKey.get(column.id!) : undefined;
- // A picked field (select / assignee) opens its choices where the cell is. It cannot use
- // glide's text overlay — a free-text editor on a constrained column is how you end up
- // with "Done", "done" and "DONE" as three different values — and glide's own dropdown
- // cell lives in a package we have deliberately not added, so this reuses the same
- // AnchoredOverlay the column and view menus already use. Wave-5: a RATING cell rides
- // the same picker surface (its choices are 1..max stars); both doors respect the
- // permissions verdict.
- if (row.kind === "data" && field && canEditField(field)
- && (isPickType(field.type) || field.type === "rating")) {
- event.preventDefault();
- // Item 6 (2026-07-31) — the click also LANDS the active cell here (preventDefault
- // stops glide from committing it), so pick → Enter walks on down the column exactly
- // like a typed edit does.
- setActiveCell(cell[0], cell[1]);
- const b = event.bounds;
- setPicker({
- pid: row.record.pid,
- fieldKey: field.key,
- anchor: {
- left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height,
- width: b.width, height: b.height,
- },
- });
- return;
- }
- // A linked-record cell is a doorway to the target database, not an opaque id list.
- // The large modal mounts the same Grid surface over exactly these pids, so its standard
- // search, Filters, Sort, Fields, and column menus keep working for every database kind.
- if (row.kind === "data" && field?.type === "link" && field.link?.table) {
- if (field.link.table.startsWith("ut_")) {
- event.preventDefault();
- setActiveCell(cell[0], cell[1]);
- setLinkAt({ pid: row.record.pid, fieldKey: field.key });
- return;
- }
- }
- // ⭐ Wave-23 C7 — a JSON cell opens the big viewer. It is the ONLY door: the cell carries
- // `allowOverlay:false`, because glide's overlay is a one-line box and one keystroke in the
- // wrong place inside a 32 KB document turns a well-formed payload into an unparseable one,
- // saved. Opened for EVERY reader (a document you may not edit is still one you must be
- // able to read) — the viewer takes `onSave` only when the permission verdict allows it,
- // and the host's write wall is the real one either way.
- if (row.kind === "data" && field?.type === "json") {
- event.preventDefault();
- setActiveCell(cell[0], cell[1]);
- setJsonAt({ pid: row.record.pid, fieldKey: field.key });
- return;
- }
- // Wave-5 item 11 — a URL cell opens its link on click (scheme-guarded: http/https only,
- // a bare domain gets https://). Editing stays with glide's overlay (Enter/double-click).
- if (row.kind === "data" && field?.type === "url") {
- const raw = String(
- overlayEdits[row.record.pid]?.[field.key] ?? row.record[field.key] ?? ""
- ).trim();
- if (raw) {
- const href = /^https?:\/\//i.test(raw)
- ? raw
- : /^[\w-]+(\.[\w-]+)+/.test(raw)
- ? `https://${raw}`
- : null;
- if (href) {
- event.preventDefault();
- window.open(href, "_blank", "noopener");
- return;
- }
- }
- }
- // Owner item 6 (2026-07-31) — CLICKING THE CUSTOMER TICKS THE CHECKBOX. The row marker
- // is a ~32px strip; the identity cell is the widest, most natural target on the row, so
- // a click there toggles the same pid-anchored set the markers write ("more surface area
- // to select individual customers into a Cohort"). No preventDefault: glide still commits
- // the cell highlight below, so reading across the row keeps working.
- if (row.kind === "data" && field && field.key === lockedKey) {
- togglePid(row.record.pid);
- }
- // Owner item 17 — A SINGLE CLICK HIGHLIGHTS. It used to open the record panel from here,
- // which meant a user could not select a cell, read across a row, or copy a value without
- // a drawer landing over the table. The highlight is glide's own doing: this handler
- // returns without `preventDefault`, so the click commits `gridSelection.current`, the
- // accent ring lands on the cell and `getRowThemeOverride` washes the row
- // (ACTIVE_ROW_NEUTRAL). Nothing is drawn here.
- //
- // ⚠ SHIPPED WITH ITEM 19, never alone. Deleting this line is what removes the ONLY way to
- // open a record; the hover Expand button below is its replacement, and half of this
- // change is a table whose records cannot be opened at all.
- },
- [displayRows, visibleCols, fieldByKey, canEditField, overlayEdits, lockedKey, togglePid,
- setActiveCell]
- );
- /**
- * Double-click / Enter. KEPT as a door to the record on purpose (owner item 17 names the
- * SINGLE click, and it is the single click that was in the way).
- *
- * Removing it too would leave the record reachable only by pointer — the hover affordance
- * cannot be reached from the keyboard at all — so a keyboard user would lose the panel
- * outright. glide routes Enter on an EDITABLE cell to its overlay editor before this fires,
- * so the two doors do not collide: this is the activation path for the read-only columns,
- * which is most of the table.
- */
- const onCellActivated = useCallback(
- (cell: Item) => {
- const row = displayRows[cell[1]];
- const column = visibleCols[cell[0]];
- const field = column ? fieldByKey.get(column.id!) : undefined;
- if (row?.kind === "data" && field?.source !== "overlay")
- setDetailPid(row.record.pid);
- },
- [displayRows, visibleCols, fieldByKey]
- );
- const onCellEdited = useCallback(
- (cell: Item, value: EditableGridCell) => {
- const row = displayRows[cell[1]];
- const column = visibleCols[cell[0]];
- const field = column ? fieldByKey.get(column.id!) : undefined;
- if (!row || row.kind !== "data" || !field || !canEditField(field)) return;
- // Wave-5 item 11 — a checkbox toggles straight through glide's BooleanCell (no overlay
- // editor); the overlay store keeps its '1'-or-empty contract.
- if (value.kind === GridCellKind.Boolean && field.type === "checkbox") {
- patchAndRecord(row.record.pid, { [field.key]: value.data ? "1" : "" }, "a tick");
- return;
- }
- if (value.kind === GridCellKind.Uri) {
- patchAndRecord(row.record.pid, { [field.key]: value.data ?? "" }, "an edit");
- return;
- }
- if (value.kind === GridCellKind.Text || value.kind === GridCellKind.Number) {
- patchAndRecord(row.record.pid, { [field.key]: value.data }, "an edit");
- }
- },
- [displayRows, visibleCols, fieldByKey, patchAndRecord, canEditField]
- );
- const validateCell = useCallback(
- (cell: Item): boolean => {
- const row = displayRows[cell[1]];
- const column = visibleCols[cell[0]];
- const field = column ? fieldByKey.get(column.id!) : undefined;
- // ONE verdict (types.mayEditField): stratum + read-only-by-nature + permissions. The
- // picked types and rating never take glide's text overlay — their pickers are the door.
- return (
- !!row &&
- row.kind === "data" &&
- !!field &&
- canEditField(field) &&
- !isPickType(field.type) &&
- field.type !== "rating" &&
- field.type !== "status"
- );
- },
- [displayRows, visibleCols, fieldByKey, canEditField]
- );
- const onGridPaste = useCallback(
- (target: Item, values: readonly (readonly string[])[]): boolean => {
- const column = visibleCols[target[0]];
- const field = column ? fieldByKey.get(column.id!) : undefined;
- if (!field) return false;
- // ⭐ Wave-15 item 3 (R8) — THE SELECTION DECIDES HOW FAR THE PASTE REACHES. Until now the
- // targets were walked down from the anchor for exactly `values.length` rows, so a
- // fifty-row selection and a one-cell clipboard wrote ONE row: the selection was painted
- // and obeyed by nothing. `pasteRowCount` is the whole rule and it is pure, so the gate
- // can drive it — this handler cannot be reached by any node test in the repo.
- const rowCount = pasteRowCount(values.length, gridSelection.current?.range,
- { col: target[0], row: target[1] });
- const targetPids = Array.from({ length: rowCount }, (_, offset) => {
- const row = displayRows[target[1] + offset];
- return row?.kind === "data" ? row.record.pid : null;
- });
- const patches = planFieldPaste({
- field,
- sourceFieldKey: copyProvenanceRef.current.fieldKey,
- editable: canEditField(field),
- values,
- targetPids,
- allowedChoices: statusValues[field.key] ?? choiceOptions(field),
- });
- if (!patches) return false;
- // R4 — ONE stack entry for the whole paste. Cell by cell would put forty entries on the
- // stack for one Ctrl+V, and undoing a paste one cell at a time is not undoing a paste.
- patchManyAndRecord(
- patches.map((patch) => ({ pid: patch.pid, updates: { [field.key]: patch.value } })),
- "a paste"
- );
- // We handled the write ourselves. Returning false tells glide not to run its cell
- // renderers' generic paste path (Bubble/rating renderers cannot enforce this contract).
- return false;
- },
- [visibleCols, fieldByKey, canEditField, statusValues, displayRows, patchManyAndRecord,
- gridSelection.current]
- );
-
- const openHeaderMenu = useCallback(
- (column: number, bounds: Rectangle) => {
- if (embedded) return;
- const definition = visibleCols[column];
- if (!definition?.id) return;
- setColumnMenu({
- fieldKey: definition.id,
- anchor: {
- left: bounds.x,
- top: bounds.y,
- right: bounds.x + bounds.width,
- bottom: bounds.y + bounds.height,
- width: bounds.width,
- height: bounds.height,
- },
- });
- },
- [embedded, visibleCols]
- );
- const onHeaderClicked = useCallback(
- (column: number, event: HeaderClickedEventArgs) => {
- if (event.isEdge) return;
- event.preventDefault();
- openHeaderMenu(column, event.bounds);
- },
- [openHeaderMenu]
- );
- const onGridKeyDown = useCallback(
- (event: GridKeyEventArgs) => {
- if (
- event.key.toLowerCase() === "c" &&
- (event.ctrlKey || event.metaKey) &&
- !event.altKey
- ) {
- const range = gridSelection.current?.range;
- const source = range?.width === 1 ? visibleCols[range.x] : undefined;
- copyProvenanceRef.current = markGridCopy(source?.id ?? null, Date.now());
- // Do not prevent default: glide still owns serialization and the OS clipboard write.
- return;
- }
- // Owner item 5 (2026-07-31) — EXCEL-GRADE ENTER. When no editor is open (an open overlay
- // editor swallows its own keys before the canvas sees them), Enter moves the active cell
- // DOWN one record and Shift+Enter moves UP — never opening the record drawer, which is
- // what made keyboard runs down a column "really clunky". Glide's own overlay editor
- // already commits-and-moves-down on Enter, so typing → Enter → typing flows like Excel;
- // this handles the BETWEEN-edits half. Group headers are skipped in the direction of
- // travel. The drawer stays reachable by double-click and the hover Expand button.
- if (event.key === "Enter" && !event.ctrlKey && !event.metaKey && !event.altKey) {
- const cur = gridSelection.current?.cell;
- if (cur) {
- event.preventDefault();
- event.stopPropagation();
- event.cancel();
- // Owner item 6 (2026-07-31) — Enter on a PICKED cell (select / multi select /
- // assignee / rating) opens its picker: the keyboard door the mouse click already
- // had. glide's text overlay cannot serve these types (validateCell refuses them),
- // so without this the keyboard run down a column dead-ends at every picked field.
- // Shift+Enter stays pure navigation, so walking UP past picked cells still works.
- const curRow = displayRows[cur[1]];
- const curColumn = visibleCols[cur[0]];
- const curField = curColumn ? fieldByKey.get(curColumn.id!) : undefined;
- if (
- !event.shiftKey &&
- curRow?.kind === "data" &&
- curField &&
- canEditField(curField) &&
- (isPickType(curField.type) || curField.type === "rating")
- ) {
- const b = gridRef.current?.getBounds(cur[0], cur[1]);
- if (b) {
- setPicker({
- pid: curRow.record.pid,
- fieldKey: curField.key,
- anchor: {
- left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height,
- width: b.width, height: b.height,
- },
- });
- return;
- }
- }
- const dir = event.shiftKey ? -1 : 1;
- let row = cur[1] + dir;
- while (row >= 0 && row < displayRows.length && displayRows[row]?.kind !== "data")
- row += dir;
- if (row >= 0 && row < displayRows.length) {
- setActiveCell(cur[0], row);
- gridRef.current?.scrollTo(cur[0], row, "vertical", 0, 0);
- }
- return;
- }
- }
- const contextMenu =
- event.key === "ContextMenu" || (event.key === "F10" && event.shiftKey);
- // DataEditor exposes public controlled selection without the row marker,
- // while GridKeyEventArgs.location comes from its internal grid and still
- // includes that marker at column zero.
- const eventColumn =
- event.location && event.location[0] > 0
- ? event.location[0] - 1
- : undefined;
- const column = gridSelection.current?.cell[0] ?? eventColumn;
- if (!contextMenu || column === undefined || !event.bounds) return;
- const gridBounds = gridBoxRef.current?.getBoundingClientRect();
- const headerTop = gridBounds?.top ?? event.bounds.y;
- event.preventDefault();
- event.stopPropagation();
- event.cancel();
- openHeaderMenu(column, {
- x: event.bounds.x,
- y: headerTop,
- width: event.bounds.width,
- height: 36,
- });
- },
- [gridSelection.current, openHeaderMenu, displayRows, setActiveCell,
- visibleCols, fieldByKey, canEditField]
- );
-
- const persistView = useCallback((view: SavedView) => {
- setViews((current) => {
- const found = current.some((item) => item.id === view.id);
- return found
- ? current.map((item) => (item.id === view.id ? view : item))
- : [...current, view];
- });
- emitHostEvent({ id: eventId("view"), type: "view_upsert", view });
- }, []);
-
- /**
- * Item 12 (C-LOCK) — set or clear a view's cohort lock, from the rail's menu.
- *
- * Works on ANY view, not only the active one, which is why it goes through `persistView`
- * rather than `updateConfig`: the rail's menu opens on whichever row you clicked. When the
- * target IS the active view, the live `config` must move too, or the table keeps showing the
- * old row set until the next select and the lock reads as ignored.
- *
- * Clearing DELETES the key rather than storing null — the no-churn rule every other optional
- * config member follows, and the shape `_clean_display`'s sibling validator expects.
- */
- const onViewCohortLock = useCallback(
- (viewId: string, cohortId: string | null) => {
- const target = views.find((v) => v.id === viewId);
- if (!target) return;
- const nextConfig = { ...target.config };
- if (cohortId) nextConfig.cohortLock = cohortId;
- else delete (nextConfig as Record).cohortLock;
- persistView({ ...target, config: nextConfig });
- if (viewId === activeViewId) {
- setConfig((live) => {
- const next = { ...live };
- if (cohortId) next.cohortLock = cohortId;
- else delete (next as Record).cohortLock;
- return next;
- });
- }
- },
- [views, activeViewId, persistView]
- );
-
- const selectView = useCallback(
- (id: string) => {
- const current = views.find((view) => view.id === activeViewId);
- if (current && !sameConfig(current.config, config))
- persistView({ ...current, config });
- const next = views.find((view) => view.id === id);
- if (!next) return;
- setActiveViewId(id);
- setConfig(normalizeConfig(next.config, fields));
- setCollapsed(new Set());
- setSearch("");
- setDetailPid(null);
- setDisplayCap(DISPLAY_PAGE);
- // Owner item 3 (2026-07-31): tell the host WHERE THE USER IS, so a fresh browser (no
- // localStorage copy) resumes on this view instead of the system default. Presentation
- // state — the host stores the id and the read side re-validates it.
- emitHostEvent({ id: eventId("view"), type: "view_select", viewId: id });
- // Owner item 10: opening a view is "I am working now" — the frame folds its nav rail.
- signal(NAV_MINIMIZE_EVENT);
- },
- [views, activeViewId, config, persistView, fields]
- );
-
- /**
- * A-S4-3 (item 25) — a notification's click-through. The SHELL routes to the table and fires
- * this; the GRID owns view selection, so neither learns the other's state.
- *
- * ⛔ IGNORE, NEVER THROW, when the view is not ours: an alert can outlive the view it watches
- * (deleted, or a share revoked), and a reader who can no longer see it must simply land on the
- * table. `selectView` already returns early on an unknown id; the topic check stops one grid
- * reacting to another's alert when both are mounted.
- */
- useEffect(() => {
- const onOpen = (e: Event) => {
- const detail = (e as CustomEvent).detail;
- if (!detail || detail.topic !== scope) return;
- if (!views.some((v) => v.id === detail.viewId)) return;
- selectView(detail.viewId);
- };
- window.addEventListener(VIEW_OPEN_EVENT, onOpen);
- return () => window.removeEventListener(VIEW_OPEN_EVENT, onOpen);
- }, [scope, views, selectView]);
- const createView = useCallback(
- (name: string, mode: DisplayMode, permissions: ViewPermissions) => {
- const acceptedName = uniqueDisplayName(name, views.map((view) => view.name));
- /**
- * ⭐⭐ WAVE 27 · OWNER ITEM 9 / RULING R4 — **A NEW VIEW IS BLANK. ALL OF IT.**
- *
- * ⛔ THIS REVERSES THE WAVE-26 COMMENT THAT STOOD HERE, so the reversal is stated rather
- * than quietly applied. That comment dropped `cohortLock` from the spread and defended
- * keeping the rest: *"their inheritance is a FEATURE (build a view, branch off it)
- * precisely because it is visible"*. The owner disagrees, in as many words — R4:
- * **"New views ALWAYS start blank — no filters/sorts inherited from anything."** The
- * branch-off use it defended gets its own door later (Duplicate view); it is not what the
- * "+" button means, and the wave-26 fix was the right diagnosis of the wrong scope — one
- * member of the spread was invisible, but ALL of them arrived unasked.
- *
- * ⛔ AND `defaultViewConfig` IS THE BLANK, not a literal assembled here. It is the same
- * function the grid's own initial state and the cohort door already use, so "blank" has
- * ONE definition in this file ([[one-evaluator-per-question]]); a second literal beside it
- * would be a place for the two to disagree the day a `ViewConfig` member is added — and
- * the one that got added last wave is exactly what caused this bug.
- *
- * WHAT STOPS CARRYING, enumerated because a reader deserves the list and not just the
- * ruling: `filters` + `filterConj`, `sorts`, `groupBy`, `colorBy`, `rowHeightMode`,
- * `order`/`visible` (hidden columns), `widths`, `memberPids`, `frozenCount` and
- * `cohortLock`. `order`/`visible` come back from the FIELD list's own default visibility,
- * which is why `fields` replaces `config` in the dependency list below.
- *
- * ⚠ THE DISPLAY REFS GO TOO, and that is the one deliberate loss. W13's carry rule kept
- * the calendar's date field / kanban's stack field / map's lat-lon across a mode switch so
- * Grid→Map→Grid→Map did not re-ask; a NEW view is not a mode switch, and inheriting the
- * previous view's stack field is inheritance of exactly the kind R4 names. A ref-less
- * kanban asks which field to stack by, which is the honest state for a view born empty.
- * `cleanDisplay` still collapses a ref-less GRID to absent, so picking "Grid" produces the
- * byte-identical shape every pre-wave-9 view has (no churn).
- */
- const nextConfig: ViewConfig = {
- ...defaultViewConfig(fields),
- display: cleanDisplay({ mode }),
- };
- const view: SavedView = {
- id: nextViewId(),
- name: acceptedName,
- kind: "custom",
- config: nextConfig,
- // I17 (C4) — sent EXPLICITLY. Absent on create means 'personal' host-side, so a user
- // who chose "Collaborative" would silently get the opposite. `createdBy` is NOT sent:
- // the host stamps it and ignores whatever the browser claims.
- permissions,
- };
- persistView(view);
- setActiveViewId(view.id);
- // The LIVE config has to move too, not just the stored one. Without this the view is
- // created as a Calendar and the user keeps staring at the Grid until they switch views
- // and back — the mode would be real in the store and invisible on screen.
- setConfig(nextConfig);
- setSaveState("saved");
- },
- // `fields`, not `config` (R4): the blank is derived from the COLUMNS, and reading the live
- // config here at all is what item 9 deletes.
- [fields, persistView, views]
- );
- const renameView = useCallback(
- (id: string, name: string) => {
- const view = views.find((item) => item.id === id);
- if (view) {
- const acceptedName = uniqueDisplayName(
- name,
- views.filter((item) => item.id !== id).map((item) => item.name)
- );
- persistView({ ...view, name: acceptedName });
- }
- },
- [views, persistView]
- );
- /**
- * I12 (contract C3) — freeze/unfreeze a view's DISPLAY MODE.
- *
- * The client re-checks the actor before emitting even though the menu entry is already
- * gated: an event can be replayed, and "hidden in the client" has never been a permission.
- * The host checks it again and that check is the wall — this one only keeps the client from
- * showing a change that will not survive the round trip.
- */
- const toggleViewLock = useCallback(
- (id: string, locked: boolean) => {
- const view = views.find((item) => item.id === id);
- if (!view || !mayToggleViewLock(view, viewer)) return;
- persistView({ ...view, locked });
- },
- [views, persistView, viewer]
- );
- // The list description. Template lists ship with prose seeded by the host, but it is
- // the USER's text once they touch it — including clearing it. An empty string is
- // persisted as an empty string (never coerced back to the seed), because the host
- // resolves a saved view OVER its template, so "" is how you delete a description.
- const setViewNote = useCallback(
- (id: string, note: string) => {
- const view = views.find((item) => item.id === id);
- if (view) persistView({ ...view, note: note.slice(0, 2000) });
- },
- [views, persistView]
- );
- const duplicateView = useCallback(
- (id: string) => {
- const source = views.find((view) => view.id === id);
- if (!source) return;
- const copy: SavedView = {
- ...source,
- id: nextViewId(),
- name: uniqueDisplayName(
- `${source.name} copy`,
- views.map((view) => view.name)
- ),
- kind: "custom",
- locked: false,
- // I17 (C4) — a COPY IS THE DUPLICATOR'S OWN, PERSONAL view. Ruled, not inherited:
- // - inheriting a 'users' grant would silently re-share the copy with a list the
- // person making it never chose;
- // - Duplicate is also the escape hatch for someone who may NOT edit the original,
- // and 'personal' + the host's fresh createdBy stamp is exactly "mine to work in".
- permissions: { edit: "personal" },
- // Never echo the SOURCE's creator: the host stamps and ignores what the browser
- // sends, but sending someone else's name is a laundering attempt on its face.
- createdBy: undefined,
- config: { ...source.config },
- };
- persistView(copy);
- setActiveViewId(copy.id);
- setConfig(copy.config);
- },
- [views, persistView]
- );
- const deleteView = useCallback(
- (id: string) => {
- const view = views.find((item) => item.id === id);
- // ⚠ This gate used to read `view.locked`, and the menu entry above it did too. C3
- // redefines `locked` as "the DISPLAY MODE is frozen" on ANY view, so leaving the gate
- // here would make a user-frozen Kanban undeletable in the client while the host would
- // delete it happily (app.py refuses only `all-customers`, by id). The MENU is just the
- // door — this is the gate, and both had to move.
- // C4 rides alongside: you may not delete a view you may not edit.
- if (!view || isUndeletableView(view) || !mayEditView(view, viewer)) return;
- // 2026-08-04 — STAMP BEFORE THE EMIT, the writeLocal-before-emit order every other
- // optimistic path here uses. The live adopt below re-reads `payload.workspace.views`
- // on every echo, and the queue sends ONE batch at a time — so an echo answering an
- // EARLIER batch still lists this view, and without the tombstone the row would come
- // back and (since that merge never removes) stay back. See liveWorkspace.ts.
- viewTombstonesRef.current = stampTombstone(viewTombstonesRef.current, id, Date.now());
- setViews((current) => current.filter((item) => item.id !== id));
- emitHostEvent({ id: eventId("view-delete"), type: "view_delete", viewId: id });
- if (activeViewId === id) {
- const all = views.find((item) => item.id === ALL_VIEW_ID) ?? allRecordsView(fields, scope);
- setActiveViewId(all.id);
- setConfig(all.config);
- }
- },
- [views, activeViewId, fields, viewer, scope]
- );
-
- /**
- * Wave-6 item 7 — PER-COHORT view state. Each cohort keeps its own view under the stable id
- * `cohort:`; a cohort with no saved view opens CLEAN (defaultViewConfig — no sort,
- * no filters), which is exactly what "first created" means. The view is added LOCALLY on
- * first open and emitted only when the user actually edits it (the autosave effect already
- * compares configs), so "absent = clean" stays true in the store — opening every cohort
- * once must not write a store full of empty views.
- */
- useEffect(() => {
- if (!cohortMode || !workspaceReady || !activeCohortId) return;
- const vid = `cohort:${activeCohortId}`;
- if (activeViewId === vid) return;
- const outgoing = views.find((view) => view.id === activeViewId);
- if (outgoing && !sameConfig(outgoing.config, config))
- persistView({ ...outgoing, config });
- const existing = views.find((view) => view.id === vid);
- const view: SavedView = existing ?? {
- id: vid,
- name: lists.find((l) => l.id === activeCohortId)?.name ?? "Cohort",
- kind: "custom",
- config: defaultViewConfig(fields),
- };
- if (!existing) setViews((current) => [...current, view]);
- setActiveViewId(vid);
- setConfig(normalizeConfig(view.config, fields));
- setCollapsed(new Set());
- setSearch("");
- }, [cohortMode, workspaceReady, activeCohortId, activeViewId, views, config,
- fields, lists, persistView]);
-
- /**
- * "Add to list" (owner item 6). Runs the SAME engine over the named view's config to learn
- * which customers it matches, then hands those pids to the host.
- *
- * ⚠ Deliberately re-runs the pipeline for THAT view rather than using what is on screen: the
- * menu is available on every view, not only the active one, and "add the rows I can see" would
- * quietly mean something different depending on which view happened to be open.
- *
- * ⚠ Refuses on a windowed table. There the client holds ONE PAGE, so "the customers this view
- * matches" is a question it cannot answer — it would add the 200 rows it happens to hold and
- * report success. Silence would be worse than the refusal (CG-3, no-unverifiable-aggregates).
- *
- * ⚠ Refuses for the SAME reason when this view has a measure condition with no answer yet
- * (CG-8). A pending condition matches nothing, so the cohort would be built from a filter that
- * is currently narrower than the one the user is reading — and a cohort is a FIXED set, so
- * that wrong membership would persist long after the answer arrived.
- */
- /**
- * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — HOW MANY RECORDS EACH ALERTED VIEW MATCHES.
- *
- * ⛔ ONE EVALUATOR, AND IT IS `runPipeline` RATHER THAN `matchFilterTree`. The question is
- * "how many records does this VIEW show", and a view is more than its condition list: a
- * `cohortLock` narrows before anything else, `memberPids` are pinned in regardless, and a
- * RANK leaf ("top 10") is only answerable against the domain the other conditions leave.
- * `matchFilterTree` answers a different question — "does this ROW match this tree" — and
- * using it here would report the whole book for a locked view and nonsense for a ranked one.
- * `addToList` below already reaches for the pipeline for the same reason
- * ([[one-evaluator-per-question]]); this is the same call with the same inputs.
- *
- * ⛔ THREE CASES WHERE THERE IS NO HONEST NUMBER, and each is ABSENT rather than zero:
- * · a WINDOWED table — the client engine is skipped by design (CG-3), so a count here
- * would be the size of one PAGE wearing the label of a scope;
- * · an unresolved MEASURE condition — the server has not answered it yet, so the view
- * matches nothing *yet*, and rendering `0` would state a fact nobody has established;
- * · a view the rail no longer holds — an alert outlives its view.
- * A missing key means no badge (the prop's own contract), which is the correct rendering of
- * "not known" and is distinguishable from `0`, which is a real and interesting answer.
- *
- * ⚠ MEMOISED ON THE ALERTED SET, not on `views`: this walks the whole book once per alerted
- * view, and on a table with no alerts it does nothing at all.
- */
- const alertedKey = alerted.join(",");
- const alertCounts = useMemo(() => {
- const out: Record = {};
- if (serverWindowed || !alerted.length) return out;
- for (const id of alerted) {
- const view = views.find((v) => v.id === id);
- if (!view) continue;
- if (pendingMeasures(view.config.filters, measureSets) > 0) continue;
- const { pidToIndex } = runPipeline({
- rawRows: computedRows,
- fields,
- filters: view.config.filters,
- search: "",
- sorts: [],
- groupBy: null,
- collapsed: new Set(),
- memberPids: view.config.memberPids,
- filterConj: view.config.filterConj ?? "and",
- measureSets,
- cohortSets,
- cohortLock: view.config.cohortLock,
- today,
- });
- // DISTINCT records, which is what `pidToIndex` is — a grouped pipeline can list one
- // record under several headings and `visibleRows.length` would then count it twice.
- out[id] = pidToIndex.size;
- }
- return out;
- // `alertedKey` is the scalar identity of the list; `alerted` itself is a fresh array on
- // every fetch even when the answer has not changed.
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [alertedKey, views, computedRows, fields, measureSets, cohortSets, today, serverWindowed]);
-
- const addToList = useCallback(
- (viewId: string, cohortId: string, name: string) => {
- if (serverWindowed) return;
- const view = views.find((v) => v.id === viewId);
- if (!view) return;
- if (pendingMeasures(view.config.filters, measureSets) > 0) return;
- const { visibleRows: matched } = runPipeline({
- rawRows: computedRows,
- fields,
- filters: view.config.filters,
- search: "",
- sorts: view.config.sorts,
- groupBy: null,
- collapsed: new Set(),
- memberPids: view.config.memberPids,
- filterConj: view.config.filterConj ?? "and",
- measureSets,
- cohortSets,
- today,
- });
- const pids = matched
- .filter((r): r is { kind: "data"; record: Row } => r.kind === "data")
- .map((r) => r.record.pid);
- emitHostEvent({
- id: eventId("addlist"),
- type: "add_to_list",
- viewId,
- cohortId,
- name,
- pids,
- });
- },
- [views, computedRows, fields, serverWindowed, measureSets, cohortSets, today]
- );
-
- // ---------------------------------------------------------------- I11c (C4)
- // Folders for the Views rail. The MODEL (echo reconcile, tombstones, dangling
- // refs) lives in folders.ts and is gated by verify_folders.py; this is only the
- // wiring: stamps for what this browser just did, and the five events.
- const [folderStamps, setFolderStamps] = useState({});
- /**
- * Folders THIS browser just created, held until the host echo returns them.
- *
- * ⚠ Without these, `reconcileFolders` was being called with the host list as
- * BOTH arguments, which makes its optimistic-create branch unreachable: a
- * folder absent from `host` is equally absent from `local`. The function was
- * right and the wiring was wrong, so "+ New folder" painted nothing until the
- * round trip landed — seconds, on a cold container, with the user clicking
- * again. The gate passed because it called reconcileFolders with two DIFFERENT
- * arrays; it now also calls it the way this component does.
- */
- const [localFolders, setLocalFolders] = useState([]);
- const workspaceFolders = payload?.workspace?.folders;
- const folders = useMemo(
- () => reconcileFolders(workspaceFolders, localFolders, folderStamps, Date.now()),
- [workspaceFolders, localFolders, folderStamps]
- );
- const folderIdOf = useCallback(
- (viewId: string) =>
- resolveFolderId(
- viewId,
- views.find((v) => v.id === viewId)?.folderId ?? null,
- folders,
- folderStamps,
- Date.now()
- ),
- [views, folders, folderStamps]
- );
- const stampFolder = useCallback((patch: (prev: FolderStamps) => FolderStamps) => {
- setFolderStamps((prev) => pruneFolderStamps(patch(prev), Date.now()));
- }, []);
-
- /* wave17 R1 / C-LOCKV — the COHORT FOLDER surface is gone. `folders['cohorts']` merged
- into `folders['views']` in the host's one-shot migration and `cohortFolders` is no longer
- emitted at all, so `reconcileFolders` here would have been reconciling an absent list
- against nothing forever. A locked view files into an ORDINARY folder now, which is what R1
- means by "native folders". */
-
- /**
- * C4 as AMENDED — the folder-level bulk add's PREVIEW. Deliberately re-runs the
- * engine per contained view, exactly as `addToList` does, and carries the SAME
- * two refusals as named skips rather than dropping them silently: a union that
- * quietly omits two views is a wrong cohort that looks right, and a cohort is a
- * FIXED set, so it stays wrong ([[no-unverifiable-aggregates]]).
- */
- const folderAddPreview = useCallback(
- (folderId: string) => {
- const pids = new Set();
- const skipped: { name: string; why: string }[] = [];
- let counted = 0;
- for (const view of views) {
- if (folderIdOf(view.id) !== folderId) continue;
- if (serverWindowed) {
- skipped.push({ name: view.name, why: "this table loads one page at a time" });
- continue;
- }
- if (pendingMeasures(view.config.filters, measureSets) > 0) {
- // C-NAME (item 9) — user-facing, so it says "metric" like every other surface.
- skipped.push({ name: view.name, why: "a metric condition has no answer yet" });
- continue;
- }
- counted += 1;
- const { visibleRows: matched } = runPipeline({
- rawRows: computedRows,
- fields,
- filters: view.config.filters,
- search: "",
- sorts: view.config.sorts,
- groupBy: null,
- collapsed: new Set(),
- memberPids: view.config.memberPids,
- filterConj: view.config.filterConj ?? "and",
- measureSets,
- cohortSets,
- today,
- });
- for (const r of matched) if (r.kind === "data") pids.add(r.record.pid);
- }
- return { pids: [...pids], counted, skipped };
- },
- [views, folderIdOf, serverWindowed, computedRows, fields, measureSets, cohortSets, today]
- );
-
- const folderAddToList = useCallback(
- (folderId: string, cohortId: string, name: string) => {
- const { pids } = folderAddPreview(folderId);
- if (!pids.length) return;
- // ONE emit with the deduped union, through the SAME guarded event a single
- // view's "Add to cohort" uses — so the host validates both identically.
- emitHostEvent({
- id: eventId("addlist"),
- type: "add_to_list",
- viewId: "",
- cohortId,
- name,
- pids,
- });
- },
- [folderAddPreview]
- );
-
- /**
- * Wave-7 item W2 (contract C2) — export "the current view", CLIENT-side. The same
- * engine run addToList uses (that is the point: the file says exactly what the count
- * says), over the same pool the named surface scopes to.
- *
- * ⚠ Same two refusals as addToList, for the same reasons: a server-windowed table
- * holds ONE PAGE (the client cannot answer "the view's matches"), and a view with a
- * PENDING measure condition currently matches nothing — an export taken then would be
- * an empty/narrower file wearing the view's name. Both windows self-resolve; the
- * refusal is silent no-op for one round trip.
- */
- const exportRows = useCallback(
- (name: string, cfg: ViewConfig, poolRows: Row[], format: ExportFormat) => {
- if (serverWindowed) return;
- if (pendingMeasures(cfg.filters, measureSets) > 0) return;
- const { visibleRows: matched } = runPipeline({
- rawRows: poolRows,
- fields,
- filters: cfg.filters,
- search: "",
- sorts: cfg.sorts,
- groupBy: null,
- collapsed: new Set(),
- memberPids: cfg.memberPids,
- filterConj: cfg.filterConj ?? "and",
- measureSets,
- cohortSets,
- today,
- });
- const records = matched
- .filter((r): r is { kind: "data"; record: Row } => r.kind === "data")
- .map((r) =>
- overlayEdits[r.record.pid]
- ? { ...r.record, ...overlayEdits[r.record.pid] }
- : r.record
- );
- // Columns = the view's VISIBLE fields in display order (the reconciled order:
- // every key present, config order first — the same shape useGridColumns paints).
- const inOrder = new Set(cfg.order);
- const keys = [
- ...cfg.order,
- ...fields.map((f) => f.key).filter((k) => !inOrder.has(k)),
- ];
- const vis = new Set(cfg.visible);
- const cols = keys
- .filter((k) => vis.has(k))
- .map((k) => fieldByKey.get(k))
- .filter((f): f is Field => !!f);
- runExport(format, name, today, cols, records);
- },
- [serverWindowed, fields, fieldByKey, measureSets, cohortSets, today, overlayEdits]
- );
-
- /**
- * ⭐ owner item 3 (2026-08-03) — THE SHEET THE TIME-SERIES VIEW IS SHOWING, published by the
- * panel. A ref rather than state on purpose: nothing renders from it, and setting state on
- * every sheet rebuild would re-render the whole grid to feed a menu nobody has opened yet.
- */
- const tsSheetRef = useRef(null);
- const onTsSheet = useCallback((s: TsSheet) => { tsSheetRef.current = s; }, []);
- /** The same channel for a SUMMARY calendar — the other mode whose content is not its rows.
- * `null` is CalendarView saying "records mode, export the rows" (see its note). */
- const calSheetRef = useRef<{ fields: Field[]; rows: Row[] } | null>(null);
- const onCalSheet = useCallback(
- (s: { fields: Field[]; rows: Row[] } | null) => { calSheetRef.current = s; },
- []
- );
-
- /**
- * W2 — the view menu's Export (Customer page): the named view over the whole pool.
- *
- * ⭐ owner item 3 (2026-08-03) — AND IT EXPORTS WHAT THE VIEW SHOWS. Every mode used to
- * export the same thing: the matched customer rows. For grid / list / kanban / calendar /
- * map / chart that is right — those modes ARRANGE rows, so the rows are what they show, and
- * a calendar's dates and a kanban's lanes are columns already in the file.
- *
- * TWO modes are not arrangements of rows, and both were wrong in the direction that matters:
- *
- * · `timeseries` — metric ROWS over period COLUMNS. Exporting it handed you a customer list
- * that shares none of its numbers. It exports the SHEET.
- * · `calendar` IN SUMMARY MODE — metric values per DAY (C-DISP item 4). Same problem, and
- * the owner named this one by hand. A calendar in RECORDS mode is genuinely an
- * arrangement of rows, so it keeps the row export; `CalendarView` says which it is by
- * publishing a sheet or publishing null.
- *
- * ⚠ ONLY FOR THE ACTIVE VIEW, and this is a real limit, not an oversight. Both sheets are
- * built by the mounted view — the time series from a server round trip, the calendar from the
- * month on screen. A view sitting unopened in the rail has neither, and this component cannot
- * conjure one without fetching it. So that case SAYS SO and downloads nothing: the
- * alternative is silently handing over the customer rows under that view's name, which is the
- * exact substitution this branch exists to stop.
- */
- const exportViewData = useCallback(
- (viewId: string, format: ExportFormat) => {
- const view = views.find((v) => v.id === viewId);
- if (!view) return;
- const cfg = normalizeConfig(view.config, fields);
- const spec = cleanDisplay(cfg.display);
- const isActive = viewId === activeViewId;
- const refuse = (what: string) =>
- signal(
- TOAST_EVENT,
- `Open "${view.name}" first: ${what} is exported from what it draws on screen.`
- );
-
- if (spec?.mode === "timeseries") {
- const sheet = isActive ? tsSheetRef.current : null;
- if (!sheet) return refuse("a time series");
- if (sheet.empty) {
- signal(TOAST_EVENT,
- `"${view.name}" has no metrics on its sheet yet — add one, then export.`);
- return;
- }
- if (format === "csv") {
- // The gated builder (verify_timeseries), kept as THE csv path so the file the owner
- // downloads is the one the gate proves — footnotes, grouped thousands and all.
- triggerDownload(
- exportFilename(view.name, today, format),
- new Blob(["" + buildTsCsv(sheet)], { type: "text/csv;charset=utf-8" })
- );
- } else {
- const { fields: tf, rows: tr } = tsSheetToTable(sheet);
- runExport(format, view.name, today, tf, tr);
- }
- return;
- }
-
- /* ═══ W18-C CATALOG ═══ A catalog is the THIRD mode that is not an arrangement of rows,
- and it is the furthest from one: its pages are authored content, and its product codes
- are a fraction of the pool in an order the user chose. Handing over the matched rows
- under a catalogue's name would be the same substitution the two branches around this
- one exist to stop. It exports as a PDF, through the browser's own print dialog (R10),
- so this branch refuses the file and names the door that works. */
- if (spec?.mode === "catalog") {
- signal(
- TOAST_EVENT,
- `"${view.name}" is a catalog — open it and use Print to save it as a PDF. ` +
- `A spreadsheet of its products is not what it draws.`
- );
- return;
- }
- /* ═══ end W18-C CATALOG ═══ */
-
- if (spec?.mode === "calendar" && spec.calendarMode === "summary") {
- const sheet = isActive ? calSheetRef.current : null;
- if (!sheet) return refuse("a calendar summary");
- runExport(format, view.name, today, sheet.fields, sheet.rows);
- return;
- }
-
- exportRows(view.name, cfg, computedRows, format);
- },
- [views, fields, computedRows, exportRows, activeViewId, today]
- );
-
- /**
- * The selection bar's "Add to locked view": exactly the CHECKED customers, not a view's
- * matches.
- * Same guarded `add_to_list` event — the host intersects the pids with the caller's pool
- * either way, so a hand-picked set and a view's match set ride one validation path.
- */
- const addSelectionToList = useCallback(
- (cohortId: string, name: string) => {
- const pids = [...selectedPids];
- if (!pids.length || serverWindowed) return;
- emitHostEvent({
- id: eventId("addlist"),
- type: "add_to_list",
- viewId: activeViewId,
- cohortId,
- name,
- pids,
- });
- setSelAddOpen(false);
- setSelListName("");
- clearSelection();
- },
- [selectedPids, serverWindowed, activeViewId, clearSelection]
- );
-
- // Item 3c: the local write is the RENDERED truth (the def half of the no-blip contract) —
- // stamped so a lagged echo cannot claw it back at the next remount. Item 9c: `scope` rides
- // the EVENT at create time only; the def itself carries `scope: 'cohort'` so the echo is
- // byte-stable ('global' stays absent — the host normalizes them to one shape).
- const scopeChoice = payload?.workspace?.scopeChoice === true;
- /**
- * Item 12 (C-LOCK) — the Filter banner's copy for a cohort-locked view.
- *
- * `undefined` when the view is not locked. When it IS locked but the set is not in `lists`,
- * this returns an EMPTY object: the banner must still appear (the reader is looking at a
- * narrowed table and deserves to know why), but with no name and no count, because a set we
- * were given no membership for is a set we must not describe. RECORD's banner reads that
- * absence and says "locked to a set you cannot see".
- *
- * The count is the cohort's OWN size, not the number of rows on screen — the conditions
- * narrow within the lock, so the two differ and the banner is stating the lock.
- */
- /**
- * ⭐ wave17 R1 / C-LOCKV — the lock in force, which is now simply the ACTIVE VIEW's.
- *
- * Wave 15 had two sources: a saved `config.cohortLock` and an ephemeral pick from the Cohorts
- * rail that outranked it while held. The second existed only because a projected cohort was
- * not a saved view, so opening one could not be allowed to write anything. Under R1 it IS a
- * saved view, so selecting it applies its own stored lock through the ordinary view path —
- * one source, and it survives a reload, which the transient one never did.
- */
- const effectiveCohortLock = config.cohortLock;
- /* wave17 R1 / C-LOCKV — `cohortRows` and the whole `cohortRail` projection are GONE. They
- turned `lists` into rail rows for a section that no longer exists; the host projects each
- cohort into `views` instead, so the rail renders them through the same path as every other
- view and there is nothing left to project. `lists` still arrives and is still load-bearing
- — it is the MEMBERSHIP channel that feeds `cohortSets`, which is what the lock resolves
- against. Its absence would blank every locked view, so it is not "unused". */
- const cohortLockChip = useMemo(() => {
- const id = effectiveCohortLock;
- if (!id) return undefined;
- const set = lists.find((l) => l.id === id);
- if (!set) return {};
- return { name: set.name, count: set.pids?.length ?? 0 };
- }, [effectiveCohortLock, lists]);
- // (Setting/clearing the lock lives in `onViewCohortLock` above — the rail's menu operates on
- // whichever view you clicked, not necessarily the active one, so it goes through
- // `persistView` and only touches the live `config` when the two are the same view.)
- /**
- * Item 5 / contract C-LAYOUT — this user's record-detail field order, for RECORD's
- * `RecordDetail` (they asked for exactly these two props in the wave mailbox).
- *
- * Read straight off the workspace stratum the host echoes, and emitted back as the
- * `record_layout` event HOST landed. Deliberately NOT reconciled or optimistically merged
- * here: the event returns False host-side (an autosave hot path like `view_upsert`), so
- * there is no rerun to race, and `RecordDetail` holds the live order while a drag is in
- * flight. Nothing else in the grid reads it — the order is the MODAL's, never the grid's
- * `config.order`.
- */
- const recordLayout = payload?.workspace?.recordLayout?.order;
- const onRecordLayout = useCallback((order: string[]) => {
- emitHostEvent({ id: eventId("rlayout"), type: "record_layout", order });
- }, []);
- const saveField = useCallback(
- (field: Field, scope?: FieldScope) => {
- const requestId = eventId("field");
- const accepted: Field = {
- ...field,
- label: uniqueDisplayName(
- field.label,
- fields.filter((item) => item.key !== field.key).map((item) => item.label)
- ),
- editRequestId: requestId,
- };
- delete accepted.labelCorrectedFrom;
- delete accepted.labelCorrectionId;
- stampFieldEdit(accepted.key);
- setFields((current) =>
- current.some((item) => item.key === accepted.key)
- ? current.map((item) => (item.key === accepted.key ? accepted : item))
- : [...current, accepted]
- );
- emitHostEvent({
- id: requestId,
- type: "field_upsert",
- field: accepted,
- ...(scope ? { scope } : {}),
- });
- },
- [stampFieldEdit, fields]
- );
- const createField = useCallback(
- (
- label: string,
- type: FieldType,
- anchorKey: string | null,
- side: "left" | "right" | "end",
- options?: string[],
- measureSpec?: { key: string; window: WindowSpec },
- extra?: FieldBuildExtra
- ) => {
- /**
- * ⚠ RENAMED FROM `scope` (wave 21, item 7). It used to SHADOW the component's own
- * `scope` prop — the surface this grid is drawing — with a completely different
- * thing: a FIELD's storage stratum (`'cohort'` or nothing). Two unrelated meanings
- * under one word, in a function that now has to speak to a per-table endpoint.
- * Caught by `tsc` only because `FieldScope` and `SurfaceScope` happen to be
- * incompatible unions; had either been a bare `string`, the C2 call below would have
- * POSTed to `/tables/undefined/fields` and read as a server bug.
- */
- const fieldScope = scopeChoice ? extra?.scope : undefined;
- const scoped = fieldScope === "cohort" ? ({ scope: "cohort" } as const) : {};
- // Owner item 7 — a FORMULA-MEASURE column. `measure_` prefixed, source:'odoo' (read-only
- // at both ends), derived (values arrive from the host per render), filterable:false (the
- // replacement is the measure CONDITION with the same measure+window — the governed path).
- // The first render after creation shows blank cells for exactly one round trip: the host
- // persists the field, then computes the values into the next payload.
- if (measureSpec) {
- const field: Field = {
- key: `measure_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`,
- label,
- type,
- source: "odoo",
- default: true,
- custom: true,
- derived: true,
- filterable: false,
- agg: ["currency", "int"].includes(type) ? "sum" : undefined,
- measure: measureSpec,
- ...scoped,
- };
- saveField(field, fieldScope);
- insertColumn(field.key, anchorKey, side);
- return;
- }
- const field = { ...buildOverlayField(label, type, options, extra), ...scoped };
- /**
- * ⭐ WAVE 21 item 7 (contract C2) — AN AUTOMATION COLUMN ON A USER DATABASE GOES TO
- * THE DEFINITION, not to this user's overlay stratum.
- *
- * The automation editor's column picker reads `user_tables` (the definition); every
- * grid-created column went to `_table_workspace` (the per-user overlay). So the
- * picker was empty by construction: the user made the column, then could not find it
- * in the automation they made it for. C2 routes exactly this one kind through
- * `POST /tables/{key}/fields`, and the ROUTE is the wall — `_field_or_refuse` allows
- * only the creator or an admin, which is the same answer the grid would have got.
- *
- * ⚠ NARROW ON PURPOSE, twice over: `type === "automation"` AND a `ut_` scope. The
- * connector surfaces have no such endpoint (their schema is their source's), and
- * every other kind keeps the overlay path this wave — moving all of them changes who
- * can SEE a column, which is a behaviour change three surfaces would feel and is
- * booked as debt rather than smuggled in here.
- *
- * ⚠ AND IT IS ASYNC, where every other create is optimistic. There is no local
- * insert first: the server re-slugs the key and can refuse outright (it 400s until
- * `UT_FIELD_TYPES` learns `automation` — C2's other half, A's), and a column shown
- * under a key the store never took is one that vanishes on the next read with no
- * explanation. The wait is one round trip; the alternative is a lie.
- */
- /**
- * ⭐⭐ 2026-08-09 — THE RELATIONAL PAIR JOINS THAT ROUTE, and for a stronger reason than
- * `automation` had. The note above says every other kind "keeps the overlay path this
- * wave", weighing it as a question of who can SEE a column. For `link` and `rollup` it is
- * not a visibility trade-off at all: their cells are computed SERVER-side by
- * `compute_relation_cells`, which walks `user_tables` definitions. A rollup parked in a
- * per-user overlay is invisible to the only code that could ever fill it, so it is not a
- * narrower feature — it is a column that cannot work, in any account, ever.
- *
- * ⛔ AND THE BAG TRAVELS. `_clean_field` returns None for a bagless `link`/`rollup`, so
- * sending `{key,label,type}` here would trade a silent blank column for a loud 400. The
- * bag is on `extra`; `field` already carries it (see `FieldBuildExtra`).
- */
- /**
- * ⭐⭐ 2026-08-10 — `formula` JOINS THE SAME ROUTE, on the visibility argument the note
- * above weighs and the relational note declines to use.
- *
- * For `link`/`rollup` visibility was not the point (their cells are computed by something
- * that reads the definition). For a formula it IS the whole point, and the trade-off falls
- * the other way from wave 21's read of it: a `ut_*` database is a SHARED database — the
- * owner builds a column on "IG profile" and expects the four people looking at that
- * database to see it. In the overlay stratum only its creator ever does. Measured on
- * nurilab: `Trimmed reel views`, the trimmed-average column the whole rollup chain exists
- * to produce, was visible to exactly one account.
- *
- * ⛔ THE VALUES STILL DO NOT MOVE, and this must not be sold as more than it is. A formula
- * is computed in this browser (`computedRows`) from the row's other cells and is stored
- * nowhere; what became shared is the EXPRESSION. An automation, a rollup and an export
- * still cannot read this column. Sharing the definition is not materialising the number.
- *
- * ⚠ `agg: "sum"` rides along so the totals row behaves identically to the overlay path
- * (`buildOverlayField` stamps it) — a column that stops totalling because of which door
- * created it is [[gate-answers-the-wrong-question]] wearing a create route.
- */
- if ((type === "automation" || type === "link" || type === "rollup"
- || type === "formula") && isUserTable) {
- void addTableField(scope, {
- key: field.key,
- label: field.label,
- type,
- ...(extra?.link ? { link: extra.link } : {}),
- ...(extra?.rollup ? { rollup: extra.rollup } : {}),
- ...(type === "formula" && extra?.formula
- ? { formula: extra.formula, agg: "sum" }
- : {}),
- }).then((made) => {
- if (!made) return; // the bridge has already said why
- setFields((current) =>
- current.some((f) => f.key === made.key) ? current : [...current, made]
- );
- insertColumn(made.key, anchorKey, side);
- /**
- * ⛔⛔ 2026-08-10 — THE WORKSPACE RE-READ USED TO FIRE HERE AND IT ATE THE LINE ABOVE.
- *
- * `insertColumn` adds the new key to the view's `visible` set and autosaves that config;
- * `signal(WORKSPACE_STALE_EVENT)` refetches `/workspace`, whose views carry the SERVER's
- * config — the one written a moment before the insert. The refetch won, every time, so
- * a column created into the shared definition was **stored correctly and rendered
- * nowhere**: present in the payload, present in `order` (which is re-derived from the
- * field list), and absent from `visible`. MEASURED on live nurilab across all four
- * views after migrating `Trimmed reel views`; it has been true for `link`, `rollup` and
- * `automation` since that route was widened on 2026-08-09.
- *
- * ⛔ AND THE COMMENT IT REPLACES WAS THE REASON NOBODY LOOKED. It claimed the signal is
- * "what makes the column survive a reload rather than living only in this browser's
- * state" — but the SERVER already stored the field; the 201 is what makes it survive.
- * The signal only refreshed a payload whose field list `setFields` had just updated by
- * hand, and `addTableField` already drops the rows cache. It bought nothing and cost
- * the one piece of state the create had just written.
- *
- * ⚠ The delete and reconfigure paths keep their signals: those change VALUES other
- * columns fold (a dropped link takes its reciprocal and every rollup over it), and
- * neither writes view config in the same breath, so neither has this race.
- */
- });
- return;
- }
- saveField(field, fieldScope);
- insertColumn(field.key, anchorKey, side);
- },
- [insertColumn, saveField, scopeChoice, isUserTable, scope]
- );
-
- /**
- * ⭐⭐ 2026-08-07 (D-79's last half) — turn a text column into THE profile column.
- *
- * The write goes to the DEFINITION (`PATCH /tables/{key}/fields/{fkey}`), never to this
- * user's overlay stratum, for the reason wave 25's amendment C3-A1 records: the automation
- * engine reads `t['rows']`/`t['fields']`, so a flag parked in one person's overlay is a flag
- * the enrich step can never see. Same door, same reasoning, as the automation column above.
- *
- * ⚠ `profile: null` CLEARS it — `_clean_field` treats an explicit null as "take the flag off"
- * rather than "leave it alone", which is what lets a column be un-marked without deleting it.
- */
- const setProfileFlag = useCallback(
- (key: string, on: boolean) => {
- if (!isUserTable) return;
- void patchTableField(scope, key, {
- profile: on ? { source: "instagram" } : null,
- }).then((made) => {
- if (!made) return; // the bridge has already said why
- setFields((current) => current.map((f) => (f.key === made.key ? made : f)));
- signal(WORKSPACE_STALE_EVENT);
- });
- },
- [isUserTable, scope]
- );
-
- /**
- * ⭐⭐ 2026-08-09 (owner) — RECONFIGURE a `link`/`rollup` column, in the SHARED definition.
- *
- * Owner: *"even when i click edit field for rollup, it doesn't actually show me the
- * configuration"* and *"No rollup field should be uneditable."* Both land here: the Edit pane
- * now renders the bag, and this is where its Save goes.
- *
- * ⛔ NOT `saveField`, and not `retypeField`. Those write the per-user overlay stratum, which is
- * precisely the defect this change exists to close — `compute_relation_cells` reads
- * `user_tables`, so a bag parked in one person's workspace can never be computed by anything.
- * ⚠ ASYNC with no optimistic insert, like the automation create: the server re-cleans the bag
- * and can refuse the whole field (`_clean_rollup` returns None for, say, a `limit` with no
- * `sortBy`), and showing a configuration the store never took is the lie the wait avoids.
- * The cells arrive on the next read — a rollup is recomputed server-side, never in the browser.
- */
- const setFieldConfig = useCallback(
- (key: string, patch: {
- label?: string; link?: unknown; rollup?: unknown; formula?: string;
- }) => {
- if (!isUserTable) return;
- void patchTableField(scope, key, patch as Record).then((made) => {
- if (!made) return; // the bridge has already said why
- setFields((current) => current.map((f) => (f.key === made.key ? made : f)));
- // The definition changed, so the wire's field list did too — and a rollup's VALUES are
- // recomputed host-side on this write, so the ROWS are a beat stale as well. The bridge
- // drops its rows cache for this table (it owns that map); these two ask for a re-read.
- signal(WORKSPACE_STALE_EVENT);
- signal(ROWS_STALE_EVENT);
- });
- },
- [isUserTable, scope]
- );
-
- /**
- * Owner item 9 — "Change field": the clicked column starts showing another field, in place.
- * The new field takes the old one's slot in `order`; the old field is HIDDEN, not lost — it
- * stays in Fields and can be swapped back. The locked primary column is the row identity and
- * cannot be changed away (ColumnMenu never offers it the control).
- */
- const changeField = useCallback(
- (oldKey: string, newKey: string) => {
- if (oldKey === lockedKey || oldKey === newKey) return;
- if (!fieldByKey.has(newKey)) return;
- // The RECONCILED order (every key present, locked first), so the splice is well-defined
- // even for a fresh view whose config.order is still the default.
- const withoutNew = order.filter((k) => k !== newKey);
- const at = withoutNew.indexOf(oldKey);
- if (at < 0) return;
- withoutNew.splice(at, 0, newKey);
- const nextVisible = new Set(visible);
- nextVisible.delete(oldKey);
- nextVisible.add(newKey);
- updateConfig({ ...config, order: withoutNew, visible: [...nextVisible] });
- },
- [config, updateConfig, order, visible, fieldByKey, lockedKey]
- );
-
- /**
- * Delete a USER-CREATED column outright (owner gap closed 2026-07-27). Only the created
- * strata qualify — `custom_` overlay fields and `measure_` formula columns, both marked
- * `custom` — so a base field can never leave the schema from here (Hide is its only exit).
- * The definition leaves local state, the ACTIVE view's config is scrubbed (order / visible /
- * widths / sorts / group / color), and the host removes it from the workspace store — other
- * views self-heal on their next autosave, the rule every stale colId already rides.
- * Two emits leave in one burst (field_delete + the config autosave); the event-log value
- * slot exists for exactly this.
- */
- const deleteField = useCallback(
- (key: string) => {
- const f = fieldByKey.get(key);
- if (!f?.custom || key === lockedKey) return;
- setFields((current) => current.filter((item) => item.key !== key));
- const nextWidths = { ...(config.widths ?? {}) };
- delete nextWidths[key];
- updateConfig({
- ...config,
- order: order.filter((k) => k !== key),
- visible: [...visible].filter((k) => k !== key),
- widths: nextWidths,
- sorts: (config.sorts ?? []).filter((s) => s.colId !== key),
- groupBy: config.groupBy === key ? null : config.groupBy,
- colorBy: config.colorBy === key ? null : config.colorBy,
- });
- stampFieldDelete(key);
- emitHostEvent({ id: eventId("fielddel"), type: "field_delete", key });
- },
- [config, updateConfig, order, visible, fieldByKey, lockedKey, stampFieldDelete]
- );
-
- /**
- * ⭐⭐ 2026-08-10 — DELETE A COLUMN FROM A USER DATABASE'S SHARED DEFINITION.
- *
- * `deleteField` above writes the per-user overlay: it emits `field_delete`, which
- * `grid_events` gates on the `custom_`/`measure_` prefix and applies to
- * `_table_workspace`. A definition column has a `custom_` key too, so that event is
- * ACCEPTED and scrubs a bucket the definition never reads — the column disappears for one
- * paint and is back on the next fetch. That is why this is a different function rather than a
- * branch: two stores, two truths, and the wrong one succeeds quietly.
- *
- * ⚠ SERVER FIRST, THEN THE SCREEN. `_field_or_refuse` can say no (a pre-set column, a
- * non-creator, the last remaining column), and the bridge has already shown its sentence —
- * removing the column optimistically would leave the user looking at a grid that disagrees
- * with the store until they reload.
- * ⚠ The VIEW config is scrubbed on the same terms as the overlay delete; a stale `colId` in
- * another view self-heals on its next autosave, the rule every other delete rides.
- */
- const deleteDefinitionField = useCallback(
- (key: string) => {
- if (!isUserTable || key === lockedKey) return;
- void deleteTableField(scope, key).then((ok) => {
- if (!ok) return; // the bridge has already said why
- setFields((current) => current.filter((item) => item.key !== key));
- const nextWidths = { ...(config.widths ?? {}) };
- delete nextWidths[key];
- updateConfig({
- ...config,
- order: order.filter((k) => k !== key),
- visible: [...visible].filter((k) => k !== key),
- widths: nextWidths,
- sorts: (config.sorts ?? []).filter((s) => s.colId !== key),
- groupBy: config.groupBy === key ? null : config.groupBy,
- colorBy: config.colorBy === key ? null : config.colorBy,
- });
- signal(WORKSPACE_STALE_EVENT);
- signal(ROWS_STALE_EVENT);
- });
- },
- [isUserTable, scope, lockedKey, config, updateConfig, order, visible]
- );
-
- /**
- * Change-field's "New field" half (wave-2 item 5): build the overlay field, then swap it into
- * the clicked column's slot — one motion, two emits (field_upsert + the config autosave),
- * which is exactly the burst the event-log value slot exists for.
- */
- const createAndSwapField = useCallback(
- (
- oldKey: string,
- label: string,
- type: FieldType,
- options?: string[],
- extra?: FieldBuildExtra
- ) => {
- const scope = scopeChoice ? extra?.scope : undefined;
- const field = {
- ...buildOverlayField(label, type, options, extra),
- ...(scope === "cohort" ? ({ scope: "cohort" } as const) : {}),
- };
- saveField(field, scope);
- changeField(oldKey, field.key);
- },
- [saveField, changeField, scopeChoice]
- );
-
- /**
- * Wave-6 item 2 — Change field on a CREATED (`custom_`) field retypes THAT field in place:
- * `field_upsert` with the SAME key and the new type/options; the host rebuilds the def and
- * keeps createdBy/scope (this optimistic def carries them too, so the echo is byte-stable).
- * Values are never converted — cells re-render per the new type, unreadable values show
- * blank (Airtable behavior, disclosed in the pane). A display `format` survives only within
- * its own family (number→number); anything else would be junk the host drops anyway.
- */
- const retypeField = useCallback(
- (
- key: string,
- type: FieldType,
- options?: string[],
- // `label` (owner item 8): rename+retype leave the Edit pane as ONE upsert — two
- // sequential emits would each read the stale def and revert the other.
- extra?: FieldBuildExtra
- ) => {
- const old = fieldByKey.get(key);
- if (!old?.custom || old.source !== "overlay") return;
- if (type === "formula" || type === "created_time") return; // a different stratum
- const numberFam = (t: FieldType) => t === "int" || t === "currency";
- const keepFormat =
- old.format && ((numberFam(old.type) && numberFam(type)) || old.type === type);
- const next: Field = {
- key,
- label: extra?.label?.trim() || old.label,
- type,
- source: "overlay",
- default: old.default,
- custom: true,
- agg: ["currency", "int"].includes(type) ? "sum" : undefined,
- ...(old.note ? { note: old.note } : {}),
- ...(old.description ? { description: old.description } : {}),
- ...(old.scope ? { scope: old.scope } : {}),
- ...(old.createdBy ? { createdBy: old.createdBy } : {}),
- ...(old.permissions ? { permissions: old.permissions } : {}),
- ...(keepFormat ? { format: old.format } : {}),
- ...((type === "select" || type === "multiselect") && options?.length
- ? { options }
- : {}),
- ...((type === "select" || type === "multiselect")
- ? {
- colorCodeOptions: extra?.colorCodeOptions !== false,
- ...(extra?.optionColors && Object.keys(extra.optionColors).length
- ? { optionColors: extra.optionColors }
- : {}),
- }
- : {}),
- ...(type === "multiselect" ? { multi: true } : {}),
- ...(type === "rating" ? { max: extra?.max ?? 5 } : {}),
- };
- saveField(next);
- /* ⭐ Owner item 15 / C-RENAME — the VALUES follow the definition.
- BESIDE the upsert, never instead of it, and AFTER it: the def is written optimistically
- here, while the rename is a host-side migration over overlay values and saved views. The
- order matters if the host processes the batch in order — the list must already offer
- "Navy" before any cell is moved onto it. */
- const renames = extra?.renames?.filter((r) => r.from && r.to && r.from !== r.to) ?? [];
- if (renames.length) {
- emitHostEvent({ id: eventId("choicerename"), type: "choice_rename", key, renames });
- // R4 — and it is undoable: Ctrl+Z emits the mapping turned around. The declared LIST is
- // restored by the same inverse (the host rewrites values and views back), so undo does
- // not need to re-send the definition.
- undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
- kind: "choiceRename", fieldKey: key, renames,
- });
- }
- },
- [fieldByKey, saveField, scope]
- );
-
- /**
- * Wave-5 item 1 — Duplicate field. The CLIENT generates the destination key (same stratum
- * prefix as the source — the ~20:20 contract amendment), emits `field_duplicate`, and slots
- * an optimistic clone right of its source SYNCHRONOUSLY. The host validates the prefix
- * pair, stamps createdBy (the clone's creator is the duplicator — the local copy drops the
- * source's), and copies overlay VALUES for `custom_` sources; those values arrive with the
- * next payload, so the clone's cells are blank for exactly one round trip.
- */
- const duplicateField = useCallback(
- (source: Field) => {
- const prefix = source.key.startsWith("measure_") ? "measure_" : "custom_";
- const label = uniqueDisplayName(
- `${source.label} copy`,
- fields.map((field) => field.label)
- );
- const key = `${prefix}${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`;
- const requestId = eventId("fielddup");
- const clone: Field = { ...source, key, label, editRequestId: requestId };
- delete clone.createdBy;
- delete clone.permissions;
- delete clone.labelCorrectedFrom;
- delete clone.labelCorrectionId;
- stampFieldEdit(key);
- setFields((current) => [...current, clone]);
- emitHostEvent({
- id: requestId,
- type: "field_duplicate",
- sourceKey: source.key,
- key,
- label,
- // Item 9c: a duplicate INHERITS its source's scope (cohort clone stays cohort-only;
- // a global source stays global = absent). Only meaningful on the cohort page.
- ...(scopeChoice && source.scope === "cohort" ? { scope: "cohort" as const } : {}),
- });
- insertColumn(key, source.key, "right");
- },
- [insertColumn, scopeChoice, stampFieldEdit, fields]
- );
-
- /**
- * Wave-2 item 8c — change the PERIOD of a measure-carrying column (user-created `measure_*`
- * and the pre-set measure fields alike). The header auto-renames to
- * ` · ` ONLY while it still reads as the auto-name for the CURRENT
- * window — a user's own title is never overwritten. Values arrive recomputed from the host on
- * the next payload; the ordinary `field_upsert` is the whole protocol.
- */
- const changeMeasurePeriod = useCallback(
- (key: string, window: WindowSpec) => {
- const f = fieldByKey.get(key);
- if (!f?.measure) return;
- const m = measures.find((item) => item.key === f.measure?.key);
- const oldAuto = m ? `${m.label} · ${windowLabel(f.measure.window)}` : null;
- const label =
- m && oldAuto && f.label === oldAuto
- ? `${m.label} · ${windowLabel(window)}`
- : f.label;
- saveField({ ...f, label, measure: { ...f.measure, window } });
- },
- [fieldByKey, measures, saveField]
- );
-
- /* wave17 R1 — `selectCohort`, `renameCohort`, `deleteCohort` and `exportCohortData` are
- gone. Every one of them was called ONLY by `CohortSidebar`, and every one now has a
- better-governed twin: the Views rail renames and deletes a locked view through the
- ordinary view path, and the HOST routes those to `cohort_mod.rename` / `.delete` on a
- projected id (C-LOCKV guards b and c). One name, one lifecycle, one set of events. */
-
- /**
- * Cohort mode — the selection bar's "Remove from cohort" (contract event `cohort_remove`).
- * The pids are intersected with the cohort's membership client-side as a courtesy; the host
- * re-validates ownership and pool anyway (a pid from the browser is untrusted input).
- */
- const removeSelectionFromCohort = useCallback(() => {
- if (!activeCohortId) return;
- const pids = [...selectedPids].filter((p) => cohortMemberSet.has(p));
- if (!pids.length) return;
- emitHostEvent({
- id: eventId("cohortrm"),
- type: "cohort_remove",
- cohortId: activeCohortId,
- pids,
- });
- clearSelection();
- }, [activeCohortId, selectedPids, cohortMemberSet, clearSelection]);
-
-
- /**
- * ⭐ Wave-20 owner item 9 — **REMOVE FROM A COHORT WITHOUT BEING INSIDE IT.**
- *
- * "Remove from cohort" existed only under `cohortMode` (you had to open the locked view
- * first), while "Add to cohort" worked from any view. So the two halves of the same idea
- * lived on different screens: you could put a customer in a cohort from wherever you found
- * them, and then had to go looking for the cohort to take them out again.
- *
- * SAME EVENT, same host validation as the cohort-mode button — `cohort_remove` with the pids
- * intersected client-side as a courtesy. What is new is only WHICH cohort: the picker names
- * it, instead of it being implied by the page you are standing on.
- */
- const removeSelectionFromList = useCallback(
- (cohortId: string) => {
- const members = cohortSets[cohortId];
- if (!members) return;
- const pids = [...selectedPids].filter((p) => members.has(p));
- if (!pids.length) return;
- emitHostEvent({ id: eventId("cohortrm"), type: "cohort_remove", cohortId, pids });
- setSelRemoveOpen(false);
- clearSelection();
- },
- [cohortSets, selectedPids, clearSelection]
- );
-
- /**
- * The cohorts the checked rows can actually be removed FROM: those holding at least one of
- * them, and only where this viewer may edit the projected view.
- *
- * ⚠ A cohort holding NONE of the checked rows is not offered. Airtable's rule and the one R8
- * states for the ghost row: an affordance that can only refuse is a fake affordance — and
- * here it would be worse than that, because "Remove from Q3 plan" that removes nothing looks
- * exactly like a write that failed.
- *
- * The edit test is the COURTESY half (`mayEditView`, like every other client-side permission
- * check on this surface); the host re-validates ownership and pool on the event regardless.
- * A cohort with no projected view is still offered — the host owns that verdict, and hiding
- * it here would silently drop sets the reader can see in their own rail.
- */
- const removableLists = useMemo(() => {
- if (cohortMode || selectedPids.size === 0) return [];
- const out: { id: string; name: string; hits: number }[] = [];
- for (const l of lists) {
- const members = cohortSets[l.id];
- if (!members) continue;
- let hits = 0;
- for (const pid of selectedPids) if (members.has(pid)) hits += 1;
- if (!hits) continue;
- const projected = views.find((v) => v.id === l.id);
- if (projected && !mayEditView(projected, viewer)) continue;
- out.push({ id: l.id, name: l.name, hits });
- }
- return out;
- }, [cohortMode, selectedPids, lists, cohortSets, views, viewer]);
-
- /** Cohort mode — confirm the "+ Add customers" picker (contract event `cohort_add`). */
- const addCustomersToCohort = useCallback(() => {
- if (!activeCohortId || addCustPicked.size === 0) return;
- emitHostEvent({
- id: eventId("cohortadd"),
- type: "cohort_add",
- cohortId: activeCohortId,
- pids: [...addCustPicked],
- });
- setAddCustPicked(new Set());
- setAddCustQuery("");
- setAddCustOpen(false);
- }, [activeCohortId, addCustPicked]);
-
- /**
- * The picker's candidates: the POOL minus the cohort's current members, narrowed by the
- * search. The pool's own order is kept (it leads with the biggest customers, which is the
- * useful default for "who am I adding").
- */
- const addCandidates = useMemo(() => {
- if (!addCustOpen) return [];
- const q = addCustQuery.trim().toLowerCase();
- const out: { pid: number; name: string }[] = [];
- for (const r of rawRows) {
- if (cohortMemberSet.has(r.pid)) continue;
- const name = String(r[lockedKey] ?? "");
- if (q && !name.toLowerCase().includes(q)) continue;
- out.push({ pid: r.pid, name });
- }
- return out;
- }, [addCustOpen, addCustQuery, rawRows, cohortMemberSet, lockedKey]);
-
- /**
- * Wave-6 item 11c — PIN. "Pin up to this field" freezes every visible column through the
- * clicked one; when the field sits past the clamp (8), it is REORDERED into the frozen
- * prefix first (the contract's wording). Pinning the identity column reads as "just the
- * identity" = the legacy 1, stored as ABSENT.
- *
- * ⚠ CORRECTED wave-14 item 15. This used to read "Grouped mode paints frozenCount 0 (spans
- * break under freezing)". **Spans do not break under freezing** — glide splits them
- * (`getSpanBounds` → `[frozenRect, contentRect]`) and draws the row's contents from the frozen
- * half. What IS true, and is the real cost of the pin, is that those contents are then clipped
- * to the frozen strip's width; `fitGroupLabel` is what makes that clip honest. The clamp is
- * gone: `freezeColumns` is `frozenN` in every mode.
- */
- const visibleKeys = useMemo(
- () => order.filter((key) => visible.has(key)),
- [order, visible]
- );
- const frozenN = frozenCountOf(config);
- const pinFieldTo = useCallback(
- (key: string) => {
- const index = visibleKeys.indexOf(key);
- if (index < 0) return;
- if (index < MAX_FROZEN) {
- updateConfig({ ...config, frozenCount: clampFrozenCount(index + 1) });
- return;
- }
- const moved = visibleKeys.filter((k) => k !== key);
- moved.splice(MAX_FROZEN - 1, 0, key);
- let cursor = 0;
- const nextOrder = order.map((k) => (visible.has(k) ? moved[cursor++] : k));
- updateConfig({ ...config, order: nextOrder, frozenCount: MAX_FROZEN });
- },
- [visibleKeys, order, visible, config, updateConfig]
- );
- const unpinFields = useCallback(
- () => updateConfig({ ...config, frozenCount: undefined }),
- [config, updateConfig]
- );
-
- // Wave-6 item 10 — display-mode plumbing. W13 (contract C4 as AMENDED 2026-07-28):
- // `config.display` is the ONE home for the field picks, so leaving a mode must NOT
- // wipe them — returning to grid keeps the refs in the draft (mode 'grid' + refs;
- // cleanDisplay's carry), which is what lets the next Kanban/Calendar restore the
- // chosen field instead of feeding the select its default. A view that never picked
- // stays byte-identical (ref-less grid normalizes to absent, the legacy rule).
- const setDisplayMode = useCallback(
- (m: DisplayMode) => {
- // I12 (C3) — a locked view's MODE is frozen. The switcher is already hidden for one,
- // so reaching here means something other than the switcher asked; refuse rather than
- // trust the UI. (The host refuses too — this only stops the local state diverging from
- // what the host will store, which would look like a working change that never saved.)
- // (Resolved from `views` here rather than the `activeView` binding below — this
- // callback is declared above it.)
- const av = views.find((v) => v.id === activeViewId);
- if (av && isModeFrozen(av)) return;
- const spec = cleanDisplay(config.display);
- if (m === "grid") {
- updateConfig({
- ...config,
- display: spec ? cleanDisplay({ ...spec, mode: "grid" }) : undefined,
- });
- return;
- }
- updateConfig({ ...config, display: { ...(spec ?? {}), mode: m } });
- },
- [config, updateConfig, views, activeViewId]
- );
- // C4 (amended): the pick is written INTO display and the selects are FED from the
- // view def (`display.`), never from component state that dies on unmount.
- const setDisplayField = useCallback(
- (k: "dateField" | "stackField" | "colorField" | "sizeField", key: string) => {
- const spec = cleanDisplay(config.display);
- if (!spec) return;
- // Wave-8 I3/I5: colour/size are OPTIONAL encodings, so "" is a real choice
- // (none) and must delete the ref rather than store an empty key that no
- // field can ever match.
- const next = { ...spec, [k]: key };
- if (!key) delete (next as Record)[k];
- updateConfig({ ...config, display: next });
- },
- [config, updateConfig]
- );
- /**
- * Item 3 (C-DISP) — the kanban card clamp.
- *
- * ⚠ `undefined` DELETES the key rather than storing `true`. Absent means clamped, so a
- * stored `true` would be the default wearing a second name — and the host's `_clean_display`
- * accepts the literal `false` only, so a `true` would be dropped on save and the toggle
- * would look like it does not persist. Delete-to-default keeps both ends agreeing.
- */
- const setKanbanClamp = useCallback(
- (next: false | undefined) => {
- const spec = cleanDisplay(config.display);
- if (!spec) return;
- const display = { ...spec };
- if (next === false) display.kanbanClamp = false;
- else delete (display as Record).kanbanClamp;
- updateConfig({ ...config, display });
- },
- [config, updateConfig]
- );
- /**
- * ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the swipe BINDING.
- *
- * ⚠ `undefined` DELETES the key, for `setKanbanClamp`'s reason one step further on. A swipe
- * binding is a single three-part thing (field + two options); both engines' `_clean_display`
- * DROP a half-binding rather than storing it, so writing one back would look like a save that
- * silently did not persist. Absent IS the unconfigured state.
- */
- const setSwipeSpec = useCallback(
- (next: SwipeSpec | undefined) => {
- const spec = cleanDisplay(config.display);
- if (!spec) return;
- const display = { ...spec };
- if (next) display.swipe = next;
- else delete (display as Record).swipe;
- updateConfig({ ...config, display });
- },
- [config, updateConfig]
- );
- /**
- * Item 4 (C-DISP) — the per-day totals. An EMPTY list deletes the key rather than storing
- * `[]`: the always-on Records count is what an absent list means, so `[]` would be that
- * state wearing a second name — and `_clean_display` drops an empty array anyway, which
- * would make un-picking the last metric look like it did not save.
- */
- const setCalendarMetrics = useCallback(
- (next: { id: string; field: string; agg: string }[]) => {
- const spec = cleanDisplay(config.display);
- if (!spec) return;
- const display = { ...spec };
- const capped = next.slice(0, MAX_CALENDAR_METRICS);
- if (capped.length) display.calendarMetrics = capped;
- else delete (display as Record).calendarMetrics;
- updateConfig({ ...config, display });
- },
- [config, updateConfig]
- );
- /** Item 4 (C-DISP) — records vs summaries. `records` is the default, so it deletes the key. */
- const setCalendarMode = useCallback(
- (next: "records" | "summary") => {
- const spec = cleanDisplay(config.display);
- if (!spec) return;
- const display = { ...spec };
- if (next === "summary") display.calendarMode = "summary";
- else delete (display as Record).calendarMode;
- updateConfig({ ...config, display });
- },
- [config, updateConfig]
- );
- // The calendar's date field: the view's pick when it is still a real date-family field,
- // else `last_order`, else the first date-family field. The kanban's stack field: status or
- // single-select ONLY (a multiselect card in two columns at once is a count lie — contract).
- const dateFieldChoices = useMemo(
- () => fields.filter((f) => isDateFamilyType(f.type)),
- [fields]
- );
- const stackFieldChoices = useMemo(
- () => fields.filter((f) => f.type === "status" || f.type === "select"),
- [fields]
- );
- const pickBy = (want: string | undefined, choices: Field[], preferred?: string): Field | undefined => {
- const wanted = want ? choices.find((f) => f.key === want) : undefined;
- if (wanted) return wanted;
- const pref = preferred ? choices.find((f) => f.key === preferred) : undefined;
- return pref ?? choices[0];
- };
- // W13 (C4 as amended): the VIEW DEF's display feeds the pickers — the refs survive
- // every mode switch via cleanDisplay's grid carry. pickBy already falls back to the
- // default when the stored key no longer names an offerable field (a dropped ref).
- // I3/I5 — the map's encodings. Deliberately NOT resolved through `pickBy`:
- // that helper falls back to the first choice, which is right for the calendar
- // (a calendar must have a date) and wrong here, where "no encoding" is a
- // legitimate, and the DEFAULT, state. An unknown stored key resolves to
- // undefined = no encoding, which is also the dropped-ref behaviour.
- // I19c (C2) — the chart list lives in `config.display.charts`, so it rides the
- // existing view autosave + viewEcho reconcile and needs no new event type.
- const charts = useMemo(
- () => cleanCharts(displaySpec?.charts) ?? [],
- [displaySpec?.charts]
- );
- const setCharts = useCallback(
- (next: ChartSpec[]) => {
- const spec = cleanDisplay(config.display);
- if (!spec) return;
- const display = { ...spec, charts: next };
- if (!next.length) delete (display as Record).charts;
- updateConfig({ ...config, display });
- },
- [config, updateConfig]
- );
- const colorFieldChoices = useMemo(
- () => fields.filter((f) => f.type === "status" || f.type === "select"),
- [fields]
- );
- const sizeFieldChoices = useMemo(
- () => fields.filter((f) => isNumericFieldType(f.type)),
- [fields]
- );
- const mapColorField = displaySpec?.colorField
- ? colorFieldChoices.find((f) => f.key === displaySpec.colorField)
- : undefined;
- const mapSizeField = displaySpec?.sizeField
- ? sizeFieldChoices.find((f) => f.key === displaySpec.sizeField)
- : undefined;
- const calendarField = pickBy(displaySpec?.dateField, dateFieldChoices, "last_order");
- const kanbanField = pickBy(displaySpec?.stackField, stackFieldChoices);
- /* ⭐ WAVE-27 item 8 (C3) — the BOUND field, resolved by key and NOT through `pickBy`. `pickBy`
- falls back to the first choice when the stored key is unknown, which is right for a kanban
- (a stack field is a preference) and wrong here: SwipeView SHOWS a rotted binding rather than
- guessing at one, so it must receive the loss instead of a silent substitute — the wave-7 trap
- [[wrong-parent-not-broken-control]] and the component's own header note both name. */
- const swipeField = displaySpec?.swipe
- ? fieldByKey.get(displaySpec.swipe.fieldKey)
- : undefined;
- // Calendar/kanban project the FULL pipeline result (a month/stack is its own bound), as
- // DISTINCT data rows — grouping can repeat a pid — with overlay edits layered so a dragged
- // card restacks and an edited title repaints without waiting on any echo (item 3c).
- const modeDataRows = useMemo(() => {
- if (
- displayMode !== "calendar" &&
- displayMode !== "kanban" &&
- displayMode !== "map" &&
- displayMode !== "chart" &&
- // C-TS close-out stitch: the panel's pids come from these rows; without this line the
- // time-series view asked its question about ZERO customers (caught by the mounted
- // judge — the node battery is blind here).
- displayMode !== "timeseries" &&
- /* ═══ W18-C CATALOG ═══ The designer's "add the products this view shows" door reads
- these rows — the same rows the toolbar counts, so the two can never disagree about
- which products the filter kept. The catalog's PAINT does not: it joins its stored
- codes against the whole pool, so a filter narrows what you can ADD and never what a
- finished page prints. ═══ end W18-C CATALOG ═══ */
- displayMode !== "catalog" &&
- /* ⭐ WAVE-27 item 8 (C3) — the swipe deck is these rows filtered to the undecided ones.
- Without this line the deck is derived from `[]`, so the view paints its "nothing left
- to decide" empty state over a table full of undecided records — the C-TS stitch above,
- repeated. */
- displayMode !== "swipe"
- )
- return [];
- const seen = new Set();
- const out: Row[] = [];
- for (const vr of visibleRows) {
- if (vr.kind !== "data" || seen.has(vr.record.pid)) continue;
- seen.add(vr.record.pid);
- const edits = overlayEdits[vr.record.pid];
- out.push(edits ? { ...vr.record, ...edits } : vr.record);
- }
- return out;
- }, [displayMode, visibleRows, overlayEdits]);
- // Owner item 1 — the CALL-SITE half of the kanban memo. `KanbanView` is `memo`'d now, but a
- // memo whose props are rebuilt every render never bails, and these two were: `cardKeys` is a
- // fresh array from `.filter().slice()`, `onMove` a fresh closure. With them stable, a
- // `detailPid` change (clicking a card) re-renders the modal and skips the board entirely —
- // which is the whole of "laggy record-open". Everything else KanbanView takes was already
- // stable: `modeDataRows`/`fieldByKey` are useMemos, `kanbanField` is a reference INTO the
- // `stackFieldChoices` useMemo, and `onOpen={setDetailPid}` is a setState identity.
- const kanbanCardKeys = useMemo(
- () =>
- kanbanField
- ? visibleKeys.filter((k) => k !== lockedKey && k !== kanbanField.key).slice(0, 3)
- : [],
- [visibleKeys, lockedKey, kanbanField]
- );
- const onKanbanMove = useCallback(
- (pid: number, value: string) => {
- if (!kanbanField) return;
- patchAndRecord(pid, { [kanbanField.key]: value }, "a card move");
- },
- [kanbanField, patchAndRecord]
- );
- /* ⭐ WAVE-27 item 8 (C3) — a swipe writes through the NORMAL cell door, the same
- `patchAndRecord` a card move uses. That is the whole reason the deck needs no permission
- logic, no undo entry and no echo handling of its own: it inherits all three. */
- const onSwipeDecide = useCallback(
- (pid: number, value: string) => {
- if (!swipeField) return;
- patchAndRecord(pid, { [swipeField.key]: value }, "a swipe");
- },
- [swipeField, patchAndRecord]
- );
- /**
- * ⭐ WAVE-26 ITEM 12 (owner ruling R8) — the kanban's "Add option" door.
- *
- * It opens the ORDINARY ColumnMenu on the stack field, straight into its Edit-field pane. So
- * the option is written into the SHARED field definition by the same `onRetype` upsert the
- * header menu uses — never a per-view list — and every guarantee that editor already carries
- * (permission wall, rename-by-row-identity, option colours, one upsert for name+choices) comes
- * along without being re-implemented. R8's "do not invent a second options editor", literally.
- *
- * ⛔ THE PERMISSION QUESTION IS NOT `canEditField`. Adding an option changes the column's
- * DEFINITION; `mayEditField` answers who may type a VALUE into it, and conflating the two is
- * the confusion [[schema-role-is-not-a-value-wall]] exists to prevent. The right predicate is
- * the one that decides whether `onRetype` is supplied at all — a created overlay field — so it
- * is written ONCE here and read at both doors.
- *
- * `null` (not "absent") when the field cannot take one: a status column is computed by the
- * source system, and an Odoo select's vocabulary lives in Odoo.
- *
- * ⭐ THE PREDICATE ADMITS AN AUTOMATION'S `stage_` COLUMN, and that is INTENDED rather than
- * incidental — worth stating because it was arrived at by matching `onRetype`'s condition, and
- * a coincidence and a decision look identical in code. It is exactly what R6 and R8 compose
- * to: R6 deleted the board's built-in terminals, so the lanes a review stage offers are now
- * only the ones a user defined — and R8 is the control that lets them define one. Landed
- * without this the pair is worse than either alone (SESSION A's A-7 says the same thing from
- * the engine side: `humanMoves` had to learn to admit user-added options, or the move door
- * would refuse the lane the product just invited you to create).
- * ⚠ CARRIED CONSEQUENCE, pre-existing and not introduced here: the pane this opens is the
- * whole Edit-field window, so it also offers the TYPE picker — one wrong click retypes a stage
- * column. The header menu has offered exactly that on the same column since wave 6; this door
- * adds a second way in, not a new hazard. Written down rather than left to be rediscovered.
- */
- const kanbanCanAddOption =
- !!kanbanField && kanbanField.type === "select" &&
- !!kanbanField.custom && kanbanField.source === "overlay";
- const onKanbanAddOption = useCallback(
- (anchor: AnchorRect) => {
- if (!kanbanField) return;
- setColumnMenu({ fieldKey: kanbanField.key, anchor, pane: "edit" });
- },
- [kanbanField]
- );
- const onListToggleGroup = useCallback((groupKey: string) => {
- setCollapsed((current) => {
- const next = new Set(current);
- if (next.has(groupKey)) next.delete(groupKey);
- else next.add(groupKey);
- return next;
- });
- }, []);
- /**
- * Item 7 (C-TS) — the pid set the time series asks about: exactly the rows the current
- * filter kept, so the panel and the toolbar count can never disagree about "who".
- *
- * `modeDataRows` is already the de-duplicated data rows for a non-grid mode, so this is a
- * projection of it and NOT a second pipeline. The server intersects with the caller's book
- * anyway — this narrows the question, it can never widen the answer.
- */
- const timeseriesPids = useMemo(
- () => (displayMode === "timeseries" ? modeDataRows.map((r) => r.pid) : []),
- [displayMode, modeDataRows]
- );
- /* ═══ W18-C CATALOG ═══ (owner item 4, contract C6)
- The catalogs this view holds, and the door that persists an edit to them.
-
- `displaySpec.catalogs` is ALREADY validated — `cleanDisplay` ran `cleanCatalogs` over it on
- read — so there is no second normalisation here and no chance of the two disagreeing.
-
- ⚠ The write goes through `updateConfig` exactly like `onTimeseriesDisplay` below, which
- means a catalog edit is a VIEW-CONFIG save. That is the point: a catalogue is authored
- content and has to survive a reload, and the display spec is the only per-view store the
- grid has. It also means the 500-code / 40-page / 12-catalog caps are enforced twice on the
- round trip (here on read, and by the host's `_clean_catalogs` on write) — the designer
- states its budget in the toolbar so the cap is never the user's first news of it. */
- const catalogs = useMemo(
- () => (displayMode === "catalog" ? (displaySpec?.catalogs ?? []) : []),
- [displayMode, displaySpec]
- );
- /** The codes the current filter kept — the designer's "add what the view shows" shortcut.
- * Derived from `modeDataRows`, so it counts what the toolbar counts. */
- const catalogFilteredCodes = useMemo(
- () =>
- displayMode === "catalog"
- ? modeDataRows
- .map((r) => r[CATALOG_CODE_FIELD])
- .filter((c): c is string => typeof c === "string" && !!c)
- : [],
- [displayMode, modeDataRows]
- );
- const onCatalogs = useCallback(
- (next: CatalogSpec[]) => {
- const spec = cleanDisplay(config.display) ?? { mode: "catalog" as const };
- updateConfig({ ...config, display: { ...spec, catalogs: next } });
- },
- [config, updateConfig]
- );
- /* ═══ end W18-C CATALOG ═══ */
- /** Persist a bucket/span/metric pick onto the view, the way every other display key rides. */
- const onTimeseriesDisplay = useCallback(
- (next: Partial) => {
- const spec = cleanDisplay(config.display) ?? { mode: "timeseries" as const };
- updateConfig({ ...config, display: { ...spec, ...next } });
- },
- [config, updateConfig]
- );
- const kanbanCanMove =
- !!kanbanField && kanbanField.type === "select" && canEditField(kanbanField);
- const kanbanReason = !kanbanField
- ? null
- : kanbanField.type === "status"
- ? `Stacked by ${kanbanField.label} — read-only, computed from the source system.`
- : !canEditField(kanbanField)
- ? "You do not have permission to edit this field, so cards cannot be moved."
- : null;
- /* ⭐ WAVE-27 item 8 (C3) — the kanban pair, said again for the deck. Same wall, same words
- shaped for the gesture: a swipe writes a VALUE, so `canEditField` is the right predicate. */
- const swipeCanWrite = !!swipeField && swipeField.type === "select" && canEditField(swipeField);
- const swipeReason = !swipeField
- ? null
- : swipeField.type === "status"
- ? `Bound to ${swipeField.label} — read-only, computed from the source system.`
- : !canEditField(swipeField)
- ? "You do not have permission to edit this field, so records cannot be decided."
- : null;
-
- if (loading || !workspaceReady) {
- /* wave17 GRID — item 3 / owner R6: one small bare rotating icon, NO WORDS. `aria-label`
- is not a word on screen and stays: `.lp-spin` is an empty span, so without it the wait
- is announced to a screen reader as nothing at all. (C-SPIN, SHELL defines the class.)
- `--lg` because this is the case C-SPIN sizes it for — the whole surface is this mark and
- nothing else, where the 14px version "reads as dust". */
- return (
-
-
-
- );
- }
-
- const activeView = views.find((view) => view.id === activeViewId);
- // Cohort mode's toolbar control (rendered by Toolbar via the `cohortAction` slot; the popover
- // it opens is with the other overlays at the bottom). Disabled without an active cohort —
- // zero cohorts is the host page's near-empty state, not this button's error to explain.
- const cohortAction = cohortMode ? (
- setAddCustOpen((o) => !o)}
- aria-expanded={addCustOpen}
- aria-haspopup="dialog"
- >
- + Add customers
-
- ) : undefined;
- // Wave-7 item W9 — the Fields panel's permanent delete rides the SAME wall as the
- // column menu's Delete: created strata only (`custom`), never the locked identity
- // column. One predicate, two doors.
- const deletableKeys = new Set(
- fields.filter((f) => f.custom && f.key !== lockedKey).map((f) => f.key)
- );
- const menuField = columnMenu ? fieldByKey.get(columnMenu.fieldKey) : undefined;
- // Wave-5 item 3 — what the CURRENT VIEW does with the menu's field, so the conditional
- // "Don't sort/filter/group" entries render exactly when they apply.
- const menuSortedDir = menuField
- ? config.sorts.find((s) => s.colId === menuField.key)?.dir ?? null
- : null;
- const menuIsFiltered = menuField
- ? treeNamesField(config.filters, menuField.key, measureCols)
- : false;
- // Item 11c — is the menu's field the current frozen boundary (its Pin would be a no-op)?
- const menuVisIndex = menuField ? visibleKeys.indexOf(menuField.key) : -1;
- const menuPinnedTo = menuVisIndex >= 0 && frozenN > 1 && menuVisIndex + 1 === frozenN;
- // Item 10 — the toolbar's mode switcher (grid-only on windowed tables, see displayMode).
- const modeControl = serverWindowed || embedded ? undefined : (
-
- );
- // ⭐ Wave-23 C7 — the open document, resolved exactly like the picker's value below it:
- // the OPTIMISTIC edit wins over the raw record, so a save the server has not echoed yet is
- // what re-opening the cell shows (the echo-suppression law — reading the raw row here would
- // make a just-saved document appear to revert).
- const jsonField = jsonAt ? fieldByKey.get(jsonAt.fieldKey) : undefined;
- const jsonRow = jsonAt ? rawRows.find((r) => r.pid === jsonAt.pid) : undefined;
- const jsonValue = String(
- (jsonAt && overlayEdits[jsonAt.pid]?.[jsonAt.fieldKey])
- ?? (jsonField && jsonRow ? jsonRow[jsonField.key] : "")
- ?? ""
- );
- const linkField = linkAt ? fieldByKey.get(linkAt.fieldKey) : undefined;
- const linkRow = linkAt ? rawRows.find((row) => row.pid === linkAt.pid) : undefined;
- const linkValue = String(
- (linkAt && overlayEdits[linkAt.pid]?.[linkAt.fieldKey])
- ?? (linkField && linkRow ? linkRow[linkField.key] : "")
- ?? ""
- );
- const linkedRecordIds = [...new Set(
- linkValue.split(",").map((part) => Number(part.trim())).filter(
- (pid) => Number.isInteger(pid) && pid > 0
- )
- )];
- const pickerField = picker ? fieldByKey.get(picker.fieldKey) : undefined;
- // A `user` field's choices come from the HOST's real user list, a `select`'s from its own
- // definition — so an assignee is always someone who can log in, and a status is always one of
- // the choices the column was created with.
- const pickerChoices = !pickerField
- ? []
- : pickerField.type === "user"
- ? payload?.userOptions ?? []
- : choiceOptions(pickerField);
- const pickerRow = picker ? rawRows.find((r) => r.pid === picker.pid) : undefined;
- const pickerValue = String(
- (picker && overlayEdits[picker.pid]?.[picker.fieldKey])
- ?? (pickerField && pickerRow ? pickerRow[pickerField.key] : "")
- ?? ""
- );
- // A multiselect cell is a SET: choices TOGGLE and the picker stays open for the next pick
- // (item 5). Single select/user keep pick-and-close.
- const pickerMulti = pickerField?.type === "multiselect";
- const pickerParts = pickerMulti ? splitMulti(pickerValue) : [];
- /** Item 6 — which choice receives focus when the picker opens: the current value where it
- * still exists in the list, else the first choice. Keyboard-only editing starts HERE. */
- const pickerFocusChoice = !pickerChoices.length
- ? null
- : pickerMulti
- ? pickerParts.find((p) => pickerChoices.includes(p)) ?? pickerChoices[0]
- : pickerChoices.includes(pickerValue)
- ? pickerValue
- : pickerChoices[0];
-
- return (
- // `data-today` is the TENANT'S day, exactly as the payload delivered it. It is here so the
- // date every relative condition resolves against is OBSERVABLE rather than inferred: a live
- // QA that reads its own clock instead compares two engines on "now" and fails whenever a run
- // straddles midnight — which is the flake harness/windows.py takes `today` as a parameter to
- // avoid, reintroduced one layer up. It cost a false failure (57 vs 55, both correct, one
- // computed either side of a date change) to notice.
- // `data-measure-rules` / `data-measure-sets` are BUG-1's handshake, published the same way
- // and for the same reason as `data-today`: a measure condition stuck on "Calculating…"
- // renders identically whether the rule carries no id, the host resolved nothing, or the two
- // sides name the rule differently — and only the first of those is ours. A console.log
- // cannot serve this: the grid runs in an iframe that is CROSS-ORIGIN on the Space, and every
- // Streamlit rerun replaces it. An attribute survives the remount and is one `get_attribute`
- // away from any harness, local or deployed.
-
- {/* ⭐ wave17 R1 / C-LOCKV — the COHORT SIDEBAR is gone, and with it the last surface that
- treated a cohort as its own kind of object. `CohortSidebar` was the retired `#/cohort`
- page's left panel: a second rail, with its own rename, delete, export and folder
- machinery, over `workspace.lists`. Under R1 a cohort IS a saved view, so the Views rail
- below renders every one of them and the second rail has nothing to switch between.
- ⚠ `lists` is NOT gone with it — it stays the membership channel that feeds `cohortSets`
- (C-LOCKV point 2), which is what the lock resolves against. */}
- {!hideViews && (
-
setSelectFromFile(true)}
- onAddToList={serverWindowed ? undefined : addToList}
- today={today}
- onExport={serverWindowed ? undefined : exportViewData}
- folders={folders}
- folderIdOf={folderIdOf}
- onFolderCreate={(name, icon) => {
- const id = newFolderId();
- // Bounded: reconcileFolders drops a local entry once the echo
- // carries it or its create stamp ages out, so this only has to not
- // grow without limit.
- // C5 (I15): the icon rides the LOCAL entry too, or the folder the user just
- // styled renders grey for one round trip and then changes colour under them.
- setLocalFolders((prev) => [...prev.slice(-31), { id, name, order: 9_999, icon }]);
- stampFolder((p) => ({ ...p, created: { ...p.created, [id]: Date.now() } }));
- emitHostEvent({
- id: eventId("foldnew"),
- type: "folder_create",
- surface: "views",
- folderId: id,
- name,
- icon,
- });
- }}
- onFolderRename={(folderId, name) => {
- stampFolder((p) => ({
- ...p,
- renamed: { ...p.renamed, [folderId]: { at: Date.now(), name } },
- }));
- emitHostEvent({
- id: eventId("foldren"),
- type: "folder_rename",
- surface: "views",
- folderId,
- name,
- });
- }}
- onFolderDelete={(folderId) => {
- // The tombstone is stamped BEFORE the emit: between the click and the
- // echo the payload still lists the folder, and rendering it again for
- // one round trip is the delete blip (folders.ts).
- stampFolder((p) => ({ ...p, deleted: { ...p.deleted, [folderId]: Date.now() } }));
- emitHostEvent({
- id: eventId("folddel"),
- type: "folder_delete",
- surface: "views",
- folderId,
- });
- }}
- onFolderDuplicate={(folderId) => {
- const newId = newFolderId();
- const src = folders.find((f) => f.id === folderId);
- setLocalFolders((prev) => [
- ...prev.slice(-31),
- { id: newId, name: `${src?.name ?? "Folder"} copy`, order: 9_999 },
- ]);
- stampFolder((p) => ({ ...p, created: { ...p.created, [newId]: Date.now() } }));
- emitHostEvent({
- id: eventId("folddup"),
- type: "folder_duplicate",
- surface: "views",
- folderId,
- newId,
- });
- }}
- onFolderReorder={(order) => {
- // A-S4-1 (item 19). S4 built the drag and the reconcile behind an OPTIONAL prop, so
- // its absence silently disabled the whole feature rather than breaking anything —
- // which is why every gate stayed green while folders would not drag at all. The
- // stamp is what stops the rail snapping back for one round trip (the echo window).
- stampFolder((p) => ({ ...p, ordered: { at: Date.now(), order } }));
- emitHostEvent({
- id: eventId("foldord"),
- type: "folder_reorder",
- surface: "views",
- order,
- });
- }}
- /**
- * ⭐ WAVE 27 · OWNER ITEM 5 (contract C7) — the rail's new VIEW order.
- *
- * Three things happen and each one is needed:
- * · the LOCAL list is reordered, so the row lands where it was dropped and stays
- * there for this render;
- * · a STAMP is written, so the next workspace echo — which is one round trip
- * behind and still carries the server's order — does not snap it back
- * (`applyViewOrder` in folders.ts holds it for the echo window and no longer);
- * · ONE event carries the WHOLE order, never a delta. A partial list cannot say
- * where an UNNAMED view went, so "moved B after C" leaves every other position
- * to be re-derived by two sides that can disagree. The server's own
- * `folder_reorder` note states this rule; the same rule, same door.
- *
- * ⚠ AMENDMENT C7-A1 (2026-08-08, SESSION C): the event is
- * `{type: "view_reorder", surface: "views", order}` — `surface`, not `workspace`.
- * The contract said `{workspace, order}`; the door `grid_events.py:1299` actually
- * opens on `surface ∈ FOLDER_SURFACES`, and the TABLE is resolved from the request
- * context rather than the payload (`_store_of(ctx)`). Emitting a key the handler
- * does not read would have shipped a drag that persists nothing, behind a green
- * gate on each side.
- */
- onViewReorder={(order) => {
- setViews((cur) => {
- const byId = new Map(cur.map((v) => [v.id, v]));
- const out: SavedView[] = [];
- const placed = new Set();
- for (const id of order) {
- const v = byId.get(id);
- if (!v || placed.has(id)) continue;
- placed.add(id);
- out.push(v);
- }
- // Anything the order does not name keeps its current place, after the named
- // ones — never dropped. A view created in another tab is not evidence the
- // drop was wrong.
- for (const v of cur) if (!placed.has(v.id)) out.push(v);
- return out;
- });
- stampFolder((p) => ({ ...p, orderedViews: { at: Date.now(), order } }));
- // ⛔ ONE CAST, AT ONE SITE, WITH A RATCHET BEHIND IT (ASK ->D, wave-27 mailbox
- // C-6). `HostEvent`'s union lives in `types.ts`, which is SESSION D's file, and
- // D owns C7's persistence half — so the member `{ id; type: "view_reorder";
- // surface: FolderSurface; order: string[] }` is theirs to add. Until it lands
- // this emit cannot be typed from here.
- // ⚠ The gap is DECLARED, not hidden: `verify_wiring.py`'s
- // `PENDING_UNION_MEMBERS` carries `view_reorder`, prints the ask, and goes RED
- // the moment `types.ts` gains the member — which forces this cast to be deleted
- // in the same change. That is the D-70 ratchet's shape, and it fired on its own
- // this wave, which is the argument for reusing it rather than a comment.
- emitHostEvent({
- id: eventId("vieword"),
- type: "view_reorder",
- surface: "views",
- order,
- } as unknown as HostEvent);
- }}
- onItemMove={(viewId, folderId) => {
- stampFolder((p) => ({
- ...p,
- moved: { ...p.moved, [viewId]: { at: Date.now(), folderId } },
- }));
- emitHostEvent({
- id: eventId("itemmove"),
- type: "item_move",
- surface: "views",
- itemId: viewId,
- folderId,
- });
- }}
- folderAddPreview={serverWindowed ? undefined : folderAddPreview}
- onFolderAddToList={serverWindowed ? undefined : folderAddToList}
- viewer={viewer}
- onToggleLock={toggleViewLock}
- userOptions={payload?.userOptions}
- />
- )}
-
- v.id !== activeViewId)}
- onCopyConfig={(patch) => updateConfig({ ...config, ...patch })}
- fields={fields}
- visible={visible}
- lockedKey={lockedKey}
- onColumnVisible={setColumnVisible}
- onColumnsVisible={(keys) =>
- updateConfig({
- ...config,
- visible: [...new Set([lockedKey, ...keys])],
- })
- }
- // Owner item 23 — the Hide-fields panel lists and reorders COLUMN order, so it is
- // handed the ordered projection, not the definition-ordered `fields` above.
- // `order` is `useGridColumns`' reconciled copy (locked key first, unknown keys
- // dropped, new keys appended), so the list can never show a field the grid does not
- // have or omit one it does.
- orderedFields={orderedFields}
- onFieldOrder={(keys) => updateConfig({ ...config, order: keys })}
- deletableKeys={deletableKeys}
- onDeleteField={deleteField}
- colorOn={config.colorBy !== null}
- onToggleColor={(on) =>
- updateConfig({
- ...config,
- colorBy: on
- ? fields.find((field) => field.type === "status")?.key ?? null
- : null,
- })
- }
- rowHeightMode={config.rowHeightMode}
- onRowHeightMode={(rowHeightMode) => updateConfig({ ...config, rowHeightMode })}
- filters={config.filters}
- onFilters={(filters) => updateConfig({ ...config, filters })}
- filterConj={config.filterConj ?? "and"}
- onFilterConj={(filterConj) => updateConfig({ ...config, filterConj })}
- sorts={config.sorts}
- onSorts={(sorts) => updateConfig({ ...config, sorts })}
- // the toolbar must not advertise a grouping the pipeline is not applying
- groupBy={serverWindowed ? null : config.groupBy}
- onGroupBy={(groupBy) => updateConfig({ ...config, groupBy })}
- statusValues={statusValues}
- measures={measures}
- unresolvedCount={unresolvedCount}
- lists={lists}
- pendingMeasureCount={pendingMeasureCount}
- search={search}
- onSearch={setSearch}
- recordCount={recordCount}
- scopeCounts={payload?.counts}
- pool={payload?.workspace?.pool}
- serverWindowed={serverWindowed}
- cohortAction={cohortAction}
- filterSeed={filterSeed}
- modeControl={modeControl}
- />
- {/* Wave-10 item 9: the view's description used to render HERE, as a full-width band
- between the toolbar and the grid — it cost every row a strip of vertical space and
- read as an alert rather than as a label. It now sits under the view's name in the
- rail (ViewSidebar), where it annotates the thing it describes. */}
- setExpandAt((prev) => (prev === null ? prev : null))}
- >
- {displayMode === "grid" && (
- <>
-
openHeaderMenu(column, bounds)}
- onHeaderClicked={onHeaderClicked}
- onHeaderContextMenu={onHeaderClicked}
- onKeyDown={onGridKeyDown}
- maxColumnWidth={600}
- getCellsForSelection={true}
- rowSelectionMode="multi"
- smoothScrollX
- smoothScrollY
- width={gridSize.width}
- height={gridSize.height}
- customRenderers={[ratingCellRenderer, userCellRenderer, imageCellRenderer]}
- headerIcons={HEADER_ICONS}
- rightElement={embedded ? undefined : (
- // Wave-6 item 4 — the "+" of the header row: the create-field form,
- // insert-at-end.
- //
- // ⚠ Wave-20 owner item 5 UNPINNED it. It used to be `sticky: true` ("so it
- // stays visible however far the columns scroll"), which parked it against the
- // right edge of the WINDOW — on a four-column table that is half a screen of
- // white between the last field and the control that adds the next one, and it
- // reads as page chrome rather than as part of the table. It now sits
- // immediately after the last column, where Airtable puts it.
- //
- // TWO changes are needed and neither works alone: `sticky:false` stops glide
- // pinning the wrapper (`infinite-scroller.js:204` sets `right` only when
- // sticky), and the CSS region kills `.dvn-spacer`'s `flex-grow: 1`, which
- // would otherwise expand to fill the scroller and push the button right back
- // to the edge. Consequence, accepted: on a table wider than the viewport the
- // "+" is off-screen until you scroll to the end of the columns — which is
- // where the new column is going to land anyway.
- {
- const r = event.currentTarget.getBoundingClientRect();
- setPlusMenu({
- left: r.left,
- top: r.top,
- right: r.right,
- bottom: r.bottom,
- width: r.width,
- height: r.height,
- });
- }}
- >
- +
-
- )}
- rightElementProps={{ sticky: false, fill: false }}
- // Owner item 4 (R8) — the trailing "+" row, USER DATABASES ONLY. Passing
- // `onRowAppended` is what makes glide paint the ghost row at all, so the
- // affordance exists exactly where a POST can succeed. See `appendRow`.
- onRowAppended={canMutateRecords ? appendRow : undefined}
- trailingRowOptions={
- canMutateRecords
- ? {
- // The hint sits in the identity column (targetColumn 0), which is the
- // one a new record is named in — the same cell the cursor lands on.
- hint: "New record",
- sticky: false,
- targetColumn: 0,
- }
- : undefined
- }
- />
- {/* ═══ W18-B VOID ═══ (wave 18, owner item 1b / ruling R12) — the two rectangles.
-
- They are SIBLINGS of the canvas, immediately after it and before everything else
- `.cg-grid-box` holds, which is the whole of their stacking story: `z-index: auto`
- plus tree order puts them over the canvas, and `.cg-selbar` / the popovers carry
- explicit `--cg-z-*` and stay over them. `pointer-events: none` keeps glide's
- scroller — which is UNDER this and owns every mouse event the grid reacts to —
- hit-testable straight through them.
-
- The right-hand one starts BELOW the header rather than at 0: in `Airtable 7.png`
- the header band continues past the last column in its own near-white (#FBFCFE at
- (1250,141)), and it is where the add-field "+" lives. The two rectangles overlap
- in the bottom-right corner; they are the same flat colour, so the overlap is not
- visible and neither one needs to know about the other. */}
- {gridVoid.below !== null && (
-
- )}
- {gridVoid.right !== null && (
-
- )}
- {/* ═══ end W18-B VOID ═══ */}
- >
- )}
- {displayMode === "list" && (
-
- )}
- {displayMode === "calendar" &&
- (calendarField ? (
-
- ) : (
-
- This table has no date field to place records by.
-
- ))}
- {displayMode === "kanban" &&
- (kanbanField ? (
-
- ) : (
-
- This table has no status or single-select field to stack by.
-
- ))}
- {/* ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the SWIPE deck.
- ⚠ No `field ? ... : empty-state` fork, and the asymmetry with kanban above is
- deliberate. A kanban with no select field cannot exist, so the host answers for it.
- A swipe deck with no binding is the ORDINARY state of a view somebody just created,
- and SwipeView owns that state along with the three ways a stored binding rots —
- each one re-opening the picker rather than being repaired by a guess. Forking here
- would put a second, dumber explanation in front of the good one. */}
- {displayMode === "swipe" && (
-
- )}
- {/* Wave-8 I19c → wave-9 I10 (C2) — CHART is a VIEW like the rest: the charts
- are computed from `modeDataRows`, the very rows the grid counts, so a filter
- change moves the charts and the toolbar count together.
- ⚠ Only 'chart' is tested, never 'dashboard': `cleanDisplay` normalises the
- legacy value on READ, so a stored 'dashboard' has already become 'chart' by
- the time it reaches here. Testing both would hide a broken normalisation. */}
- {displayMode === "chart" && (
-
- )}
- {/* Owner item 7 (C-TS) — Time series is a VIEW: metric rows x bucket columns over
- the pids the current filter kept, so the table and the series answer the same
- question. The panel is HOST-AGNOSTIC by contract (RECORD re-mounts it inside the
- record modal for item 13), which is why it takes pids and fields rather than
- reaching into anything here. */}
- {displayMode === "timeseries" && (
-
- )}
- {/* ═══ W18-C CATALOG ═══ Owner item 4 (C6) — Catalog is a VIEW, and the only one
- whose payload is authored CONTENT rather than an arrangement of the rows. It
- joins its stored product codes against the WHOLE pool (`rawRows`), not the
- filtered rows: a finished page must keep printing the products it names after
- somebody narrows the view. The filter still has a job — `catalogFilteredCodes`
- is the "add what this view shows" door — but it narrows what you can ADD, never
- what a page prints.
-
- ⚠ Gated on the product surface. A catalog joins products to their images by
- `code`, which is `product_data`'s business key AND the C2-ASSET key; on the
- customer table the primary field is a customer NAME, and joining on it would ask
- the asset route for a picture of a person. The refusal says so in words rather
- than rendering an empty designer. */}
- {displayMode === "catalog" &&
- (scope === "product" ? (
-
- ) : (
-
- A catalog prints products. Open the Products table to build one.
-
- ))}
- {/* ═══ end W18-C CATALOG ═══ */}
- {/* Wave-7 item W11c (C3) — Map is a VIEW: same pipeline rows, pins for rows
- with host-supplied coords, click-through to the same record detail, and
- an honest no-location chip. */}
- {displayMode === "map" && fieldByKey.get(lockedKey) && (
-
- )}
- {/* Owner item 19 — the hover-only Expand. The COUNTERPART to item 17: the single
- click that used to open a record now highlights it, and this is where opening
- moved to. `position: fixed` because glide reports viewport coordinates (the same
- space `.cg-header-tip` is placed in).
-
- ⚠ `onMouseDown` preventDefault, not just onClick: mousedown would otherwise reach
- the canvas underneath, move the selection to the primary cell, and repaint the
- row a frame before the panel opens — a visible flicker on every expand.
-
- Rendered only in `grid` mode: list / calendar / kanban / map have their own
- open affordances and glide is not mounted to report bounds for them. */}
- {!embedded && displayMode === "grid" && expandAt && (
- event.preventDefault()}
- onClick={() => setDetailPid(expandAt.pid)}
- >
- {/* Airtable's expand mark: two corner brackets pushing apart. SVG, never a
- glyph — no emojis in this UI, and a unicode arrow renders differently on
- every platform. Sized in the parent's em so it tracks the button. */}
-
-
-
-
- )}
- {/* Wave-5 item 6 — the header description tip. pointer-events:none + aria-hidden:
- it can never become a click target, so the control under it stays clickable
- (the tooltip-swallows-the-next-click trap, paid for once already). */}
- {headerTip && (
-
- {headerTip.text}
-
- )}
- {/* I2 — the cell tip. `pointer-events: none` + `aria-hidden` are not styling here,
- they are the contract: the control under the tip must stay clickable, and a
- screen reader already has the cell's own value. */}
- {cellTip && (
-
- {cellTip.text}
-
- )}
- {/* Owner item 2's point: checked customers accumulate, and the selection DOES
- something. FLOATED over the grid, never in the flow: a bar that reflows the grid
- on first selection shifts every row under the user's cursor mid-click-run (and
- broke this gate's own coordinate math before it broke a user). It states the
- count and offers "Add to cohort" over exactly the checked pids — the same guarded
- add_to_list event the view menu uses, so the host treats both alike. */}
- {!embedded && selectedPids.size > 0 && !serverWindowed &&
- (displayMode === "grid" || displayMode === "map") && (
-
-
- {selectedPids.size.toLocaleString()} selected
-
- {/* ⭐ 2026-08-06 (owner) — *"right now it only shows 'Add to cohort'"*. On a user
- database the checked rows are RECORDS the reader owns, and there was no door to
- remove one; the whole delete path existed server-side with nothing calling it.
- ⚠ ONLY on a `ut_` database: the customer and product grids are projections of
- Odoo, where a "delete" would be a write to somebody else's system of record —
- the server refuses it, and a button that must be refused is worse than none. */}
- {canMutateRecords && (
- void deleteRecords([...selectedPids])}
- >
- {delArmed && delArmed === [...selectedPids].join(",")
- ? "Delete anyway?"
- : `Delete ${selectedPids.size === 1 ? "record" : "records"}`}
-
- )}
- {cohortMode ? (
- // Item 2c: on the Cohort page the selection EDITS MEMBERSHIP of the fixed set —
- // "Remove from cohort" replaces "Add to cohort". Direct (no popover): the
- // checked set is explicit, the button says what it does, and re-adding is one
- // picker away.
-
- Remove from cohort
-
- ) : (
- {
- // Item 4: seed the new-cohort name as ` · ` — today from
- // the PAYLOAD verbatim, never the browser clock; no date when absent.
- if (!selAddOpen) {
- const vn = activeView?.name ?? "Customers";
- setSelListName(today ? `${vn} · ${today}` : vn);
- }
- setSelAddOpen(!selAddOpen);
- }}
- aria-expanded={selAddOpen}
- aria-haspopup="dialog"
- >
- Add to cohort
-
- )}
- {/* ⭐ Owner item 9 — the counterpart, and it renders ONLY when it can do
- something: the checked rows have to be in a cohort this viewer may edit for
- "Remove from cohort…" to mean anything. Same bar, same selection, opposite
- direction — the ellipsis says a picker follows, because unlike cohort mode
- the target is not implied by where you are standing. */}
- {!cohortMode && removableLists.length > 0 && (
- setSelRemoveOpen((open) => !open)}
- aria-expanded={selRemoveOpen}
- aria-haspopup="dialog"
- >
- Remove from cohort…
-
- )}
-
- Clear
-
-
- )}
-
- {/* Owner item 8 — the cap is VISIBLE, beside the control that lifts it. The toolbar
- count above still states the full matched count; this bar says what is painted.
- Grid + list only: calendar/kanban project the FULL set with their own stated
- bounds (a month, a stack's Show-all). */}
- {capped && (displayMode === "grid" || displayMode === "list") && (
-
-
- Showing first {shownRecords.toLocaleString()} of{" "}
- {recordCount.toLocaleString()}
-
- setDisplayCap((c) => c + DISPLAY_STEP)}
- >
- See more
-
- setDisplayCap(visibleRows.length)}
- >
- Show all
-
-
- )}
-
-
- {menuField && columnMenu && (
-
- * t.key !== scope)` and said so honestly: "a legal relation in Airtable and a confusing
- * one to explain in a one-line picker … left out until somebody asks". It was a taste
- * call, not a constraint, and nothing below the picker ever needed it — `_clean_link`
- * only requires a `ut_*` key, and `sync_reciprocal_link` puts the backlink on the target
- * whichever table that is. "This lead is a duplicate of that lead" and "this record's
- * parent" are the ordinary cases, and both were unbuildable.
- *
- * ⚠ THE UNFILTERED LIST ALSO REPAIRS THE ROLLUP EDITOR, which is the half a one-line
- * change hides: it resolves the fold's source columns with
- * `linkTargets.find(t => t.key === chosenLink.link.table)`, so a rollup over a self-link
- * would have offered an EMPTY column picker and looked like the rollup was broken.
- * ⚠ Both `` renders take the same list — the full menu and the "+" create
- * menu. One of the two is the half-landing this repo keeps paying for.
- */
- linkTargets={linkTargets}
- rollupSourceOffer={rollupSourceOffer}
- locked={menuField.key === lockedKey}
- schemaLocked={isSchemaLocked(menuField)}
- viewer={viewer}
- sortedDir={menuSortedDir}
- isFiltered={menuIsFiltered}
- isGrouped={config.groupBy === menuField.key}
- groupable={isGroupableField(menuField, lockedKey)}
- // Item 12 (R8) — the kanban's "Add option" opens THIS menu straight into its Edit-field
- // pane. Absent for every other opening, which is the action list as before.
- initialPane={columnMenu.pane}
- onClose={() => setColumnMenu(null)}
- onNote={(note) => saveField({ ...menuField, note })}
- onHide={() => {
- setColumnVisible(menuField.key, false);
- setColumnMenu(null);
- }}
- onCreate={createField}
- onChangeField={(newKey) => changeField(menuField.key, newKey)}
- onCreateAndSwap={(label, type, options, extra) =>
- createAndSwapField(menuField.key, label, type, options, extra)
- }
- onProfileFlag={isUserTable ? setProfileFlag : undefined}
- onFieldConfig={isUserTable ? (patch) => setFieldConfig(menuField.key, patch) : undefined}
- onRetype={
- // Item 2 — a CREATED (`custom_`) field retypes IN PLACE from the pane.
- menuField.custom && menuField.source === "overlay"
- ? (type, options, extra) =>
- retypeField(menuField.key, type, options, extra)
- : undefined
- }
- onRename={
- // Item 5 — rename is a def edit on created strata (host accepts the label
- // change via the ordinary field_upsert).
- menuField.custom
- ? (label) => saveField({ ...menuField, label })
- : undefined
- }
- onFormula={
- // Owner item 8 — edit a created FORMULA field's source in place; the optional
- // label rides the SAME upsert (see ColumnMenu's one-upsert rule).
- //
- // ⭐⭐ 2026-08-10 — TWO STRATA, TWO DOORS, and the gate is `custom` rather than the
- // type. A formula column on a `ut_*` database now lives in the SHARED definition, and
- // definition fields carry no `custom` flag (`_clean_field` never stamps one) — so the
- // old single condition made the expression editor DISAPPEAR for exactly the columns
- // this change creates: creatable and then permanently frozen, which is a worse state
- // than the one it replaces. `setFieldConfig` PATCHes the definition; `saveField`
- // writes this user's overlay, and sending a definition field down that path would
- // fork the column into a private second copy of itself.
- menuField.type !== "formula"
- ? undefined
- : isUserTable && !menuField.custom
- ? (formula, label) =>
- setFieldConfig(menuField.key, {
- formula,
- ...(label ? { label } : {}),
- })
- : menuField.custom
- ? (formula, label) =>
- saveField({
- ...menuField,
- formula,
- ...(label ? { label } : {}),
- })
- : undefined
- }
- onPeriod={
- menuField.measure
- ? (w) => changeMeasurePeriod(menuField.key, w)
- : undefined
- }
- onDelete={
- menuField.custom && menuField.key !== lockedKey
- ? () => {
- deleteField(menuField.key);
- setColumnMenu(null);
- }
- : /**
- * ⭐⭐ 2026-08-10 — the DEFINITION stratum's own delete.
- *
- * ⛔ NARROWED TO THE THREE KINDS THAT HOLD NO DATA OF THEIR OWN, which is
- * `preset_editable`'s argument reused rather than a new one: a link, a rollup and
- * a formula are all QUESTIONS asked of other cells, so removing one destroys no
- * measurement and can be undone by asking it again. Every other definition column
- * on a `ut_*` database holds real values, and offering a type-blind Delete over
- * those is a destructive widening this change has no mandate for (booked as
- * D-114). It is also exactly the three kinds `createField` now routes to the
- * definition — so what a user can make here, a user can unmake.
- * ⚠ `schemaLocked` still gates the button inside ColumnMenu, so a pre-set link
- * never shows it; a pre-set ROLLUP does, by the same owner ruling that made it
- * editable, and the confirm says it will come back.
- */
- isUserTable &&
- !menuField.custom &&
- menuField.key !== lockedKey &&
- (menuField.type === "link" || menuField.type === "rollup" ||
- menuField.type === "formula")
- ? () => {
- deleteDefinitionField(menuField.key);
- setColumnMenu(null);
- }
- : undefined
- }
- onDuplicate={
- // Creatable strata only — a base Odoo field offers no Duplicate (contract).
- menuField.custom ? () => duplicateField(menuField) : undefined
- }
- onPermissions={
- // Creatable strata only; ColumnMenu further gates on the viewer (creator/admin).
- menuField.custom
- ? (edit) => saveField({ ...menuField, permissions: { edit } })
- : undefined
- }
- onFormat={(format) => saveField({ ...menuField, format })}
- onSort={(dir) =>
- updateConfig({ ...config, sorts: [{ colId: menuField.key, dir }] })
- }
- onClearSort={() =>
- updateConfig({
- ...config,
- sorts: config.sorts.filter((s) => s.colId !== menuField.key),
- })
- }
- onFilterBy={() =>
- // Item 6c — a MEASURE-carrying column seeds the equivalent MEASURE condition
- // (same measure key + window, value empty); every other filterable column seeds
- // an ordinary column condition. Same signal, one consumer (the Toolbar).
- setFilterSeed((prev) => ({
- n: (prev?.n ?? 0) + 1,
- ...(menuField.measure
- ? { measure: { key: menuField.measure.key, window: menuField.measure.window } }
- : { key: menuField.key }),
- }))
- }
- pinnedTo={menuPinnedTo}
- onPinTo={
- // Item 11c — pinning the identity column is the same as unpinning; skip it.
- menuVisIndex > 0 && !menuPinnedTo
- ? () => pinFieldTo(menuField.key)
- : undefined
- }
- onUnpin={frozenN > 1 ? unpinFields : undefined}
- scopeChoice={scopeChoice}
- onClearFilter={() =>
- updateConfig({
- ...config,
- filters: dropFieldFromTree(config.filters, menuField.key, measureCols),
- })
- }
- onGroupByField={() => updateConfig({ ...config, groupBy: menuField.key })}
- onClearGroup={() => updateConfig({ ...config, groupBy: null })}
- userOptions={payload?.userOptions ?? []}
- measures={measures}
- />
- )}
-
- {/* Wave-6 item 4 — the header "+": the SAME ColumnMenu, opened straight into its
- create form in insert-at-end mode. Anchored on the button; the nominal field is the
- locked identity column (only its position matters — the create form never shows it).
- The action/pane handlers below are unreachable in this mode (create-only; Cancel
- closes), supplied inert to satisfy the surface. */}
- {plusMenu && fieldByKey.get(lockedKey) && (
- setPlusMenu(null)}
- onNote={() => undefined}
- onHide={() => undefined}
- onCreate={createField}
- onChangeField={() => undefined}
- onCreateAndSwap={() => undefined}
- onSort={() => undefined}
- onClearSort={() => undefined}
- onFilterBy={() => undefined}
- onClearFilter={() => undefined}
- onGroupByField={() => undefined}
- onClearGroup={() => undefined}
- scopeChoice={scopeChoice}
- userOptions={payload?.userOptions ?? []}
- measures={measures}
- // ⛔⛔ 2026-08-09 — THE SECOND HALF OF THE SAME DEFECT. The sibling instance above has
- // passed these since the relational types shipped; this one never did, so a rollup
- // created through "+" had no databases to resolve its link against and no source offer
- // to switch modes with — the Column picker held its placeholder and nothing else.
- // ⚠ TWO THINGS WERE BOTH WRONG (the fetch was gated on `columnMenu` and these props
- // were absent), so fixing either alone would still have shipped an empty picker and
- // read as "the fix did not work" — [[defects-that-mask-each-other]].
- // ⚠ 2026-08-10 — THE SELF-EXCLUSION IS GONE FROM BOTH SITES (D-113). The line this
- // replaces said the two call sites "must not disagree about that", and it was right for
- // a reason it did not anticipate: the exclusion was removed from the sibling first and
- // this copy kept it, so for one edit the "+" menu and the column menu offered DIFFERENT
- // databases — caught by grepping the prop rather than by the build, which is happy
- // either way. See the sibling's note for why self-links are offered now.
- linkTargets={linkTargets}
- rollupSourceOffer={rollupSourceOffer}
- />
- )}
-
- {/* ⭐ WAVE 21 item 11 (R10 / contract C5) — "Select records from a list".
- ⚠ `inView` is `visibleRows`, NOT `displayRows`. `displayRows` is a capped slice
- (`DISPLAY_PAGE`) of what the view matched; matching over it would report every
- record past the cap as "not found" — a lie that grows with the table. The whole
- pool goes in beside it so a value the FILTER hides is reported as filtered, not
- as missing. */}
- {selectFromFile && (
- (vr.kind === "data" ? [vr.record] : []))}
- wholeTable={computedRows}
- viewName={activeView?.name ?? "this view"}
- onSelect={(pids) => selectPids(pids, "replace")}
- onClose={() => setSelectFromFile(false)}
- />
- )}
-
- {/* The selection bar's cohort picker: existing cohorts, or a new one by name. Mirrors the
- view menu's Add-to-list surface so the two ways of building a cohort read identically. */}
- {selAddOpen && selAddRef.current && selectedPids.size > 0 && (
- setSelAddOpen(false)}
- dataKind="selection-add-to-list"
- >
- Add to locked view
-
- Adds the {selectedPids.size.toLocaleString()} checked customer
- {selectedPids.size === 1 ? "" : "s"}. The cohort stays fixed as the data changes.
-
- {lists.map((l) => (
- addSelectionToList(l.id, l.name)}
- >
- {l.name}
-
- ))}
-
-
Lock records into a new view
-
setSelListName(event.target.value)}
- onKeyDown={(event) => {
- if (event.key === "Enter" && selListName.trim())
- addSelectionToList("", selListName.trim());
- }}
- />
-
- addSelectionToList("", selListName.trim())}
- >
- Create and add
-
-
-
-
- )}
-
- {/* ⭐ Owner item 9 — the remove picker. Every row states how many of the CHECKED rows it
- would take out ("3 of 4"), because the selection and the cohort are two different sets
- and the difference is the whole reason this needs a picker rather than a button. */}
- {selRemoveOpen && selRemoveRef.current && removableLists.length > 0 && (
- setSelRemoveOpen(false)}
- dataKind="selection-remove-from-list"
- >
- Remove from locked view
-
- Only the locked views holding one of the {selectedPids.size.toLocaleString()} checked
- customer{selectedPids.size === 1 ? "" : "s"} are listed. The records themselves are
- not touched.
-
- {removableLists.map((l) => (
- removeSelectionFromList(l.id)}
- >
- {l.name}
-
- {l.hits.toLocaleString()} of {selectedPids.size.toLocaleString()}
-
-
- ))}
-
- )}
-
- {/* Cohort mode's "+ Add customers" picker (item 2c): search the POOL, tick many, confirm
- once. Candidates exclude current members; picks ACCUMULATE across searches, so the
- confirm button carries the running total. The 50-row render cap is stated beside the
- list — nothing silently truncated. */}
- {cohortMode && addCustOpen && addCustRef.current && activeCohortId && (
- setAddCustOpen(false)}
- dataKind="cohort-add-customers"
- >
- Add customers
-
- Search your customer pool and tick who to add to{" "}
- {activeCohort?.name ?? "this locked view"} .
-
- setAddCustQuery(e.target.value)}
- />
-
- {addCandidates.slice(0, 50).map((c) => (
-
-
- setAddCustPicked((prev) => {
- const next = new Set(prev);
- if (next.has(c.pid)) next.delete(c.pid);
- else next.add(c.pid);
- return next;
- })
- }
- />
- {c.name}
-
- ))}
- {addCandidates.length === 0 && (
-
- {addCustQuery.trim()
- ? "No customer outside this locked view matches your search."
- : "Every customer in your pool is already in this locked view."}
-
- )}
-
- {addCandidates.length > 50 && (
-
- Showing the first 50 of {addCandidates.length.toLocaleString()} matches — keep
- typing to narrow.
-
- )}
-
-
- {addCustPicked.size === 0
- ? "Add customers"
- : `Add ${addCustPicked.size.toLocaleString()} customer${
- addCustPicked.size === 1 ? "" : "s"
- }`}
-
- setAddCustOpen(false)}
- >
- Cancel
-
-
-
- )}
-
- {/* ⭐ Wave-23 C7 — the document viewer. `onSave` is OMITTED, not disabled, when the reader
- may not edit: an absent handler is what makes the raw tab a `` instead of a
- textarea, so there is no editor to be refused by. */}
- {jsonField && jsonAt && (
- patchAndRecord(jsonAt.pid, { [jsonField.key]: next }, "a document")
- : undefined
- }
- onClose={closeJson}
- />
- )}
- {linkField?.type === "link" && linkField.link?.table.startsWith("ut_") && linkAt && (
- patchAndRecord(
- linkAt.pid,
- { [linkField.key]: ids.join(",") },
- "linked records"
- )}
- onClose={closeLink}
- />
- )}
- {pickerField && picker && (
- {
- const fwd = e.key === "ArrowDown" || e.key === "ArrowRight";
- const back = e.key === "ArrowUp" || e.key === "ArrowLeft";
- if (!fwd && !back) return;
- const items = Array.from(
- e.currentTarget.querySelectorAll(
- ".cg-pick-row, .cg-star-btn"
- )
- );
- if (!items.length) return;
- e.preventDefault();
- const at = items.indexOf(document.activeElement as HTMLButtonElement);
- const next = fwd
- ? (at + 1) % items.length
- : (at - 1 + items.length) % items.length;
- items[next]?.focus();
- }}
- >
- {/* Wave-5 item 11 — a RATING cell's picker is the star row itself. */}
- {pickerField.type === "rating" && (
-
- {Array.from({ length: ratingMax(pickerField) }, (_, i) => i + 1).map(
- (n) => (
- {
- patchAndRecord(picker.pid, { [pickerField.key]: String(n) }, "a rating");
- closePicker();
- }}
- >
- = n} />
-
- )
- )}
-
- )}
- {pickerField.type !== "rating" && pickerChoices.length === 0 && (
-
- {pickerField.type === "user"
- ? "No assignable people were supplied for this workspace."
- : "This field has no choices yet. Add them from the column menu."}
-
- )}
-
- {pickerChoices.map((choice) => {
- const tint = optionTint(pickerField, choice);
- const on = pickerMulti
- ? pickerParts.includes(choice)
- : pickerValue === choice;
- return (
-
{
- if (pickerMulti) {
- // Toggle membership; each toggle persists (the event log absorbs the
- // burst), and the picker STAYS OPEN so a set is built in one visit.
- const next = on
- ? pickerParts.filter((p) => p !== choice)
- : [...pickerParts, choice];
- patchAndRecord(picker.pid, {
- [pickerField.key]: next.join(","),
- }, "a choice");
- return;
- }
- patchAndRecord(picker.pid, { [pickerField.key]: choice }, "a choice");
- closePicker();
- }}
- >
- {/* Wave-14 item 11 / R6 — a PICKER shows the avatar AND the name. The grid
- cell shows the face alone (you already know who your people are); this is
- where you find out which face is whose, so dropping the name here would
- make the feature unusable the first time two colleagues share initials.
- ⚠ Inline styles, no new class rules: `index.css` is PANEL's fence this
- wave. The class name is a QA hook only and styles nothing. */}
- {pickerField.type === "user" && (
-
- {/* ⚠ The initials are ALWAYS rendered and the photo is laid OVER them, so
- a corrupt data URL hides itself and the fallback is already underneath.
- C-AVATAR's rule is "never a broken-image glyph, anywhere" — the canvas
- path honours it via `img.onerror`, and this is the DOM path's version
- of the same promise. It cannot fire until HOST serves a photo, which is
- exactly why it is easy to ship without. */}
- {avatarInitials(choice)}
- {userAvatars?.[choice] && (
- {
- e.currentTarget.style.visibility = "hidden";
- }}
- style={{
- position: "absolute",
- inset: 0,
- width: "100%",
- height: "100%",
- objectFit: "cover",
- }}
- />
- )}
-
- )}
-
- {choice}
-
-
- );
- })}
-
- {pickerMulti && (
-
-
- Done
-
-
- )}
- {/* Clearing must be possible, and it must write "" rather than delete the key —
- the overlay patch contract is a value per field, and a missing key would leave
- the previous value standing on the next render. */}
- {pickerValue !== "" && (
- {
- patchAndRecord(picker.pid, { [pickerField.key]: "" }, "a choice");
- closePicker();
- }}
- >
- Clear
-
- )}
-
- )}
-
- {!embedded && detailRecord && detailPid !== null && (
- go(-1)}
- onNext={() => go(1)}
- onClose={() => setDetailPid(null)}
- // Item 5 / C-LAYOUT — see `recordLayout` above. Both optional on RECORD's side, so
- // this compiles whether or not their reorder UI has landed yet.
- recordLayout={recordLayout}
- onRecordLayout={onRecordLayout}
- // Item 13 / C-EMBED — the host names WHICH question Insights asks. Without this the
- // tab deliberately does not render (RECORD's no-safe-default rule: a wrong scope
- // succeeds with plausible wrong numbers, so absence must mean absent, not guessed).
- scope={scope}
- docs={payload?.docs?.[String(detailPid)] ?? []}
- docPayload={payload?.docPayload}
- // C5 — documents ride the ordinary event log. The host answers a fetch
- // on the NEXT render via `payload.docPayload`; Documents.tsx matches it
- // to its own pending request before touching it.
- onDocAdd={
- payload?.docs
- ? (file) =>
- emitHostEvent({
- id: eventId("docadd"),
- type: "doc_add",
- pid: detailPid,
- docId: `doc_${eventId("d").slice(2, 14).replace(/[^a-z0-9]/gi, "")}`,
- ...file,
- })
- : undefined
- }
- onDocFetch={
- payload?.docs
- ? (docId) =>
- emitHostEvent({
- id: eventId("docget"),
- type: "doc_fetch",
- pid: detailPid,
- docId,
- })
- : undefined
- }
- onDocDelete={
- payload?.docs
- ? (docId) =>
- emitHostEvent({
- id: eventId("docdel"),
- type: "doc_delete",
- pid: detailPid,
- docId,
- })
- : undefined
- }
- onNotesChange={(key, value) =>
- setOverlayEdits((current) => ({
- ...current,
- [detailPid]: { ...current[detailPid], [key]: value },
- }))
- }
- onNotesCommit={(key, value) => patchAndRecord(detailPid, { [key]: value })}
- viewer={viewer}
- // Wave-14 close-out stitch (HOST): TSREC's two light-switch props, unwired when
- // the GRID session ended — item 9's picker vocabulary + item 11's photos. Same
- // values the grid's own surfaces already read (the wave-13 sixth-prop precedent).
- userOptions={payload?.userOptions ?? []}
- userAvatars={userAvatars}
- />
- )}
-
- );
-}
-
-export default memo(CustomerGrid);
+import {
+ memo,
+ useCallback,
+ useEffect,
+ useLayoutEffect,
+ useMemo,
+ useRef,
+ useState,
+} from "react";
+import { DataEditor, GridCellKind } from "@glideapps/glide-data-grid";
+import type {
+ CellClickedEventArgs,
+ DataEditorRef,
+ EditableGridCell,
+ GridKeyEventArgs,
+ GridMouseEventArgs,
+ GridSelection,
+ DrawHeaderCallback,
+ HeaderClickedEventArgs,
+ Item,
+ Rectangle,
+ Theme,
+} from "@glideapps/glide-data-grid";
+import "@glideapps/glide-data-grid/dist/index.css";
+
+import { useCustomerData } from "./useCustomerData";
+import type { SurfaceScope } from "./apiBridge";
+import { defaultViewConfig, useGridColumns } from "./useGridColumns";
+import { activeMeasureRuleIds, pendingMeasures, runPipeline, sliceForDisplay,
+ unresolvedConditions, useVisibleRows } from "./useVisibleRows";
+import { computeAggs } from "./aggregations";
+import type { CohortSets } from "./useVisibleRows";
+import type { MeasureSets } from "./useVisibleRows";
+import { useGetCellContent } from "./useGetCellContent";
+import { useGridSelection } from "./useGridSelection";
+import Toolbar from "./Toolbar";
+import JsonViewer from "./JsonViewer";
+import RecordDetail from "./RecordDetail";
+import ViewSidebar from "./ViewSidebar";
+import ColumnMenu from "./ColumnMenu";
+import type { ColumnMenuState } from "./ColumnMenu";
+import { HEADER_ICONS } from "./iconShapes";
+import { emitHostEvent, eventId } from "./hostBridge";
+// ⭐ WAVE 27 item 21 (R14) — which of this table's views carry an alert. The SERVER's
+// list is the only answer; the rail's alert door is a one-way create.
+import { fetchAlerts } from "../alerts/alertsApi";
+import { NAV_MINIMIZE_EVENT, ROWS_STALE_EVENT, TOAST_EVENT, VIEW_OPEN_EVENT,
+ WORKSPACE_STALE_EVENT, signal }
+ from "../apiContract";
+import type { ViewOpenDetail } from "../apiContract";
+import { addTableField, addTableRow, deleteTableField, deleteTableRow, fetchLinkTargets,
+ fetchRollupSources, patchTableField } from "./apiBridge";
+import type { LinkTarget, RollupSourceOffer } from "./apiBridge";
+import SelectFromFile from "./SelectFromFile";
+import ImportDialog from "./ImportDialog";
+import { FormInterface } from "./FormInterface";
+import type { FormSpec } from "./FormInterface";
+// Owner item 16 / R4 / C-UNDO — the stack and every inverse. Pure, so a node gate can run it.
+import { describe, directed, popRedo, popUndo, pushUndo, stackFor } from "./undoStack";
+import type { CellChange, UndoBook, UndoEntry, UndoRow, UndoValue } from "./undoStack";
+import { exportFilename, runExport, triggerDownload } from "./export";
+import type { ExportFormat } from "./export";
+// owner item 3 (2026-08-03) — a time-series view exports its SHEET, not the rows under it.
+import { buildTsCsv, tsSheetToTable } from "./timeSeriesData";
+import type { TsSheet } from "./timeSeriesData";
+import { echoReemit, reconcileEchoView } from "./viewEcho";
+import { pruneStamps, reconcileFields } from "./optimism";
+import { applyViewOrder, newFolderId, pruneFolderStamps, reconcileFolders,
+ resolveFolderId } from "./folders";
+import type { FolderStamps } from "./folders";
+// ⭐ WAVE 27 item 22 — the cells the active view's filter forces on a record added under it.
+import { filterSeedValues } from "./filterSeed";
+import { adoptNewFields, adoptNewViews, pruneTombstones, seedLocalViews, stampTombstone }
+ from "./liveWorkspace";
+import type { Tombstones } from "./liveWorkspace";
+import type { GridFolder, ViewPermissions } from "./types";
+import type { FieldStamps } from "./optimism";
+import { evalFormula, orderFormulas, parseFormula } from "./formulaEngine";
+import type { FormulaAst } from "./formulaEngine";
+import { CalendarView, KanbanView, ListView, ModeSwitch } from "./viewModes";
+import { SwipeView } from "./SwipeView";
+import type { SwipeSpec } from "./SwipeView";
+// The chart engine lives in `viz/` since EXIT wave 2 (W2-5/Y3) — the grid is now
+// one of its two callers, the Y1 page envelope being the other.
+import { DashboardView } from "../viz/DashboardView";
+import { cleanCharts } from "../viz/chartData";
+import type { ChartSpec } from "../viz/chartData";
+import { MapView } from "./MapView";
+/* ═══ W18-C CATALOG ═══ (owner item 4, contract C6) */
+import { CatalogView } from "./CatalogView";
+import { CATALOG_CODE_FIELD } from "./catalogData";
+import type { CatalogSpec } from "./types";
+/* ═══ end W18-C CATALOG ═══ */
+import { lightTheme, STATUS_ROW_THEME, HOVER_ROW_THEME, HOVER_NEUTRAL,
+ ACTIVE_ROW_NEUTRAL, CUSTOM_FIELD_MARK } from "./theme";
+import { avatarInitials, formatDisplay, imageCellRenderer, ratingCellRenderer, setAvatarRepaint,
+ userCellRenderer } from "./cells";
+import { optionTint, pickTint } from "./choiceColors";
+import {
+ EMPTY_GRID_COPY_PROVENANCE,
+ markGridCopy,
+ observeCopyEvent,
+ planFieldPaste,
+ pasteRowCount,
+} from "./clipboard";
+import { cellTipText, expandButtonRect, GROUP_HEADER_FONT, GROUP_LABEL_PAD, headerMarkLayout,
+ headerMarkSizes, tipLeft } from "./overlayPlacement";
+import { AnchoredOverlay, BodyPortal, useOverlayLayer } from "./OverlaySurface";
+import type { AnchorRect } from "./OverlaySurface";
+import { StarIcon } from "./Stars";
+import { ALL_VIEW_ID, MAX_CALENDAR_METRICS, allViewName,
+ MAX_FROZEN, choiceOptions, choiceVocabulary, clampFrozenCount, cleanDisplay, formulaOf,
+ topicForScope,
+ isDateFamilyType, isFilterGroup, isGroupableField, isMachineOwned,
+ isDerivedLink, isUserSchemaField, TOTAL_GROUP_KEY,
+ isNumericFieldType,
+ machineFoundRows, reFindConsequences,
+ isPickType, mayEditField,
+ isModeFrozen, isUndeletableView, mayEditView, mayToggleViewLock,
+ measureColumnIndex, ratingMax, ruleColumnKeys,
+ tableMode, uniqueDisplayName } from "./types";
+import type {
+ DisplayMode,
+ DisplaySpec,
+ Field,
+ HostEvent,
+ FieldScope,
+ FieldType,
+ FilterNode,
+ FilterRule,
+ Row,
+ RowHeightMode,
+ SavedView,
+ ViewConfig,
+} from "./types";
+// Item 7 (C-TS). Default-exported because it is a leaf VIEW like MapView, and because RECORD
+// imports the same default for the record-detail Insights tab (C-EMBED).
+import TimeSeriesPanel from "./TimeSeriesPanel";
+import type { WindowSpec } from "./windows";
+import { windowLabel } from "./windows";
+
+const ROW_PX: Record = { short: 28, medium: 34, tall: 48 };
+
+/**
+ * Wave-14 R7 — **a button label is ONE LINE.** Every button row in this file is a flex row with
+ * no width reservation, so on a narrow grid box the items shrink and their labels wrap:
+ * "Remove from cohort" becomes two lines, the row grows, and the bar stops being a bar. It only
+ * happens below a width no fixed-size screenshot is taken at, which is why it survived.
+ *
+ * ⚠ Applied INLINE rather than on `.cg-btn`, where it belongs: `index.css` is PANEL's fence this
+ * wave. A mailbox line asks them to hoist it onto `.cg-btn` globally — after which this is
+ * redundant, and harmless, because it says exactly the same thing.
+ */
+const ONE_LINE = { whiteSpace: "nowrap" } as const;
+
+/**
+ * How many VISIBLE columns glide freezes. Module-level and used twice on purpose (item 15): the
+ * `freezeColumns` prop and the group bar's label fit must read ONE answer, because the label is
+ * clipped to exactly this strip and a second copy of the clamp would let the two disagree by a
+ * column — which shows up as a label cut early for no visible reason.
+ */
+function frozenCountOf(config: ViewConfig): number {
+ return Math.min(MAX_FROZEN, Math.max(1, config.frozenCount ?? 1));
+}
+/* WAVE 21 item 3 (R6): the id and the NAME both moved to `types.ts` — the id because a second
+ copy of a pinned literal is the drift class this repo gates against, the name because it is
+ now topic-derived and the host mints the same string. */
+
+/**
+ * owner item 2 (2026-08-03) — the measure-cell skeleton's pulse, and its ceiling.
+ *
+ * 140ms against the 4-step colour ramp is a ~0.6s cycle: a wait, not a strobe. The ceiling is
+ * ~60s, after which the cells fall back to ordinary blanks — a measure that has not resolved in
+ * a minute is not "still loading", and a shimmer that never ends promises a number that is not
+ * coming. Shared constant so the two are read in one place rather than tuned apart.
+ */
+const PULSE_MS = 140;
+const PULSE_MAX_TICKS = Math.round(60_000 / PULSE_MS);
+/** Referentially stable, so the fallback does not change `getCellContent`'s identity per render
+ * and repaint the canvas forever. */
+const NO_PENDING_KEYS: ReadonlySet = new Set();
+
+/**
+ * I2 — measure a cell's text in GLIDE'S OWN font, so "is this cut off?" is a fact rather than
+ * a character-count guess (a guess is wrong in both directions: "IIIII" is narrow, "WWWWW" is
+ * wide, and being wrong means either a missing tip or a tip over text you can already read).
+ * One lazily-built offscreen context for the whole module — `measureText` is cheap, but
+ * creating a canvas per mouse-move would not be.
+ */
+let _tipCtx: CanvasRenderingContext2D | null | undefined;
+function measureCellText(text: string): number {
+ if (_tipCtx === undefined) {
+ _tipCtx = document.createElement("canvas").getContext("2d");
+ // glide's default `baseFontStyle` is 13px; the family comes from our own theme.
+ if (_tipCtx) _tipCtx.font = `13px ${lightTheme.fontFamily ?? "Inter, sans-serif"}`;
+ }
+ return _tipCtx ? _tipCtx.measureText(text).width : 0;
+}
+/**
+ * Item 15 — the GROUP BAR's measurer, and it is a SECOND context on purpose.
+ *
+ * `measureCellText` above is set to plain "13px", which is right for ordinary cells and WRONG for
+ * a group bar: that row carries `baseFontStyle: GROUP_HEADER_FONT` (semibold), and glide paints a
+ * cell with the merged theme's font. Measuring the fit with the lighter weight under-truncates
+ * and the label overflows into the frozen strip's hard clip — the exact failure `fitGroupLabel`
+ * exists to prevent. Font string built from the SAME constant the cell's themeOverride uses, so
+ * the two cannot drift.
+ */
+let _groupCtx: CanvasRenderingContext2D | null | undefined;
+function measureGroupText(text: string): number {
+ if (_groupCtx === undefined) {
+ _groupCtx = document.createElement("canvas").getContext("2d");
+ if (_groupCtx)
+ _groupCtx.font = `${GROUP_HEADER_FONT} ${lightTheme.fontFamily ?? "Inter, sans-serif"}`;
+ }
+ return _groupCtx ? _groupCtx.measureText(text).width : 0;
+}
+/**
+ * Owner item 8 (2026-07-27): the first render shows this many rows; "See more" reveals the rest.
+ * A DISPLAY cap only — the pipeline still runs over the whole book, the toolbar count is still
+ * the full matched count, and the cap is stated beside the control that lifts it, which is what
+ * keeps [[no-unverifiable-aggregates]] satisfied: nothing is silently truncated.
+ */
+const DISPLAY_PAGE = 50;
+const DISPLAY_STEP = 250;
+
+/* ═══════════════════════════════════════════════════════════════════════════════════════════
+ ═══ W18-B VOID ═══ (wave 18, owner item 1b) — the two constants the void geometry needs
+ that belong to the BROWSER and to GLIDE rather than to us. Both are mirrors, and both say
+ here what they are mirroring, because a mirror that does not name its original is how the
+ two copies stop agreeing.
+ ═══════════════════════════════════════════════════════════════════════════════════════════ */
+
+/** The header band. `headerHeight={36}` at the DataEditor mount is the original.
+ * Not `+ groupHeaderHeight`: glide adds that only when `enableGroups` is on, which it turns on
+ * when a COLUMN carries a `group` — `useGridColumns` never sets one (scrolling-data-grid.js:13). */
+const HEADER_PX = 36;
+
+/** glide's own row-marker width ladder, `data-editor.js:103`, verbatim.
+ *
+ * ⚠ It is passed BACK to glide as `rowMarkerWidth` rather than merely predicted here. The
+ * marker column is part of the content width the void's left edge is measured from, so a
+ * four-pixel disagreement between what glide draws and what we compute is a four-pixel seam of
+ * white against the tint — visible, and invisible to every gate. Pinning makes the two the same
+ * number by construction; mirroring the ladder (rather than pinning one constant) is what keeps
+ * the pin from changing the marker column's width on tables of more than 100 rows. */
+function rowMarkerPx(rows: number): number {
+ return rows > 10_000 ? 48 : rows > 1000 ? 44 : rows > 100 ? 36 : 32;
+}
+
+/** The scrollbar gutter this browser steals from a scrollable box, measured once.
+ *
+ * ⛔ WHY THE VOID HAS TO KNOW. The void rectangles sit ON TOP of glide's scroller, so a
+ * rectangle drawn to the box's own edge paints over the scrollbar and hides it. The two cases
+ * are CROSSED, which is the part that is easy to get backwards: the BOTTOM void has to stop
+ * short when the columns overflow (a horizontal scrollbar), and the RIGHT void has to stop
+ * short when the rows overflow (a vertical one). Both happen in ordinary use — twelve columns
+ * filtered to five rows is exactly the owner's case.
+ *
+ * 0 on overlay-scrollbar platforms (macOS, touch), which is correct: nothing is stolen there. */
+let scrollbarGutterPx: number | null = null;
+function scrollbarGutter(): number {
+ if (scrollbarGutterPx !== null) return scrollbarGutterPx;
+ if (typeof document === "undefined") return 0;
+ const probe = document.createElement("div");
+ probe.style.cssText =
+ "position:absolute;top:-9999px;width:100px;height:100px;overflow:scroll";
+ document.body.append(probe);
+ scrollbarGutterPx = probe.offsetWidth - probe.clientWidth;
+ probe.remove();
+ return scrollbarGutterPx;
+}
+/* ═══ end W18-B VOID (module scope) ═══ */
+
+interface LocalWorkspace {
+ fields: Field[];
+ views: SavedView[];
+ activeViewId: string;
+ /** When this copy was written (browser clock — same-machine freshness only,
+ * never date semantics). Absent on copies from before the echo reconcile. */
+ savedAt?: number;
+ /** Wave-6 item 3c — per-key freshness for this browser's own field-def
+ * writes (edits + tombstones), consumed by reconcileFields at init. */
+ fieldStamps?: FieldStamps;
+ /** BUG-1 (wave 11) — viewId → the filter tree already pushed back to the host, so the
+ * echo re-emit happens once per user edit rather than once per remount. NOT a timestamp:
+ * `savedAt` beside it is rewritten on every init, so any time-based bound is refreshed by
+ * the remount it is meant to bound. See viewEcho.echoReemit. */
+ reemitted?: Record;
+ /** 2026-08-04 — viewId → when THIS browser deleted it. The views half of the tombstone
+ * rule fields and folders already had, needed the moment the rail adopts host views
+ * after mount (liveWorkspace.ts). Persisted, not a ref, because a remount inside the
+ * echo window would otherwise re-adopt a view the user deleted a second ago. */
+ viewTombstones?: Tombstones;
+ /** D-19 (wave 20) — viewId → when THIS browser last WROTE it. The counterpart of the
+ * tombstones above: they stop a deleted view coming back, this stops a just-created one
+ * being dropped by the host-list rule at init, before its upsert has round-tripped.
+ * Same window, same prune, same browser-clock caveat. */
+ viewWrites?: Tombstones;
+}
+
+function sameConfig(a: ViewConfig, b: ViewConfig): boolean {
+ return JSON.stringify(a) === JSON.stringify(b);
+}
+
+function normalizeConfig(config: Partial | undefined, fields: Field[]): ViewConfig {
+ const base = defaultViewConfig(fields);
+ return {
+ ...base,
+ ...config,
+ filters: Array.isArray(config?.filters) ? config.filters : [],
+ // Legacy views persisted a flat rule array with an implicit AND — anything
+ // other than an explicit "or" normalizes to "and", so old views are unchanged.
+ filterConj: config?.filterConj === "or" ? "or" : "and",
+ sorts: Array.isArray(config?.sorts) ? config.sorts : [],
+ order: Array.isArray(config?.order) && config.order.length ? config.order : base.order,
+ visible:
+ Array.isArray(config?.visible) && config.visible.length
+ ? config.visible
+ : base.visible,
+ widths: config?.widths && typeof config.widths === "object" ? config.widths : {},
+ memberPids: Array.isArray(config?.memberPids) ? config.memberPids : [],
+ // Wave-6 items 10/11. Both normalize junk to ABSENT (grid / 1 frozen), so
+ // a legacy view is byte-identical to itself after a round trip. W13 (C4 as
+ // amended): cleanDisplay keeps grid-carried field picks.
+ display: cleanDisplay(config?.display),
+ frozenCount: clampFrozenCount(config?.frozenCount),
+ };
+}
+
+/**
+ * The pinned system view, as this browser mints it when no host copy has arrived.
+ *
+ * WAVE 21 item 3 (R6) — the NAME is the scope's now (`allViewName`), not the literal "All
+ * customers" this minted on every topic including `ut_*` databases of anything at all. The
+ * ID is unchanged and deliberately so (see `ALL_VIEW_ID`).
+ */
+function allRecordsView(fields: Field[], scope: string): SavedView {
+ return {
+ id: ALL_VIEW_ID,
+ name: allViewName(scope),
+ kind: "system",
+ locked: true,
+ config: defaultViewConfig(fields),
+ };
+}
+
+function readLocal(storageKey: string): LocalWorkspace | null {
+ try {
+ const raw = localStorage.getItem(`aios-grid:${storageKey}`);
+ return raw ? (JSON.parse(raw) as LocalWorkspace) : null;
+ } catch {
+ return null;
+ }
+}
+
+function writeLocal(storageKey: string, value: LocalWorkspace): void {
+ try {
+ localStorage.setItem(`aios-grid:${storageKey}`, JSON.stringify(value));
+ } catch {
+ // Storage can be blocked in hardened/opaque iframes. The host bridge remains durable.
+ }
+}
+
+function nextViewId(): string {
+ return `view_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
+}
+
+/** A multiselect cell's SET, out of its comma-joined string (the `multi` contract). */
+function splitMulti(v: string): string[] {
+ return v
+ .split(",")
+ .map((s) => s.trim())
+ .filter((s) => s !== "");
+}
+
+/** Wave-5 item 3 — does any leaf of the filter tree name this column?
+ *
+ * ⚠ Wave-20 item 2 RETARGETED the note that stood here ("measure leaves carry measure keys,
+ * which are never field keys, so they cannot false-positive"). True about false POSITIVES,
+ * and precisely why it was a false NEGATIVE: a measure column's own condition names the
+ * MEASURE, so this answered "not filtered" for a column the user had visibly filtered, and
+ * the menu never offered "Don't filter by this field" on it. Both doors now go through
+ * `ruleColumnKeys` — the same resolution the tint uses, so the menu and the colour can never
+ * disagree about which columns a filter is about. */
+function treeNamesField(
+ nodes: FilterNode[],
+ key: string,
+ measureCols: Map
+): boolean {
+ for (const n of nodes) {
+ if (isFilterGroup(n)) {
+ if (treeNamesField(n.children, key, measureCols)) return true;
+ } else if (ruleColumnKeys(n as FilterRule, measureCols).includes(key)) return true;
+ }
+ return false;
+}
+
+/** Wave-5 item 3 — "Don't filter by this field": drop every leaf naming the column, prune
+ * groups that end up empty. Returns new arrays throughout (the config is state). */
+function dropFieldFromTree(
+ nodes: FilterNode[],
+ key: string,
+ measureCols: Map
+): FilterNode[] {
+ const out: FilterNode[] = [];
+ for (const n of nodes) {
+ if (isFilterGroup(n)) {
+ const children = dropFieldFromTree(n.children, key, measureCols);
+ if (children.length) out.push({ ...n, children });
+ } else if (!ruleColumnKeys(n as FilterRule, measureCols).includes(key)) {
+ out.push(n);
+ }
+ }
+ return out;
+}
+
+// The header sprite map moved to iconShapes.ts (wave-8 I20/I21) — it is data +
+// SVG-source builders with no React in it, which is what lets verify_icons.py
+// import it under node and assert the (i) actually contrasts.
+
+function slugify(label: string): string {
+ return (
+ label
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, "_")
+ .replace(/^_|_$/g, "")
+ .slice(0, 28) || "field"
+ );
+}
+
+interface FieldBuildExtra {
+ formula?: string;
+ max?: number;
+ scope?: FieldScope;
+ label?: string;
+ colorCodeOptions?: boolean;
+ optionColors?: Record;
+ /**
+ * ⭐⭐ 2026-08-09 — THE RELATIONAL BAGS, WHICH THIS INTERFACE USED TO DROP ON THE FLOOR.
+ *
+ * `ColumnMenu.FieldConfigExtra` has carried them since 2026-08-07 and `extraFor` builds a
+ * complete bag; this type — the receiving end of the same `extra` argument — never declared
+ * either key, so `buildOverlayField` could not copy what it could not see. The measured result
+ * on nurilab: a column stored as
+ * `{key:'custom_video_views_90j26', label:'Video Views', type:'rollup', source:'overlay'}`
+ * with NO bag, in one user's workspace stratum, which `compute_relation_cells` never reads.
+ * Named, configured, rendering, and permanently blank — the owner's *"the Rollup doesn't
+ * work"*. ⛔ Both halves were needed: this type, and the `createField` route below.
+ */
+ link?: Record;
+ rollup?: Record;
+ /** Item 15 (C-RENAME) — the option renames this save carries, by row identity. Consumed by
+ * `retypeField` (which emits `choice_rename`) and ignored by every create path: a field
+ * being CREATED has no values to migrate. */
+ renames?: { from: string; to: string }[];
+}
+
+/**
+ * One constructor for a user-created field, shared by "insert field" and Change-field's
+ * "New field" half (wave-2 item 5) so the two doors cannot drift. A `multiselect` carries its
+ * declared options AND `multi: true` — the cell is a comma-joined SET, and the flag is what the
+ * grouping contract keys on (belt to the type-derived brace in groupRows).
+ *
+ * Wave-5: `formula` and `created_time` are built in the HOST's OWN emission shape (source
+ * 'odoo' + derived — aios_grid.READONLY_CUSTOM_TYPES), so the optimistic local def and the
+ * next payload's echo are byte-identical and nothing restyles on the round trip. `rating`
+ * carries top-level `max`, `formula` carries top-level `formula` (the ~20:20 contract
+ * amendment: options stays the select-family list). Wave-6 item 6: both types are now
+ * `filterable: true` — the host flipped its emission, and conditions on them evaluate in the
+ * client engine over computedRows (this table's counts are client-mode; a windowed table's
+ * columns come from the semantic model, so the SQL engine never sees these types).
+ */
+/**
+ * ⭐⭐ 2026-08-09 (owner ruling) — DOES THE PRE-SET LOCK CLOSE THIS COLUMN'S SCHEMA?
+ *
+ * Owner: *"No rollup field should be uneditable, everything is custom and changeable always."*
+ * A ROLLUP holds no data of its own — it is a question asked of other rows, re-askable at any
+ * time, and re-asking it costs nothing because the answer is recomputed from the authoritative
+ * store on the next pass. `preset` still closes every column that HOLDS something: retyping a
+ * pre-set `followers` column would strand real measurements in a column that can no longer read
+ * them.
+ *
+ * ⛔ THE CLIENT MIRROR OF `core.user_tables.preset_editable`, and `verify_rollup_editor.py`
+ * holds the two in step. This check has THREE enforcement points (here, `may_edit_field`, and
+ * `routes_tables._field_or_refuse`'s sentence); a client that kept hiding the editor while the
+ * route allowed the PATCH would read as "you fixed nothing", which is the failure mode a fix in
+ * one of three places always wears.
+ */
+function isSchemaLocked(field: Field): boolean {
+ return field.automation?.preset === true && field.type !== "rollup";
+}
+
+function buildOverlayField(
+ label: string,
+ type: FieldType,
+ options?: string[],
+ extra?: FieldBuildExtra
+): Field {
+ const key = `custom_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`;
+ if (type === "formula" || type === "created_time") {
+ return {
+ key,
+ label,
+ type,
+ source: "odoo",
+ derived: true,
+ filterable: true,
+ default: true,
+ custom: true,
+ ...(type === "formula"
+ ? { agg: "sum", ...(extra?.formula ? { formula: extra.formula } : {}) }
+ : {}),
+ };
+ }
+ return {
+ key,
+ label,
+ type,
+ source: "overlay",
+ default: true,
+ custom: true,
+ agg: ["currency", "int"].includes(type) ? "sum" : undefined,
+ // The relational bags ride the local definition too, so the optimistic field and the
+ // server's echo describe the same column. Without them the grid held a bagless twin of a
+ // field the store had configured, and `mayEditField`/`isDerivedLink` read the twin.
+ ...(extra?.link ? { link: extra.link as Field["link"] } : {}),
+ ...(extra?.rollup ? { rollup: extra.rollup as Field["rollup"] } : {}),
+ ...((type === "select" || type === "multiselect") && options?.length ? { options } : {}),
+ ...((type === "select" || type === "multiselect")
+ ? {
+ colorCodeOptions: extra?.colorCodeOptions !== false,
+ ...(extra?.optionColors && Object.keys(extra.optionColors).length
+ ? { optionColors: extra.optionColors }
+ : {}),
+ }
+ : {}),
+ ...(type === "multiselect" ? { multi: true } : {}),
+ ...(type === "rating" ? { max: extra?.max ?? 5 } : {}),
+ };
+}
+
+/** The surface this grid draws. `cohort` is the same table over hand-curated SETS — the server
+ * confirms it by stamping `workspace.cohortMode`, which is the only thing the body below reads.
+ *
+ * MEMOIZED (owner item 1, 2026-07-31): the shell re-renders on every chrome state flip — nav
+ * collapse, toast, settings — and an unmemoized grid re-rendered its whole 3,400-line tree
+ * each time. Measured cost: 200ms-5s of main-thread block, which ate the 240ms rail-fold
+ * transition whole (the fold froze, then SNAPPED — the exact "static" the owner named). The
+ * props surface is one stable string, so memo makes chrome state changes free; the grid still
+ * re-renders for its own state (edits, resize) and remounts on route change via `key`. */
+interface CustomerGridProps {
+ scope?: SurfaceScope;
+ /** Read-only Grid view embedded in a linked-record modal. It keeps the standard
+ * filter/sort/search toolbar while withdrawing schema and row mutations. */
+ embedded?: boolean;
+ /** Exact linked pids to project from the target database. */
+ embeddedRecordIds?: readonly number[];
+ /** Selection mode used by an editable ordinary-Link modal. */
+ embeddedSelectable?: boolean;
+ embeddedSelectedIds?: readonly number[];
+ onEmbeddedSelectionChange?: (recordIds: number[]) => void;
+}
+
+interface LinkGridModalProps {
+ label: string;
+ table: SurfaceScope;
+ recordIds: readonly number[];
+ editable?: boolean;
+ single?: boolean;
+ onSave?: (recordIds: number[]) => void;
+ onClose: () => void;
+}
+
+function LinkGridModal({ label, table, recordIds, editable = false, single = false,
+ onSave, onClose }: LinkGridModalProps) {
+ const panelRef = useRef(null);
+ const [selectedIds, setSelectedIds] = useState(() => [...recordIds]);
+ const changeSelectedIds = useCallback((ids: number[]) => {
+ const next = single ? ids.slice(-1) : ids;
+ setSelectedIds((current) =>
+ current.join(",") === next.join(",") ? current : next
+ );
+ }, [single]);
+ useOverlayLayer({
+ panelRef,
+ onDismiss: onClose,
+ dismissOnOutside: true,
+ initialFocus: "[data-overlay-autofocus]",
+ trapFocus: true,
+ });
+ return (
+
+
+
+
+
+
+
+ {editable ? (
+
+ Cancel
+ { onSave?.(selectedIds); onClose(); }}
+ >
+ Save links
+
+
+ ) : null}
+
+
+
+ );
+}
+
+function CustomerGrid({
+ scope = "customer",
+ embedded = false,
+ embeddedRecordIds,
+ embeddedSelectable = false,
+ embeddedSelectedIds = [],
+ onEmbeddedSelectionChange,
+}: CustomerGridProps = {}) {
+ // Wave 16 C-TOPIC: which TABLE this tree is drawing, derived from the one scope prop.
+ const topic = topicForScope(scope);
+ const {
+ fields: payloadFields,
+ rawRows: fetchedRows,
+ payload,
+ loading,
+ overlayEdits,
+ setOverlayEdits,
+ patchOverlay,
+ } = useCustomerData(scope, {
+ bindSurface: !embedded,
+ includeWorkspace: !embedded,
+ writable: !embedded,
+ });
+ const embeddedIdsKey = embeddedRecordIds?.join(",") ?? "";
+ /**
+ * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — **ROWS THIS BROWSER JUST CREATED**, held locally
+ * until the server's own copy comes back.
+ *
+ * ⛔ THE OWNER'S WORDS: *"adding a new record visually takes too long, I need to be able to
+ * spam it."* The old path awaited the POST, then fired `ROWS_STALE_EVENT` and re-read the
+ * WHOLE table — so every "+" cost a round trip plus a full refetch before anything moved, and
+ * pressing it five times queued five full refetches of a table that grew by five rows.
+ *
+ * ⚠ ADD-ONLY AND SELF-PRUNING, which is `liveWorkspace.ts`'s discipline applied to rows: a
+ * pending row is merged in only while the fetched payload does NOT carry its pid, and the
+ * effect below drops it the moment the server's copy arrives. There is no third state and no
+ * merge of values — the server's row wins outright, because by then it IS this row.
+ */
+ const [pendingRows, setPendingRows] = useState([]);
+ const rawRows = useMemo(() => {
+ const base = embeddedRecordIds
+ ? fetchedRows.filter((row) => new Set(embeddedRecordIds).has(row.pid))
+ : fetchedRows;
+ if (!pendingRows.length) return base;
+ const known = new Set(base.map((r) => r.pid));
+ const extra = pendingRows.filter((r) => !known.has(r.pid));
+ return extra.length ? [...base, ...extra] : base;
+ // The scalar key keeps this stable when a caller reconstructs the same id list.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [fetchedRows, embeddedIdsKey, pendingRows]);
+ useEffect(() => {
+ // ⚠ PRUNED AGAINST THE FETCHED PAYLOAD, not against `rawRows` — `rawRows` contains the
+ // pending rows, so it can never report that one has been absorbed. Pruning against the
+ // memo's own output is the loop that keeps a placeholder alive forever.
+ if (!pendingRows.length) return;
+ const known = new Set(fetchedRows.map((r) => r.pid));
+ if (!pendingRows.some((r) => known.has(r.pid))) return;
+ setPendingRows((cur) => cur.filter((r) => !known.has(r.pid)));
+ }, [fetchedRows, pendingRows]);
+ /**
+ * The rows as of the last render, for callbacks that must not be rebuilt per payload.
+ * `appendRow` reads it to mint the next row id the way the server does (`max + 1`); putting
+ * `rawRows` in that callback's dependency list would give the trailing "+" a new identity on
+ * every refetch, which is a re-render of the grid for a value only used at click time.
+ */
+ const rowsRef = useRef(rawRows);
+ rowsRef.current = rawRows;
+ /**
+ * ⭐ WAVE 27 · ITEM 2 — THE NEXT ROW ID, ADVANCED SYNCHRONOUSLY AT MINT TIME.
+ *
+ * ⛔ A RENDER-TIME SNAPSHOT IS NOT ENOUGH FOR THE GESTURE THIS ITEM IS ABOUT. `rowsRef.current`
+ * is assigned during RENDER, so two "+" presses that land before React commits the first
+ * `setPendingRows` both read the same list and both mint the same id. The owner's words were
+ * *"I need to be able to spam it"* — that IS the failing case, and the damage is not a
+ * duplicate row: `minted` is the identity key the failure and re-anchor branches filter and
+ * map on, so ONE refused write would withdraw BOTH rows and one taken id would rewrite both.
+ *
+ * ⚠ SEEDED FROM THE ROWS ON EVERY MINT, then advanced: `Math.max(seen, storeMax) + 1`. Reading
+ * the store's max each time is what keeps the client in step after a refetch brings rows in
+ * from elsewhere; the ref is what stops a burst from colliding with itself in between.
+ */
+ const nextRidRef = useRef(0);
+ const mintRid = useCallback(() => {
+ const storeMax = rowsRef.current.reduce((m, r) => Math.max(m, r.pid), 0);
+ nextRidRef.current = Math.max(nextRidRef.current, storeMax) + 1;
+ return nextRidRef.current;
+ }, []);
+ /**
+ * Withdraw rows this browser is still holding locally, because they have been DELETED.
+ *
+ * ⛔ THE PRUNE EFFECT CANNOT DO THIS AND THAT IS THE WHOLE BUG IT FIXES. Pruning is "the
+ * server's payload now CONTAINS this row, so drop my copy" — a deleted row will never appear
+ * in a payload again, so a pending row that is deleted before its first refetch stays merged
+ * into `rawRows` for the life of the mount. Add a record, press Ctrl+Z, and the row sits there
+ * looking undeleted while the store has already lost it.
+ * ⚠ CALLED AT EVERY `deleteTableRow` SITE (the selection delete and undo's `rowDelete`
+ * inverse), because those are the only two doors that remove a row — and the undo one is
+ * exactly the path that produces a still-pending victim.
+ */
+ const withdrawPending = useCallback((rids: (string | number)[]) => {
+ if (!rids.length) return;
+ const gone = new Set(rids.map((r) => Number(r)));
+ setPendingRows((cur) => (cur.some((r) => gone.has(r.pid))
+ ? cur.filter((r) => !gone.has(r.pid))
+ : cur));
+ }, []);
+ /**
+ * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — WHICH VIEWS ON THIS TABLE CARRY AN ALERT.
+ *
+ * ⛔ THE SERVER'S LIST IS THE ONLY ANSWER. The rail's "Alert me about new records…" is a
+ * one-way CREATE (it emits an event the shell turns into a POST) and nothing in the grid
+ * hears the outcome — a `no_filter` refusal is a real answer, so an optimistic local set
+ * would paint a badge on a view that has no alert. `GET /alerts` is the state.
+ *
+ * ⚠ RE-READ ON FOCUS, the same idiom the shell's own inbox uses. An alert created or deleted
+ * in another surface (the Alerts pane owns the delete door) reaches this rail on the next
+ * focus rather than instantly; the alternative was for the grid to guess at the outcome of a
+ * request another component made.
+ * ⚠ EMBEDDED GRIDS DO NOT ASK. A linked-record grid inside a modal is not a table anyone
+ * alerts on, and a second fetch per relation cell would be a workspace read per click.
+ */
+ const [alerted, setAlerted] = useState([]);
+ useEffect(() => {
+ if (embedded) return;
+ let live = true;
+ const pull = () => {
+ void fetchAlerts().then((r) => {
+ if (!live || !r.ok) return; // a failed read leaves the badges as they were
+ setAlerted(r.value.filter((a) => a.topic === scope).map((a) => a.viewId));
+ });
+ };
+ pull();
+ window.addEventListener("focus", pull);
+ return () => {
+ live = false;
+ window.removeEventListener("focus", pull);
+ };
+ }, [embedded, scope]);
+
+ const [fields, setFields] = useState([]);
+ const [views, setViews] = useState([]);
+ const [activeViewId, setActiveViewId] = useState(ALL_VIEW_ID);
+ const [config, setConfig] = useState(() => defaultViewConfig([]));
+ const [workspaceReady, setWorkspaceReady] = useState(false);
+ const [saveState, setSaveState] = useState<"saved" | "saving">("saved");
+ const [search, setSearch] = useState("");
+ const [collapsed, setCollapsed] = useState>(new Set());
+ const [hoverRow, setHoverRow] = useState(undefined);
+ /**
+ * Owner item 19 — the hover-only Expand button: which record it opens and where it sits, in
+ * VIEWPORT coordinates (glide's `getBounds` space, the same one `.cg-header-tip` uses).
+ *
+ * A real DOM button rather than a mark painted into the canvas, and that is the whole design:
+ * a drawn affordance needs a second hit-test that drifts out of step with the drawing, and
+ * neither `tsc` nor a screenshot can see the drift — the button is painted, and clicking it
+ * does nothing ([[ui-invisible-to-assertions]]). An element IS its own hit test.
+ */
+ const [expandAt, setExpandAt] = useState<
+ { pid: number; x: number; y: number; size: number } | null
+ >(null);
+ const [detailPid, setDetailPid] = useState(null);
+ const [columnMenu, setColumnMenu] = useState(null);
+ /** Open choice list for a `select` / `user` cell — see onCellClicked. */
+ const [picker, setPicker] = useState<{
+ pid: number;
+ fieldKey: string;
+ anchor: AnchorRect;
+ } | null>(null);
+ /** Owner item 5 (2026-07-31) — closing a picker RETURNS FOCUS TO THE GRID, so Enter keeps
+ * navigating (pick → Enter → next record) instead of stranding focus on a dead overlay. */
+ const closePicker = useCallback(() => {
+ setPicker(null);
+ requestAnimationFrame(() => gridRef.current?.focus());
+ }, []);
+ /** ⭐ Wave-23 C7 — which `json` cell the big viewer is open on. NOT anchored like the picker
+ * above: a document is not a choice list, so it opens as a centred modal (the record drawer's
+ * surface) rather than a popover the size of the cell it came from. */
+ const [jsonAt, setJsonAt] = useState<{ pid: number; fieldKey: string } | null>(null);
+ const closeJson = useCallback(() => {
+ setJsonAt(null);
+ requestAnimationFrame(() => gridRef.current?.focus());
+ }, []);
+ /** A relation opens as the target database's real Grid view, projected to the linked ids.
+ * This is intentionally distinct from JsonViewer: filters, sorts, search, column visibility,
+ * and the standard cell renderers all remain available inside the large modal. */
+ const [linkAt, setLinkAt] = useState<{ pid: number; fieldKey: string } | null>(null);
+ const closeLink = useCallback(() => {
+ setLinkAt(null);
+ requestAnimationFrame(() => gridRef.current?.focus());
+ }, []);
+ /** WAVE 21 item 11 (R10) — is "Select records from a list" open? Opened from the view
+ * rail's "…" and closed by the dialog; the SELECTION it produces outlives it. */
+ const [selectFromFile, setSelectFromFile] = useState(false);
+ const [importOpen, setImportOpen] = useState(false);
+ /** The selection bar's "Add to cohort" popover (owner item 2's purpose for the checkboxes). */
+ const [selAddOpen, setSelAddOpen] = useState(false);
+ const [selListName, setSelListName] = useState("");
+ const selAddRef = useRef(null);
+ /** Owner item 9 (wave 20) — its counterpart: "Remove from cohort…", in ORDINARY views. */
+ const [selRemoveOpen, setSelRemoveOpen] = useState(false);
+ const selRemoveRef = useRef(null);
+ /** Owner item 4 / C-ADDROW — the pid the trailing "+" just created, so the cursor can land on
+ * it once the re-read actually brings it back (the row does not exist locally before that). */
+ const [newRowPid, setNewRowPid] = useState(null);
+ /**
+ * ⭐ WARN-THEN-ALLOW on delete (owner, 2026-08-06) — the SIGNATURE of the selection currently
+ * armed, or `""`.
+ *
+ * A signature and not a boolean, deliberately: it is the pid list itself, so changing the
+ * selection DISARMS automatically. A boolean would stay true while you re-selected, and the
+ * second Delete would then destroy a set nobody had been warned about — which is worse than no
+ * warning at all, because the warning is what made it feel checked.
+ */
+ const [delArmed, setDelArmed] = useState("");
+ /** Wave-2 item 2c — cohort mode: the active cohort and the "+ Add customers" picker. Picks
+ * ACCUMULATE across searches (search, tick, search again, confirm once). */
+ const [activeCohortId, setActiveCohortId] = useState(null);
+ const [addCustOpen, setAddCustOpen] = useState(false);
+ const [addCustQuery, setAddCustQuery] = useState("");
+ const [addCustPicked, setAddCustPicked] = useState>(new Set());
+ const addCustRef = useRef(null);
+ /** Wave-5 item 1 — "Filter by this field": a monotonic signal the Toolbar consumes (it
+ * appends the seeded condition and opens the builder). Wave-6 item 6c: a MEASURE-carrying
+ * column seeds the equivalent measure condition instead (same measure key + window). */
+ const [filterSeed, setFilterSeed] = useState<
+ | { key?: string; measure?: { key: string; window: WindowSpec }; n: number }
+ | null
+ >(null);
+ /** Wave-6 item 4 — the header-row "+" (insert at end): anchors the create-only menu. */
+ const [plusMenu, setPlusMenu] = useState(null);
+ /**
+ * Wave-6 item 3c — per-key freshness of this browser's own field-def writes, persisted with
+ * the workspace copy and consumed by reconcileFields at init. A ref, not state: stamps ride
+ * along with the setFields that caused them; nothing renders from a stamp.
+ */
+ const fieldStampsRef = useRef<{ edited: Record; deleted: Record }>(
+ { edited: {}, deleted: {} }
+ );
+ /** Wave-5 item 6 — the header description tip. Floated, pointer-events:none — a tooltip
+ * that can receive the pointer swallows the NEXT click ([[ui-invisible-to-assertions]]). */
+ const [headerTip, setHeaderTip] = useState<{ text: string; x: number; y: number } | null>(
+ null
+ );
+ /** Wave-9 I2 — the CELL tip: hovering a cell whose text is cut off reveals it in full.
+ * Same contract as the header tip above (floated, pointer-events:none, aria-hidden) for
+ * the same reason — a tooltip that can receive the pointer swallows the NEXT click, which
+ * this surface has already paid for once ([[ui-invisible-to-assertions]]). */
+ const [cellTip, setCellTip] = useState<{ text: string; x: number; y: number } | null>(null);
+
+ /** BUG-1 — viewId → the filter tree we have already pushed back to the host. Persisted with
+ * the local workspace because the remount it guards against destroys any ref. */
+ const reemittedRef = useRef>({});
+ /** 2026-08-04 — views THIS browser deleted, by id. Read by the live adopt below AND by
+ * init, so the mount path and the after-mount path answer "was this deleted here?" the
+ * same way. Persisted with the local workspace for the same reason `reemitted` is. */
+ const viewTombstonesRef = useRef({});
+ /** D-19 — viewId → when this browser last wrote it (see LocalWorkspace.viewWrites). */
+ const viewWritesRef = useRef({});
+ const initializedKey = useRef(null);
+ const saveTimer = useRef(null);
+ const gridRef = useRef(null);
+ const gridBoxRef = useRef(null);
+ /** Internal clipboard provenance. The browser exposes pasted text, not its source column;
+ * recording the grid's own copy keeps cross-field paste fail-closed without blocking
+ * ordinary external text pasted into an editable field. */
+ const copyProvenanceRef = useRef(EMPTY_GRID_COPY_PROVENANCE);
+ useEffect(() => {
+ const onCopy = () => {
+ // The native copy event follows our grid key event. Preserve that provenance once;
+ // every later copy (address bar, another app surface, another input) invalidates it.
+ copyProvenanceRef.current = observeCopyEvent(copyProvenanceRef.current, Date.now());
+ };
+ const onWindowBlur = () => {
+ copyProvenanceRef.current = EMPTY_GRID_COPY_PROVENANCE;
+ };
+ document.addEventListener("copy", onCopy, true);
+ window.addEventListener("blur", onWindowBlur);
+ return () => {
+ document.removeEventListener("copy", onCopy, true);
+ window.removeEventListener("blur", onWindowBlur);
+ };
+ }, []);
+ const [gridSize, setGridSize] = useState({ width: 800, height: 600 });
+
+ /**
+ * ⛔ WAVE 21 item 3 (3c) — THE FALLBACK KEY IS THE SCOPE'S, and the literal it replaces is
+ * the cross-table bleed the owner reported as "RI fields on a new database".
+ *
+ * MEASURED, not theorised. `fetchWorkspace` collapses 403 / 5xx / network / unreadable into
+ * one silent `null` — deliberately, because the workspace is an ENHANCEMENT of the table and
+ * an older host simply does not serve one. But `null` also meant "no `storageKey`", and the
+ * fallback was the LITERAL `"customer-grid-standalone"` — ONE localStorage bucket for every
+ * scope this component can be mounted at. So on a tenant whose `/workspace` 403s for `ut_*`
+ * (routes_grid gated the route on `module_gate("customer_data")` — C4 fixes that half), the
+ * FIRST user table to be opened writes its views and custom fields into that bucket and the
+ * SECOND one reads them back as its own. Two databases with nothing to do with each other,
+ * sharing a schema, with no error anywhere.
+ *
+ * Scoping the key makes that structurally impossible: a bucket belongs to one surface, so
+ * the worst a failed workspace read can now do is show THIS table its own last-known local
+ * copy. The prefix marks it as the local-only stratum — a key the server never issues, so it
+ * can never collide with a real `storageKey` (those are `_table_workspace`-shaped).
+ *
+ * ⚠ Existing browsers lose the contents of the old shared bucket. That is the point: every
+ * byte in it is a workspace some other surface persisted, and there is no way to tell whose.
+ */
+ const storageKey = payload?.workspace?.storageKey ?? `local:${scope}`;
+
+ // Initialize once per permission/data scope. Host state wins; local state
+ // fills only missing objects and keeps the standalone path useful.
+ useEffect(() => {
+ if (!payload || payloadFields.length === 0 || initializedKey.current === storageKey) return;
+ const local = embedded ? null : readLocal(storageKey);
+ // Item 3c: the def half of the no-blip layer. A RECENT local stamp beats a
+ // lagged host echo (rename survives, retype holds, a delete stays deleted);
+ // a caught-up echo returns host objects byte-identical (see optimism.ts).
+ const stamps = pruneStamps(local?.fieldStamps, Date.now());
+ fieldStampsRef.current = {
+ edited: { ...(stamps.edited ?? {}) },
+ deleted: { ...(stamps.deleted ?? {}) },
+ };
+ const initialFields = reconcileFields(
+ payloadFields,
+ local?.fields ?? [],
+ stamps,
+ Date.now(),
+ payload.workspace != null
+ );
+ const hostViews = embedded ? [] : payload.workspace?.views ?? [];
+ const byId = new Map();
+ byId.set(ALL_VIEW_ID, allRecordsView(initialFields, scope));
+ // D-19 — the HOST'S LIST DECIDES WHICH VIEWS EXIST. A local copy the host no longer
+ // names is a ghost (deleted elsewhere, share revoked, store moved) and is dropped here,
+ // guarded by this browser's own recent writes so an optimistic create survives.
+ viewWritesRef.current = pruneTombstones(local?.viewWrites, Date.now());
+ for (const view of seedLocalViews(local?.views ?? [], hostViews, viewWritesRef.current,
+ Date.now(), payload.workspace != null))
+ byId.set(view.id, view);
+ // Host state wins — EXCEPT when it is a lagged echo of this browser's own
+ // in-flight edit, where taking it would turn a just-completed measure rule
+ // valueless: inactive, no pending marker, whole-book count. A rerun replaces
+ // the iframe, so this init runs after every host round trip; the reconcile
+ // is what keeps a rule with a typed rhs PENDING rather than inactive across
+ // that remount (see viewEcho.ts).
+ //
+ // BUG-1 (wave 11) — which views carry a FILTER TREE the host copy does not have, and have
+ // not already been told about. `echoReemit` owns both guards (see viewEcho.ts): identity
+ // alone would also fire on a display-only reconcile, and a time-based bound cannot work
+ // because the local-workspace effect below restamps `savedAt` on every init.
+ const reemit = new Map();
+ // 2026-08-04 — the views tombstone map, seeded from the persisted copy so a remount
+ // INSIDE the echo window cannot walk back a view this browser just deleted. Init had
+ // no such guard before (fields and folders both did), which was survivable only while
+ // every host round trip was itself a remount.
+ const tombstones = pruneTombstones(local?.viewTombstones, Date.now());
+ viewTombstonesRef.current = tombstones;
+ for (const view of hostViews) {
+ // Both doors, the `reconcileFields` rule: a tombstone drops the host copy AND any
+ // local copy, so a delete that raced localStorage cannot re-enter through either.
+ if (tombstones[view.id] !== undefined) {
+ byId.delete(view.id);
+ continue;
+ }
+ const merged = reconcileEchoView(
+ view, byId.get(view.id), local?.savedAt, Date.now()
+ );
+ const key = echoReemit(merged, view, local?.reemitted?.[view.id]);
+ if (key !== null) reemit.set(view.id, key);
+ byId.set(view.id, merged);
+ }
+ const initialViews = [...byId.values()].map((view) => ({
+ ...view,
+ config: normalizeConfig(view.config, initialFields),
+ }));
+ const requested =
+ local?.activeViewId ?? payload.workspace?.activeViewId ?? ALL_VIEW_ID;
+ const active = initialViews.find((view) => view.id === requested) ?? initialViews[0];
+ setFields(initialFields);
+ setViews(initialViews);
+ setActiveViewId(active.id);
+ setConfig(active.config);
+ setWorkspaceReady(true);
+ initializedKey.current = storageKey;
+
+ /**
+ * BUG-1 (wave 11) — TELL THE HOST. The reconcile above was right about the state and silent
+ * about the consequence: it restored a rule this browser completed and the host copy does
+ * not have, and then nothing sent it. The autosave effect cannot: init seeds `views` AND
+ * `config` from the same reconciled object, so `sameConfig` reports "saved" and no upsert
+ * is ever emitted. The rule stays ACTIVE client-side, PENDING forever, and unknown to
+ * `_cl_resolve_measures` — which is exactly W11-HOST's live trace, a pending view carrying
+ * no measure rule at all while the saved-view path resolves correctly.
+ *
+ * ⚠ THIS TERMINATES, and NOT because the host will store what we send. It might not: a
+ * measure key outside this user's BU offer is stripped by `clean_filter_tree`, so the host
+ * copy never gains the rule and `lagged` stays true forever. Termination comes from the
+ * IDEMPOTENCE KEY — the filter tree itself, persisted per view — so we tell the host about
+ * a given tree exactly once and only speak again when the USER edits the conditions.
+ * ⚠ A ref cannot hold that: the remount this exists to survive destroys it. It rides in
+ * localStorage beside the views, and `writeLocal` below carries it forward.
+ *
+ * Emitted from the NORMALIZED copy (`initialViews`), never from the raw host view: that is
+ * the config the client is actually filtering with, and sending anything else would ask the
+ * host to resolve a question nobody on screen is asking.
+ */
+ if (!embedded && reemit.size > 0) {
+ const stamped = { ...(local?.reemitted ?? {}) };
+ for (const view of initialViews) {
+ const key = reemit.get(view.id);
+ if (key === undefined) continue;
+ viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id,
+ Date.now());
+ viewWritesRef.current = stampTombstone(viewWritesRef.current, view.id, Date.now());
+ emitHostEvent({ id: eventId("view"), type: "view_upsert", view });
+ stamped[view.id] = key;
+ }
+ reemittedRef.current = stamped;
+ } else {
+ reemittedRef.current = { ...(local?.reemitted ?? {}) };
+ }
+ }, [payload, payloadFields, storageKey, embedded, scope]);
+
+ useEffect(() => {
+ if (!workspaceReady || embedded) return;
+ writeLocal(storageKey, {
+ fields,
+ views,
+ activeViewId,
+ savedAt: Date.now(),
+ fieldStamps: pruneStamps(fieldStampsRef.current, Date.now()),
+ // BUG-1 — carried forward, never recomputed here. Dropping it would restore the loop:
+ // the next init would find no record of what it already told the host and say it again.
+ reemitted: reemittedRef.current,
+ // Pruned on the way out, like every other stamp map here: the persisted blob is a
+ // recent window, never an archive of everything this tab ever deleted.
+ viewTombstones: pruneTombstones(viewTombstonesRef.current, Date.now()),
+ // D-19's other half — without persisting these, a reload inside the echo window would
+ // drop a view this browser created seconds ago.
+ viewWrites: pruneTombstones(viewWritesRef.current, Date.now()),
+ });
+ }, [workspaceReady, storageKey, fields, views, activeViewId, embedded]);
+
+ /**
+ * ⭐ THE LIVE WORKSPACE (owner report, 2026-08-04) — what has APPEARED since we mounted.
+ *
+ * The init effect above runs ONCE per mount (`initializedKey`), which under Streamlit was
+ * indistinguishable from "once per host round trip" because a rerun replaces the iframe.
+ * The standalone shell has no rerun: this component stays mounted until `key={active.key}`
+ * changes in Shell.tsx — switching modules. So a cohort created here reached the store,
+ * came back in the very next `/workspace` re-read, sat in `payload.workspace.views`, and
+ * never reached the rail. That is the owner's report, exactly: it appears after you visit
+ * another module and come back, because that is the only thing that remounts this tree.
+ *
+ * ⚠ FIELDS FIRST, AND THE ORDER IS LOAD-BEARING. `fields_from_workspace(ws,
+ * cohorts=bool(cohort_lists))` emits the derived "Locked views" column only once a cohort
+ * EXISTS — so a user's FIRST cohort changes the field contract in the same payload that
+ * carries the new view. The adopted views are normalized against `nextFields`, not against
+ * the `fields` state (which React has not committed yet), or a projected `config.order`
+ * naming the new column would be normalized against a list that does not have it.
+ *
+ * ⚠ Add-only, tombstoned, and identity-stable — see liveWorkspace.ts for why each of those
+ * is a correctness property rather than a nicety.
+ */
+ const hostWorkspaceViews = payload?.workspace?.views;
+ useEffect(() => {
+ if (!workspaceReady || embedded) return;
+ const now = Date.now();
+ const nextFields = adoptNewFields(
+ fields, payloadFields, fieldStampsRef.current.deleted, now
+ );
+ if (nextFields !== fields) setFields(nextFields);
+ setViews((current) =>
+ adoptNewViews(current, hostWorkspaceViews, viewTombstonesRef.current, now,
+ (config) => normalizeConfig(config, nextFields))
+ );
+ }, [workspaceReady, hostWorkspaceViews, payloadFields, fields, embedded]);
+
+ /** Item 3c — stamp a def write / a delete. Pruned at every touch so the persisted blob
+ * stays a recent window, never an archive. */
+ const stampFieldEdit = useCallback((key: string) => {
+ const s = pruneStamps(fieldStampsRef.current, Date.now());
+ fieldStampsRef.current = {
+ edited: { ...(s.edited ?? {}), [key]: Date.now() },
+ deleted: { ...(s.deleted ?? {}) },
+ };
+ }, []);
+ const stampFieldDelete = useCallback((key: string) => {
+ const s = pruneStamps(fieldStampsRef.current, Date.now());
+ const edited = { ...(s.edited ?? {}) };
+ delete edited[key];
+ fieldStampsRef.current = {
+ edited,
+ deleted: { ...(s.deleted ?? {}), [key]: Date.now() },
+ };
+ }, []);
+
+ const updateConfig = useCallback(
+ (next: ViewConfig) => {
+ if (next.groupBy !== config.groupBy) setCollapsed(new Set());
+ setConfig(next);
+ },
+ [config.groupBy]
+ );
+
+ const {
+ visibleCols,
+ fieldByKey,
+ order,
+ visible,
+ lockedKey,
+ onColumnResize,
+ onColumnMoved,
+ onColumnProposeMove,
+ setColumnVisible,
+ insertColumn,
+ } = useGridColumns(fields, config, updateConfig);
+
+ /**
+ * ⭐ Wave-20 owner item 4 (ruling R8, contract C-ADDROW) — **THE GHOST ROW.**
+ *
+ * A trailing "+" row at the bottom of the grid, the way Airtable grows a table, replacing the
+ * "Add record" button that used to sit in the shell's header bar (S4 deleted it this wave).
+ *
+ * ⛔ USER DATABASES ONLY. A connector's rows are read-synced from Odoo; a "+" there could only
+ * refuse, and R8 names that a fake affordance. The test is the SCOPE (`ut_*`), which is also
+ * the only scope with a rows endpoint to POST to — so the affordance and the capability come
+ * from the same fact rather than from two lists that can drift.
+ *
+ * ⚠ THE ROW IS NOT ADDED LOCALLY. `rawRows` is the server's answer, and a client-invented row
+ * would have no rid, no defaults and no place in anyone else's copy. The POST clears the rows
+ * cache and fires `ROWS_STALE_EVENT`, `useCustomerData` re-reads, and the row arrives with the
+ * id the store gave it. The cursor then follows it (`newRowPid` + the effect below) — which is
+ * why the pid is remembered instead of glide's `"bottom"` being returned here: at the moment
+ * this resolves, the appended row does not exist yet, so "bottom" would land on the last OLD
+ * row.
+ */
+ const isUserTable = !embedded && scope.startsWith("ut_");
+ const recordsMutable = payload?.recordsMutable !== false;
+ const canMutateRecords = isUserTable && recordsMutable;
+ /**
+ * ⭐ 2026-08-07 — the databases a `link` column may point at, fetched when the column menu
+ * OPENS rather than on every render of the grid.
+ *
+ * ⚠ ON DEMAND IS THE WHOLE DESIGN. `GET /tables` walks every table in the tenant and returns
+ * their full field lists; hanging that off the grid's mount would put a workspace-wide read
+ * behind every page view for a picker almost nobody opens. The column menu is the only surface
+ * that needs it, so it is the surface that asks.
+ * ⚠ The list is left standing once fetched — a database created in another tab mid-session is
+ * a staleness a menu re-open corrects, and re-fetching per open would spend the same read
+ * repeatedly for a list that changes about once a week.
+ */
+ const [linkTargets, setLinkTargets] = useState([]);
+ useEffect(() => {
+ // ⛔⛔ 2026-08-09 — `plusMenu` TOO, AND THAT OMISSION WAS THE WHOLE BUG. Owner: *"I am not
+ // able to see Choose Column for Post rows link"*. There are TWO doors into this editor —
+ // a column's own menu (`columnMenu`) and the "+" Add-field button (`plusMenu`) — and this
+ // effect knew about one. Via "+", `linkTargets` stayed `[]`, so `target` never resolved and
+ // the rollup Column picker held nothing but its placeholder, permanently. Editing an
+ // EXISTING column worked, which is what made it look like a rollup bug rather than a fetch
+ // that never fired. MEASURED on the live app: 5 selects rendered, "Column to roll up" with
+ // exactly 1 option while `GET /tables` returns `ut_ig_posts` with 22 fields.
+ // ⚠ A picker that is EMPTY and a picker whose data never loaded look identical, which is
+ // why this survived: both render "Choose a column…" over nothing.
+ if ((!columnMenu && !plusMenu) || linkTargets.length) return;
+ let live = true;
+ void fetchLinkTargets().then((t) => {
+ if (live) setLinkTargets(t);
+ });
+ return () => {
+ live = false;
+ };
+ }, [columnMenu, plusMenu, linkTargets.length]);
+ /**
+ * ⭐⭐ 2026-08-09 — the READ-THROUGH rollup's offer, fetched on the same terms as the link
+ * targets above: only when the column menu opens, and once per session.
+ *
+ * ⚠ THE GUARD IS A SEPARATE `asked` FLAG, NOT `topics.length`. An empty offer is the CORRECT
+ * and common answer (a tenant with nothing connected), so guarding on the length would re-ask
+ * on every single menu open for exactly the workspaces where the answer can never change —
+ * the one case the "fetch once" rule exists for.
+ */
+ const [rollupSourceOffer, setRollupSourceOffer] =
+ useState({ topics: [], windows: [] });
+ const rollupOfferAsked = useRef(false);
+ useEffect(() => {
+ // ⛔ SAME TWO DOORS as the link targets above. This one is less visible because an empty
+ // offer is the correct answer for a tenant with nothing connected — so a source offer that
+ // never loaded is indistinguishable from one that is legitimately empty, and the mode switch
+ // simply never appears. Fixing only the link half would have left that asymmetry in place.
+ if ((!columnMenu && !plusMenu) || rollupOfferAsked.current) return;
+ rollupOfferAsked.current = true;
+ let live = true;
+ void fetchRollupSources().then((o) => {
+ if (live) setRollupSourceOffer(o);
+ });
+ return () => {
+ live = false;
+ };
+ }, [columnMenu, plusMenu]);
+ /**
+ * ⭐ WAVE 27 · OWNER ITEM 22 — the values the ACTIVE VIEW forces on a new record.
+ *
+ * ⛔ THE DEFECT: "+" posted `{values: {}}` on every view, so a record added while a filter was
+ * on was created, was real, and was invisible — the button read as broken and the row the user
+ * then typed into did not exist as far as they could see. `filterSeed.ts` owns the derivation
+ * and its whole design is how much it REFUSES to derive (negations, ranges, `or` branches,
+ * contradictions, machine columns — each with its reason at the clause).
+ *
+ * ⚠ ONE FUNCTION, TWO CALLERS, deliberately: item 2's optimistic add must insert the row
+ * carrying these same cells, or the row it paints locally fails the filter it was added under
+ * and disappears on the next read — the exact blip C2 exists to prevent.
+ */
+ const seedValues = useCallback(
+ () => filterSeedValues(config.filters, config.filterConj, fields, payload?.viewer),
+ [config.filters, config.filterConj, fields, payload?.viewer]
+ );
+ /**
+ * ⭐ WAVE 27 · OWNER ITEM 2 (contract C2) — the "+" paints its row NOW.
+ *
+ * ⛔ THE ID IS MINTED THE WAY THE SERVER MINTS IT — `max(numeric row id) + 1`, which is
+ * `core.user_tables.add_row`'s own rule — and then POSTed as `{rid, values}` through the door
+ * the undo path already opened (C-ADDROW). That is what makes the optimism honest rather than
+ * hopeful: on the overwhelmingly common single-writer path the server stores exactly the id
+ * this browser drew, so the row on screen and the row in the store are the same record from
+ * the first frame.
+ *
+ * ⚠ AND WHEN THEY ARE NOT, IT RE-ANCHORS. Two people adding at once means the second POST
+ * finds the id taken; `add_row` falls back to `max+1` and ANSWERS with what it actually wrote
+ * (its own note says so), so the local row is re-keyed to the returned pid. Assuming the
+ * requested id came back is the one way this could leave a row on screen that no longer
+ * matches anything in the store.
+ *
+ * ⛔ NO `ROWS_STALE_EVENT` ON THE HAPPY PATH. That event re-reads the whole table, which is
+ * the cost item 2 exists to remove — and re-reading would also un-paint the row for the
+ * duration of the fetch, which is the NO-BLIP law's exact subject. The pending row is pruned
+ * by the effect above when a payload that already contains it arrives, from whatever cause.
+ *
+ * ⚠ THE NARROW RACE, STATED: a cell PATCHed within the POST's round trip could reach the
+ * server before the row exists. The window is now ~20ms (A moved the relation refresh off
+ * this path), it needs a keystroke inside it, and the PATCH's failure is a rolled-back
+ * overlay edit rather than lost data — the alternative, blocking the paint until the POST
+ * resolves, is the defect being fixed.
+ */
+ const appendRow = useCallback(async (): Promise => {
+ if (!canMutateRecords) return undefined;
+ const seeded = seedValues();
+ const minted = mintRid();
+ const optimistic: Row = { ...seeded, pid: minted };
+ setPendingRows((cur) => [...cur, optimistic]);
+ setNewRowPid(minted);
+ const made = await addTableRow(scope, seeded, minted);
+ if (!made) {
+ // `addTableRow` already said WHY, in the server's own words (a row cap, a refused profile
+ // cell, an unreachable host). The row is withdrawn rather than left standing: a row that
+ // survives its own failed write is the lie this whole path has to avoid.
+ setPendingRows((cur) => cur.filter((r) => r.pid !== minted));
+ setNewRowPid(null);
+ return undefined;
+ }
+ if (made.pid !== minted) {
+ setPendingRows((cur) =>
+ cur.map((r) => (r.pid === minted ? { ...r, pid: made.pid } : r))
+ );
+ setNewRowPid(made.pid);
+ }
+ // R4 — the append is undoable: Ctrl+Z deletes the row it just created, and a redo restores
+ // it under the SAME rid (the server's `{rid}` passthrough, C-ADDROW).
+ // ⛔ ITEM 22 MOVED THIS LINE. It read `values: {}` with a comment saying "an empty new record
+ // carries no values" — true until this wave, and now false: a row added on a filtered view is
+ // born holding the cells that filter forces. `redo` replays `r.values` verbatim
+ // (`:1569`), so leaving the literal would have made Ctrl+Z → Ctrl+Y restore the row under the
+ // right id with its seeded cells silently dropped — [[undo-capture-before-the-write]] in its
+ // quietest form, since the id and the row count would both be right.
+ undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
+ kind: "rowAdd", table: scope, rows: [{ rid: made.rid, values: seeded }],
+ });
+ // ⛔ NO `signal(ROWS_STALE_EVENT)` HERE ANY MORE (item 2 / C2). It re-read the whole table
+ // on every "+", which is the whole cost the owner reported — and every OTHER caller of it
+ // still fires, so a row created by an automation, another tab or an undo still arrives.
+ // The one thing that used to depend on this refetch was the cursor landing on the new row,
+ // and the row is now on screen before the POST resolves, so it lands immediately.
+ // `undefined` is returned rather than glide's "bottom": "bottom" is resolved against the
+ // rows glide knew about when the gesture started, which is the row BEFORE this one.
+ return undefined;
+ }, [canMutateRecords, scope, seedValues, mintRid]);
+
+ /* wave20 item 2 — measure key + window -> the columns that display it. Built ONCE per field
+ list and handed to every consumer of "which column is this rule about", so the tint
+ (`columnTones`, inside the hook above) and the column menu's filter doors resolve a
+ measure condition the same way. See `types.ruleColumnKeys`. */
+ const measureCols = useMemo(() => measureColumnIndex(fields), [fields]);
+
+ // Airtable behavior: configuration changes to the active view autosave.
+ useEffect(() => {
+ if (!workspaceReady || embedded) return;
+ const active = views.find((view) => view.id === activeViewId);
+ if (!active || sameConfig(active.config, config)) {
+ setSaveState("saved");
+ return;
+ }
+ setSaveState("saving");
+ if (saveTimer.current !== null) window.clearTimeout(saveTimer.current);
+ saveTimer.current = window.setTimeout(() => {
+ const updated = { ...active, config };
+ setViews((current) =>
+ current.map((view) => (view.id === updated.id ? updated : view))
+ );
+ viewWritesRef.current = stampTombstone(viewWritesRef.current, updated.id,
+ Date.now());
+ emitHostEvent({ id: eventId("view"), type: "view_upsert", view: updated });
+ setSaveState("saved");
+ saveTimer.current = null;
+ }, 420);
+ return () => {
+ if (saveTimer.current !== null) window.clearTimeout(saveTimer.current);
+ };
+ }, [workspaceReady, activeViewId, config, views, embedded]);
+
+ // Numeric dimensions force a glide relayout when either the component frame
+ // or Streamlit's main column changes width (notably sidebar collapse).
+ useLayoutEffect(() => {
+ const element = gridBoxRef.current;
+ if (!element) return;
+ const measure = () => {
+ const rect = element.getBoundingClientRect();
+ if (rect.width > 0 && rect.height > 0) {
+ setGridSize({
+ width: Math.floor(rect.width),
+ height: Math.floor(rect.height),
+ });
+ }
+ };
+ measure();
+ const observer = new ResizeObserver(measure);
+ observer.observe(element);
+ window.addEventListener("resize", measure);
+ return () => {
+ observer.disconnect();
+ window.removeEventListener("resize", measure);
+ };
+ }, [workspaceReady]);
+
+ // CG-3: one mode per table, decided by whether the payload is a window. See types.ts
+ // `TableMode` for why both engines running at once is the failure this prevents.
+ const mode = tableMode(payload?.counts);
+ const serverWindowed = mode === "server-windowed";
+ // Owner items 4+6 — the Cohort page's grid renders without the Views sidebar.
+ const hideViews = embedded || payload?.workspace?.hideViews === true;
+ // Wave-6 item 10 — how this view displays. A WINDOWED table is always the grid: list/
+ // calendar/kanban compute over the whole matched set, and one page is not it (CG-3's rule,
+ // the same reason grouping is off there).
+ const displaySpec = cleanDisplay(config.display);
+ const displayMode: DisplayMode = serverWindowed ? "grid" : displaySpec?.mode ?? "grid";
+
+ // CG-8 — measure conditions, answered by the host as `{ruleId: pid[]}`. Converted to Sets
+ // once here rather than per row: the pipeline runs this over every row of the book.
+ //
+ // ⚠ REPLACED, not merged. Merging looks like the safer choice — "hold the previous result"
+ // is the design — but it holds it FOREVER: the host omits a rule's id when it could not
+ // resolve it (a warming store, a scope change mid-flight), and a merged map keeps yesterday's
+ // answer under today's question with `pendingMeasures` reporting zero, so the marker never
+ // shows. That is the widening sin wearing a confident count.
+ //
+ // Holding the previous result across a LOCAL edit does not need the merge and never did: the
+ // memo is keyed on `payload.measureSets` identity, which does not change while the user is
+ // typing, so the ref already carries the last answers through to the next payload. Merge and
+ // replace therefore differ only in the dangerous case.
+ const measures = useMemo(() => payload?.measures ?? [], [payload?.measures]);
+ const measureSetsRef = useRef({});
+ const measureSets = useMemo(() => {
+ const incoming = payload?.measureSets;
+ if (incoming) {
+ const next: MeasureSets = {};
+ for (const [ruleId, pids] of Object.entries(incoming)) next[ruleId] = new Set(pids);
+ measureSetsRef.current = next;
+ }
+ return measureSetsRef.current;
+ }, [payload?.measureSets]);
+ const pendingMeasureCount = useMemo(
+ () => pendingMeasures(config.filters, measureSets),
+ [config.filters, measureSets]
+ );
+ // BUG-1 (wave 11) — the two key sets, published for the handshake. See the `.cg-shell`
+ // attributes below for why they are DOM attributes and not console logs.
+ const measureRuleKeys = useMemo(
+ () => activeMeasureRuleIds(config.filters).join(","),
+ [config.filters]
+ );
+ const measureSetKeys = useMemo(() => Object.keys(measureSets).join(","), [measureSets]);
+
+ // Owner item 5. Cohort membership rides `workspace.lists`, the same array the "Add to list"
+ // picker already reads, so the names offered and the sets tested come from one place. A
+ // cohort with no `pids` is a target you can add to but cannot filter on; the engine treats it
+ // as unanswerable (matches nothing) rather than absent (matches everything).
+ const lists = useMemo(() => payload?.workspace?.lists ?? [], [payload?.workspace?.lists]);
+ /* wave17 R1 / C-LOCKV — `railCohortId` is gone. It held the TRANSIENT lock the retired
+ Cohorts section applied on click: a lock that lived in component state and was never
+ persisted. A locked view now carries its own `config.cohortLock`, so SELECTING the view
+ is the lock and the saved config is the only source. One mechanism, and it survives a
+ reload — which the transient one never did. */
+ const cohortSets = useMemo(() => {
+ const out: CohortSets = {};
+ for (const l of lists) if (l.pids) out[l.id] = new Set(l.pids);
+ return out;
+ }, [lists]);
+ // The TENANT'S today. Never `new Date()`: a browser a day ahead of the server would resolve
+ // "the past month" to a different month than the host summed, and the count would deny the
+ // rows with nothing erroring.
+ const today = payload?.today;
+ // Wave-5 item 1 — who is looking (host-computed). ONE editability verdict for every edit
+ // door (cells, pickers, drawer): types.mayEditField — stratum + read-only-by-nature +
+ // permissions vs the viewer, fail-closed on restricted fields when the viewer is unknown.
+ const viewer = payload?.viewer;
+ const canEditField = useCallback(
+ (f: Field): boolean => !embedded && recordsMutable && mayEditField(f, viewer),
+ [embedded, recordsMutable, viewer]
+ );
+ const unresolvedCount = useMemo(
+ () => unresolvedConditions(config.filters, { cohortSets, today }),
+ [config.filters, cohortSets, today]
+ );
+
+ // Wave-5 items 9/11 — CLIENT-COMPUTED cells, injected AT their field keys so the whole
+ // pipeline (filter/search/sort/group), the cells and the drawer read them like any other
+ // value. Formulas parse ONCE per definition change (never per row); evaluation reads the
+ // row WITH this session's overlay edits layered, so editing a referenced field recomputes
+ // live. A formula that does not parse, or errors on a row, yields BLANK — never a wrong
+ // number (formulaEngine.ts). `created_time` copies the row's `_created`.
+ // 2026-07-31 (owner item 2): formulas may reference OTHER formulas now, so parse order is
+ // TOPOLOGICAL (orderFormulas) — a formula runs after the formulas it reads, cycle members
+ // never run (blank, never a stale number), and each row's results feed the next formula
+ // through a per-row scope.
+ const formulaAsts = useMemo(() => {
+ const sources = new Map();
+ for (const f of fields) {
+ if (f.type !== "formula") continue;
+ const src = formulaOf(f);
+ if (src) sources.set(f.key, src);
+ }
+ const { order, cyclic } = orderFormulas(sources);
+ const out: { key: string; ast: FormulaAst }[] = [];
+ for (const key of order) {
+ if (cyclic.has(key)) continue;
+ const p = parseFormula(sources.get(key)!);
+ if (p.ok) out.push({ key, ast: p.ast });
+ }
+ return out;
+ }, [fields]);
+ const createdTimeKeys = useMemo(
+ () => fields.filter((f) => f.type === "created_time").map((f) => f.key),
+ [fields]
+ );
+ const computedRows = useMemo(() => {
+ if (formulaAsts.length === 0 && createdTimeKeys.length === 0) return rawRows;
+ const env = { today: payload?.today };
+ return rawRows.map((r) => {
+ const edits = overlayEdits[r.pid];
+ const scope: Row = edits ? { ...r, ...edits } : { ...r };
+ const out: Row = { ...r };
+ for (const k of createdTimeKeys) {
+ out[k] = (r._created as string | undefined) ?? null;
+ scope[k] = out[k];
+ }
+ for (const { key, ast } of formulaAsts) {
+ const v = evalFormula(ast, (k) => scope[k], env);
+ out[key] = v;
+ scope[key] = v; // later formulas read this one's result — the topo order above
+ }
+ return out;
+ });
+ }, [rawRows, overlayEdits, formulaAsts, createdTimeKeys, payload?.today]);
+
+ // Wave-2 item 2c — COHORT MODE (the Cohort page). The host serves the WHOLE pool (rows +
+ // derived values over the pool); the ACTIVE cohort scopes the table to its pids CLIENT-side.
+ const cohortMode = payload?.workspace?.cohortMode === true;
+ useEffect(() => {
+ if (!cohortMode) return;
+ if (activeCohortId && lists.some((l) => l.id === activeCohortId)) return;
+ setActiveCohortId(lists[0]?.id ?? null);
+ }, [cohortMode, lists, activeCohortId]);
+ const activeCohort = cohortMode
+ ? lists.find((l) => l.id === activeCohortId) ?? null
+ : null;
+ const cohortMemberSet = useMemo(
+ () => new Set(activeCohort?.pids ?? []),
+ [activeCohort]
+ );
+ // The FIXED subset is the scope. A cohort with no membership shows nothing — falling back to
+ // the whole pool would put 1,500 rows under a rail entry that says 12, which is the widening
+ // sin wearing a sidebar. Everything downstream (pipeline, counts, selection, detail) runs
+ // over this scope, so the toolbar count is the cohort's matched count by construction.
+ const scopedRows = useMemo(
+ () =>
+ cohortMode
+ ? computedRows.filter((r) => cohortMemberSet.has(r.pid))
+ : computedRows,
+ [cohortMode, computedRows, cohortMemberSet]
+ );
+
+ const { visibleRows, pidToIndex } = useVisibleRows(
+ scopedRows,
+ fields,
+ config.filters,
+ search,
+ config.sorts,
+ // grouping a WINDOW would headline a page's count/subtotals as the group's
+ serverWindowed ? null : config.groupBy,
+ collapsed,
+ config.memberPids,
+ config.filterConj,
+ serverWindowed,
+ measureSets,
+ cohortSets,
+ today,
+ // Item 12 (C-LOCK) — the 14th positional, RECORD's engine input. The lock intersects
+ // FIRST, so everything downstream (conditions, memberPids, the ranking domain) operates
+ // inside the cohort. Absent = today's behaviour exactly.
+ // wave17 R1 / C-LOCKV: the view's SAVED lock is now the only source. A locked view is a
+ // saved view whose `config.cohortLock` names its own id, so opening it IS applying the
+ // lock — the wave-15 transient rail pick that used to outrank this is gone with its rail.
+ config.cohortLock
+ );
+
+ // Owner item 8 — the DISPLAY cap. The pipeline above still ran over the whole book (counts,
+ // "Add to list" and selection-by-pid all depend on that); only what glide PAINTS is sliced.
+ // pidToIndex is rebuilt over the slice because every index consumer (selection, detail
+ // navigation, hover) is positional against what is actually on screen.
+ const [displayCap, setDisplayCap] = useState(DISPLAY_PAGE);
+ const capped = visibleRows.length > displayCap;
+ /**
+ * ⭐⭐ WAVE-29 T33 (owner item 17) — THE TOTALS ROW, and the cap fix it forced.
+ *
+ * ⛔ THE SUMMARY IS OVER ALL M MATCHED ROWS, NEVER THE N PAINTED. The pipeline above already
+ * ran over the whole book; only what glide paints is sliced. A total computed from the slice
+ * would silently describe the first `DISPLAY_PAGE` records while sitting under a bar that says
+ * "Showing first N of M" — an answer to a question nobody asked, wearing the right label.
+ *
+ * ⛔ AND THE SLICE ITSELF WAS ALREADY WRONG FOR GROUPS. `visibleRows` is FLATTENED (header,
+ * rows, footer, header, …), so a straight `slice(0, cap)` could cut a group between its last
+ * row and its footer — the subtotal simply vanished, for the groups furthest down, with no
+ * marker of any kind. `sliceForDisplay` counts DATA rows toward the cap and keeps the
+ * structural rows of every group it admits, so a group is whole or absent.
+ */
+ const totalsAggs = useMemo(
+ () => computeAggs(
+ visibleRows.flatMap((vr) => (vr.kind === "data" ? [vr.record] : [])),
+ fields
+ ),
+ [visibleRows, fields]
+ );
+ const showTotals = useMemo(
+ () => !config.groupBy && Object.keys(totalsAggs).length > 0,
+ [config.groupBy, totalsAggs]
+ );
+ const displayRows = useMemo(() => {
+ const shown = capped ? sliceForDisplay(visibleRows, displayCap) : visibleRows;
+ // ⚠ APPENDED AFTER THE SLICE, so the cap can never eat the totals row itself — and it is the
+ // LAST row, which is what `freezeTrailingRows={1}` pins.
+ return showTotals
+ ? [...shown,
+ { kind: "group-footer" as const, groupKey: TOTAL_GROUP_KEY, aggs: totalsAggs }]
+ : shown;
+ }, [capped, visibleRows, displayCap, showTotals, totalsAggs]);
+ const displayPidToIndex = useMemo(() => {
+ if (!capped) return pidToIndex;
+ const m = new Map();
+ displayRows.forEach((vr, i) => {
+ if (vr.kind === "data" && !m.has(vr.record.pid)) m.set(vr.record.pid, i);
+ });
+ return m;
+ }, [capped, displayRows, pidToIndex]);
+
+ // C-AVATAR (wave-14 item 11) — profile photos, absent until HOST serves them. The tick is the
+ // async half: `cells.setAvatarRepaint` fires it when an Image finishes decoding, and it is a
+ // dep of `getCellContent`, which is the only thing glide watches. See useGetCellContent.
+ const [avatarTick, setAvatarTick] = useState(0);
+ useEffect(() => {
+ setAvatarRepaint(() => setAvatarTick((t) => t + 1));
+ return () => setAvatarRepaint(undefined);
+ }, []);
+ const userAvatars = payload?.workspace?.userAvatars;
+ /**
+ * Item 15 — the frozen strip's usable width, which is what a GROUP BAR's label is clipped to
+ * now that the first column is pinned in every mode. `null` when nothing is grouped (no bar to
+ * clip) — see `fitGroupLabel` for why the clip exists and why the count outranks the name.
+ */
+ const groupLabelSpace = useMemo(() => {
+ if (!config.groupBy) return null;
+ const n = Math.min(frozenCountOf(config), visibleCols.length);
+ let px = 0;
+ // `GridColumn` is a union — only its SIZED member declares `width`, and `useGridColumns`
+ // always builds that one. Narrowed rather than asserted so an auto-sized column (which we
+ // do not create) would read 0 and simply not be counted, instead of throwing in a memo.
+ for (let i = 0; i < n; i += 1)
+ px += (visibleCols[i] as { width?: number } | undefined)?.width ?? 0;
+ return Math.max(0, px - GROUP_LABEL_PAD);
+ }, [config, visibleCols]);
+ /**
+ * ⭐ owner item 2 (2026-08-03) — WHICH MEASURE COLUMNS ARE STILL BEING CALCULATED.
+ *
+ * A measure column's numbers are resolved by the server (one aggregate over the whole book)
+ * and reach the browser on the workspace re-read that follows the create — seconds later.
+ * In between, the column is on screen with nothing in it, which the owner correctly read as
+ * an error rather than as a wait.
+ *
+ * ⛔ `editRequestId` IS THE TEST, and it is the honest one. It is set optimistically at the
+ * moment the create is emitted and cleared by `reconcileFields` when the host's own copy of
+ * that field comes back — and the host's copy travels in the SAME `/workspace` response as
+ * `derived`, which is where the values are. So the flag is true across exactly the window
+ * where the column exists and its numbers do not, and false the instant they land.
+ *
+ * The alternative — "no value anywhere in the column" — cannot tell a pending column from
+ * one that permanently failed to resolve (a BU-scoped caller on a company-level measure,
+ * say), and would spin forever on the second. This one resolves either way: when the echo
+ * arrives with no values, the flag clears and the cells go honestly blank.
+ */
+ const pendingMeasureKeys = useMemo(() => {
+ const out = new Set();
+ for (const f of fields)
+ if (f.editRequestId && f.key.startsWith("measure_")) out.add(f.key);
+ return out;
+ }, [fields]);
+ /**
+ * The skeleton's pulse. A canvas cell cannot hold a CSS animation, so the motion is repaints:
+ * this counter is a dependency of `getCellContent`, which is the only thing glide watches.
+ *
+ * ⚠ IT RUNS ONLY WHILE SOMETHING IS PENDING, and the effect's own guard is what stops it —
+ * an interval left running would repaint the whole canvas ~7×/s forever, on every grid, to
+ * animate nothing. 140ms × the 4-step ramp is a ~0.6s cycle: a wait, not a strobe.
+ */
+ const [pulse, setPulse] = useState(0);
+ /**
+ * ⛔ AND IT GIVES UP. A skeleton that never resolves is worse than the blank it replaced: a
+ * blank cell is at least honest about having no number, while a permanent shimmer promises
+ * one that is never coming.
+ *
+ * The window it guards is narrow but real. `editRequestId` clears when `reconcileFields` takes
+ * the host's copy — and that function has a branch (`hostAuthoritative: false`, i.e. a payload
+ * with no workspace) whose `{...local, ...host}` spread would PRESERVE the flag forever. Today
+ * that branch cannot be reached with a measure column on screen (measures are offered through
+ * the workspace, so a payload without one cannot have produced this field), which is an
+ * argument about the current call graph and not a property of the code. This bound holds
+ * whether or not the argument stays true, and it costs one boolean.
+ */
+ const [pendingGaveUp, setPendingGaveUp] = useState(false);
+ useEffect(() => {
+ setPendingGaveUp(false);
+ if (pendingMeasureKeys.size === 0) return;
+ let n = 0;
+ const id = window.setInterval(() => {
+ n += 1;
+ if (n > PULSE_MAX_TICKS) {
+ window.clearInterval(id);
+ setPendingGaveUp(true); // fall back to ordinary blank cells
+ return;
+ }
+ setPulse((p) => p + 1);
+ }, PULSE_MS);
+ return () => window.clearInterval(id);
+ }, [pendingMeasureKeys]);
+ const activePendingKeys = pendingGaveUp ? NO_PENDING_KEYS : pendingMeasureKeys;
+ const getCellContent = useGetCellContent(
+ displayRows,
+ visibleCols,
+ fieldByKey,
+ overlayEdits,
+ canEditField,
+ userAvatars,
+ avatarTick,
+ groupLabelSpace,
+ measureGroupText,
+ activePendingKeys,
+ pulse
+ );
+ const { gridSelection, selectedPids, onGridSelectionChange, selectPids, togglePid,
+ setActiveCell, clearSelection } =
+ useGridSelection(
+ displayRows,
+ displayPidToIndex,
+ visibleCols.length,
+ embeddedSelectable ? embeddedSelectedIds : []
+ );
+ const embeddedSelectedKey = embeddedSelectedIds.join(",");
+ const selectedPidKey = [...selectedPids].join(",");
+ useEffect(() => {
+ if (!embeddedSelectable) return;
+ if (selectedPidKey !== embeddedSelectedKey)
+ selectPids(embeddedSelectedIds as number[], "replace");
+ }, [embeddedSelectable, embeddedSelectedKey, embeddedSelectedIds, selectedPidKey, selectPids]);
+ useEffect(() => {
+ if (!embeddedSelectable) return;
+ onEmbeddedSelectionChange?.([...selectedPids]);
+ }, [embeddedSelectable, onEmbeddedSelectionChange, selectedPids]);
+ /* ════════════════════════ owner item 16 / R4 / C-UNDO ════════════════════════
+ THE RECORDING LAYER. `undoStack.ts` owns the stack and every inverse; this owns the one
+ thing it cannot: reading the value a cell held BEFORE the write, which only exists at the
+ call site. Everything below funnels through `patchAndRecord` / `patchManyAndRecord`, so a
+ write path that forgets to record is a write path that does not reach the store either.
+
+ ⚠ A REF, NOT STATE. Nothing on screen depends on the stack in v1 (no undo button), so
+ keeping it in state would repaint the grid on every keystroke of a paste for no pixels.
+ Per SCOPE (a Ctrl+Z on the Customer grid must never rewrite a user table's cell) and per
+ tab (nothing persists it — a stack restored into a session that did not make those edits
+ would undo somebody else's work). */
+ const undoBook = useRef({});
+ const applyingUndo = useRef(false);
+
+ /** The value a cell holds RIGHT NOW: the overlay stratum wins over the payload row, exactly
+ * as `useGetCellContent` renders it — so what undo restores is what was on screen. */
+ const rowByPid = useMemo(() => {
+ const map = new Map();
+ for (const r of rawRows) map.set(r.pid, r);
+ return map;
+ }, [rawRows]);
+ const currentValue = useCallback(
+ (pid: number, key: string): UndoValue => {
+ const edited = overlayEdits[pid]?.[key];
+ const raw = edited !== undefined ? edited : rowByPid.get(pid)?.[key];
+ return raw == null ? null : (raw as UndoValue);
+ },
+ [overlayEdits, rowByPid]
+ );
+
+ const recordCells = useCallback(
+ (changes: CellChange[], label: string) => {
+ if (applyingUndo.current || changes.length === 0) return;
+ undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
+ kind: "cells", label, changes,
+ });
+ },
+ [scope]
+ );
+
+ /** ONE cell write, recorded. Every editor, picker and drag goes through this. */
+ const patchAndRecord = useCallback(
+ (pid: number, updates: Partial, label = "an edit") => {
+ const changes: CellChange[] = Object.entries(updates).map(([key, value]) => ({
+ pid, key,
+ before: currentValue(pid, key),
+ after: (value ?? null) as UndoValue,
+ }));
+ patchOverlay(pid, updates);
+ recordCells(changes, label);
+ },
+ [currentValue, patchOverlay, recordCells]
+ );
+
+ /** MANY cells, ONE stack entry — a paste and a bulk clear are each one user action (R4). */
+ const patchManyAndRecord = useCallback(
+ (writes: { pid: number; updates: Partial }[], label: string) => {
+ const changes: CellChange[] = [];
+ for (const w of writes)
+ for (const [key, value] of Object.entries(w.updates))
+ changes.push({
+ pid: w.pid, key,
+ before: currentValue(w.pid, key),
+ after: (value ?? null) as UndoValue,
+ });
+ for (const w of writes) patchOverlay(w.pid, w.updates);
+ recordCells(changes, label);
+ },
+ [currentValue, patchOverlay, recordCells]
+ );
+
+ /**
+ * Apply one stack entry in one direction. The INVERSE lives in `undoStack.directed` — this
+ * only knows how to WRITE each op, and it writes through the same doors the user does
+ * (`patchOverlay`, the rows endpoint), so an undone edit is persisted exactly like the edit
+ * was. Nothing here is optimistic-only: a Ctrl+Z that reverted the screen and not the store
+ * would come back on the next reload.
+ */
+ const applyEntry = useCallback(
+ async (entry: UndoEntry, dir: "back" | "forward") => {
+ const op = directed(entry, dir);
+ if (op.kind === "cells") {
+ const byPid = new Map>();
+ for (const c of op.changes) {
+ const at = byPid.get(c.pid) ?? {};
+ at[c.key] = (c.after ?? "") as Row[string];
+ byPid.set(c.pid, at);
+ }
+ applyingUndo.current = true;
+ try {
+ for (const [pid, updates] of byPid) patchOverlay(pid, updates);
+ } finally {
+ applyingUndo.current = false;
+ }
+ } else if (op.kind === "rowAdd") {
+ // Restore under the OLD id where the store still has it free; the server answers with
+ // what it actually wrote and the re-read is what puts the rows back on screen.
+ //
+ // ⚠ SEQUENTIAL, not `Promise.all`. `add_row` picks `max(id)+1` when the requested id is
+ // taken, and it reads the store to do it — firing ten restores concurrently is ten
+ // readers racing one counter, which is how two rows end up sharing an id.
+ let any = false;
+ let moved = 0;
+ for (const r of op.rows) {
+ const made = await addTableRow(op.table, r.values as Record, r.rid);
+ any = any || !!made; // a refusal already said why, in the server's words
+ // ⚠ THE ID IS PART OF WHAT IS BEING UNDONE, and `add_row` falls back to `max(id)+1`
+ // when the one it was asked for is no longer free — which happens if anything created
+ // a row in the gap. The restore is still the right thing to do, but it is no longer
+ // the SAME record to anything that named the old id (a cohort, a comment, a filter),
+ // and a redo would then look for an id that is not there. Said out loud rather than
+ // discovered later.
+ if (made && String(made.rid) !== String(r.rid)) moved += 1;
+ }
+ if (!any) return;
+ if (moved)
+ signal(
+ TOAST_EVENT,
+ `${moved} restored record${moved === 1 ? "" : "s"} came back under a new id — ` +
+ `something had taken the original while it was gone.`
+ );
+ signal(ROWS_STALE_EVENT);
+ } else if (op.kind === "rowDelete") {
+ let any = false;
+ for (const r of op.rows) any = (await deleteTableRow(op.table, r.rid)) || any;
+ if (!any) return;
+ // ⭐ ITEM 2 — the row may still be one of THIS browser's pending copies (add, then
+ // Ctrl+Z). A deleted row never reappears in a payload, so the prune effect can never
+ // absorb it and it would sit on screen looking undeleted. See `withdrawPending`.
+ withdrawPending(op.rows.map((r) => r.rid));
+ signal(ROWS_STALE_EVENT);
+ } else if (op.kind === "choiceRename") {
+ // Item 15 — the inverse mapping. The host rewrites the values and the saved views that
+ // name them, exactly as it did on the way out; `directed` already turned the pairs
+ // around, so this emits what it is given.
+ emitHostEvent({
+ id: eventId("choicerename"),
+ type: "choice_rename",
+ key: op.fieldKey,
+ renames: op.renames,
+ });
+ }
+ signal(TOAST_EVENT, describe(entry, dir));
+ },
+ [patchOverlay, withdrawPending]
+ );
+
+ /**
+ * Ctrl+Z / Ctrl+Shift+Z (and Ctrl+Y, which is the same request on Windows).
+ *
+ * ⚠ ON `window`, IN CAPTURE, and it steps aside for real text fields. The grid is a canvas —
+ * glide's key handling only fires while the canvas has focus, so a Ctrl+Z after clicking the
+ * toolbar would do nothing, which is exactly the "sometimes it works" the owner would report
+ * next. But an with a cursor in it has its OWN undo that belongs to the browser, and
+ * stealing that would be worse than not having ours: the search box, the rename field and
+ * glide's own cell editor are all inputs, so `activeElement` decides.
+ *
+ * Nothing to undo says so out loud rather than silently ignoring the key — an undo that
+ * appears to do nothing is indistinguishable from one that is broken.
+ */
+ useEffect(() => {
+ const onKey = (event: KeyboardEvent) => {
+ if (!(event.ctrlKey || event.metaKey) || event.altKey) return;
+ const key = event.key.toLowerCase();
+ if (key !== "z" && key !== "y") return;
+ const el = document.activeElement as HTMLElement | null;
+ if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable))
+ return;
+ const forward = key === "y" || event.shiftKey;
+ event.preventDefault();
+ const book = undoBook.current;
+ const state = stackFor(book, scope);
+ const { state: next, entry } = forward ? popRedo(state) : popUndo(state);
+ if (!entry) {
+ signal(TOAST_EVENT, forward ? "Nothing to redo." : "Nothing to undo.");
+ return;
+ }
+ book[scope] = next;
+ void applyEntry(entry, forward ? "forward" : "back");
+ };
+ window.addEventListener("keydown", onKey, true);
+ return () => window.removeEventListener("keydown", onKey, true);
+ }, [scope, applyEntry]);
+
+
+ /**
+ * R4 — **BULK BACKSPACE/DELETE IS ONE ACTION.** glide's own delete walks the selection and
+ * calls `onCellEdited` per cell, which would put forty entries on the stack for one keypress;
+ * returning `false` takes the whole operation over so it lands as one.
+ *
+ * ⚠ PRESETS ARE NEVER ATTEMPTED (the contract says so, and it is also the only honest
+ * behaviour): `canEditField` is the same verdict the editor and the paste path use, so a
+ * selection spanning read-only columns clears the editable ones and leaves the rest exactly
+ * as they were — rather than firing writes the server will refuse one by one.
+ */
+ /**
+ * ⭐ 2026-08-06 (owner) — DELETE RECORDS, with Ctrl+Z.
+ *
+ * Owner, verbatim: *"I should always be able to delete records that I created myself manually
+ * (not from automation), because right now it only shows 'Add to cohort'… make sure I can use
+ * 'Delete/Backspace' to delete the record and use Ctrl Z if i want to undo."*
+ *
+ * Everything under this was already built and had no door: `DELETE /tables/{key}/rows/{rid}`,
+ * `deleteTableRow`, and `undoStack`'s `rowDelete` — whose inverse (`rowAdd` under the SAME rid)
+ * is exactly "put it back". The only missing piece was something that calls them.
+ *
+ * ⛔ THE VALUES ARE CAPTURED BEFORE THE DELETE, and this is the line the whole feature rests on.
+ * `deleteTableRow` answers a boolean; it does not hand the row back. Push the undo entry after
+ * the round trip and there is nothing left to read, so Ctrl+Z would faithfully restore an EMPTY
+ * row under the right id — a silent data loss wearing the costume of a working undo.
+ *
+ * ⚠ ONE ENTRY FOR THE WHOLE GESTURE (R4's "bulk Backspace = ONE grouped stack entry"), which is
+ * why `UndoEntry` carries a row LIST now. Ten rows deleted must be one Ctrl+Z, not ten.
+ *
+ * ⚠ NO CONFIRM DIALOG, deliberately. The owner named Ctrl+Z as the safety net in the same
+ * sentence as the delete; a modal on top of a working undo is friction that teaches people to
+ * dismiss modals.
+ */
+ const deleteRecords = useCallback(
+ async (pids: number[]): Promise => {
+ if (!canMutateRecords || !pids.length) return false;
+ // The row as it stands NOW, straight off the rendered records — the same values the
+ // reader can see, so a restore puts back what they watched disappear.
+ const byPid = new Map();
+ for (const vr of displayRows)
+ if (vr.kind === "data") byPid.set(vr.record.pid, vr.record);
+ const captured: UndoRow[] = [];
+ /*
+ * ⛔ A SKIPPED ROW IS COUNTED AND NAMED, never dropped quietly ([[no-unverifiable-aggregates]]).
+ * `displayRows` is FILTERED and CAPPED, and the selection is not: Select-from-file matches
+ * on a value and can tick pids the current view does not render — `fileSelect` has a whole
+ * bucket for exactly that ("not in this view"). Capturing only what is on screen and
+ * `continue`ing past the rest would delete two rows and report three, which is the silent-cap
+ * defect in the one place it is least forgivable.
+ *
+ * REFUSED WHOLE rather than partially applied: a delete that half-happened leaves the reader
+ * reconciling a count against a table, and the fix (scroll or clear the filter, then select
+ * again) is one sentence away.
+ */
+ const offscreen: number[] = [];
+ for (const pid of pids) {
+ const rec = byPid.get(pid);
+ if (!rec) {
+ offscreen.push(pid);
+ continue;
+ }
+ const values: Record = {};
+ for (const f of fields) {
+ const v = rec[f.key];
+ if (v !== undefined && v !== null && v !== "")
+ values[f.key] = v as UndoValue;
+ }
+ captured.push({ rid: pid, values });
+ }
+ if (offscreen.length) {
+ signal(
+ TOAST_EVENT,
+ `Nothing was deleted — ${offscreen.length} of the ${pids.length} selected ` +
+ `record${pids.length === 1 ? " is" : "s are"} not shown in this view. Clear the ` +
+ `filter (or scroll them into view) and select again.`
+ );
+ return false;
+ }
+ if (!captured.length) return false;
+
+ /*
+ * ⭐ WARN THEN ALLOW (owner ruling, 2026-08-06) — the arm fires ONLY where something is
+ * actually at stake, so deleting your own scratch rows stays one keypress.
+ *
+ * ⛔ THE PREDICATE IS "WHAT WOULD BE LOST", NOT "WHO CREATED THIS". There is no stored
+ * creator on a row, and deriving one from a stage column would flip meaning as the
+ * automation runs. But the question the warning answers is a different and answerable one:
+ * *has an automation written to this row* — because that is exactly what a re-find cannot
+ * give back. MEASURED in `run_discover_instagram`: the seen-before test reads the rows
+ * CURRENTLY IN THE TABLE, so a deleted candidate returns as NEW — `found_count` back to 1,
+ * `first_found` re-stamped, and its stage reset to Review. Deleting a candidate somebody
+ * had already judged throws that judgement away.
+ *
+ * ⚠ This paragraph used to name a "Declined" stage. WAVE 26 / R6 DELETED THE BOARD'S
+ * BUILT-IN TERMINALS — the lanes are whatever the user defined now — so the warning names
+ * the mechanism (the stage resets to Review) and no longer a stage that does not ship.
+ */
+ /**
+ * ⭐⭐ WAVE-29 T23 (owner item 2b) — **THE QUESTION IS AUTHORSHIP, AND IT HAS AN ANSWER.**
+ *
+ * The note above this used to argue there is no stored creator on a row, so the warning
+ * asked "has a machine written here" instead. There IS one: `automation_engine` stamps
+ * `created_by` ("Found by") on every candidate row it discovers, and `add_row` stamps
+ * nothing — so the two authors are distinguishable, and `machineFoundRows` (types.ts) is
+ * that distinction. Everything the old predicate got wrong followed from asking the
+ * answerable-but-different question: every preset column is machine-tagged, a hand-added
+ * row is born holding materialised rollups, and the STAGE cell a human drives was
+ * force-included — so pressing "+" and then Delete produced a loud warning about losing
+ * an automation's work on a row the person had just made themselves.
+ *
+ * ⚠ The wave-27 note that lived here (`stageField` survives the stage deletion, so a
+ * legacy stage column stays read-only) is about the READ-ONLY readers — `isMachineWritten`
+ * / `mayEditField` in types.ts, which are untouched. It never described this predicate; the
+ * risk test is the one place where including the stage column is the defect rather than the
+ * protection.
+ *
+ * ⛔ AND THE SENTENCE IS BUILT FROM THE COLUMNS THIS TABLE HAS. It used to name first-found,
+ * times-found and a stage reset unconditionally while all three live in `CANDIDATE_FIELDS`
+ * — so on a user-named IG Profile database it named the loss of three columns that are not
+ * there. `reFindConsequences` reads the field list.
+ */
+ const riskyIds = new Set(machineFoundRows(fields, captured));
+ const risky = captured.filter((r) => riskyIds.has(r.rid));
+ const sig = captured.map((r) => r.rid).join(",");
+ if (risky.length && delArmed !== sig) {
+ setDelArmed(sig);
+ const losses = reFindConsequences(fields);
+ const one = risky.length === 1;
+ signal(
+ TOAST_EVENT,
+ `${risky.length} of these ${one ? "records was" : "records were"} FOUND by an ` +
+ `automation. Deleting ${one ? "it" : "them"} is not undone by a re-find` +
+ (losses.length
+ ? `: ${losses.join(", ")} — so a card somebody had already moved on comes back ` +
+ `undecided`
+ : ` — ${one ? "it" : "they"} would come back as a new record with no history`) +
+ `. Press Delete again (or click again) to confirm.`
+ );
+ return false;
+ }
+ setDelArmed("");
+ // SEQUENTIAL, matching the restore path: the store is one document and ten concurrent
+ // read-modify-writes against it is how a delete silently misses a row.
+ const gone: UndoRow[] = [];
+ for (const r of captured)
+ if (await deleteTableRow(scope, r.rid)) gone.push(r);
+ if (!gone.length) {
+ signal(TOAST_EVENT, "Nothing was deleted — the server refused.");
+ return false;
+ }
+ undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
+ kind: "rowDelete", table: scope, rows: gone,
+ });
+ // ⭐ ITEM 2 — same reason as the undo path: a row added and then deleted in one session
+ // is still a PENDING copy here, and no future payload will ever prune it.
+ withdrawPending(gone.map((r) => r.rid));
+ clearSelection();
+ signal(ROWS_STALE_EVENT);
+ // ⚠ THE CONSEQUENCE IS NAMED IN FULL, and the first version of this sentence undersold it.
+ // It said only that a matching record "will return", which is true and reassuring and
+ // leaves out the part that actually costs something.
+ //
+ // MEASURED in `run_discover_instagram`: the seen-before test is built from the rows
+ // CURRENTLY IN THE TABLE (`seen` comes from `existing2`). A deleted profile is therefore
+ // not "seen again" on the next run — it is NEW. `found_count` resets to 1, `first_found`
+ // is re-stamped, and the stage goes back to Review, because a new candidate's card starts
+ // at the human gate. So deleting a candidate somebody had already moved out of Review puts
+ // it back there undecided — the human judgement is the thing the re-find cannot give back.
+ // (W26/R6 deleted the board's built-in terminals, so there is no named stage to cite here.)
+ //
+ // (The follower SNAPSHOT history is safe either way — it lives in the platform master keyed
+ // by handle, a different store, which only `purge_handle` touches.)
+ // Same survival, same reason as `machineKeys` above (item 12 / R3, B's ASK ->C): the flag
+ // is no longer WRITTEN, and a table that has not been migrated yet still carries it.
+ const fedByAutomation = fields.some((f) => f.automation?.stageField);
+ signal(
+ TOAST_EVENT,
+ `Deleted ${gone.length} record${gone.length === 1 ? "" : "s"}. Ctrl+Z to undo.` +
+ (fedByAutomation
+ ? " If this automation finds one again it comes back as a NEW candidate — first-found" +
+ " and times-found reset, and its stage back to Review. Undo keeps all of that."
+ : "")
+ );
+ return true;
+ },
+ [canMutateRecords, displayRows, fields, scope, clearSelection, delArmed,
+ withdrawPending]
+ );
+
+ const onGridDelete = useCallback(
+ (sel: GridSelection): boolean => {
+ /*
+ * ⭐ 2026-08-06 (owner) — A ROW SELECTION + Delete/Backspace DELETES THE RECORDS.
+ *
+ * Checking rows and pressing Delete used to CLEAR every editable cell in them, which on the
+ * owner's Instagram table now clears nothing at all: every column but one is machine-written
+ * and refuses the write. So the key appeared to do nothing, which is what they reported.
+ *
+ * ⚠ ROWS ONLY, AND ONLY WHEN NO **MULTI-CELL** RANGE IS DRAWN. Checking rows is an explicit
+ * gesture about RECORDS; DRAGGING a range is a gesture about CELLS, and a Delete that
+ * destroyed records because some rows happened to be checked would feel unrecoverable even
+ * with an undo behind it. When both are present, the narrower reading (clear the cells) wins.
+ *
+ * ⛔ A SINGLE ACTIVE CELL IS NOT A RANGE, and the first version of this got it wrong.
+ * `sel.current.range` is ALWAYS set when a cell is merely focused — a 1×1 rect — so
+ * `!sel.current?.range` meant the feature worked only if you had never clicked a cell.
+ * Caught by driving it live rather than by the gate: click a cell, then check a row, then
+ * press Delete, and the key silently went back to clearing cells. The gesture the rule is
+ * actually about is a DRAG, which is width or height greater than one.
+ */
+ const dragged = !!sel.current
+ && (sel.current.range.width > 1 || sel.current.range.height > 1);
+ if (canMutateRecords && sel.rows.length > 0 && !dragged) {
+ const pids: number[] = [];
+ for (const rowIndex of sel.rows) {
+ const vr = displayRows[rowIndex];
+ if (vr && vr.kind === "data") pids.push(vr.record.pid);
+ }
+ // ⛔ `true`, NOT `false`, WHEN THIS BRANCH DOES NOT DO THE WORK — the scar the comment at
+ // the bottom of this handler already records, pointed at a second case. `false` cancels
+ // glide's own delete, so bailing with it would leave Delete doing nothing at all on a
+ // selection this branch declined. `deleteRecords` is async and the keypress cannot wait
+ // for it, so the branch commits here and reports the outcome through its own toast.
+ if (!pids.length) return true;
+ void deleteRecords(pids);
+ return false;
+ }
+ const byPid = new Map>();
+ const clear = (rowIndex: number, colIndex: number) => {
+ const vr = displayRows[rowIndex];
+ if (!vr || vr.kind !== "data") return;
+ const column = visibleCols[colIndex];
+ const field = column ? fieldByKey.get(column.id!) : undefined;
+ if (!field || !canEditField(field)) return;
+ const at = byPid.get(vr.record.pid) ?? {};
+ at[field.key] = "";
+ byPid.set(vr.record.pid, at);
+ };
+ for (const rowIndex of sel.rows)
+ for (let c = 0; c < visibleCols.length; c++) clear(rowIndex, c);
+ const range = sel.current?.range;
+ if (range)
+ for (let y = range.y; y < range.y + range.height; y++)
+ for (let x = range.x; x < range.x + range.width; x++) clear(y, x);
+ const writes = [...byPid.entries()].map(([pid, updates]) => ({ pid, updates }));
+ if (!writes.length) return true;
+ patchManyAndRecord(writes, "clearing cells");
+ // ⚠ `false` ONLY WHEN THIS ACTUALLY DID THE WORK. Returning it unconditionally cancels
+ // glide's own delete for cases this handler does not cover — a COLUMN selection, which
+ // glide deletes from `toDelete.columns` and the loops above never look at — so Delete
+ // would silently clear nothing. Handing the keypress back when there is nothing to
+ // group is strictly safer than swallowing it: the per-cell path still refuses read-only
+ // fields (`onCellEdited`'s own `canEditField`), it just does not arrive as one entry.
+ return false;
+ },
+ [canMutateRecords, deleteRecords, displayRows, visibleCols, fieldByKey, canEditField,
+ patchManyAndRecord]
+ );
+
+ /**
+ * Owner item 4 / C-ADDROW — the cursor FOLLOWS the appended row, once it exists.
+ *
+ * The append is a server round trip, so at click time there is nothing to focus; this waits
+ * for the re-read to bring the pid back and then lands the active cell on its first column,
+ * scrolled into view. `newRowPid` is cleared either way — a row the re-read never produced
+ * (a refused write, a filter that excludes it) must not leave a cursor waiting forever.
+ */
+ useEffect(() => {
+ if (newRowPid === null) return;
+ const index = displayPidToIndex.get(newRowPid);
+ if (index === undefined) return;
+ setActiveCell(0, index);
+ gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" });
+ setNewRowPid(null);
+ }, [newRowPid, displayPidToIndex, setActiveCell]);
+
+ /** Owner item 23 — the fields in the view's column order, for the Hide-fields panel. Built
+ * from `order` (already reconciled by useGridColumns) rather than from `config.order` so the
+ * panel and the grid can never disagree about which fields exist or where they sit. */
+ const orderedFields = useMemo(
+ () => order.map((key) => fieldByKey.get(key)).filter((f): f is Field => !!f),
+ [order, fieldByKey]
+ );
+ /** Owner item 17 — the row of the ACTIVE cell, i.e. what the last click highlighted. Read off
+ * `gridSelection` rather than tracked separately so keyboard navigation moves the wash too;
+ * `useGridSelection` already guards this index against a grid that shrank this render. */
+ const activeRow = gridSelection.current?.cell[1];
+
+ const statusValues = useMemo(() => {
+ const map: Record = {};
+ for (const field of fields) {
+ // A `user`'s people come from the HOST, never from the rows — an assignee nobody has been
+ // given yet is still assignable.
+ if (field.type === "user") {
+ map[field.key] = payload?.userOptions ?? [];
+ continue;
+ }
+ // ⭐ Owner item 24 — every other choice column goes through ONE rule (`choiceVocabulary`):
+ // the DECLARED list when the field has one, the values seen in the data when it does not.
+ // This used to branch on the type, which sent `stock_bucket` — a `select` whose vocabulary
+ // is computed server-side and declares no `options` — down the declared path to an empty
+ // list, and an empty supplied list WINS in the filter panel. See the function's own note;
+ // it is pure so the gate can run it, which nothing inside this file can be.
+ if (field.type === "select" || field.type === "multiselect" || field.type === "status")
+ map[field.key] = choiceVocabulary(field, rawRows);
+ }
+ return map;
+ }, [fields, rawRows, payload?.userOptions]);
+
+ const rowPx = ROW_PX[config.rowHeightMode];
+ // DISTINCT customers, not painted data rows. Grouping a MULTI field (Cohorts) puts the same
+ // customer under every group it belongs to, so counting rows would report more records than
+ // there are customers — a number nobody could reconcile against the book. `pidToIndex` is
+ // first-wins per pid, so its size IS the distinct count, and it equals the row count for every
+ // non-multi grouping and for no grouping at all.
+ //
+ // ⚠ The FULL pipeline's map, never the display slice's: the toolbar count must state what the
+ // view MATCHES. What is painted is the "Showing first N" bar's job (owner item 8).
+ const recordCount = pidToIndex.size;
+ const shownRecords = displayPidToIndex.size;
+ /* wave17 GRID — item 2 / owner R5. The whole-table row band is GONE, and with it the
+ `bandTone` / `bandMask` pair that used to sit here: the winning control's tone, and the
+ alternating-row mask built from the DATA-ROW ORDINAL so group headers could not flip the
+ stripe at a boundary. What follows is now three row states, not four.
+ ⛔ wave-29 R8 (2026-08-11) — R5 also kept an involved-COLUMN cell wash; that is now gone too,
+ so `COLUMN_TONE_THEME` carries HEADER keys only. This callback and `cells.AUTOMATION_TINT`
+ are therefore the ONLY writers of a body `bgCell` on this canvas: a sorted or filtered column
+ can no longer layer anything over a status wash, which is exactly what R8 asked for. */
+ const getRowThemeOverride = useCallback(
+ (row: number): Partial | undefined => {
+ const visibleRow = displayRows[row];
+ if (!visibleRow || visibleRow.kind !== "data") return undefined;
+ // Owner item 17 — the row holding the ACTIVE cell stays washed after the pointer moves
+ // on. Ordered AFTER hover on purpose: the pointer is the more immediate signal, and a
+ // hovered-and-active row reading as merely active would make hover look broken.
+ if (row === hoverRow) {
+ if (!config.colorBy) return HOVER_NEUTRAL;
+ return (
+ HOVER_ROW_THEME[String(visibleRow.record[config.colorBy] ?? "").toLowerCase()] ??
+ HOVER_NEUTRAL
+ );
+ }
+ // A colour-by wash already marks this row with meaning the user chose; overlaying the
+ // active tint on top would blend two hues into a third that means neither. There, glide's
+ // accent ring is the active marker and this stays out of the way.
+ if (!config.colorBy) {
+ if (row === activeRow) return ACTIVE_ROW_NEUTRAL;
+ return undefined;
+ }
+ return STATUS_ROW_THEME[
+ String(visibleRow.record[config.colorBy] ?? "").toLowerCase()
+ ];
+ },
+ [displayRows, config.colorBy, hoverRow, activeRow]
+ );
+ const onItemHovered = useCallback(
+ (args: GridMouseEventArgs) => {
+ const row = args.kind === "cell" ? args.location[1] : undefined;
+ setHoverRow((previous) => (previous === row ? previous : row));
+ // Owner item 19 — the hover-only Expand. Placed from glide's OWN bounds for the primary
+ // cell of this row, so freeze, horizontal scroll and row-height mode are handled by the
+ // component that owns them rather than re-derived here.
+ //
+ // ⚠ Only ever CLEARED from here for a different row, never for "the pointer left the
+ // canvas". Moving the pointer ONTO the button leaves the canvas, so clearing on
+ // out-of-bounds would unmount the control between the mouse arriving and the click
+ // landing — an affordance that vanishes exactly when you reach for it, and one that
+ // every assertion still sees because it exists in every state except the one that
+ // matters. `.cg-grid-box`'s own onMouseLeave is what dismisses it.
+ const vrow = row !== undefined ? displayRows[row] : undefined;
+ if (args.kind === "cell" && vrow?.kind === "data" && row !== undefined) {
+ const pid = vrow.record.pid;
+ const b = gridRef.current?.getBounds(0, row);
+ // Clamped to the grid's own box. ⚠ RETARGETED wave-14 item 15: this used to say "with a
+ // grouping active `freezeColumns` is 0, so the primary column scrolls away". It no
+ // longer does — the first column is now frozen in EVERY mode. The clamp stays because
+ // its other half is still live: glide's bounds are VIEWPORT coordinates, so a row
+ // scrolled under the header or past the bottom still reports a rect outside the grid,
+ // and a `position: fixed` button with no clamp would paint over the chrome on a row
+ // nobody can see. The horizontal legs are now defence in depth (a column drag, a box
+ // narrower than the frozen strip) rather than the everyday case.
+ const at = b
+ ? expandButtonRect(b, gridBoxRef.current?.getBoundingClientRect())
+ : null;
+ // Recomputed EVERY move, not memoised on the pid: a scroll can leave the pointer over
+ // the same record at a new y, and a button that keeps the pid but not the position
+ // floats over the wrong row. Referential stability is preserved by comparing values,
+ // which is the cheap half — `getBounds` is arithmetic over glide's own layout.
+ setExpandAt((prev) =>
+ !at
+ ? null
+ : prev && prev.pid === pid && prev.x === at.x && prev.y === at.y
+ ? prev
+ : { pid, x: at.x, y: at.y, size: at.size }
+ );
+ }
+ // Wave-5 item 6 — hovering a header whose field carries a description floats the text
+ // under the header. The tip element is pointer-events:none and aria-hidden: it can
+ // NEVER become the click target, which is the tooltip trap this page has already paid
+ // for once ([[ui-invisible-to-assertions]]).
+ if (args.kind === "header") {
+ const colDef = visibleCols[args.location[0]];
+ const field = colDef ? fieldByKey.get(colDef.id!) : undefined;
+ const note = field ? field.note || field.description : undefined;
+ // Owner item 16 — a title the fit SHORTENED must still be readable somewhere, and the
+ // tip is where. `colDef.title !== field.label` IS the truncation test: `fitHeaderTitle`
+ // returns the label by identity when it fits, so this asks the renderer what it did
+ // rather than re-running the measurement and hoping the two agree.
+ const cut = !!field && !!colDef && colDef.title !== field.label;
+ const text = cut && field ? (note ? `${field.label} — ${note}` : field.label) : note;
+ const bounds = args.bounds;
+ if (text && bounds) {
+ setHeaderTip((prev) =>
+ prev && prev.text === text && prev.x === bounds.x
+ ? prev
+ : { text, x: bounds.x, y: bounds.y + bounds.height + 4 }
+ );
+ return;
+ }
+ }
+ setHeaderTip((prev) => (prev === null ? prev : null));
+
+ // I2 — the cell half. Only text that is genuinely CUT OFF gets a tip: a tooltip on
+ // every cell just repeats what is already legible and covers the row under it.
+ // Truncation is MEASURED with glide's own font rather than guessed from a character
+ // count, which is what makes "e.g. a Note" work and a short currency cell stay quiet.
+ if (args.kind === "cell" && row !== undefined) {
+ const vr = displayRows[row];
+ const cellCol = visibleCols[args.location[0]];
+ // Group headers and footers carry no cell text of their own — a tip over the group
+ // bar would repeat the label glide has already drawn across it.
+ if (vr?.kind === "data" && cellCol) {
+ const field = fieldByKey.get(cellCol.id!);
+ const text = field ? formatDisplay(field, vr.record[field.key]) : "";
+ const bounds = args.bounds;
+ const tip = bounds ? cellTipText(text, measureCellText(text), bounds.width) : null;
+ if (tip && bounds) {
+ const x = tipLeft(bounds.x, window.innerWidth, measureCellText(tip) + 20);
+ const y = bounds.y + bounds.height + 4;
+ setCellTip((prev) =>
+ prev && prev.text === tip && prev.x === x && prev.y === y
+ ? prev
+ : { text: tip, x, y }
+ );
+ return;
+ }
+ }
+ }
+ setCellTip((prev) => (prev === null ? prev : null));
+ },
+ [visibleCols, fieldByKey, displayRows]
+ );
+ const rowHeight = useMemo(
+ () =>
+ config.groupBy
+ ? (row: number) => (displayRows[row]?.kind === "group-header" ? 32 : rowPx)
+ : rowPx,
+ [config.groupBy, rowPx, displayRows]
+ );
+
+ /* ═══ W18-B VOID ═══ (wave 18, owner item 1b) — WHERE THE TABLE ENDS, in canvas pixels.
+ Consumed by the two `.cg-grid-void` rectangles at the DataEditor mount; see the CSS region
+ of the same name for why the void is painted in the DOM rather than in the glide theme.
+
+ ⭐ THE PROPERTY THAT MAKES THIS ARITHMETIC AND NOT SCROLL-TRACKING, and the reason there is
+ no `onVisibleRegionChanged` handler anywhere near it: **a void can only exist in an axis the
+ content does not overflow.** glide's scroll extent is EXACTLY the content — `scrollWidth =
+ nonGrowWidth (+ overscrollX)`, `scrollHeight = header + rows (+ overscrollY)`, and we pass
+ neither overscroll prop (scrolling-data-grid.js:12-24) — so at maximum scroll the last
+ row/column lands flush on the client edge. Wherever there is something to paint, the scroll
+ offset in that axis is 0 and cannot become anything else. The rectangles therefore depend on
+ the columns, the rows and the box, and on nothing that moves while the user drags.
+
+ Every column carries an explicit `width` (`useGridColumns` sets `config.widths[key] ??
+ DEFAULT_WIDTH`) and none carries `grow`, so glide's column sizer passes them through
+ untouched and this sum is the width it actually lays out. */
+ const gridVoid = useMemo(() => {
+ const gutter = scrollbarGutter();
+ let colsPx = rowMarkerPx(displayRows.length);
+ for (const c of visibleCols) colsPx += (c as { width?: number }).width ?? 0;
+ let rowsPx = HEADER_PX;
+ if (typeof rowHeight === "number") rowsPx += displayRows.length * rowHeight;
+ else for (let i = 0; i < displayRows.length; i++) rowsPx += rowHeight(i);
+ // ⚠ Owner item 4 — THE GHOST ROW IS PART OF THE TABLE, and this line is what makes it
+ // visible. glide draws its trailing row AFTER the last data row, but `displayRows` (our
+ // rows) does not contain it, so the void started exactly where the ghost row does and
+ // painted flat #F6F8FC straight over it. The "+" was still clickable the whole time
+ // (`.cg-grid-void` is `pointer-events: none`), which is the worst version of this bug:
+ // the gate went green on an affordance nobody could see. Found by READING THE SCREENSHOT
+ // ([[ui-invisible-to-assertions]], [[finalize-visual-review-sop]]).
+ if (canMutateRecords)
+ rowsPx += typeof rowHeight === "number" ? rowHeight : rowHeight(displayRows.length);
+
+ /* Fit is tested against the client box the OTHER axis's scrollbar leaves behind — the same
+ `clientWidth`/`clientHeight` glide's own scroll handler reads (infinite-scroller.js:
+ 110-116), so this branch and glide's cannot disagree about whether a bar is there. The
+ two `if`s resolve the circularity in the only direction it can run: a bar in one axis can
+ CREATE one in the other, but two bars can never un-create each other. */
+ let vBar = rowsPx > gridSize.height;
+ let hBar = colsPx > gridSize.width;
+ if (vBar && !hBar) hBar = colsPx > gridSize.width - gutter;
+ if (hBar && !vBar) vBar = rowsPx > gridSize.height - gutter;
+ const clientW = gridSize.width - (vBar ? gutter : 0);
+ const clientH = gridSize.height - (hBar ? gutter : 0);
+
+ /* `null` = the content reaches that edge, so there is no void and no rectangle. The
+ comparison is strict: a table ending exactly on the edge has nothing past it. */
+ return {
+ clientW,
+ clientH,
+ below: rowsPx < clientH ? rowsPx : null,
+ right: colsPx < clientW ? colsPx : null,
+ };
+ }, [visibleCols, displayRows, rowHeight, gridSize, canMutateRecords]);
+ /* ═══ end W18-B VOID (geometry) ═══ */
+
+ // The record drawer resolves positions against what the MODE paints: the display slice for
+ // grid/list (the cap is real there), the FULL pipeline for calendar/kanban/map (a month, a
+ // stack or a pin reaches past the cap by design — a drawer that refused those pids would
+ // close itself on a card the user can plainly see).
+ // ⭐ WAVE-27 item 8 (C3) — `swipe` belongs here for calendar/kanban/map's own reason: the deck
+ // is derived from `modeDataRows` (the FULL pipeline), so a card past the display cap is one
+ // the user can plainly see, and a drawer that refused its pid would close itself on open.
+ const fullSetMode =
+ displayMode === "calendar" || displayMode === "kanban" || displayMode === "map" ||
+ displayMode === "swipe";
+ const detailRows = fullSetMode ? visibleRows : displayRows;
+ const detailIdxMap = fullSetMode ? pidToIndex : displayPidToIndex;
+ const detailIndex = detailPid !== null ? detailIdxMap.get(detailPid) : undefined;
+ useEffect(() => {
+ if (detailPid !== null && detailIndex === undefined) setDetailPid(null);
+ }, [detailPid, detailIndex]);
+ const detailRecord = useMemo(() => {
+ if (detailIndex === undefined) return null;
+ const row = detailRows[detailIndex];
+ if (!row || row.kind !== "data") return null;
+ return overlayEdits[row.record.pid]
+ ? { ...row.record, ...overlayEdits[row.record.pid] }
+ : row.record;
+ }, [detailIndex, detailRows, overlayEdits]);
+ const dataPosition = useMemo(() => {
+ if (detailIndex === undefined) return 0;
+ return detailRows
+ .slice(0, detailIndex + 1)
+ .filter((row) => row.kind === "data").length;
+ }, [detailIndex, detailRows]);
+ const neighborExists = useCallback(
+ (delta: -1 | 1) => {
+ if (detailIndex === undefined) return false;
+ for (
+ let index = detailIndex + delta;
+ index >= 0 && index < detailRows.length;
+ index += delta
+ )
+ if (detailRows[index]?.kind === "data") return true;
+ return false;
+ },
+ [detailIndex, detailRows]
+ );
+ const go = useCallback(
+ (delta: -1 | 1) => {
+ if (detailIndex === undefined) return;
+ for (
+ let index = detailIndex + delta;
+ index >= 0 && index < detailRows.length;
+ index += delta
+ ) {
+ const row = detailRows[index];
+ if (row?.kind !== "data") continue;
+ setDetailPid(row.record.pid);
+ if (displayMode === "grid")
+ gridRef.current?.scrollTo(0, index, "vertical", 0, 0, { vAlign: "center" });
+ return;
+ }
+ },
+ [detailIndex, detailRows, displayMode]
+ );
+
+ /**
+ * Wave-9 I3 — the description "(i)", drawn RIGHT-ALIGNED and vertically centred with the
+ * field name.
+ *
+ * ⚠ This is the header-draw alternative, taken because glide's `overlayIcon` PROVABLY
+ * cannot do it: `drawHeaderInner` paints an overlay at a hard-coded `drawX + 9` /
+ * `(height - 18) / 2 + 6` — a badge on the bottom-right corner of the TYPE mark at the far
+ * LEFT of the header — and no prop moves it. `drawHeader` is glide's supported escape
+ * hatch: it hands over the ctx, the rect, the menu bounds and the sprite manager, plus a
+ * `drawContent()` that runs its own rendering first. So glide still draws the header it
+ * always drew (type mark, title, menu); only the (i) is ours, and `overlayIcon` is no
+ * longer set on any column.
+ *
+ * The geometry is `infoMarkRect()` in overlayPlacement.ts, so "right-aligned and centred"
+ * is asserted numerically by a gate rather than judged from one screenshot at one width.
+ */
+ const drawGridHeader = useCallback(
+ (args, drawContent) => {
+ const field = args.column.id ? fieldByKey.get(args.column.id) : undefined;
+ // ⚠⚠ WAVE-14 ITEM 2 — THE ONE LINE THAT MAKES AN INVOLVED HEADER BOLD, and it is not a
+ // font setting anywhere near where you would look for one.
+ //
+ // glide's `drawGridHeaders` does, verbatim:
+ // if (theme !== outerTheme) ctx.font = theme.baseFontFull; // :38-40
+ // i.e. the moment a column carries ANY `themeOverride` its header title is painted in the
+ // CELL font ("13px") instead of the header font ("600 13px"). That is why the owner's
+ // filtered/sorted/grouped headers had already lost their weight — nothing in our code
+ // asked for it, and setting `headerFontStyle` on the involved theme alone would have
+ // changed precisely nothing, because glide never reads it on that path. Re-asserting the
+ // MERGED theme's header font on the ctx here is what puts it back, for every column,
+ // uniformly: on a column with no override this is a no-op (glide set the same value at
+ // :21), and glide's own save/restore around each column stops it leaking to the next.
+ //
+ // Built from `headerFontStyle` + `fontFamily` rather than reading `headerFontFull`,
+ // which is real at runtime but is declared on `FullTheme`, not the public `Theme` the
+ // callback is typed with — same string, no cast, and it is exactly how
+ // `mergeAndRealizeTheme` composes it.
+ //
+ // Bonus fix, worth knowing: `drawHeaderInner` passes `theme.headerFontFull` to
+ // `getMiddleCenterBias`, which measures with the CURRENT ctx font but CACHES under the
+ // string it was handed. Before this line, an overridden column measured its baseline bias
+ // in the 13px font and filed it under the "600 13px" key, so whichever column drew first
+ // decided the vertical centring for all of them.
+ args.ctx.font = `${args.theme.headerFontStyle} ${args.theme.fontFamily}`;
+ drawContent();
+ if (!field) return;
+ const hasInfo = !!(field.note || field.description);
+ // ⭐ OWNER ITEM 4 (2026-08-06): *"why is the Field still have the Dot at the header to
+ // mark its a custom editable field?"* Because the predicate was `source === "overlay"`
+ // alone — true of columns a person added BESIDE the Odoo ones, and true of literally
+ // every column in a `ut_*` database, including the ones an automation spawns and fills.
+ // So on the owner's Instagram table the dot marked all 23 columns as "yours", which is
+ // both meaningless (it never varies) and wrong (they are not yours to edit).
+ const isCustom = field.source === "overlay" && !isMachineOwned(field);
+ // ONE layout for both marks, from the same `headerMarkSizes` order `useGridColumns` used
+ // to reserve the label's room — so the number of marks drawn and the number reserved for
+ // cannot disagree. null = the column is too narrow to carry them without covering its own
+ // label, and then NOTHING is drawn (all-or-nothing; see headerMarkLayout).
+ const marks = headerMarkLayout(
+ args.rect,
+ args.menuBounds.width,
+ headerMarkSizes(hasInfo, isCustom)
+ );
+ if (!marks) return;
+ let slot = 0;
+ if (hasInfo) {
+ const at = marks[slot++];
+ args.spriteManager.drawSprite(
+ "aiosInfo",
+ "normal",
+ args.ctx,
+ at.x,
+ at.y,
+ at.size,
+ args.theme
+ );
+ }
+ if (isCustom) {
+ // Wave-14 item 1 / R11 — the user-created-field marker that REPLACES the yellow header
+ // wash. A muted dot: quiet enough to ignore while reading, present enough to answer
+ // "which of these columns are mine?" at a glance. Never a background — the owner killed
+ // the wash, and a paler wash would have been the same answer in a lower voice.
+ const at = marks[slot++];
+ const ctx = args.ctx;
+ ctx.save();
+ ctx.beginPath();
+ ctx.arc(at.x + at.size / 2, at.y + at.size / 2, at.size / 2, 0, Math.PI * 2);
+ ctx.fillStyle = CUSTOM_FIELD_MARK;
+ ctx.fill();
+ ctx.restore();
+ }
+ },
+ [fieldByKey]
+ );
+
+ const onCellClicked = useCallback(
+ (cell: Item, event: CellClickedEventArgs) => {
+ const row = displayRows[cell[1]];
+ if (!row) return;
+ // Owner item 10: clicking into the cells is "I am working now" — the frame folds its
+ // navigation rail to the slim strip (a no-op in the embed; the shell listens).
+ signal(NAV_MINIMIZE_EVENT);
+ if (row.kind === "group-header") {
+ event.preventDefault();
+ setCollapsed((current) => {
+ const next = new Set(current);
+ if (next.has(row.groupKey)) next.delete(row.groupKey);
+ else next.add(row.groupKey);
+ return next;
+ });
+ return;
+ }
+ const column = visibleCols[cell[0]];
+ const field = column ? fieldByKey.get(column.id!) : undefined;
+ // A picked field (select / assignee) opens its choices where the cell is. It cannot use
+ // glide's text overlay — a free-text editor on a constrained column is how you end up
+ // with "Done", "done" and "DONE" as three different values — and glide's own dropdown
+ // cell lives in a package we have deliberately not added, so this reuses the same
+ // AnchoredOverlay the column and view menus already use. Wave-5: a RATING cell rides
+ // the same picker surface (its choices are 1..max stars); both doors respect the
+ // permissions verdict.
+ if (row.kind === "data" && field && canEditField(field)
+ && (isPickType(field.type) || field.type === "rating")) {
+ event.preventDefault();
+ // Item 6 (2026-07-31) — the click also LANDS the active cell here (preventDefault
+ // stops glide from committing it), so pick → Enter walks on down the column exactly
+ // like a typed edit does.
+ setActiveCell(cell[0], cell[1]);
+ const b = event.bounds;
+ setPicker({
+ pid: row.record.pid,
+ fieldKey: field.key,
+ anchor: {
+ left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height,
+ width: b.width, height: b.height,
+ },
+ });
+ return;
+ }
+ // A linked-record cell is a doorway to the target database, not an opaque id list.
+ // The large modal mounts the same Grid surface over exactly these pids, so its standard
+ // search, Filters, Sort, Fields, and column menus keep working for every database kind.
+ if (row.kind === "data" && field?.type === "link" && field.link?.table) {
+ if (field.link.table.startsWith("ut_")) {
+ event.preventDefault();
+ setActiveCell(cell[0], cell[1]);
+ setLinkAt({ pid: row.record.pid, fieldKey: field.key });
+ return;
+ }
+ }
+ // ⭐ Wave-23 C7 — a JSON cell opens the big viewer. It is the ONLY door: the cell carries
+ // `allowOverlay:false`, because glide's overlay is a one-line box and one keystroke in the
+ // wrong place inside a 32 KB document turns a well-formed payload into an unparseable one,
+ // saved. Opened for EVERY reader (a document you may not edit is still one you must be
+ // able to read) — the viewer takes `onSave` only when the permission verdict allows it,
+ // and the host's write wall is the real one either way.
+ if (row.kind === "data" && field?.type === "json") {
+ event.preventDefault();
+ setActiveCell(cell[0], cell[1]);
+ setJsonAt({ pid: row.record.pid, fieldKey: field.key });
+ return;
+ }
+ // Wave-5 item 11 — a URL cell opens its link on click (scheme-guarded: http/https only,
+ // a bare domain gets https://). Editing stays with glide's overlay (Enter/double-click).
+ if (row.kind === "data" && field?.type === "url") {
+ const raw = String(
+ overlayEdits[row.record.pid]?.[field.key] ?? row.record[field.key] ?? ""
+ ).trim();
+ if (raw) {
+ const href = /^https?:\/\//i.test(raw)
+ ? raw
+ : /^[\w-]+(\.[\w-]+)+/.test(raw)
+ ? `https://${raw}`
+ : null;
+ if (href) {
+ event.preventDefault();
+ window.open(href, "_blank", "noopener");
+ return;
+ }
+ }
+ }
+ // Owner item 6 (2026-07-31) — CLICKING THE CUSTOMER TICKS THE CHECKBOX. The row marker
+ // is a ~32px strip; the identity cell is the widest, most natural target on the row, so
+ // a click there toggles the same pid-anchored set the markers write ("more surface area
+ // to select individual customers into a Cohort"). No preventDefault: glide still commits
+ // the cell highlight below, so reading across the row keeps working.
+ if (row.kind === "data" && field && field.key === lockedKey) {
+ togglePid(row.record.pid);
+ }
+ // Owner item 17 — A SINGLE CLICK HIGHLIGHTS. It used to open the record panel from here,
+ // which meant a user could not select a cell, read across a row, or copy a value without
+ // a drawer landing over the table. The highlight is glide's own doing: this handler
+ // returns without `preventDefault`, so the click commits `gridSelection.current`, the
+ // accent ring lands on the cell and `getRowThemeOverride` washes the row
+ // (ACTIVE_ROW_NEUTRAL). Nothing is drawn here.
+ //
+ // ⚠ SHIPPED WITH ITEM 19, never alone. Deleting this line is what removes the ONLY way to
+ // open a record; the hover Expand button below is its replacement, and half of this
+ // change is a table whose records cannot be opened at all.
+ },
+ [displayRows, visibleCols, fieldByKey, canEditField, overlayEdits, lockedKey, togglePid,
+ setActiveCell]
+ );
+ /**
+ * Double-click / Enter. KEPT as a door to the record on purpose (owner item 17 names the
+ * SINGLE click, and it is the single click that was in the way).
+ *
+ * Removing it too would leave the record reachable only by pointer — the hover affordance
+ * cannot be reached from the keyboard at all — so a keyboard user would lose the panel
+ * outright. glide routes Enter on an EDITABLE cell to its overlay editor before this fires,
+ * so the two doors do not collide: this is the activation path for the read-only columns,
+ * which is most of the table.
+ */
+ const onCellActivated = useCallback(
+ (cell: Item) => {
+ const row = displayRows[cell[1]];
+ const column = visibleCols[cell[0]];
+ const field = column ? fieldByKey.get(column.id!) : undefined;
+ if (row?.kind === "data" && field?.source !== "overlay")
+ setDetailPid(row.record.pid);
+ },
+ [displayRows, visibleCols, fieldByKey]
+ );
+ const onCellEdited = useCallback(
+ (cell: Item, value: EditableGridCell) => {
+ const row = displayRows[cell[1]];
+ const column = visibleCols[cell[0]];
+ const field = column ? fieldByKey.get(column.id!) : undefined;
+ if (!row || row.kind !== "data" || !field || !canEditField(field)) return;
+ // Wave-5 item 11 — a checkbox toggles straight through glide's BooleanCell (no overlay
+ // editor); the overlay store keeps its '1'-or-empty contract.
+ if (value.kind === GridCellKind.Boolean && field.type === "checkbox") {
+ patchAndRecord(row.record.pid, { [field.key]: value.data ? "1" : "" }, "a tick");
+ return;
+ }
+ if (value.kind === GridCellKind.Uri) {
+ patchAndRecord(row.record.pid, { [field.key]: value.data ?? "" }, "an edit");
+ return;
+ }
+ if (value.kind === GridCellKind.Text || value.kind === GridCellKind.Number) {
+ patchAndRecord(row.record.pid, { [field.key]: value.data }, "an edit");
+ }
+ },
+ [displayRows, visibleCols, fieldByKey, patchAndRecord, canEditField]
+ );
+ const validateCell = useCallback(
+ (cell: Item): boolean => {
+ const row = displayRows[cell[1]];
+ const column = visibleCols[cell[0]];
+ const field = column ? fieldByKey.get(column.id!) : undefined;
+ // ONE verdict (types.mayEditField): stratum + read-only-by-nature + permissions. The
+ // picked types and rating never take glide's text overlay — their pickers are the door.
+ return (
+ !!row &&
+ row.kind === "data" &&
+ !!field &&
+ canEditField(field) &&
+ !isPickType(field.type) &&
+ field.type !== "rating" &&
+ field.type !== "status"
+ );
+ },
+ [displayRows, visibleCols, fieldByKey, canEditField]
+ );
+ const onGridPaste = useCallback(
+ (target: Item, values: readonly (readonly string[])[]): boolean => {
+ const column = visibleCols[target[0]];
+ const field = column ? fieldByKey.get(column.id!) : undefined;
+ if (!field) return false;
+ // ⭐ Wave-15 item 3 (R8) — THE SELECTION DECIDES HOW FAR THE PASTE REACHES. Until now the
+ // targets were walked down from the anchor for exactly `values.length` rows, so a
+ // fifty-row selection and a one-cell clipboard wrote ONE row: the selection was painted
+ // and obeyed by nothing. `pasteRowCount` is the whole rule and it is pure, so the gate
+ // can drive it — this handler cannot be reached by any node test in the repo.
+ const rowCount = pasteRowCount(values.length, gridSelection.current?.range,
+ { col: target[0], row: target[1] });
+ const targetPids = Array.from({ length: rowCount }, (_, offset) => {
+ const row = displayRows[target[1] + offset];
+ return row?.kind === "data" ? row.record.pid : null;
+ });
+ const patches = planFieldPaste({
+ field,
+ sourceFieldKey: copyProvenanceRef.current.fieldKey,
+ editable: canEditField(field),
+ values,
+ targetPids,
+ allowedChoices: statusValues[field.key] ?? choiceOptions(field),
+ });
+ if (!patches) return false;
+ // R4 — ONE stack entry for the whole paste. Cell by cell would put forty entries on the
+ // stack for one Ctrl+V, and undoing a paste one cell at a time is not undoing a paste.
+ patchManyAndRecord(
+ patches.map((patch) => ({ pid: patch.pid, updates: { [field.key]: patch.value } })),
+ "a paste"
+ );
+ // We handled the write ourselves. Returning false tells glide not to run its cell
+ // renderers' generic paste path (Bubble/rating renderers cannot enforce this contract).
+ return false;
+ },
+ [visibleCols, fieldByKey, canEditField, statusValues, displayRows, patchManyAndRecord,
+ gridSelection.current]
+ );
+
+ const openHeaderMenu = useCallback(
+ (column: number, bounds: Rectangle) => {
+ if (embedded) return;
+ const definition = visibleCols[column];
+ if (!definition?.id) return;
+ setColumnMenu({
+ fieldKey: definition.id,
+ anchor: {
+ left: bounds.x,
+ top: bounds.y,
+ right: bounds.x + bounds.width,
+ bottom: bounds.y + bounds.height,
+ width: bounds.width,
+ height: bounds.height,
+ },
+ });
+ },
+ [embedded, visibleCols]
+ );
+ const onHeaderClicked = useCallback(
+ (column: number, event: HeaderClickedEventArgs) => {
+ if (event.isEdge) return;
+ event.preventDefault();
+ openHeaderMenu(column, event.bounds);
+ },
+ [openHeaderMenu]
+ );
+ const onGridKeyDown = useCallback(
+ (event: GridKeyEventArgs) => {
+ if (
+ event.key.toLowerCase() === "c" &&
+ (event.ctrlKey || event.metaKey) &&
+ !event.altKey
+ ) {
+ const range = gridSelection.current?.range;
+ const source = range?.width === 1 ? visibleCols[range.x] : undefined;
+ copyProvenanceRef.current = markGridCopy(source?.id ?? null, Date.now());
+ // Do not prevent default: glide still owns serialization and the OS clipboard write.
+ return;
+ }
+ // Owner item 5 (2026-07-31) — EXCEL-GRADE ENTER. When no editor is open (an open overlay
+ // editor swallows its own keys before the canvas sees them), Enter moves the active cell
+ // DOWN one record and Shift+Enter moves UP — never opening the record drawer, which is
+ // what made keyboard runs down a column "really clunky". Glide's own overlay editor
+ // already commits-and-moves-down on Enter, so typing → Enter → typing flows like Excel;
+ // this handles the BETWEEN-edits half. Group headers are skipped in the direction of
+ // travel. The drawer stays reachable by double-click and the hover Expand button.
+ if (event.key === "Enter" && !event.ctrlKey && !event.metaKey && !event.altKey) {
+ const cur = gridSelection.current?.cell;
+ if (cur) {
+ event.preventDefault();
+ event.stopPropagation();
+ event.cancel();
+ // Owner item 6 (2026-07-31) — Enter on a PICKED cell (select / multi select /
+ // assignee / rating) opens its picker: the keyboard door the mouse click already
+ // had. glide's text overlay cannot serve these types (validateCell refuses them),
+ // so without this the keyboard run down a column dead-ends at every picked field.
+ // Shift+Enter stays pure navigation, so walking UP past picked cells still works.
+ const curRow = displayRows[cur[1]];
+ const curColumn = visibleCols[cur[0]];
+ const curField = curColumn ? fieldByKey.get(curColumn.id!) : undefined;
+ if (
+ !event.shiftKey &&
+ curRow?.kind === "data" &&
+ curField &&
+ canEditField(curField) &&
+ (isPickType(curField.type) || curField.type === "rating")
+ ) {
+ const b = gridRef.current?.getBounds(cur[0], cur[1]);
+ if (b) {
+ setPicker({
+ pid: curRow.record.pid,
+ fieldKey: curField.key,
+ anchor: {
+ left: b.x, top: b.y, right: b.x + b.width, bottom: b.y + b.height,
+ width: b.width, height: b.height,
+ },
+ });
+ return;
+ }
+ }
+ const dir = event.shiftKey ? -1 : 1;
+ let row = cur[1] + dir;
+ while (row >= 0 && row < displayRows.length && displayRows[row]?.kind !== "data")
+ row += dir;
+ if (row >= 0 && row < displayRows.length) {
+ setActiveCell(cur[0], row);
+ gridRef.current?.scrollTo(cur[0], row, "vertical", 0, 0);
+ }
+ return;
+ }
+ }
+ const contextMenu =
+ event.key === "ContextMenu" || (event.key === "F10" && event.shiftKey);
+ // DataEditor exposes public controlled selection without the row marker,
+ // while GridKeyEventArgs.location comes from its internal grid and still
+ // includes that marker at column zero.
+ const eventColumn =
+ event.location && event.location[0] > 0
+ ? event.location[0] - 1
+ : undefined;
+ const column = gridSelection.current?.cell[0] ?? eventColumn;
+ if (!contextMenu || column === undefined || !event.bounds) return;
+ const gridBounds = gridBoxRef.current?.getBoundingClientRect();
+ const headerTop = gridBounds?.top ?? event.bounds.y;
+ event.preventDefault();
+ event.stopPropagation();
+ event.cancel();
+ openHeaderMenu(column, {
+ x: event.bounds.x,
+ y: headerTop,
+ width: event.bounds.width,
+ height: 36,
+ });
+ },
+ [gridSelection.current, openHeaderMenu, displayRows, setActiveCell,
+ visibleCols, fieldByKey, canEditField]
+ );
+
+ const persistView = useCallback((view: SavedView) => {
+ setViews((current) => {
+ const found = current.some((item) => item.id === view.id);
+ return found
+ ? current.map((item) => (item.id === view.id ? view : item))
+ : [...current, view];
+ });
+ emitHostEvent({ id: eventId("view"), type: "view_upsert", view });
+ }, []);
+
+ /**
+ * Item 12 (C-LOCK) — set or clear a view's cohort lock, from the rail's menu.
+ *
+ * Works on ANY view, not only the active one, which is why it goes through `persistView`
+ * rather than `updateConfig`: the rail's menu opens on whichever row you clicked. When the
+ * target IS the active view, the live `config` must move too, or the table keeps showing the
+ * old row set until the next select and the lock reads as ignored.
+ *
+ * Clearing DELETES the key rather than storing null — the no-churn rule every other optional
+ * config member follows, and the shape `_clean_display`'s sibling validator expects.
+ */
+ const onViewCohortLock = useCallback(
+ (viewId: string, cohortId: string | null) => {
+ const target = views.find((v) => v.id === viewId);
+ if (!target) return;
+ const nextConfig = { ...target.config };
+ if (cohortId) nextConfig.cohortLock = cohortId;
+ else delete (nextConfig as Record).cohortLock;
+ persistView({ ...target, config: nextConfig });
+ if (viewId === activeViewId) {
+ setConfig((live) => {
+ const next = { ...live };
+ if (cohortId) next.cohortLock = cohortId;
+ else delete (next as Record).cohortLock;
+ return next;
+ });
+ }
+ },
+ [views, activeViewId, persistView]
+ );
+
+ const selectView = useCallback(
+ (id: string) => {
+ const current = views.find((view) => view.id === activeViewId);
+ if (current && !sameConfig(current.config, config))
+ persistView({ ...current, config });
+ const next = views.find((view) => view.id === id);
+ if (!next) return;
+ setActiveViewId(id);
+ setConfig(normalizeConfig(next.config, fields));
+ setCollapsed(new Set());
+ setSearch("");
+ setDetailPid(null);
+ setDisplayCap(DISPLAY_PAGE);
+ // Owner item 3 (2026-07-31): tell the host WHERE THE USER IS, so a fresh browser (no
+ // localStorage copy) resumes on this view instead of the system default. Presentation
+ // state — the host stores the id and the read side re-validates it.
+ emitHostEvent({ id: eventId("view"), type: "view_select", viewId: id });
+ // Owner item 10: opening a view is "I am working now" — the frame folds its nav rail.
+ signal(NAV_MINIMIZE_EVENT);
+ },
+ [views, activeViewId, config, persistView, fields]
+ );
+
+ /**
+ * A-S4-3 (item 25) — a notification's click-through. The SHELL routes to the table and fires
+ * this; the GRID owns view selection, so neither learns the other's state.
+ *
+ * ⛔ IGNORE, NEVER THROW, when the view is not ours: an alert can outlive the view it watches
+ * (deleted, or a share revoked), and a reader who can no longer see it must simply land on the
+ * table. `selectView` already returns early on an unknown id; the topic check stops one grid
+ * reacting to another's alert when both are mounted.
+ */
+ useEffect(() => {
+ const onOpen = (e: Event) => {
+ const detail = (e as CustomEvent).detail;
+ if (!detail || detail.topic !== scope) return;
+ if (!views.some((v) => v.id === detail.viewId)) return;
+ selectView(detail.viewId);
+ };
+ window.addEventListener(VIEW_OPEN_EVENT, onOpen);
+ return () => window.removeEventListener(VIEW_OPEN_EVENT, onOpen);
+ }, [scope, views, selectView]);
+ const createView = useCallback(
+ (name: string, mode: DisplayMode, permissions: ViewPermissions) => {
+ const acceptedName = uniqueDisplayName(name, views.map((view) => view.name));
+ /**
+ * ⭐⭐ WAVE 27 · OWNER ITEM 9 / RULING R4 — **A NEW VIEW IS BLANK. ALL OF IT.**
+ *
+ * ⛔ THIS REVERSES THE WAVE-26 COMMENT THAT STOOD HERE, so the reversal is stated rather
+ * than quietly applied. That comment dropped `cohortLock` from the spread and defended
+ * keeping the rest: *"their inheritance is a FEATURE (build a view, branch off it)
+ * precisely because it is visible"*. The owner disagrees, in as many words — R4:
+ * **"New views ALWAYS start blank — no filters/sorts inherited from anything."** The
+ * branch-off use it defended gets its own door later (Duplicate view); it is not what the
+ * "+" button means, and the wave-26 fix was the right diagnosis of the wrong scope — one
+ * member of the spread was invisible, but ALL of them arrived unasked.
+ *
+ * ⛔ AND `defaultViewConfig` IS THE BLANK, not a literal assembled here. It is the same
+ * function the grid's own initial state and the cohort door already use, so "blank" has
+ * ONE definition in this file ([[one-evaluator-per-question]]); a second literal beside it
+ * would be a place for the two to disagree the day a `ViewConfig` member is added — and
+ * the one that got added last wave is exactly what caused this bug.
+ *
+ * WHAT STOPS CARRYING, enumerated because a reader deserves the list and not just the
+ * ruling: `filters` + `filterConj`, `sorts`, `groupBy`, `colorBy`, `rowHeightMode`,
+ * `order`/`visible` (hidden columns), `widths`, `memberPids`, `frozenCount` and
+ * `cohortLock`. `order`/`visible` come back from the FIELD list's own default visibility,
+ * which is why `fields` replaces `config` in the dependency list below.
+ *
+ * ⚠ THE DISPLAY REFS GO TOO, and that is the one deliberate loss. W13's carry rule kept
+ * the calendar's date field / kanban's stack field / map's lat-lon across a mode switch so
+ * Grid→Map→Grid→Map did not re-ask; a NEW view is not a mode switch, and inheriting the
+ * previous view's stack field is inheritance of exactly the kind R4 names. A ref-less
+ * kanban asks which field to stack by, which is the honest state for a view born empty.
+ * `cleanDisplay` still collapses a ref-less GRID to absent, so picking "Grid" produces the
+ * byte-identical shape every pre-wave-9 view has (no churn).
+ */
+ const nextConfig: ViewConfig = {
+ ...defaultViewConfig(fields),
+ display: cleanDisplay({ mode }),
+ };
+ const view: SavedView = {
+ id: nextViewId(),
+ name: acceptedName,
+ kind: "custom",
+ config: nextConfig,
+ // I17 (C4) — sent EXPLICITLY. Absent on create means 'personal' host-side, so a user
+ // who chose "Collaborative" would silently get the opposite. `createdBy` is NOT sent:
+ // the host stamps it and ignores whatever the browser claims.
+ permissions,
+ };
+ persistView(view);
+ setActiveViewId(view.id);
+ // The LIVE config has to move too, not just the stored one. Without this the view is
+ // created as a Calendar and the user keeps staring at the Grid until they switch views
+ // and back — the mode would be real in the store and invisible on screen.
+ setConfig(nextConfig);
+ setSaveState("saved");
+ },
+ // `fields`, not `config` (R4): the blank is derived from the COLUMNS, and reading the live
+ // config here at all is what item 9 deletes.
+ [fields, persistView, views]
+ );
+ const renameView = useCallback(
+ (id: string, name: string) => {
+ const view = views.find((item) => item.id === id);
+ if (view) {
+ const acceptedName = uniqueDisplayName(
+ name,
+ views.filter((item) => item.id !== id).map((item) => item.name)
+ );
+ persistView({ ...view, name: acceptedName });
+ }
+ },
+ [views, persistView]
+ );
+ /**
+ * I12 (contract C3) — freeze/unfreeze a view's DISPLAY MODE.
+ *
+ * The client re-checks the actor before emitting even though the menu entry is already
+ * gated: an event can be replayed, and "hidden in the client" has never been a permission.
+ * The host checks it again and that check is the wall — this one only keeps the client from
+ * showing a change that will not survive the round trip.
+ */
+ const toggleViewLock = useCallback(
+ (id: string, locked: boolean) => {
+ const view = views.find((item) => item.id === id);
+ if (!view || !mayToggleViewLock(view, viewer)) return;
+ persistView({ ...view, locked });
+ },
+ [views, persistView, viewer]
+ );
+ // The list description. Template lists ship with prose seeded by the host, but it is
+ // the USER's text once they touch it — including clearing it. An empty string is
+ // persisted as an empty string (never coerced back to the seed), because the host
+ // resolves a saved view OVER its template, so "" is how you delete a description.
+ const setViewNote = useCallback(
+ (id: string, note: string) => {
+ const view = views.find((item) => item.id === id);
+ if (view) persistView({ ...view, note: note.slice(0, 2000) });
+ },
+ [views, persistView]
+ );
+ const duplicateView = useCallback(
+ (id: string) => {
+ const source = views.find((view) => view.id === id);
+ if (!source) return;
+ const copy: SavedView = {
+ ...source,
+ id: nextViewId(),
+ name: uniqueDisplayName(
+ `${source.name} copy`,
+ views.map((view) => view.name)
+ ),
+ kind: "custom",
+ locked: false,
+ // I17 (C4) — a COPY IS THE DUPLICATOR'S OWN, PERSONAL view. Ruled, not inherited:
+ // - inheriting a 'users' grant would silently re-share the copy with a list the
+ // person making it never chose;
+ // - Duplicate is also the escape hatch for someone who may NOT edit the original,
+ // and 'personal' + the host's fresh createdBy stamp is exactly "mine to work in".
+ permissions: { edit: "personal" },
+ // Never echo the SOURCE's creator: the host stamps and ignores what the browser
+ // sends, but sending someone else's name is a laundering attempt on its face.
+ createdBy: undefined,
+ config: { ...source.config },
+ };
+ persistView(copy);
+ setActiveViewId(copy.id);
+ setConfig(copy.config);
+ },
+ [views, persistView]
+ );
+ const deleteView = useCallback(
+ (id: string) => {
+ const view = views.find((item) => item.id === id);
+ // ⚠ This gate used to read `view.locked`, and the menu entry above it did too. C3
+ // redefines `locked` as "the DISPLAY MODE is frozen" on ANY view, so leaving the gate
+ // here would make a user-frozen Kanban undeletable in the client while the host would
+ // delete it happily (app.py refuses only `all-customers`, by id). The MENU is just the
+ // door — this is the gate, and both had to move.
+ // C4 rides alongside: you may not delete a view you may not edit.
+ if (!view || isUndeletableView(view) || !mayEditView(view, viewer)) return;
+ // 2026-08-04 — STAMP BEFORE THE EMIT, the writeLocal-before-emit order every other
+ // optimistic path here uses. The live adopt below re-reads `payload.workspace.views`
+ // on every echo, and the queue sends ONE batch at a time — so an echo answering an
+ // EARLIER batch still lists this view, and without the tombstone the row would come
+ // back and (since that merge never removes) stay back. See liveWorkspace.ts.
+ viewTombstonesRef.current = stampTombstone(viewTombstonesRef.current, id, Date.now());
+ setViews((current) => current.filter((item) => item.id !== id));
+ emitHostEvent({ id: eventId("view-delete"), type: "view_delete", viewId: id });
+ if (activeViewId === id) {
+ const all = views.find((item) => item.id === ALL_VIEW_ID) ?? allRecordsView(fields, scope);
+ setActiveViewId(all.id);
+ setConfig(all.config);
+ }
+ },
+ [views, activeViewId, fields, viewer, scope]
+ );
+
+ /**
+ * Wave-6 item 7 — PER-COHORT view state. Each cohort keeps its own view under the stable id
+ * `cohort:`; a cohort with no saved view opens CLEAN (defaultViewConfig — no sort,
+ * no filters), which is exactly what "first created" means. The view is added LOCALLY on
+ * first open and emitted only when the user actually edits it (the autosave effect already
+ * compares configs), so "absent = clean" stays true in the store — opening every cohort
+ * once must not write a store full of empty views.
+ */
+ useEffect(() => {
+ if (!cohortMode || !workspaceReady || !activeCohortId) return;
+ const vid = `cohort:${activeCohortId}`;
+ if (activeViewId === vid) return;
+ const outgoing = views.find((view) => view.id === activeViewId);
+ if (outgoing && !sameConfig(outgoing.config, config))
+ persistView({ ...outgoing, config });
+ const existing = views.find((view) => view.id === vid);
+ const view: SavedView = existing ?? {
+ id: vid,
+ name: lists.find((l) => l.id === activeCohortId)?.name ?? "Cohort",
+ kind: "custom",
+ config: defaultViewConfig(fields),
+ };
+ if (!existing) setViews((current) => [...current, view]);
+ setActiveViewId(vid);
+ setConfig(normalizeConfig(view.config, fields));
+ setCollapsed(new Set());
+ setSearch("");
+ }, [cohortMode, workspaceReady, activeCohortId, activeViewId, views, config,
+ fields, lists, persistView]);
+
+ /**
+ * "Add to list" (owner item 6). Runs the SAME engine over the named view's config to learn
+ * which customers it matches, then hands those pids to the host.
+ *
+ * ⚠ Deliberately re-runs the pipeline for THAT view rather than using what is on screen: the
+ * menu is available on every view, not only the active one, and "add the rows I can see" would
+ * quietly mean something different depending on which view happened to be open.
+ *
+ * ⚠ Refuses on a windowed table. There the client holds ONE PAGE, so "the customers this view
+ * matches" is a question it cannot answer — it would add the 200 rows it happens to hold and
+ * report success. Silence would be worse than the refusal (CG-3, no-unverifiable-aggregates).
+ *
+ * ⚠ Refuses for the SAME reason when this view has a measure condition with no answer yet
+ * (CG-8). A pending condition matches nothing, so the cohort would be built from a filter that
+ * is currently narrower than the one the user is reading — and a cohort is a FIXED set, so
+ * that wrong membership would persist long after the answer arrived.
+ */
+ /**
+ * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — HOW MANY RECORDS EACH ALERTED VIEW MATCHES.
+ *
+ * ⛔ ONE EVALUATOR, AND IT IS `runPipeline` RATHER THAN `matchFilterTree`. The question is
+ * "how many records does this VIEW show", and a view is more than its condition list: a
+ * `cohortLock` narrows before anything else, `memberPids` are pinned in regardless, and a
+ * RANK leaf ("top 10") is only answerable against the domain the other conditions leave.
+ * `matchFilterTree` answers a different question — "does this ROW match this tree" — and
+ * using it here would report the whole book for a locked view and nonsense for a ranked one.
+ * `addToList` below already reaches for the pipeline for the same reason
+ * ([[one-evaluator-per-question]]); this is the same call with the same inputs.
+ *
+ * ⛔ THREE CASES WHERE THERE IS NO HONEST NUMBER, and each is ABSENT rather than zero:
+ * · a WINDOWED table — the client engine is skipped by design (CG-3), so a count here
+ * would be the size of one PAGE wearing the label of a scope;
+ * · an unresolved MEASURE condition — the server has not answered it yet, so the view
+ * matches nothing *yet*, and rendering `0` would state a fact nobody has established;
+ * · a view the rail no longer holds — an alert outlives its view.
+ * A missing key means no badge (the prop's own contract), which is the correct rendering of
+ * "not known" and is distinguishable from `0`, which is a real and interesting answer.
+ *
+ * ⚠ MEMOISED ON THE ALERTED SET, not on `views`: this walks the whole book once per alerted
+ * view, and on a table with no alerts it does nothing at all.
+ */
+ const alertedKey = alerted.join(",");
+ const alertCounts = useMemo(() => {
+ const out: Record = {};
+ if (serverWindowed || !alerted.length) return out;
+ for (const id of alerted) {
+ const view = views.find((v) => v.id === id);
+ if (!view) continue;
+ if (pendingMeasures(view.config.filters, measureSets) > 0) continue;
+ const { pidToIndex } = runPipeline({
+ rawRows: computedRows,
+ fields,
+ filters: view.config.filters,
+ search: "",
+ sorts: [],
+ groupBy: null,
+ collapsed: new Set(),
+ memberPids: view.config.memberPids,
+ filterConj: view.config.filterConj ?? "and",
+ measureSets,
+ cohortSets,
+ cohortLock: view.config.cohortLock,
+ today,
+ });
+ // DISTINCT records, which is what `pidToIndex` is — a grouped pipeline can list one
+ // record under several headings and `visibleRows.length` would then count it twice.
+ out[id] = pidToIndex.size;
+ }
+ return out;
+ // `alertedKey` is the scalar identity of the list; `alerted` itself is a fresh array on
+ // every fetch even when the answer has not changed.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [alertedKey, views, computedRows, fields, measureSets, cohortSets, today, serverWindowed]);
+
+ const addToList = useCallback(
+ (viewId: string, cohortId: string, name: string) => {
+ if (serverWindowed) return;
+ const view = views.find((v) => v.id === viewId);
+ if (!view) return;
+ if (pendingMeasures(view.config.filters, measureSets) > 0) return;
+ const { visibleRows: matched } = runPipeline({
+ rawRows: computedRows,
+ fields,
+ filters: view.config.filters,
+ search: "",
+ sorts: view.config.sorts,
+ groupBy: null,
+ collapsed: new Set(),
+ memberPids: view.config.memberPids,
+ filterConj: view.config.filterConj ?? "and",
+ measureSets,
+ cohortSets,
+ today,
+ });
+ const pids = matched
+ .filter((r): r is { kind: "data"; record: Row } => r.kind === "data")
+ .map((r) => r.record.pid);
+ emitHostEvent({
+ id: eventId("addlist"),
+ type: "add_to_list",
+ viewId,
+ cohortId,
+ name,
+ pids,
+ });
+ },
+ [views, computedRows, fields, serverWindowed, measureSets, cohortSets, today]
+ );
+
+ // ---------------------------------------------------------------- I11c (C4)
+ // Folders for the Views rail. The MODEL (echo reconcile, tombstones, dangling
+ // refs) lives in folders.ts and is gated by verify_folders.py; this is only the
+ // wiring: stamps for what this browser just did, and the five events.
+ const [folderStamps, setFolderStamps] = useState({});
+ /**
+ * Folders THIS browser just created, held until the host echo returns them.
+ *
+ * ⚠ Without these, `reconcileFolders` was being called with the host list as
+ * BOTH arguments, which makes its optimistic-create branch unreachable: a
+ * folder absent from `host` is equally absent from `local`. The function was
+ * right and the wiring was wrong, so "+ New folder" painted nothing until the
+ * round trip landed — seconds, on a cold container, with the user clicking
+ * again. The gate passed because it called reconcileFolders with two DIFFERENT
+ * arrays; it now also calls it the way this component does.
+ */
+ const [localFolders, setLocalFolders] = useState([]);
+ const workspaceFolders = payload?.workspace?.folders;
+ const folders = useMemo(
+ () => reconcileFolders(workspaceFolders, localFolders, folderStamps, Date.now()),
+ [workspaceFolders, localFolders, folderStamps]
+ );
+ const folderIdOf = useCallback(
+ (viewId: string) =>
+ resolveFolderId(
+ viewId,
+ views.find((v) => v.id === viewId)?.folderId ?? null,
+ folders,
+ folderStamps,
+ Date.now()
+ ),
+ [views, folders, folderStamps]
+ );
+ const stampFolder = useCallback((patch: (prev: FolderStamps) => FolderStamps) => {
+ setFolderStamps((prev) => pruneFolderStamps(patch(prev), Date.now()));
+ }, []);
+
+ /* wave17 R1 / C-LOCKV — the COHORT FOLDER surface is gone. `folders['cohorts']` merged
+ into `folders['views']` in the host's one-shot migration and `cohortFolders` is no longer
+ emitted at all, so `reconcileFolders` here would have been reconciling an absent list
+ against nothing forever. A locked view files into an ORDINARY folder now, which is what R1
+ means by "native folders". */
+
+ /**
+ * C4 as AMENDED — the folder-level bulk add's PREVIEW. Deliberately re-runs the
+ * engine per contained view, exactly as `addToList` does, and carries the SAME
+ * two refusals as named skips rather than dropping them silently: a union that
+ * quietly omits two views is a wrong cohort that looks right, and a cohort is a
+ * FIXED set, so it stays wrong ([[no-unverifiable-aggregates]]).
+ */
+ const folderAddPreview = useCallback(
+ (folderId: string) => {
+ const pids = new Set();
+ const skipped: { name: string; why: string }[] = [];
+ let counted = 0;
+ for (const view of views) {
+ if (folderIdOf(view.id) !== folderId) continue;
+ if (serverWindowed) {
+ skipped.push({ name: view.name, why: "this table loads one page at a time" });
+ continue;
+ }
+ if (pendingMeasures(view.config.filters, measureSets) > 0) {
+ // C-NAME (item 9) — user-facing, so it says "metric" like every other surface.
+ skipped.push({ name: view.name, why: "a metric condition has no answer yet" });
+ continue;
+ }
+ counted += 1;
+ const { visibleRows: matched } = runPipeline({
+ rawRows: computedRows,
+ fields,
+ filters: view.config.filters,
+ search: "",
+ sorts: view.config.sorts,
+ groupBy: null,
+ collapsed: new Set(),
+ memberPids: view.config.memberPids,
+ filterConj: view.config.filterConj ?? "and",
+ measureSets,
+ cohortSets,
+ today,
+ });
+ for (const r of matched) if (r.kind === "data") pids.add(r.record.pid);
+ }
+ return { pids: [...pids], counted, skipped };
+ },
+ [views, folderIdOf, serverWindowed, computedRows, fields, measureSets, cohortSets, today]
+ );
+
+ const folderAddToList = useCallback(
+ (folderId: string, cohortId: string, name: string) => {
+ const { pids } = folderAddPreview(folderId);
+ if (!pids.length) return;
+ // ONE emit with the deduped union, through the SAME guarded event a single
+ // view's "Add to cohort" uses — so the host validates both identically.
+ emitHostEvent({
+ id: eventId("addlist"),
+ type: "add_to_list",
+ viewId: "",
+ cohortId,
+ name,
+ pids,
+ });
+ },
+ [folderAddPreview]
+ );
+
+ /**
+ * Wave-7 item W2 (contract C2) — export "the current view", CLIENT-side. The same
+ * engine run addToList uses (that is the point: the file says exactly what the count
+ * says), over the same pool the named surface scopes to.
+ *
+ * ⚠ Same two refusals as addToList, for the same reasons: a server-windowed table
+ * holds ONE PAGE (the client cannot answer "the view's matches"), and a view with a
+ * PENDING measure condition currently matches nothing — an export taken then would be
+ * an empty/narrower file wearing the view's name. Both windows self-resolve; the
+ * refusal is silent no-op for one round trip.
+ */
+ const exportRows = useCallback(
+ (name: string, cfg: ViewConfig, poolRows: Row[], format: ExportFormat) => {
+ if (serverWindowed) return;
+ if (pendingMeasures(cfg.filters, measureSets) > 0) return;
+ const { visibleRows: matched } = runPipeline({
+ rawRows: poolRows,
+ fields,
+ filters: cfg.filters,
+ search: "",
+ sorts: cfg.sorts,
+ groupBy: null,
+ collapsed: new Set(),
+ memberPids: cfg.memberPids,
+ filterConj: cfg.filterConj ?? "and",
+ measureSets,
+ cohortSets,
+ today,
+ });
+ const records = matched
+ .filter((r): r is { kind: "data"; record: Row } => r.kind === "data")
+ .map((r) =>
+ overlayEdits[r.record.pid]
+ ? { ...r.record, ...overlayEdits[r.record.pid] }
+ : r.record
+ );
+ // Columns = the view's VISIBLE fields in display order (the reconciled order:
+ // every key present, config order first — the same shape useGridColumns paints).
+ const inOrder = new Set(cfg.order);
+ const keys = [
+ ...cfg.order,
+ ...fields.map((f) => f.key).filter((k) => !inOrder.has(k)),
+ ];
+ const vis = new Set(cfg.visible);
+ const cols = keys
+ .filter((k) => vis.has(k))
+ .map((k) => fieldByKey.get(k))
+ .filter((f): f is Field => !!f);
+ runExport(format, name, today, cols, records);
+ },
+ [serverWindowed, fields, fieldByKey, measureSets, cohortSets, today, overlayEdits]
+ );
+
+ /**
+ * ⭐ owner item 3 (2026-08-03) — THE SHEET THE TIME-SERIES VIEW IS SHOWING, published by the
+ * panel. A ref rather than state on purpose: nothing renders from it, and setting state on
+ * every sheet rebuild would re-render the whole grid to feed a menu nobody has opened yet.
+ */
+ const tsSheetRef = useRef(null);
+ const onTsSheet = useCallback((s: TsSheet) => { tsSheetRef.current = s; }, []);
+ /** The same channel for a SUMMARY calendar — the other mode whose content is not its rows.
+ * `null` is CalendarView saying "records mode, export the rows" (see its note). */
+ const calSheetRef = useRef<{ fields: Field[]; rows: Row[] } | null>(null);
+ const onCalSheet = useCallback(
+ (s: { fields: Field[]; rows: Row[] } | null) => { calSheetRef.current = s; },
+ []
+ );
+
+ /**
+ * W2 — the view menu's Export (Customer page): the named view over the whole pool.
+ *
+ * ⭐ owner item 3 (2026-08-03) — AND IT EXPORTS WHAT THE VIEW SHOWS. Every mode used to
+ * export the same thing: the matched customer rows. For grid / list / kanban / calendar /
+ * map / chart that is right — those modes ARRANGE rows, so the rows are what they show, and
+ * a calendar's dates and a kanban's lanes are columns already in the file.
+ *
+ * TWO modes are not arrangements of rows, and both were wrong in the direction that matters:
+ *
+ * · `timeseries` — metric ROWS over period COLUMNS. Exporting it handed you a customer list
+ * that shares none of its numbers. It exports the SHEET.
+ * · `calendar` IN SUMMARY MODE — metric values per DAY (C-DISP item 4). Same problem, and
+ * the owner named this one by hand. A calendar in RECORDS mode is genuinely an
+ * arrangement of rows, so it keeps the row export; `CalendarView` says which it is by
+ * publishing a sheet or publishing null.
+ *
+ * ⚠ ONLY FOR THE ACTIVE VIEW, and this is a real limit, not an oversight. Both sheets are
+ * built by the mounted view — the time series from a server round trip, the calendar from the
+ * month on screen. A view sitting unopened in the rail has neither, and this component cannot
+ * conjure one without fetching it. So that case SAYS SO and downloads nothing: the
+ * alternative is silently handing over the customer rows under that view's name, which is the
+ * exact substitution this branch exists to stop.
+ */
+ const exportViewData = useCallback(
+ (viewId: string, format: ExportFormat) => {
+ const view = views.find((v) => v.id === viewId);
+ if (!view) return;
+ const cfg = normalizeConfig(view.config, fields);
+ const spec = cleanDisplay(cfg.display);
+ const isActive = viewId === activeViewId;
+ const refuse = (what: string) =>
+ signal(
+ TOAST_EVENT,
+ `Open "${view.name}" first: ${what} is exported from what it draws on screen.`
+ );
+
+ if (spec?.mode === "timeseries") {
+ const sheet = isActive ? tsSheetRef.current : null;
+ if (!sheet) return refuse("a time series");
+ if (sheet.empty) {
+ signal(TOAST_EVENT,
+ `"${view.name}" has no metrics on its sheet yet — add one, then export.`);
+ return;
+ }
+ if (format === "csv") {
+ // The gated builder (verify_timeseries), kept as THE csv path so the file the owner
+ // downloads is the one the gate proves — footnotes, grouped thousands and all.
+ triggerDownload(
+ exportFilename(view.name, today, format),
+ new Blob(["" + buildTsCsv(sheet)], { type: "text/csv;charset=utf-8" })
+ );
+ } else {
+ const { fields: tf, rows: tr } = tsSheetToTable(sheet);
+ runExport(format, view.name, today, tf, tr);
+ }
+ return;
+ }
+
+ /* ═══ W18-C CATALOG ═══ A catalog is the THIRD mode that is not an arrangement of rows,
+ and it is the furthest from one: its pages are authored content, and its product codes
+ are a fraction of the pool in an order the user chose. Handing over the matched rows
+ under a catalogue's name would be the same substitution the two branches around this
+ one exist to stop. It exports as a PDF, through the browser's own print dialog (R10),
+ so this branch refuses the file and names the door that works. */
+ if (spec?.mode === "catalog") {
+ signal(
+ TOAST_EVENT,
+ `"${view.name}" is a catalog — open it and use Print to save it as a PDF. ` +
+ `A spreadsheet of its products is not what it draws.`
+ );
+ return;
+ }
+ /* ═══ end W18-C CATALOG ═══ */
+
+ if (spec?.mode === "calendar" && spec.calendarMode === "summary") {
+ const sheet = isActive ? calSheetRef.current : null;
+ if (!sheet) return refuse("a calendar summary");
+ runExport(format, view.name, today, sheet.fields, sheet.rows);
+ return;
+ }
+
+ exportRows(view.name, cfg, computedRows, format);
+ },
+ [views, fields, computedRows, exportRows, activeViewId, today]
+ );
+
+ /**
+ * The selection bar's "Add to locked view": exactly the CHECKED customers, not a view's
+ * matches.
+ * Same guarded `add_to_list` event — the host intersects the pids with the caller's pool
+ * either way, so a hand-picked set and a view's match set ride one validation path.
+ */
+ const addSelectionToList = useCallback(
+ (cohortId: string, name: string) => {
+ const pids = [...selectedPids];
+ if (!pids.length || serverWindowed) return;
+ emitHostEvent({
+ id: eventId("addlist"),
+ type: "add_to_list",
+ viewId: activeViewId,
+ cohortId,
+ name,
+ pids,
+ });
+ setSelAddOpen(false);
+ setSelListName("");
+ clearSelection();
+ },
+ [selectedPids, serverWindowed, activeViewId, clearSelection]
+ );
+
+ // Item 3c: the local write is the RENDERED truth (the def half of the no-blip contract) —
+ // stamped so a lagged echo cannot claw it back at the next remount. Item 9c: `scope` rides
+ // the EVENT at create time only; the def itself carries `scope: 'cohort'` so the echo is
+ // byte-stable ('global' stays absent — the host normalizes them to one shape).
+ const scopeChoice = payload?.workspace?.scopeChoice === true;
+ /**
+ * Item 12 (C-LOCK) — the Filter banner's copy for a cohort-locked view.
+ *
+ * `undefined` when the view is not locked. When it IS locked but the set is not in `lists`,
+ * this returns an EMPTY object: the banner must still appear (the reader is looking at a
+ * narrowed table and deserves to know why), but with no name and no count, because a set we
+ * were given no membership for is a set we must not describe. RECORD's banner reads that
+ * absence and says "locked to a set you cannot see".
+ *
+ * The count is the cohort's OWN size, not the number of rows on screen — the conditions
+ * narrow within the lock, so the two differ and the banner is stating the lock.
+ */
+ /**
+ * ⭐ wave17 R1 / C-LOCKV — the lock in force, which is now simply the ACTIVE VIEW's.
+ *
+ * Wave 15 had two sources: a saved `config.cohortLock` and an ephemeral pick from the Cohorts
+ * rail that outranked it while held. The second existed only because a projected cohort was
+ * not a saved view, so opening one could not be allowed to write anything. Under R1 it IS a
+ * saved view, so selecting it applies its own stored lock through the ordinary view path —
+ * one source, and it survives a reload, which the transient one never did.
+ */
+ const effectiveCohortLock = config.cohortLock;
+ /* wave17 R1 / C-LOCKV — `cohortRows` and the whole `cohortRail` projection are GONE. They
+ turned `lists` into rail rows for a section that no longer exists; the host projects each
+ cohort into `views` instead, so the rail renders them through the same path as every other
+ view and there is nothing left to project. `lists` still arrives and is still load-bearing
+ — it is the MEMBERSHIP channel that feeds `cohortSets`, which is what the lock resolves
+ against. Its absence would blank every locked view, so it is not "unused". */
+ const cohortLockChip = useMemo(() => {
+ const id = effectiveCohortLock;
+ if (!id) return undefined;
+ const set = lists.find((l) => l.id === id);
+ if (!set) return {};
+ return { name: set.name, count: set.pids?.length ?? 0 };
+ }, [effectiveCohortLock, lists]);
+ // (Setting/clearing the lock lives in `onViewCohortLock` above — the rail's menu operates on
+ // whichever view you clicked, not necessarily the active one, so it goes through
+ // `persistView` and only touches the live `config` when the two are the same view.)
+ /**
+ * Item 5 / contract C-LAYOUT — this user's record-detail field order, for RECORD's
+ * `RecordDetail` (they asked for exactly these two props in the wave mailbox).
+ *
+ * Read straight off the workspace stratum the host echoes, and emitted back as the
+ * `record_layout` event HOST landed. Deliberately NOT reconciled or optimistically merged
+ * here: the event returns False host-side (an autosave hot path like `view_upsert`), so
+ * there is no rerun to race, and `RecordDetail` holds the live order while a drag is in
+ * flight. Nothing else in the grid reads it — the order is the MODAL's, never the grid's
+ * `config.order`.
+ */
+ const recordLayout = payload?.workspace?.recordLayout?.order;
+ const onRecordLayout = useCallback((order: string[]) => {
+ emitHostEvent({ id: eventId("rlayout"), type: "record_layout", order });
+ }, []);
+ const saveField = useCallback(
+ (field: Field, scope?: FieldScope) => {
+ const requestId = eventId("field");
+ const accepted: Field = {
+ ...field,
+ label: uniqueDisplayName(
+ field.label,
+ fields.filter((item) => item.key !== field.key).map((item) => item.label)
+ ),
+ editRequestId: requestId,
+ };
+ delete accepted.labelCorrectedFrom;
+ delete accepted.labelCorrectionId;
+ stampFieldEdit(accepted.key);
+ setFields((current) =>
+ current.some((item) => item.key === accepted.key)
+ ? current.map((item) => (item.key === accepted.key ? accepted : item))
+ : [...current, accepted]
+ );
+ emitHostEvent({
+ id: requestId,
+ type: "field_upsert",
+ field: accepted,
+ ...(scope ? { scope } : {}),
+ });
+ },
+ [stampFieldEdit, fields]
+ );
+ const createField = useCallback(
+ (
+ label: string,
+ type: FieldType,
+ anchorKey: string | null,
+ side: "left" | "right" | "end",
+ options?: string[],
+ measureSpec?: { key: string; window: WindowSpec },
+ extra?: FieldBuildExtra
+ ) => {
+ /**
+ * ⚠ RENAMED FROM `scope` (wave 21, item 7). It used to SHADOW the component's own
+ * `scope` prop — the surface this grid is drawing — with a completely different
+ * thing: a FIELD's storage stratum (`'cohort'` or nothing). Two unrelated meanings
+ * under one word, in a function that now has to speak to a per-table endpoint.
+ * Caught by `tsc` only because `FieldScope` and `SurfaceScope` happen to be
+ * incompatible unions; had either been a bare `string`, the C2 call below would have
+ * POSTed to `/tables/undefined/fields` and read as a server bug.
+ */
+ const fieldScope = scopeChoice ? extra?.scope : undefined;
+ const scoped = fieldScope === "cohort" ? ({ scope: "cohort" } as const) : {};
+ // Owner item 7 — a FORMULA-MEASURE column. `measure_` prefixed, source:'odoo' (read-only
+ // at both ends), derived (values arrive from the host per render), filterable:false (the
+ // replacement is the measure CONDITION with the same measure+window — the governed path).
+ // The first render after creation shows blank cells for exactly one round trip: the host
+ // persists the field, then computes the values into the next payload.
+ if (measureSpec) {
+ const field: Field = {
+ key: `measure_${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`,
+ label,
+ type,
+ source: "odoo",
+ default: true,
+ custom: true,
+ derived: true,
+ filterable: false,
+ agg: ["currency", "int"].includes(type) ? "sum" : undefined,
+ measure: measureSpec,
+ ...scoped,
+ };
+ saveField(field, fieldScope);
+ insertColumn(field.key, anchorKey, side);
+ return;
+ }
+ const field = { ...buildOverlayField(label, type, options, extra), ...scoped };
+ /**
+ * ⭐ WAVE 21 item 7 (contract C2) — AN AUTOMATION COLUMN ON A USER DATABASE GOES TO
+ * THE DEFINITION, not to this user's overlay stratum.
+ *
+ * The automation editor's column picker reads `user_tables` (the definition); every
+ * grid-created column went to `_table_workspace` (the per-user overlay). So the
+ * picker was empty by construction: the user made the column, then could not find it
+ * in the automation they made it for. C2 routes exactly this one kind through
+ * `POST /tables/{key}/fields`, and the ROUTE is the wall — `_field_or_refuse` allows
+ * only the creator or an admin, which is the same answer the grid would have got.
+ *
+ * ⚠ NARROW ON PURPOSE, twice over: `type === "automation"` AND a `ut_` scope. The
+ * connector surfaces have no such endpoint (their schema is their source's), and
+ * every other kind keeps the overlay path this wave — moving all of them changes who
+ * can SEE a column, which is a behaviour change three surfaces would feel and is
+ * booked as debt rather than smuggled in here.
+ *
+ * ⚠ AND IT IS ASYNC, where every other create is optimistic. There is no local
+ * insert first: the server re-slugs the key and can refuse outright (it 400s until
+ * `UT_FIELD_TYPES` learns `automation` — C2's other half, A's), and a column shown
+ * under a key the store never took is one that vanishes on the next read with no
+ * explanation. The wait is one round trip; the alternative is a lie.
+ */
+ /**
+ * ⭐⭐ 2026-08-09 — THE RELATIONAL PAIR JOINS THAT ROUTE, and for a stronger reason than
+ * `automation` had. The note above says every other kind "keeps the overlay path this
+ * wave", weighing it as a question of who can SEE a column. For `link` and `rollup` it is
+ * not a visibility trade-off at all: their cells are computed SERVER-side by
+ * `compute_relation_cells`, which walks `user_tables` definitions. A rollup parked in a
+ * per-user overlay is invisible to the only code that could ever fill it, so it is not a
+ * narrower feature — it is a column that cannot work, in any account, ever.
+ *
+ * ⛔ AND THE BAG TRAVELS. `_clean_field` returns None for a bagless `link`/`rollup`, so
+ * sending `{key,label,type}` here would trade a silent blank column for a loud 400. The
+ * bag is on `extra`; `field` already carries it (see `FieldBuildExtra`).
+ */
+ /**
+ * ⭐⭐ 2026-08-10 — `formula` JOINS THE SAME ROUTE, on the visibility argument the note
+ * above weighs and the relational note declines to use.
+ *
+ * For `link`/`rollup` visibility was not the point (their cells are computed by something
+ * that reads the definition). For a formula it IS the whole point, and the trade-off falls
+ * the other way from wave 21's read of it: a `ut_*` database is a SHARED database — the
+ * owner builds a column on "IG profile" and expects the four people looking at that
+ * database to see it. In the overlay stratum only its creator ever does. Measured on
+ * nurilab: `Trimmed reel views`, the trimmed-average column the whole rollup chain exists
+ * to produce, was visible to exactly one account.
+ *
+ * ⛔ THE VALUES STILL DO NOT MOVE, and this must not be sold as more than it is. A formula
+ * is computed in this browser (`computedRows`) from the row's other cells and is stored
+ * nowhere; what became shared is the EXPRESSION. An automation, a rollup and an export
+ * still cannot read this column. Sharing the definition is not materialising the number.
+ *
+ * ⚠ `agg: "sum"` rides along so the totals row behaves identically to the overlay path
+ * (`buildOverlayField` stamps it) — a column that stops totalling because of which door
+ * created it is [[gate-answers-the-wrong-question]] wearing a create route.
+ */
+ /**
+ * ⭐⭐ WAVE-29 T22 (owner item 2a) — **EVERY grid-created column on a `ut_*` database goes
+ * to the DEFINITION.** The three notes above widened this set one kind at a time
+ * (`automation`, then `link`/`rollup`, then `formula`), each ending by asking whoever needed
+ * a fifth to come back here and decide. This is that decision, and it goes to ALL of them
+ * rather than to `select` alone, because the defect the owner reported is not about select:
+ *
+ * ⛔ **ON A `ut_` SCOPE THERE IS NO PER-USER CELL STRATUM LEFT TO WRITE TO.** Every accepted
+ * cell edit is routed to `user_tables.patch_cells` — the shared DEFINITION rows
+ * (`grid_events.py:1979-1984`) — while both `ut_` row doors project rows through the
+ * definition's own field keys (`routes_tables.scoped_pool:176`, `user_tables.add_row:678`).
+ * So a column living only in `_table_workspace` accepts a value, stores it under a
+ * key the projection does not know, and reads back BLANK. **The write is not lost; the read
+ * cannot see it** — which is why the server reports the write as accepted and the cell is
+ * empty after any refetch. That is item 2a, and it applied to `text`, `int`, `date`, `user`,
+ * `image` and `checkbox` exactly as much as to `select`. The owner reported the one they
+ * happened to build.
+ *
+ * ⛔ **`created_time` is the ONE exclusion, and it is the SERVER's, not a preference:**
+ * `UT_FIELD_TYPES` does not contain it, so this POST would 400 (measured — CREATABLE_TYPES
+ * minus UT_FIELD_TYPES is exactly `{created_time}`). It keeps the overlay path, where it is
+ * harmless: it is `derived`, so it has no cell of its own to lose.
+ *
+ * ⚠ **The body is a PROJECTION of the field the constructor already built**, not a
+ * per-kind list of keys. The old spelling enumerated what each kind needed, which is how
+ * `options` came to be missing for the kind that needed it most — a select POSTed without
+ * its options is degraded to `text` on read (`aios_grid.py:664-668`). `buildOverlayField`
+ * decides the shape; this sends what it decided, so a new kind cannot arrive half-described.
+ * (`_clean_field` ignores what it does not know, so `custom`/`derived`/`multi` staying
+ * client-side costs nothing; `max` on a `rating` is the one key it drops — booked, not lost.)
+ */
+ if (isUserTable && type !== "created_time") {
+ void addTableField(scope, {
+ key: field.key,
+ label: field.label,
+ type,
+ ...(field.options?.length ? { options: field.options } : {}),
+ ...(field.colorCodeOptions !== undefined
+ ? { colorCodeOptions: field.colorCodeOptions }
+ : {}),
+ ...(field.optionColors ? { optionColors: field.optionColors } : {}),
+ ...(extra?.link ? { link: extra.link } : {}),
+ ...(extra?.rollup ? { rollup: extra.rollup } : {}),
+ ...(type === "formula" && extra?.formula ? { formula: extra.formula } : {}),
+ ...(field.agg ? { agg: field.agg } : {}),
+ }).then((made) => {
+ if (!made) return; // the bridge has already said why
+ setFields((current) =>
+ current.some((f) => f.key === made.key) ? current : [...current, made]
+ );
+ insertColumn(made.key, anchorKey, side);
+ /**
+ * ⛔⛔ 2026-08-10 — THE WORKSPACE RE-READ USED TO FIRE HERE AND IT ATE THE LINE ABOVE.
+ *
+ * `insertColumn` adds the new key to the view's `visible` set and autosaves that config;
+ * `signal(WORKSPACE_STALE_EVENT)` refetches `/workspace`, whose views carry the SERVER's
+ * config — the one written a moment before the insert. The refetch won, every time, so
+ * a column created into the shared definition was **stored correctly and rendered
+ * nowhere**: present in the payload, present in `order` (which is re-derived from the
+ * field list), and absent from `visible`. MEASURED on live nurilab across all four
+ * views after migrating `Trimmed reel views`; it has been true for `link`, `rollup` and
+ * `automation` since that route was widened on 2026-08-09.
+ *
+ * ⛔ AND THE COMMENT IT REPLACES WAS THE REASON NOBODY LOOKED. It claimed the signal is
+ * "what makes the column survive a reload rather than living only in this browser's
+ * state" — but the SERVER already stored the field; the 201 is what makes it survive.
+ * The signal only refreshed a payload whose field list `setFields` had just updated by
+ * hand, and `addTableField` already drops the rows cache. It bought nothing and cost
+ * the one piece of state the create had just written.
+ *
+ * ⚠ The delete and reconfigure paths keep their signals: those change VALUES other
+ * columns fold (a dropped link takes its reciprocal and every rollup over it), and
+ * neither writes view config in the same breath, so neither has this race.
+ */
+ });
+ return;
+ }
+ saveField(field, fieldScope);
+ insertColumn(field.key, anchorKey, side);
+ },
+ [insertColumn, saveField, scopeChoice, isUserTable, scope]
+ );
+
+ /**
+ * ⭐⭐ 2026-08-07 (D-79's last half) — turn a text column into THE profile column.
+ *
+ * The write goes to the DEFINITION (`PATCH /tables/{key}/fields/{fkey}`), never to this
+ * user's overlay stratum, for the reason wave 25's amendment C3-A1 records: the automation
+ * engine reads `t['rows']`/`t['fields']`, so a flag parked in one person's overlay is a flag
+ * the enrich step can never see. Same door, same reasoning, as the automation column above.
+ *
+ * ⚠ `profile: null` CLEARS it — `_clean_field` treats an explicit null as "take the flag off"
+ * rather than "leave it alone", which is what lets a column be un-marked without deleting it.
+ */
+ const setProfileFlag = useCallback(
+ (key: string, on: boolean) => {
+ if (!isUserTable) return;
+ void patchTableField(scope, key, {
+ profile: on ? { source: "instagram" } : null,
+ }).then((made) => {
+ if (!made) return; // the bridge has already said why
+ setFields((current) => current.map((f) => (f.key === made.key ? made : f)));
+ signal(WORKSPACE_STALE_EVENT);
+ });
+ },
+ [isUserTable, scope]
+ );
+
+ /**
+ * ⭐⭐ 2026-08-09 (owner) — RECONFIGURE a `link`/`rollup` column, in the SHARED definition.
+ *
+ * Owner: *"even when i click edit field for rollup, it doesn't actually show me the
+ * configuration"* and *"No rollup field should be uneditable."* Both land here: the Edit pane
+ * now renders the bag, and this is where its Save goes.
+ *
+ * ⛔ NOT `saveField`, and not `retypeField`. Those write the per-user overlay stratum, which is
+ * precisely the defect this change exists to close — `compute_relation_cells` reads
+ * `user_tables`, so a bag parked in one person's workspace can never be computed by anything.
+ * ⚠ ASYNC with no optimistic insert, like the automation create: the server re-cleans the bag
+ * and can refuse the whole field (`_clean_rollup` returns None for, say, a `limit` with no
+ * `sortBy`), and showing a configuration the store never took is the lie the wait avoids.
+ * The cells arrive on the next read — a rollup is recomputed server-side, never in the browser.
+ */
+ const setFieldConfig = useCallback(
+ (key: string, patch: {
+ label?: string; link?: unknown; rollup?: unknown; formula?: string; agg?: string;
+ }) => {
+ if (!isUserTable) return;
+ void patchTableField(scope, key, patch as Record).then((made) => {
+ if (!made) return; // the bridge has already said why
+ setFields((current) => current.map((f) => (f.key === made.key ? made : f)));
+ // The definition changed, so the wire's field list did too — and a rollup's VALUES are
+ // recomputed host-side on this write, so the ROWS are a beat stale as well. The bridge
+ // drops its rows cache for this table (it owns that map); these two ask for a re-read.
+ signal(WORKSPACE_STALE_EVENT);
+ signal(ROWS_STALE_EVENT);
+ });
+ },
+ [isUserTable, scope]
+ );
+
+ /**
+ * Owner item 9 — "Change field": the clicked column starts showing another field, in place.
+ * The new field takes the old one's slot in `order`; the old field is HIDDEN, not lost — it
+ * stays in Fields and can be swapped back. The locked primary column is the row identity and
+ * cannot be changed away (ColumnMenu never offers it the control).
+ */
+ const changeField = useCallback(
+ (oldKey: string, newKey: string) => {
+ if (oldKey === lockedKey || oldKey === newKey) return;
+ if (!fieldByKey.has(newKey)) return;
+ // The RECONCILED order (every key present, locked first), so the splice is well-defined
+ // even for a fresh view whose config.order is still the default.
+ const withoutNew = order.filter((k) => k !== newKey);
+ const at = withoutNew.indexOf(oldKey);
+ if (at < 0) return;
+ withoutNew.splice(at, 0, newKey);
+ const nextVisible = new Set(visible);
+ nextVisible.delete(oldKey);
+ nextVisible.add(newKey);
+ updateConfig({ ...config, order: withoutNew, visible: [...nextVisible] });
+ },
+ [config, updateConfig, order, visible, fieldByKey, lockedKey]
+ );
+
+ /**
+ * Delete a USER-CREATED column outright (owner gap closed 2026-07-27). Only the created
+ * strata qualify — `custom_` overlay fields and `measure_` formula columns, both marked
+ * `custom` — so a base field can never leave the schema from here (Hide is its only exit).
+ * The definition leaves local state, the ACTIVE view's config is scrubbed (order / visible /
+ * widths / sorts / group / color), and the host removes it from the workspace store — other
+ * views self-heal on their next autosave, the rule every stale colId already rides.
+ * Two emits leave in one burst (field_delete + the config autosave); the event-log value
+ * slot exists for exactly this.
+ */
+ const deleteField = useCallback(
+ (key: string) => {
+ const f = fieldByKey.get(key);
+ if (!f?.custom || key === lockedKey) return;
+ setFields((current) => current.filter((item) => item.key !== key));
+ const nextWidths = { ...(config.widths ?? {}) };
+ delete nextWidths[key];
+ updateConfig({
+ ...config,
+ order: order.filter((k) => k !== key),
+ visible: [...visible].filter((k) => k !== key),
+ widths: nextWidths,
+ sorts: (config.sorts ?? []).filter((s) => s.colId !== key),
+ groupBy: config.groupBy === key ? null : config.groupBy,
+ colorBy: config.colorBy === key ? null : config.colorBy,
+ });
+ stampFieldDelete(key);
+ emitHostEvent({ id: eventId("fielddel"), type: "field_delete", key });
+ },
+ [config, updateConfig, order, visible, fieldByKey, lockedKey, stampFieldDelete]
+ );
+
+ /**
+ * ⭐⭐ 2026-08-10 — DELETE A COLUMN FROM A USER DATABASE'S SHARED DEFINITION.
+ *
+ * `deleteField` above writes the per-user overlay: it emits `field_delete`, which
+ * `grid_events` gates on the `custom_`/`measure_` prefix and applies to
+ * `_table_workspace`. A definition column has a `custom_` key too, so that event is
+ * ACCEPTED and scrubs a bucket the definition never reads — the column disappears for one
+ * paint and is back on the next fetch. That is why this is a different function rather than a
+ * branch: two stores, two truths, and the wrong one succeeds quietly.
+ *
+ * ⚠ SERVER FIRST, THEN THE SCREEN. `_field_or_refuse` can say no (a pre-set column, a
+ * non-creator, the last remaining column), and the bridge has already shown its sentence —
+ * removing the column optimistically would leave the user looking at a grid that disagrees
+ * with the store until they reload.
+ * ⚠ The VIEW config is scrubbed on the same terms as the overlay delete; a stale `colId` in
+ * another view self-heals on its next autosave, the rule every other delete rides.
+ */
+ const deleteDefinitionField = useCallback(
+ (key: string) => {
+ if (!isUserTable || key === lockedKey) return;
+ void deleteTableField(scope, key).then((ok) => {
+ if (!ok) return; // the bridge has already said why
+ setFields((current) => current.filter((item) => item.key !== key));
+ const nextWidths = { ...(config.widths ?? {}) };
+ delete nextWidths[key];
+ updateConfig({
+ ...config,
+ order: order.filter((k) => k !== key),
+ visible: [...visible].filter((k) => k !== key),
+ widths: nextWidths,
+ sorts: (config.sorts ?? []).filter((s) => s.colId !== key),
+ groupBy: config.groupBy === key ? null : config.groupBy,
+ colorBy: config.colorBy === key ? null : config.colorBy,
+ });
+ signal(WORKSPACE_STALE_EVENT);
+ signal(ROWS_STALE_EVENT);
+ });
+ },
+ [isUserTable, scope, lockedKey, config, updateConfig, order, visible]
+ );
+
+ /**
+ * Change-field's "New field" half (wave-2 item 5): build the overlay field, then swap it into
+ * the clicked column's slot — one motion, two emits (field_upsert + the config autosave),
+ * which is exactly the burst the event-log value slot exists for.
+ */
+ const createAndSwapField = useCallback(
+ (
+ oldKey: string,
+ label: string,
+ type: FieldType,
+ options?: string[],
+ extra?: FieldBuildExtra
+ ) => {
+ const scope = scopeChoice ? extra?.scope : undefined;
+ const field = {
+ ...buildOverlayField(label, type, options, extra),
+ ...(scope === "cohort" ? ({ scope: "cohort" } as const) : {}),
+ };
+ saveField(field, scope);
+ changeField(oldKey, field.key);
+ },
+ [saveField, changeField, scopeChoice]
+ );
+
+ /**
+ * Wave-6 item 2 — Change field on a CREATED (`custom_`) field retypes THAT field in place:
+ * `field_upsert` with the SAME key and the new type/options; the host rebuilds the def and
+ * keeps createdBy/scope (this optimistic def carries them too, so the echo is byte-stable).
+ * Values are never converted — cells re-render per the new type, unreadable values show
+ * blank (Airtable behavior, disclosed in the pane). A display `format` survives only within
+ * its own family (number→number); anything else would be junk the host drops anyway.
+ */
+ const retypeField = useCallback(
+ (
+ key: string,
+ type: FieldType,
+ options?: string[],
+ // `label` (owner item 8): rename+retype leave the Edit pane as ONE upsert — two
+ // sequential emits would each read the stale def and revert the other.
+ extra?: FieldBuildExtra
+ ) => {
+ const old = fieldByKey.get(key);
+ /**
+ * ⭐⭐ WAVE-29 T27 — the same widening as the pane above, and it is load-bearing rather
+ * than cosmetic: supplying `onRetype` while this function still bailed would render a
+ * Field-type picker that silently does nothing, which is worse than the missing control
+ * the owner reported.
+ *
+ * ⭐ AND IT IS ALSO T24's SECOND HALF. Option colours have a validator now, but the EDIT
+ * path for them is this function: on a `ut_*` database the field lives in the shared
+ * DEFINITION, so `saveField` (this user's overlay stratum) would fork the column into a
+ * private copy of itself and the colours would never reach the door that stores them.
+ */
+ if (!old || !isUserSchemaField(old, isUserTable)) return;
+ if (type === "formula" || type === "created_time") return; // a different stratum
+ const numberFam = (t: FieldType) => t === "int" || t === "currency";
+ const keepFormat =
+ old.format && ((numberFam(old.type) && numberFam(type)) || old.type === type);
+ const next: Field = {
+ key,
+ label: extra?.label?.trim() || old.label,
+ type,
+ source: "overlay",
+ default: old.default,
+ custom: true,
+ agg: ["currency", "int"].includes(type) ? "sum" : undefined,
+ ...(old.note ? { note: old.note } : {}),
+ ...(old.description ? { description: old.description } : {}),
+ ...(old.scope ? { scope: old.scope } : {}),
+ ...(old.createdBy ? { createdBy: old.createdBy } : {}),
+ ...(old.permissions ? { permissions: old.permissions } : {}),
+ ...(keepFormat ? { format: old.format } : {}),
+ ...((type === "select" || type === "multiselect") && options?.length
+ ? { options }
+ : {}),
+ ...((type === "select" || type === "multiselect")
+ ? {
+ colorCodeOptions: extra?.colorCodeOptions !== false,
+ ...(extra?.optionColors && Object.keys(extra.optionColors).length
+ ? { optionColors: extra.optionColors }
+ : {}),
+ }
+ : {}),
+ ...(type === "multiselect" ? { multi: true } : {}),
+ ...(type === "rating" ? { max: extra?.max ?? 5 } : {}),
+ };
+ if (isUserTable && !old.custom) {
+ /* THE DEFINITION DOOR. `patch_field` merges the body through the same `_clean_field` the
+ create used, so the type, the options, the colours and a rating's max all land in the
+ one validator — and the SERVER's echo replaces the local field, exactly as the create
+ path does, because it re-cleans what it was given and may refuse. */
+ void patchTableField(scope, key, {
+ label: next.label,
+ type,
+ ...(next.options ? { options: next.options } : {}),
+ ...(next.optionColors ? { optionColors: next.optionColors } : {}),
+ ...(next.colorCodeOptions !== undefined
+ ? { colorCodeOptions: next.colorCodeOptions }
+ : {}),
+ ...(next.max !== undefined ? { max: next.max } : {}),
+ ...(next.agg ? { agg: next.agg } : {}),
+ ...(next.format ? { format: next.format } : {}),
+ }).then((made) => {
+ if (!made) return; // the bridge has already said why
+ setFields((current) => current.map((f) => (f.key === made.key ? made : f)));
+ signal(WORKSPACE_STALE_EVENT);
+ signal(ROWS_STALE_EVENT);
+ });
+ } else {
+ saveField(next);
+ }
+ /* ⭐ Owner item 15 / C-RENAME — the VALUES follow the definition.
+ BESIDE the upsert, never instead of it, and AFTER it: the def is written optimistically
+ here, while the rename is a host-side migration over overlay values and saved views. The
+ order matters if the host processes the batch in order — the list must already offer
+ "Navy" before any cell is moved onto it. */
+ const renames = extra?.renames?.filter((r) => r.from && r.to && r.from !== r.to) ?? [];
+ if (renames.length) {
+ emitHostEvent({ id: eventId("choicerename"), type: "choice_rename", key, renames });
+ // R4 — and it is undoable: Ctrl+Z emits the mapping turned around. The declared LIST is
+ // restored by the same inverse (the host rewrites values and views back), so undo does
+ // not need to re-send the definition.
+ undoBook.current[scope] = pushUndo(stackFor(undoBook.current, scope), {
+ kind: "choiceRename", fieldKey: key, renames,
+ });
+ }
+ },
+ // ⚠ `isUserTable` is in the list because T27 made this function BRANCH on it. A callback that
+ // closed over a stale `isUserTable` would route a definition retype to the overlay on the
+ // first render after a scope change — the exact defect this ticket removes, reintroduced by
+ // a memo rather than by a predicate.
+ [fieldByKey, saveField, scope, isUserTable]
+ );
+
+ /**
+ * Wave-5 item 1 — Duplicate field. The CLIENT generates the destination key (same stratum
+ * prefix as the source — the ~20:20 contract amendment), emits `field_duplicate`, and slots
+ * an optimistic clone right of its source SYNCHRONOUSLY. The host validates the prefix
+ * pair, stamps createdBy (the clone's creator is the duplicator — the local copy drops the
+ * source's), and copies overlay VALUES for `custom_` sources; those values arrive with the
+ * next payload, so the clone's cells are blank for exactly one round trip.
+ */
+ const duplicateField = useCallback(
+ (source: Field) => {
+ const prefix = source.key.startsWith("measure_") ? "measure_" : "custom_";
+ const label = uniqueDisplayName(
+ `${source.label} copy`,
+ fields.map((field) => field.label)
+ );
+ const key = `${prefix}${slugify(label)}_${Math.random().toString(36).slice(2, 7)}`;
+ const requestId = eventId("fielddup");
+ const clone: Field = { ...source, key, label, editRequestId: requestId };
+ delete clone.createdBy;
+ delete clone.permissions;
+ delete clone.labelCorrectedFrom;
+ delete clone.labelCorrectionId;
+ stampFieldEdit(key);
+ setFields((current) => [...current, clone]);
+ emitHostEvent({
+ id: requestId,
+ type: "field_duplicate",
+ sourceKey: source.key,
+ key,
+ label,
+ // Item 9c: a duplicate INHERITS its source's scope (cohort clone stays cohort-only;
+ // a global source stays global = absent). Only meaningful on the cohort page.
+ ...(scopeChoice && source.scope === "cohort" ? { scope: "cohort" as const } : {}),
+ });
+ insertColumn(key, source.key, "right");
+ },
+ [insertColumn, scopeChoice, stampFieldEdit, fields]
+ );
+
+ /**
+ * Wave-2 item 8c — change the PERIOD of a measure-carrying column (user-created `measure_*`
+ * and the pre-set measure fields alike). The header auto-renames to
+ * ` · ` ONLY while it still reads as the auto-name for the CURRENT
+ * window — a user's own title is never overwritten. Values arrive recomputed from the host on
+ * the next payload; the ordinary `field_upsert` is the whole protocol.
+ */
+ const changeMeasurePeriod = useCallback(
+ (key: string, window: WindowSpec) => {
+ const f = fieldByKey.get(key);
+ if (!f?.measure) return;
+ const m = measures.find((item) => item.key === f.measure?.key);
+ const oldAuto = m ? `${m.label} · ${windowLabel(f.measure.window)}` : null;
+ const label =
+ m && oldAuto && f.label === oldAuto
+ ? `${m.label} · ${windowLabel(window)}`
+ : f.label;
+ saveField({ ...f, label, measure: { ...f.measure, window } });
+ },
+ [fieldByKey, measures, saveField]
+ );
+
+ /* wave17 R1 — `selectCohort`, `renameCohort`, `deleteCohort` and `exportCohortData` are
+ gone. Every one of them was called ONLY by `CohortSidebar`, and every one now has a
+ better-governed twin: the Views rail renames and deletes a locked view through the
+ ordinary view path, and the HOST routes those to `cohort_mod.rename` / `.delete` on a
+ projected id (C-LOCKV guards b and c). One name, one lifecycle, one set of events. */
+
+ /**
+ * Cohort mode — the selection bar's "Remove from cohort" (contract event `cohort_remove`).
+ * The pids are intersected with the cohort's membership client-side as a courtesy; the host
+ * re-validates ownership and pool anyway (a pid from the browser is untrusted input).
+ */
+ const removeSelectionFromCohort = useCallback(() => {
+ if (!activeCohortId) return;
+ const pids = [...selectedPids].filter((p) => cohortMemberSet.has(p));
+ if (!pids.length) return;
+ emitHostEvent({
+ id: eventId("cohortrm"),
+ type: "cohort_remove",
+ cohortId: activeCohortId,
+ pids,
+ });
+ clearSelection();
+ }, [activeCohortId, selectedPids, cohortMemberSet, clearSelection]);
+
+
+ /**
+ * ⭐ Wave-20 owner item 9 — **REMOVE FROM A COHORT WITHOUT BEING INSIDE IT.**
+ *
+ * "Remove from cohort" existed only under `cohortMode` (you had to open the locked view
+ * first), while "Add to cohort" worked from any view. So the two halves of the same idea
+ * lived on different screens: you could put a customer in a cohort from wherever you found
+ * them, and then had to go looking for the cohort to take them out again.
+ *
+ * SAME EVENT, same host validation as the cohort-mode button — `cohort_remove` with the pids
+ * intersected client-side as a courtesy. What is new is only WHICH cohort: the picker names
+ * it, instead of it being implied by the page you are standing on.
+ */
+ const removeSelectionFromList = useCallback(
+ (cohortId: string) => {
+ const members = cohortSets[cohortId];
+ if (!members) return;
+ const pids = [...selectedPids].filter((p) => members.has(p));
+ if (!pids.length) return;
+ emitHostEvent({ id: eventId("cohortrm"), type: "cohort_remove", cohortId, pids });
+ setSelRemoveOpen(false);
+ clearSelection();
+ },
+ [cohortSets, selectedPids, clearSelection]
+ );
+
+ /**
+ * The cohorts the checked rows can actually be removed FROM: those holding at least one of
+ * them, and only where this viewer may edit the projected view.
+ *
+ * ⚠ A cohort holding NONE of the checked rows is not offered. Airtable's rule and the one R8
+ * states for the ghost row: an affordance that can only refuse is a fake affordance — and
+ * here it would be worse than that, because "Remove from Q3 plan" that removes nothing looks
+ * exactly like a write that failed.
+ *
+ * The edit test is the COURTESY half (`mayEditView`, like every other client-side permission
+ * check on this surface); the host re-validates ownership and pool on the event regardless.
+ * A cohort with no projected view is still offered — the host owns that verdict, and hiding
+ * it here would silently drop sets the reader can see in their own rail.
+ */
+ const removableLists = useMemo(() => {
+ if (cohortMode || selectedPids.size === 0) return [];
+ const out: { id: string; name: string; hits: number }[] = [];
+ for (const l of lists) {
+ const members = cohortSets[l.id];
+ if (!members) continue;
+ let hits = 0;
+ for (const pid of selectedPids) if (members.has(pid)) hits += 1;
+ if (!hits) continue;
+ const projected = views.find((v) => v.id === l.id);
+ if (projected && !mayEditView(projected, viewer)) continue;
+ out.push({ id: l.id, name: l.name, hits });
+ }
+ return out;
+ }, [cohortMode, selectedPids, lists, cohortSets, views, viewer]);
+
+ /** Cohort mode — confirm the "+ Add customers" picker (contract event `cohort_add`). */
+ const addCustomersToCohort = useCallback(() => {
+ if (!activeCohortId || addCustPicked.size === 0) return;
+ emitHostEvent({
+ id: eventId("cohortadd"),
+ type: "cohort_add",
+ cohortId: activeCohortId,
+ pids: [...addCustPicked],
+ });
+ setAddCustPicked(new Set());
+ setAddCustQuery("");
+ setAddCustOpen(false);
+ }, [activeCohortId, addCustPicked]);
+
+ /**
+ * The picker's candidates: the POOL minus the cohort's current members, narrowed by the
+ * search. The pool's own order is kept (it leads with the biggest customers, which is the
+ * useful default for "who am I adding").
+ */
+ const addCandidates = useMemo(() => {
+ if (!addCustOpen) return [];
+ const q = addCustQuery.trim().toLowerCase();
+ const out: { pid: number; name: string }[] = [];
+ for (const r of rawRows) {
+ if (cohortMemberSet.has(r.pid)) continue;
+ const name = String(r[lockedKey] ?? "");
+ if (q && !name.toLowerCase().includes(q)) continue;
+ out.push({ pid: r.pid, name });
+ }
+ return out;
+ }, [addCustOpen, addCustQuery, rawRows, cohortMemberSet, lockedKey]);
+
+ /**
+ * Wave-6 item 11c — PIN. "Pin up to this field" freezes every visible column through the
+ * clicked one; when the field sits past the clamp (8), it is REORDERED into the frozen
+ * prefix first (the contract's wording). Pinning the identity column reads as "just the
+ * identity" = the legacy 1, stored as ABSENT.
+ *
+ * ⚠ CORRECTED wave-14 item 15. This used to read "Grouped mode paints frozenCount 0 (spans
+ * break under freezing)". **Spans do not break under freezing** — glide splits them
+ * (`getSpanBounds` → `[frozenRect, contentRect]`) and draws the row's contents from the frozen
+ * half. What IS true, and is the real cost of the pin, is that those contents are then clipped
+ * to the frozen strip's width; `fitGroupLabel` is what makes that clip honest. The clamp is
+ * gone: `freezeColumns` is `frozenN` in every mode.
+ */
+ const visibleKeys = useMemo(
+ () => order.filter((key) => visible.has(key)),
+ [order, visible]
+ );
+ const frozenN = frozenCountOf(config);
+ const pinFieldTo = useCallback(
+ (key: string) => {
+ const index = visibleKeys.indexOf(key);
+ if (index < 0) return;
+ if (index < MAX_FROZEN) {
+ updateConfig({ ...config, frozenCount: clampFrozenCount(index + 1) });
+ return;
+ }
+ const moved = visibleKeys.filter((k) => k !== key);
+ moved.splice(MAX_FROZEN - 1, 0, key);
+ let cursor = 0;
+ const nextOrder = order.map((k) => (visible.has(k) ? moved[cursor++] : k));
+ updateConfig({ ...config, order: nextOrder, frozenCount: MAX_FROZEN });
+ },
+ [visibleKeys, order, visible, config, updateConfig]
+ );
+ const unpinFields = useCallback(
+ () => updateConfig({ ...config, frozenCount: undefined }),
+ [config, updateConfig]
+ );
+
+ // Wave-6 item 10 — display-mode plumbing. W13 (contract C4 as AMENDED 2026-07-28):
+ // `config.display` is the ONE home for the field picks, so leaving a mode must NOT
+ // wipe them — returning to grid keeps the refs in the draft (mode 'grid' + refs;
+ // cleanDisplay's carry), which is what lets the next Kanban/Calendar restore the
+ // chosen field instead of feeding the select its default. A view that never picked
+ // stays byte-identical (ref-less grid normalizes to absent, the legacy rule).
+ const setDisplayMode = useCallback(
+ (m: DisplayMode) => {
+ // I12 (C3) — a locked view's MODE is frozen. The switcher is already hidden for one,
+ // so reaching here means something other than the switcher asked; refuse rather than
+ // trust the UI. (The host refuses too — this only stops the local state diverging from
+ // what the host will store, which would look like a working change that never saved.)
+ // (Resolved from `views` here rather than the `activeView` binding below — this
+ // callback is declared above it.)
+ const av = views.find((v) => v.id === activeViewId);
+ if (av && isModeFrozen(av)) return;
+ const spec = cleanDisplay(config.display);
+ if (m === "grid") {
+ updateConfig({
+ ...config,
+ display: spec ? cleanDisplay({ ...spec, mode: "grid" }) : undefined,
+ });
+ return;
+ }
+ updateConfig({ ...config, display: { ...(spec ?? {}), mode: m } });
+ },
+ [config, updateConfig, views, activeViewId]
+ );
+ // C4 (amended): the pick is written INTO display and the selects are FED from the
+ // view def (`display.`), never from component state that dies on unmount.
+ const setDisplayField = useCallback(
+ (k: "dateField" | "stackField" | "colorField" | "sizeField", key: string) => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ // Wave-8 I3/I5: colour/size are OPTIONAL encodings, so "" is a real choice
+ // (none) and must delete the ref rather than store an empty key that no
+ // field can ever match.
+ const next = { ...spec, [k]: key };
+ if (!key) delete (next as Record)[k];
+ updateConfig({ ...config, display: next });
+ },
+ [config, updateConfig]
+ );
+ /**
+ * Item 3 (C-DISP) — the kanban card clamp.
+ *
+ * ⚠ `undefined` DELETES the key rather than storing `true`. Absent means clamped, so a
+ * stored `true` would be the default wearing a second name — and the host's `_clean_display`
+ * accepts the literal `false` only, so a `true` would be dropped on save and the toggle
+ * would look like it does not persist. Delete-to-default keeps both ends agreeing.
+ */
+ const setKanbanClamp = useCallback(
+ (next: false | undefined) => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ const display = { ...spec };
+ if (next === false) display.kanbanClamp = false;
+ else delete (display as Record).kanbanClamp;
+ updateConfig({ ...config, display });
+ },
+ [config, updateConfig]
+ );
+ /**
+ * ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the swipe BINDING.
+ *
+ * ⚠ `undefined` DELETES the key, for `setKanbanClamp`'s reason one step further on. A swipe
+ * binding is a single three-part thing (field + two options); both engines' `_clean_display`
+ * DROP a half-binding rather than storing it, so writing one back would look like a save that
+ * silently did not persist. Absent IS the unconfigured state.
+ */
+ const setSwipeSpec = useCallback(
+ (next: SwipeSpec | undefined) => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ const display = { ...spec };
+ if (next) display.swipe = next;
+ else delete (display as Record).swipe;
+ updateConfig({ ...config, display });
+ },
+ [config, updateConfig]
+ );
+ /**
+ * ⭐⭐ WAVE-29 C4 / T29 (owner R7) — the FORM spec's writer, shaped exactly like `setSwipeSpec`
+ * above it: `null` DELETES the key rather than storing an empty object, because "this view has
+ * no form" and "this view has a form with nothing in it" are different states and only one of
+ * them should survive a reload. `_clean_display` drops an empty bag anyway, so storing one
+ * would make clearing a form look like it did not save.
+ */
+ const setFormSpec = useCallback(
+ (next: FormSpec | null) => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ const display = { ...spec };
+ if (next) display.form = next;
+ else delete (display as Record).form;
+ updateConfig({ ...config, display });
+ },
+ [config, updateConfig]
+ );
+ /**
+ * Item 4 (C-DISP) — the per-day totals. An EMPTY list deletes the key rather than storing
+ * `[]`: the always-on Records count is what an absent list means, so `[]` would be that
+ * state wearing a second name — and `_clean_display` drops an empty array anyway, which
+ * would make un-picking the last metric look like it did not save.
+ */
+ const setCalendarMetrics = useCallback(
+ (next: { id: string; field: string; agg: string }[]) => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ const display = { ...spec };
+ const capped = next.slice(0, MAX_CALENDAR_METRICS);
+ if (capped.length) display.calendarMetrics = capped;
+ else delete (display as Record).calendarMetrics;
+ updateConfig({ ...config, display });
+ },
+ [config, updateConfig]
+ );
+ /** Item 4 (C-DISP) — records vs summaries. `records` is the default, so it deletes the key. */
+ const setCalendarMode = useCallback(
+ (next: "records" | "summary") => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ const display = { ...spec };
+ if (next === "summary") display.calendarMode = "summary";
+ else delete (display as Record).calendarMode;
+ updateConfig({ ...config, display });
+ },
+ [config, updateConfig]
+ );
+ // The calendar's date field: the view's pick when it is still a real date-family field,
+ // else `last_order`, else the first date-family field. The kanban's stack field: status or
+ // single-select ONLY (a multiselect card in two columns at once is a count lie — contract).
+ const dateFieldChoices = useMemo(
+ () => fields.filter((f) => isDateFamilyType(f.type)),
+ [fields]
+ );
+ const stackFieldChoices = useMemo(
+ () => fields.filter((f) => f.type === "status" || f.type === "select"),
+ [fields]
+ );
+ const pickBy = (want: string | undefined, choices: Field[], preferred?: string): Field | undefined => {
+ const wanted = want ? choices.find((f) => f.key === want) : undefined;
+ if (wanted) return wanted;
+ const pref = preferred ? choices.find((f) => f.key === preferred) : undefined;
+ return pref ?? choices[0];
+ };
+ // W13 (C4 as amended): the VIEW DEF's display feeds the pickers — the refs survive
+ // every mode switch via cleanDisplay's grid carry. pickBy already falls back to the
+ // default when the stored key no longer names an offerable field (a dropped ref).
+ // I3/I5 — the map's encodings. Deliberately NOT resolved through `pickBy`:
+ // that helper falls back to the first choice, which is right for the calendar
+ // (a calendar must have a date) and wrong here, where "no encoding" is a
+ // legitimate, and the DEFAULT, state. An unknown stored key resolves to
+ // undefined = no encoding, which is also the dropped-ref behaviour.
+ // I19c (C2) — the chart list lives in `config.display.charts`, so it rides the
+ // existing view autosave + viewEcho reconcile and needs no new event type.
+ const charts = useMemo(
+ () => cleanCharts(displaySpec?.charts) ?? [],
+ [displaySpec?.charts]
+ );
+ const setCharts = useCallback(
+ (next: ChartSpec[]) => {
+ const spec = cleanDisplay(config.display);
+ if (!spec) return;
+ const display = { ...spec, charts: next };
+ if (!next.length) delete (display as Record).charts;
+ updateConfig({ ...config, display });
+ },
+ [config, updateConfig]
+ );
+ const colorFieldChoices = useMemo(
+ () => fields.filter((f) => f.type === "status" || f.type === "select"),
+ [fields]
+ );
+ const sizeFieldChoices = useMemo(
+ () => fields.filter((f) => isNumericFieldType(f.type)),
+ [fields]
+ );
+ const mapColorField = displaySpec?.colorField
+ ? colorFieldChoices.find((f) => f.key === displaySpec.colorField)
+ : undefined;
+ const mapSizeField = displaySpec?.sizeField
+ ? sizeFieldChoices.find((f) => f.key === displaySpec.sizeField)
+ : undefined;
+ const calendarField = pickBy(displaySpec?.dateField, dateFieldChoices, "last_order");
+ const kanbanField = pickBy(displaySpec?.stackField, stackFieldChoices);
+ /* ⭐ WAVE-27 item 8 (C3) — the BOUND field, resolved by key and NOT through `pickBy`. `pickBy`
+ falls back to the first choice when the stored key is unknown, which is right for a kanban
+ (a stack field is a preference) and wrong here: SwipeView SHOWS a rotted binding rather than
+ guessing at one, so it must receive the loss instead of a silent substitute — the wave-7 trap
+ [[wrong-parent-not-broken-control]] and the component's own header note both name. */
+ const swipeField = displaySpec?.swipe
+ ? fieldByKey.get(displaySpec.swipe.fieldKey)
+ : undefined;
+ // Calendar/kanban project the FULL pipeline result (a month/stack is its own bound), as
+ // DISTINCT data rows — grouping can repeat a pid — with overlay edits layered so a dragged
+ // card restacks and an edited title repaints without waiting on any echo (item 3c).
+ const modeDataRows = useMemo(() => {
+ if (
+ displayMode !== "calendar" &&
+ displayMode !== "kanban" &&
+ displayMode !== "map" &&
+ displayMode !== "chart" &&
+ // C-TS close-out stitch: the panel's pids come from these rows; without this line the
+ // time-series view asked its question about ZERO customers (caught by the mounted
+ // judge — the node battery is blind here).
+ displayMode !== "timeseries" &&
+ /* ═══ W18-C CATALOG ═══ The designer's "add the products this view shows" door reads
+ these rows — the same rows the toolbar counts, so the two can never disagree about
+ which products the filter kept. The catalog's PAINT does not: it joins its stored
+ codes against the whole pool, so a filter narrows what you can ADD and never what a
+ finished page prints. ═══ end W18-C CATALOG ═══ */
+ displayMode !== "catalog" &&
+ /* ⭐ WAVE-27 item 8 (C3) — the swipe deck is these rows filtered to the undecided ones.
+ Without this line the deck is derived from `[]`, so the view paints its "nothing left
+ to decide" empty state over a table full of undecided records — the C-TS stitch above,
+ repeated. */
+ displayMode !== "swipe"
+ )
+ return [];
+ const seen = new Set();
+ const out: Row[] = [];
+ for (const vr of visibleRows) {
+ if (vr.kind !== "data" || seen.has(vr.record.pid)) continue;
+ seen.add(vr.record.pid);
+ const edits = overlayEdits[vr.record.pid];
+ out.push(edits ? { ...vr.record, ...edits } : vr.record);
+ }
+ return out;
+ }, [displayMode, visibleRows, overlayEdits]);
+ // Owner item 1 — the CALL-SITE half of the kanban memo. `KanbanView` is `memo`'d now, but a
+ // memo whose props are rebuilt every render never bails, and these two were: `cardKeys` is a
+ // fresh array from `.filter().slice()`, `onMove` a fresh closure. With them stable, a
+ // `detailPid` change (clicking a card) re-renders the modal and skips the board entirely —
+ // which is the whole of "laggy record-open". Everything else KanbanView takes was already
+ // stable: `modeDataRows`/`fieldByKey` are useMemos, `kanbanField` is a reference INTO the
+ // `stackFieldChoices` useMemo, and `onOpen={setDetailPid}` is a setState identity.
+ const kanbanCardKeys = useMemo(
+ () =>
+ kanbanField
+ ? visibleKeys.filter((k) => k !== lockedKey && k !== kanbanField.key).slice(0, 3)
+ : [],
+ [visibleKeys, lockedKey, kanbanField]
+ );
+ const onKanbanMove = useCallback(
+ (pid: number, value: string) => {
+ if (!kanbanField) return;
+ patchAndRecord(pid, { [kanbanField.key]: value }, "a card move");
+ },
+ [kanbanField, patchAndRecord]
+ );
+ /* ⭐ WAVE-27 item 8 (C3) — a swipe writes through the NORMAL cell door, the same
+ `patchAndRecord` a card move uses. That is the whole reason the deck needs no permission
+ logic, no undo entry and no echo handling of its own: it inherits all three. */
+ const onSwipeDecide = useCallback(
+ (pid: number, value: string) => {
+ if (!swipeField) return;
+ patchAndRecord(pid, { [swipeField.key]: value }, "a swipe");
+ },
+ [swipeField, patchAndRecord]
+ );
+ /**
+ * ⭐ WAVE-26 ITEM 12 (owner ruling R8) — the kanban's "Add option" door.
+ *
+ * It opens the ORDINARY ColumnMenu on the stack field, straight into its Edit-field pane. So
+ * the option is written into the SHARED field definition by the same `onRetype` upsert the
+ * header menu uses — never a per-view list — and every guarantee that editor already carries
+ * (permission wall, rename-by-row-identity, option colours, one upsert for name+choices) comes
+ * along without being re-implemented. R8's "do not invent a second options editor", literally.
+ *
+ * ⛔ THE PERMISSION QUESTION IS NOT `canEditField`. Adding an option changes the column's
+ * DEFINITION; `mayEditField` answers who may type a VALUE into it, and conflating the two is
+ * the confusion [[schema-role-is-not-a-value-wall]] exists to prevent. The right predicate is
+ * the one that decides whether `onRetype` is supplied at all — a created overlay field — so it
+ * is written ONCE here and read at both doors.
+ *
+ * `null` (not "absent") when the field cannot take one: a status column is computed by the
+ * source system, and an Odoo select's vocabulary lives in Odoo.
+ *
+ * ⭐ THE PREDICATE ADMITS AN AUTOMATION'S `stage_` COLUMN, and that is INTENDED rather than
+ * incidental — worth stating because it was arrived at by matching `onRetype`'s condition, and
+ * a coincidence and a decision look identical in code. It is exactly what R6 and R8 compose
+ * to: R6 deleted the board's built-in terminals, so the lanes a review stage offers are now
+ * only the ones a user defined — and R8 is the control that lets them define one. Landed
+ * without this the pair is worse than either alone (SESSION A's A-7 says the same thing from
+ * the engine side: `humanMoves` had to learn to admit user-added options, or the move door
+ * would refuse the lane the product just invited you to create).
+ * ⚠ CARRIED CONSEQUENCE, pre-existing and not introduced here: the pane this opens is the
+ * whole Edit-field window, so it also offers the TYPE picker — one wrong click retypes a stage
+ * column. The header menu has offered exactly that on the same column since wave 6; this door
+ * adds a second way in, not a new hazard. Written down rather than left to be rediscovered.
+ */
+ /* ⭐ WAVE-29 T27: the same stratum predicate as the Edit-field pane, for the same reason — this
+ door OPENS that pane, so gating the two differently means offering an add-option control that
+ leads to a pane with no editor in it. On a `ut_*` kanban (every automation board) the old
+ `custom` test was false for every column, so the lane header offered nothing. */
+ const kanbanCanAddOption =
+ !!kanbanField && kanbanField.type === "select" &&
+ isUserSchemaField(kanbanField, isUserTable);
+ const onKanbanAddOption = useCallback(
+ (anchor: AnchorRect) => {
+ if (!kanbanField) return;
+ setColumnMenu({ fieldKey: kanbanField.key, anchor, pane: "edit" });
+ },
+ [kanbanField]
+ );
+ const onListToggleGroup = useCallback((groupKey: string) => {
+ setCollapsed((current) => {
+ const next = new Set(current);
+ if (next.has(groupKey)) next.delete(groupKey);
+ else next.add(groupKey);
+ return next;
+ });
+ }, []);
+ /**
+ * Item 7 (C-TS) — the pid set the time series asks about: exactly the rows the current
+ * filter kept, so the panel and the toolbar count can never disagree about "who".
+ *
+ * `modeDataRows` is already the de-duplicated data rows for a non-grid mode, so this is a
+ * projection of it and NOT a second pipeline. The server intersects with the caller's book
+ * anyway — this narrows the question, it can never widen the answer.
+ */
+ const timeseriesPids = useMemo(
+ () => (displayMode === "timeseries" ? modeDataRows.map((r) => r.pid) : []),
+ [displayMode, modeDataRows]
+ );
+ /* ═══ W18-C CATALOG ═══ (owner item 4, contract C6)
+ The catalogs this view holds, and the door that persists an edit to them.
+
+ `displaySpec.catalogs` is ALREADY validated — `cleanDisplay` ran `cleanCatalogs` over it on
+ read — so there is no second normalisation here and no chance of the two disagreeing.
+
+ ⚠ The write goes through `updateConfig` exactly like `onTimeseriesDisplay` below, which
+ means a catalog edit is a VIEW-CONFIG save. That is the point: a catalogue is authored
+ content and has to survive a reload, and the display spec is the only per-view store the
+ grid has. It also means the 500-code / 40-page / 12-catalog caps are enforced twice on the
+ round trip (here on read, and by the host's `_clean_catalogs` on write) — the designer
+ states its budget in the toolbar so the cap is never the user's first news of it. */
+ const catalogs = useMemo(
+ () => (displayMode === "catalog" ? (displaySpec?.catalogs ?? []) : []),
+ [displayMode, displaySpec]
+ );
+ /** The codes the current filter kept — the designer's "add what the view shows" shortcut.
+ * Derived from `modeDataRows`, so it counts what the toolbar counts. */
+ const catalogFilteredCodes = useMemo(
+ () =>
+ displayMode === "catalog"
+ ? modeDataRows
+ .map((r) => r[CATALOG_CODE_FIELD])
+ .filter((c): c is string => typeof c === "string" && !!c)
+ : [],
+ [displayMode, modeDataRows]
+ );
+ const onCatalogs = useCallback(
+ (next: CatalogSpec[]) => {
+ const spec = cleanDisplay(config.display) ?? { mode: "catalog" as const };
+ updateConfig({ ...config, display: { ...spec, catalogs: next } });
+ },
+ [config, updateConfig]
+ );
+ /* ═══ end W18-C CATALOG ═══ */
+ /** Persist a bucket/span/metric pick onto the view, the way every other display key rides. */
+ const onTimeseriesDisplay = useCallback(
+ (next: Partial) => {
+ const spec = cleanDisplay(config.display) ?? { mode: "timeseries" as const };
+ updateConfig({ ...config, display: { ...spec, ...next } });
+ },
+ [config, updateConfig]
+ );
+ const kanbanCanMove =
+ !!kanbanField && kanbanField.type === "select" && canEditField(kanbanField);
+ const kanbanReason = !kanbanField
+ ? null
+ : kanbanField.type === "status"
+ ? `Stacked by ${kanbanField.label} — read-only, computed from the source system.`
+ : !canEditField(kanbanField)
+ ? "You do not have permission to edit this field, so cards cannot be moved."
+ : null;
+ /* ⭐ WAVE-27 item 8 (C3) — the kanban pair, said again for the deck. Same wall, same words
+ shaped for the gesture: a swipe writes a VALUE, so `canEditField` is the right predicate. */
+ const swipeCanWrite = !!swipeField && swipeField.type === "select" && canEditField(swipeField);
+ const swipeReason = !swipeField
+ ? null
+ : swipeField.type === "status"
+ ? `Bound to ${swipeField.label} — read-only, computed from the source system.`
+ : !canEditField(swipeField)
+ ? "You do not have permission to edit this field, so records cannot be decided."
+ : null;
+
+ if (loading || !workspaceReady) {
+ /* wave17 GRID — item 3 / owner R6: one small bare rotating icon, NO WORDS. `aria-label`
+ is not a word on screen and stays: `.lp-spin` is an empty span, so without it the wait
+ is announced to a screen reader as nothing at all. (C-SPIN, SHELL defines the class.)
+ `--lg` because this is the case C-SPIN sizes it for — the whole surface is this mark and
+ nothing else, where the 14px version "reads as dust". */
+ return (
+
+
+
+ );
+ }
+
+ const activeView = views.find((view) => view.id === activeViewId);
+ // Cohort mode's toolbar control (rendered by Toolbar via the `cohortAction` slot; the popover
+ // it opens is with the other overlays at the bottom). Disabled without an active cohort —
+ // zero cohorts is the host page's near-empty state, not this button's error to explain.
+ const cohortAction = cohortMode ? (
+ setAddCustOpen((o) => !o)}
+ aria-expanded={addCustOpen}
+ aria-haspopup="dialog"
+ >
+ + Add customers
+
+ ) : undefined;
+ // Wave-7 item W9 — the Fields panel's permanent delete rides the SAME wall as the
+ // column menu's Delete: created strata only (`custom`), never the locked identity
+ // column. One predicate, two doors.
+ //
+ // ⚠ WAVE-29 T22 NARROWED WHAT THIS SET CONTAINS, and the narrowing is CORRECT rather than an
+ // oversight — said out loud because it looks like a regression. A grid-created column on a
+ // `ut_*` database now lives in the shared DEFINITION and carries no `custom` flag, so it drops
+ // out of this set. It has to: this door emits `field_delete`, which scrubs the per-user
+ // workspace overlay — a bucket the definition never reads — so the column would vanish for one
+ // paint and be back on the next fetch (`deleteDefinitionField`'s note says the same thing from
+ // the other side). ⛔ The consequence is real and is NOT silently widened here: a definition
+ // column can only be deleted through the column menu, which offers it for `link`/`rollup`/
+ // `formula` alone (D-114's deliberate narrowness — every other kind holds real values). Booked
+ // as pending work rather than fixed by handing a type-blind delete to a panel.
+ const deletableKeys = new Set(
+ fields.filter((f) => f.custom && f.key !== lockedKey).map((f) => f.key)
+ );
+ const menuField = columnMenu ? fieldByKey.get(columnMenu.fieldKey) : undefined;
+ // Wave-5 item 3 — what the CURRENT VIEW does with the menu's field, so the conditional
+ // "Don't sort/filter/group" entries render exactly when they apply.
+ const menuSortedDir = menuField
+ ? config.sorts.find((s) => s.colId === menuField.key)?.dir ?? null
+ : null;
+ const menuIsFiltered = menuField
+ ? treeNamesField(config.filters, menuField.key, measureCols)
+ : false;
+ // Item 11c — is the menu's field the current frozen boundary (its Pin would be a no-op)?
+ const menuVisIndex = menuField ? visibleKeys.indexOf(menuField.key) : -1;
+ const menuPinnedTo = menuVisIndex >= 0 && frozenN > 1 && menuVisIndex + 1 === frozenN;
+ // Item 10 — the toolbar's mode switcher (grid-only on windowed tables, see displayMode).
+ const modeControl = serverWindowed || embedded ? undefined : (
+
+ );
+ // ⭐ Wave-23 C7 — the open document, resolved exactly like the picker's value below it:
+ // the OPTIMISTIC edit wins over the raw record, so a save the server has not echoed yet is
+ // what re-opening the cell shows (the echo-suppression law — reading the raw row here would
+ // make a just-saved document appear to revert).
+ const jsonField = jsonAt ? fieldByKey.get(jsonAt.fieldKey) : undefined;
+ const jsonRow = jsonAt ? rawRows.find((r) => r.pid === jsonAt.pid) : undefined;
+ const jsonValue = String(
+ (jsonAt && overlayEdits[jsonAt.pid]?.[jsonAt.fieldKey])
+ ?? (jsonField && jsonRow ? jsonRow[jsonField.key] : "")
+ ?? ""
+ );
+ const linkField = linkAt ? fieldByKey.get(linkAt.fieldKey) : undefined;
+ const linkRow = linkAt ? rawRows.find((row) => row.pid === linkAt.pid) : undefined;
+ const linkValue = String(
+ (linkAt && overlayEdits[linkAt.pid]?.[linkAt.fieldKey])
+ ?? (linkField && linkRow ? linkRow[linkField.key] : "")
+ ?? ""
+ );
+ const linkedRecordIds = [...new Set(
+ linkValue.split(",").map((part) => Number(part.trim())).filter(
+ (pid) => Number.isInteger(pid) && pid > 0
+ )
+ )];
+ const pickerField = picker ? fieldByKey.get(picker.fieldKey) : undefined;
+ // A `user` field's choices come from the HOST's real user list, a `select`'s from its own
+ // definition — so an assignee is always someone who can log in, and a status is always one of
+ // the choices the column was created with.
+ const pickerChoices = !pickerField
+ ? []
+ : pickerField.type === "user"
+ ? payload?.userOptions ?? []
+ : choiceOptions(pickerField);
+ const pickerRow = picker ? rawRows.find((r) => r.pid === picker.pid) : undefined;
+ const pickerValue = String(
+ (picker && overlayEdits[picker.pid]?.[picker.fieldKey])
+ ?? (pickerField && pickerRow ? pickerRow[pickerField.key] : "")
+ ?? ""
+ );
+ // A multiselect cell is a SET: choices TOGGLE and the picker stays open for the next pick
+ // (item 5). Single select/user keep pick-and-close.
+ const pickerMulti = pickerField?.type === "multiselect";
+ const pickerParts = pickerMulti ? splitMulti(pickerValue) : [];
+ /** Item 6 — which choice receives focus when the picker opens: the current value where it
+ * still exists in the list, else the first choice. Keyboard-only editing starts HERE. */
+ const pickerFocusChoice = !pickerChoices.length
+ ? null
+ : pickerMulti
+ ? pickerParts.find((p) => pickerChoices.includes(p)) ?? pickerChoices[0]
+ : pickerChoices.includes(pickerValue)
+ ? pickerValue
+ : pickerChoices[0];
+
+ return (
+ // `data-today` is the TENANT'S day, exactly as the payload delivered it. It is here so the
+ // date every relative condition resolves against is OBSERVABLE rather than inferred: a live
+ // QA that reads its own clock instead compares two engines on "now" and fails whenever a run
+ // straddles midnight — which is the flake harness/windows.py takes `today` as a parameter to
+ // avoid, reintroduced one layer up. It cost a false failure (57 vs 55, both correct, one
+ // computed either side of a date change) to notice.
+ // `data-measure-rules` / `data-measure-sets` are BUG-1's handshake, published the same way
+ // and for the same reason as `data-today`: a measure condition stuck on "Calculating…"
+ // renders identically whether the rule carries no id, the host resolved nothing, or the two
+ // sides name the rule differently — and only the first of those is ours. A console.log
+ // cannot serve this: the grid runs in an iframe that is CROSS-ORIGIN on the Space, and every
+ // Streamlit rerun replaces it. An attribute survives the remount and is one `get_attribute`
+ // away from any harness, local or deployed.
+
+ {/* ⭐ wave17 R1 / C-LOCKV — the COHORT SIDEBAR is gone, and with it the last surface that
+ treated a cohort as its own kind of object. `CohortSidebar` was the retired `#/cohort`
+ page's left panel: a second rail, with its own rename, delete, export and folder
+ machinery, over `workspace.lists`. Under R1 a cohort IS a saved view, so the Views rail
+ below renders every one of them and the second rail has nothing to switch between.
+ ⚠ `lists` is NOT gone with it — it stays the membership channel that feeds `cohortSets`
+ (C-LOCKV point 2), which is what the lock resolves against. */}
+ {!hideViews && (
+
setSelectFromFile(true)}
+ /* ⭐ WAVE-29 T25 (owner item 6) — the Import door, offered ONLY where records can be
+ added. `canMutateRecords` is `isUserTable && recordsMutable`: the same predicate the
+ "+" row and the delete path read, so a locked database (an Odoo mirror, an
+ automation-owned child table) has no Import row at all rather than an Import that
+ fails at the server. */
+ onImport={canMutateRecords ? () => setImportOpen(true) : undefined}
+ onAddToList={serverWindowed ? undefined : addToList}
+ today={today}
+ onExport={serverWindowed ? undefined : exportViewData}
+ folders={folders}
+ folderIdOf={folderIdOf}
+ onFolderCreate={(name, icon) => {
+ const id = newFolderId();
+ // Bounded: reconcileFolders drops a local entry once the echo
+ // carries it or its create stamp ages out, so this only has to not
+ // grow without limit.
+ // C5 (I15): the icon rides the LOCAL entry too, or the folder the user just
+ // styled renders grey for one round trip and then changes colour under them.
+ setLocalFolders((prev) => [...prev.slice(-31), { id, name, order: 9_999, icon }]);
+ stampFolder((p) => ({ ...p, created: { ...p.created, [id]: Date.now() } }));
+ emitHostEvent({
+ id: eventId("foldnew"),
+ type: "folder_create",
+ surface: "views",
+ folderId: id,
+ name,
+ icon,
+ });
+ }}
+ onFolderRename={(folderId, name) => {
+ stampFolder((p) => ({
+ ...p,
+ renamed: { ...p.renamed, [folderId]: { at: Date.now(), name } },
+ }));
+ emitHostEvent({
+ id: eventId("foldren"),
+ type: "folder_rename",
+ surface: "views",
+ folderId,
+ name,
+ });
+ }}
+ onFolderDelete={(folderId) => {
+ // The tombstone is stamped BEFORE the emit: between the click and the
+ // echo the payload still lists the folder, and rendering it again for
+ // one round trip is the delete blip (folders.ts).
+ stampFolder((p) => ({ ...p, deleted: { ...p.deleted, [folderId]: Date.now() } }));
+ emitHostEvent({
+ id: eventId("folddel"),
+ type: "folder_delete",
+ surface: "views",
+ folderId,
+ });
+ }}
+ onFolderDuplicate={(folderId) => {
+ const newId = newFolderId();
+ const src = folders.find((f) => f.id === folderId);
+ setLocalFolders((prev) => [
+ ...prev.slice(-31),
+ { id: newId, name: `${src?.name ?? "Folder"} copy`, order: 9_999 },
+ ]);
+ stampFolder((p) => ({ ...p, created: { ...p.created, [newId]: Date.now() } }));
+ emitHostEvent({
+ id: eventId("folddup"),
+ type: "folder_duplicate",
+ surface: "views",
+ folderId,
+ newId,
+ });
+ }}
+ onFolderReorder={(order) => {
+ // A-S4-1 (item 19). S4 built the drag and the reconcile behind an OPTIONAL prop, so
+ // its absence silently disabled the whole feature rather than breaking anything —
+ // which is why every gate stayed green while folders would not drag at all. The
+ // stamp is what stops the rail snapping back for one round trip (the echo window).
+ stampFolder((p) => ({ ...p, ordered: { at: Date.now(), order } }));
+ emitHostEvent({
+ id: eventId("foldord"),
+ type: "folder_reorder",
+ surface: "views",
+ order,
+ });
+ }}
+ /**
+ * ⭐ WAVE 27 · OWNER ITEM 5 (contract C7) — the rail's new VIEW order.
+ *
+ * Three things happen and each one is needed:
+ * · the LOCAL list is reordered, so the row lands where it was dropped and stays
+ * there for this render;
+ * · a STAMP is written, so the next workspace echo — which is one round trip
+ * behind and still carries the server's order — does not snap it back
+ * (`applyViewOrder` in folders.ts holds it for the echo window and no longer);
+ * · ONE event carries the WHOLE order, never a delta. A partial list cannot say
+ * where an UNNAMED view went, so "moved B after C" leaves every other position
+ * to be re-derived by two sides that can disagree. The server's own
+ * `folder_reorder` note states this rule; the same rule, same door.
+ *
+ * ⚠ AMENDMENT C7-A1 (2026-08-08, SESSION C): the event is
+ * `{type: "view_reorder", surface: "views", order}` — `surface`, not `workspace`.
+ * The contract said `{workspace, order}`; the door `grid_events.py:1299` actually
+ * opens on `surface ∈ FOLDER_SURFACES`, and the TABLE is resolved from the request
+ * context rather than the payload (`_store_of(ctx)`). Emitting a key the handler
+ * does not read would have shipped a drag that persists nothing, behind a green
+ * gate on each side.
+ */
+ onViewReorder={(order) => {
+ setViews((cur) => {
+ const byId = new Map(cur.map((v) => [v.id, v]));
+ const out: SavedView[] = [];
+ const placed = new Set();
+ for (const id of order) {
+ const v = byId.get(id);
+ if (!v || placed.has(id)) continue;
+ placed.add(id);
+ out.push(v);
+ }
+ // Anything the order does not name keeps its current place, after the named
+ // ones — never dropped. A view created in another tab is not evidence the
+ // drop was wrong.
+ for (const v of cur) if (!placed.has(v.id)) out.push(v);
+ return out;
+ });
+ stampFolder((p) => ({ ...p, orderedViews: { at: Date.now(), order } }));
+ // ⛔ ONE CAST, AT ONE SITE, WITH A RATCHET BEHIND IT (ASK ->D, wave-27 mailbox
+ // C-6). `HostEvent`'s union lives in `types.ts`, which is SESSION D's file, and
+ // D owns C7's persistence half — so the member `{ id; type: "view_reorder";
+ // surface: FolderSurface; order: string[] }` is theirs to add. Until it lands
+ // this emit cannot be typed from here.
+ // ⚠ The gap is DECLARED, not hidden: `verify_wiring.py`'s
+ // `PENDING_UNION_MEMBERS` carries `view_reorder`, prints the ask, and goes RED
+ // the moment `types.ts` gains the member — which forces this cast to be deleted
+ // in the same change. That is the D-70 ratchet's shape, and it fired on its own
+ // this wave, which is the argument for reusing it rather than a comment.
+ emitHostEvent({
+ id: eventId("vieword"),
+ type: "view_reorder",
+ surface: "views",
+ order,
+ } as unknown as HostEvent);
+ }}
+ onItemMove={(viewId, folderId) => {
+ stampFolder((p) => ({
+ ...p,
+ moved: { ...p.moved, [viewId]: { at: Date.now(), folderId } },
+ }));
+ emitHostEvent({
+ id: eventId("itemmove"),
+ type: "item_move",
+ surface: "views",
+ itemId: viewId,
+ folderId,
+ });
+ }}
+ folderAddPreview={serverWindowed ? undefined : folderAddPreview}
+ onFolderAddToList={serverWindowed ? undefined : folderAddToList}
+ viewer={viewer}
+ onToggleLock={toggleViewLock}
+ userOptions={payload?.userOptions}
+ />
+ )}
+
+ v.id !== activeViewId)}
+ onCopyConfig={(patch) => updateConfig({ ...config, ...patch })}
+ fields={fields}
+ visible={visible}
+ lockedKey={lockedKey}
+ onColumnVisible={setColumnVisible}
+ onColumnsVisible={(keys) =>
+ updateConfig({
+ ...config,
+ visible: [...new Set([lockedKey, ...keys])],
+ })
+ }
+ // Owner item 23 — the Hide-fields panel lists and reorders COLUMN order, so it is
+ // handed the ordered projection, not the definition-ordered `fields` above.
+ // `order` is `useGridColumns`' reconciled copy (locked key first, unknown keys
+ // dropped, new keys appended), so the list can never show a field the grid does not
+ // have or omit one it does.
+ orderedFields={orderedFields}
+ onFieldOrder={(keys) => updateConfig({ ...config, order: keys })}
+ deletableKeys={deletableKeys}
+ onDeleteField={deleteField}
+ colorOn={config.colorBy !== null}
+ onToggleColor={(on) =>
+ updateConfig({
+ ...config,
+ colorBy: on
+ ? fields.find((field) => field.type === "status")?.key ?? null
+ : null,
+ })
+ }
+ rowHeightMode={config.rowHeightMode}
+ onRowHeightMode={(rowHeightMode) => updateConfig({ ...config, rowHeightMode })}
+ filters={config.filters}
+ onFilters={(filters) => updateConfig({ ...config, filters })}
+ filterConj={config.filterConj ?? "and"}
+ onFilterConj={(filterConj) => updateConfig({ ...config, filterConj })}
+ sorts={config.sorts}
+ onSorts={(sorts) => updateConfig({ ...config, sorts })}
+ // the toolbar must not advertise a grouping the pipeline is not applying
+ groupBy={serverWindowed ? null : config.groupBy}
+ onGroupBy={(groupBy) => updateConfig({ ...config, groupBy })}
+ statusValues={statusValues}
+ measures={measures}
+ unresolvedCount={unresolvedCount}
+ lists={lists}
+ pendingMeasureCount={pendingMeasureCount}
+ search={search}
+ onSearch={setSearch}
+ recordCount={recordCount}
+ scopeCounts={payload?.counts}
+ pool={payload?.workspace?.pool}
+ serverWindowed={serverWindowed}
+ cohortAction={cohortAction}
+ filterSeed={filterSeed}
+ modeControl={modeControl}
+ />
+ {/* Wave-10 item 9: the view's description used to render HERE, as a full-width band
+ between the toolbar and the grid — it cost every row a strip of vertical space and
+ read as an alert rather than as a label. It now sits under the view's name in the
+ rail (ViewSidebar), where it annotates the thing it describes. */}
+ setExpandAt((prev) => (prev === null ? prev : null))}
+ >
+ {displayMode === "grid" && (
+ <>
+
openHeaderMenu(column, bounds)}
+ onHeaderClicked={onHeaderClicked}
+ onHeaderContextMenu={onHeaderClicked}
+ onKeyDown={onGridKeyDown}
+ maxColumnWidth={600}
+ getCellsForSelection={true}
+ rowSelectionMode="multi"
+ smoothScrollX
+ smoothScrollY
+ width={gridSize.width}
+ height={gridSize.height}
+ customRenderers={[ratingCellRenderer, userCellRenderer, imageCellRenderer]}
+ headerIcons={HEADER_ICONS}
+ rightElement={embedded ? undefined : (
+ // Wave-6 item 4 — the "+" of the header row: the create-field form,
+ // insert-at-end.
+ //
+ // ⚠ Wave-20 owner item 5 UNPINNED it. It used to be `sticky: true` ("so it
+ // stays visible however far the columns scroll"), which parked it against the
+ // right edge of the WINDOW — on a four-column table that is half a screen of
+ // white between the last field and the control that adds the next one, and it
+ // reads as page chrome rather than as part of the table. It now sits
+ // immediately after the last column, where Airtable puts it.
+ //
+ // TWO changes are needed and neither works alone: `sticky:false` stops glide
+ // pinning the wrapper (`infinite-scroller.js:204` sets `right` only when
+ // sticky), and the CSS region kills `.dvn-spacer`'s `flex-grow: 1`, which
+ // would otherwise expand to fill the scroller and push the button right back
+ // to the edge. Consequence, accepted: on a table wider than the viewport the
+ // "+" is off-screen until you scroll to the end of the columns — which is
+ // where the new column is going to land anyway.
+ {
+ const r = event.currentTarget.getBoundingClientRect();
+ setPlusMenu({
+ left: r.left,
+ top: r.top,
+ right: r.right,
+ bottom: r.bottom,
+ width: r.width,
+ height: r.height,
+ });
+ }}
+ >
+ +
+
+ )}
+ rightElementProps={{ sticky: false, fill: false }}
+ // Owner item 4 (R8) — the trailing "+" row, USER DATABASES ONLY. Passing
+ // `onRowAppended` is what makes glide paint the ghost row at all, so the
+ // affordance exists exactly where a POST can succeed. See `appendRow`.
+ onRowAppended={canMutateRecords ? appendRow : undefined}
+ trailingRowOptions={
+ canMutateRecords
+ ? {
+ // The hint sits in the identity column (targetColumn 0), which is the
+ // one a new record is named in — the same cell the cursor lands on.
+ hint: "New record",
+ sticky: false,
+ targetColumn: 0,
+ }
+ : undefined
+ }
+ />
+ {/* ═══ W18-B VOID ═══ (wave 18, owner item 1b / ruling R12) — the two rectangles.
+
+ They are SIBLINGS of the canvas, immediately after it and before everything else
+ `.cg-grid-box` holds, which is the whole of their stacking story: `z-index: auto`
+ plus tree order puts them over the canvas, and `.cg-selbar` / the popovers carry
+ explicit `--cg-z-*` and stay over them. `pointer-events: none` keeps glide's
+ scroller — which is UNDER this and owns every mouse event the grid reacts to —
+ hit-testable straight through them.
+
+ The right-hand one starts BELOW the header rather than at 0: in `Airtable 7.png`
+ the header band continues past the last column in its own near-white (#FBFCFE at
+ (1250,141)), and it is where the add-field "+" lives. The two rectangles overlap
+ in the bottom-right corner; they are the same flat colour, so the overlap is not
+ visible and neither one needs to know about the other. */}
+ {gridVoid.below !== null && (
+
+ )}
+ {gridVoid.right !== null && (
+
+ )}
+ {/* ═══ end W18-B VOID ═══ */}
+ >
+ )}
+ {displayMode === "list" && (
+
+ )}
+ {displayMode === "calendar" &&
+ (calendarField ? (
+
+ ) : (
+
+ This table has no date field to place records by.
+
+ ))}
+ {displayMode === "kanban" &&
+ (kanbanField ? (
+
+ ) : (
+
+ This table has no status or single-select field to stack by.
+
+ ))}
+ {/* ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the SWIPE deck.
+ ⚠ No `field ? ... : empty-state` fork, and the asymmetry with kanban above is
+ deliberate. A kanban with no select field cannot exist, so the host answers for it.
+ A swipe deck with no binding is the ORDINARY state of a view somebody just created,
+ and SwipeView owns that state along with the three ways a stored binding rots —
+ each one re-opening the picker rather than being repaired by a guess. Forking here
+ would put a second, dumber explanation in front of the good one. */}
+ {displayMode === "swipe" && (
+ k !== lockedKey)}
+ scope={scope}
+ viewer={viewer}
+ docs={payload?.docs}
+ docPayload={payload?.docPayload}
+ onDocAdd={
+ payload?.docs
+ ? (pid, file) =>
+ emitHostEvent({
+ id: eventId("docadd"),
+ type: "doc_add",
+ pid,
+ docId: `doc_${eventId("d").slice(2, 14).replace(/[^a-z0-9]/gi, "")}`,
+ ...file,
+ })
+ : undefined
+ }
+ onDocFetch={
+ payload?.docs
+ ? (pid, docId) =>
+ emitHostEvent({ id: eventId("docget"), type: "doc_fetch", pid, docId })
+ : undefined
+ }
+ onDocDelete={
+ payload?.docs
+ ? (pid, docId) =>
+ emitHostEvent({ id: eventId("docdel"), type: "doc_delete", pid, docId })
+ : undefined
+ }
+ />
+ )}
+ {/**
+ * ⭐⭐ WAVE-29 T29 / CONTRACT C4 (owner R7) — THE FORM INTERFACE, mounted.
+ *
+ * ⛔ THE WAVE'S ONLY CROSS-FENCE WIRING, and the single most reliably-dropped artifact
+ * in this protocol: four consecutive waves lost one, and one wave shipped four dead
+ * features behind 43 green gates. C declared it, A carries the `verify_wiring` row,
+ * and this is the mount — three parties, so the session that REQUESTS a wiring cannot
+ * be the one that certifies it.
+ *
+ * ⛔ EIGHT PROPS, ALL REQUIRED, and none of them guessed — C published the exact names
+ * and the exact expressions in its mailbox before this ticket started. `spec` is
+ * required-but-NULLABLE and the `?? null` is the contract: `displaySpec?.form` is
+ * `undefined`, and widening the prop to accept it would be the optional-prop
+ * degradation wearing another hat.
+ * ⚠ `readOnlyReason` carries the `canEdit` reason ONLY. A locked database is a
+ * different sentence and `FormInterface` owns it, because it is a property of the
+ * database rather than of this viewer's grant.
+ */}
+ {displayMode === "form" && (
+
+ )}
+ {/* Wave-8 I19c → wave-9 I10 (C2) — CHART is a VIEW like the rest: the charts
+ are computed from `modeDataRows`, the very rows the grid counts, so a filter
+ change moves the charts and the toolbar count together.
+ ⚠ Only 'chart' is tested, never 'dashboard': `cleanDisplay` normalises the
+ legacy value on READ, so a stored 'dashboard' has already become 'chart' by
+ the time it reaches here. Testing both would hide a broken normalisation. */}
+ {displayMode === "chart" && (
+
+ )}
+ {/* Owner item 7 (C-TS) — Time series is a VIEW: metric rows x bucket columns over
+ the pids the current filter kept, so the table and the series answer the same
+ question. The panel is HOST-AGNOSTIC by contract (RECORD re-mounts it inside the
+ record modal for item 13), which is why it takes pids and fields rather than
+ reaching into anything here. */}
+ {displayMode === "timeseries" && (
+
+ )}
+ {/* ═══ W18-C CATALOG ═══ Owner item 4 (C6) — Catalog is a VIEW, and the only one
+ whose payload is authored CONTENT rather than an arrangement of the rows. It
+ joins its stored product codes against the WHOLE pool (`rawRows`), not the
+ filtered rows: a finished page must keep printing the products it names after
+ somebody narrows the view. The filter still has a job — `catalogFilteredCodes`
+ is the "add what this view shows" door — but it narrows what you can ADD, never
+ what a page prints.
+
+ ⚠ Gated on the product surface. A catalog joins products to their images by
+ `code`, which is `product_data`'s business key AND the C2-ASSET key; on the
+ customer table the primary field is a customer NAME, and joining on it would ask
+ the asset route for a picture of a person. The refusal says so in words rather
+ than rendering an empty designer. */}
+ {displayMode === "catalog" &&
+ (scope === "product" ? (
+
+ ) : (
+
+ A catalog prints products. Open the Products table to build one.
+
+ ))}
+ {/* ═══ end W18-C CATALOG ═══ */}
+ {/* Wave-7 item W11c (C3) — Map is a VIEW: same pipeline rows, pins for rows
+ with host-supplied coords, click-through to the same record detail, and
+ an honest no-location chip. */}
+ {displayMode === "map" && fieldByKey.get(lockedKey) && (
+
+ )}
+ {/* Owner item 19 — the hover-only Expand. The COUNTERPART to item 17: the single
+ click that used to open a record now highlights it, and this is where opening
+ moved to. `position: fixed` because glide reports viewport coordinates (the same
+ space `.cg-header-tip` is placed in).
+
+ ⚠ `onMouseDown` preventDefault, not just onClick: mousedown would otherwise reach
+ the canvas underneath, move the selection to the primary cell, and repaint the
+ row a frame before the panel opens — a visible flicker on every expand.
+
+ Rendered only in `grid` mode: list / calendar / kanban / map have their own
+ open affordances and glide is not mounted to report bounds for them. */}
+ {!embedded && displayMode === "grid" && expandAt && (
+ event.preventDefault()}
+ onClick={() => setDetailPid(expandAt.pid)}
+ >
+ {/* Airtable's expand mark: two corner brackets pushing apart. SVG, never a
+ glyph — no emojis in this UI, and a unicode arrow renders differently on
+ every platform. Sized in the parent's em so it tracks the button. */}
+
+
+
+
+ )}
+ {/* Wave-5 item 6 — the header description tip. pointer-events:none + aria-hidden:
+ it can never become a click target, so the control under it stays clickable
+ (the tooltip-swallows-the-next-click trap, paid for once already). */}
+ {headerTip && (
+
+ {headerTip.text}
+
+ )}
+ {/* I2 — the cell tip. `pointer-events: none` + `aria-hidden` are not styling here,
+ they are the contract: the control under the tip must stay clickable, and a
+ screen reader already has the cell's own value. */}
+ {cellTip && (
+
+ {cellTip.text}
+
+ )}
+ {/* Owner item 2's point: checked customers accumulate, and the selection DOES
+ something. FLOATED over the grid, never in the flow: a bar that reflows the grid
+ on first selection shifts every row under the user's cursor mid-click-run (and
+ broke this gate's own coordinate math before it broke a user). It states the
+ count and offers "Add to cohort" over exactly the checked pids — the same guarded
+ add_to_list event the view menu uses, so the host treats both alike. */}
+ {!embedded && selectedPids.size > 0 && !serverWindowed &&
+ (displayMode === "grid" || displayMode === "map") && (
+
+
+ {selectedPids.size.toLocaleString()} selected
+
+ {/* ⭐ 2026-08-06 (owner) — *"right now it only shows 'Add to cohort'"*. On a user
+ database the checked rows are RECORDS the reader owns, and there was no door to
+ remove one; the whole delete path existed server-side with nothing calling it.
+ ⚠ ONLY on a `ut_` database: the customer and product grids are projections of
+ Odoo, where a "delete" would be a write to somebody else's system of record —
+ the server refuses it, and a button that must be refused is worse than none. */}
+ {canMutateRecords && (
+ void deleteRecords([...selectedPids])}
+ >
+ {delArmed && delArmed === [...selectedPids].join(",")
+ ? "Delete anyway?"
+ : `Delete ${selectedPids.size === 1 ? "record" : "records"}`}
+
+ )}
+ {cohortMode ? (
+ // Item 2c: on the Cohort page the selection EDITS MEMBERSHIP of the fixed set —
+ // "Remove from cohort" replaces "Add to cohort". Direct (no popover): the
+ // checked set is explicit, the button says what it does, and re-adding is one
+ // picker away.
+
+ Remove from cohort
+
+ ) : (
+ {
+ // Item 4: seed the new-cohort name as ` · ` — today from
+ // the PAYLOAD verbatim, never the browser clock; no date when absent.
+ if (!selAddOpen) {
+ const vn = activeView?.name ?? "Customers";
+ setSelListName(today ? `${vn} · ${today}` : vn);
+ }
+ setSelAddOpen(!selAddOpen);
+ }}
+ aria-expanded={selAddOpen}
+ aria-haspopup="dialog"
+ >
+ Add to cohort
+
+ )}
+ {/* ⭐ Owner item 9 — the counterpart, and it renders ONLY when it can do
+ something: the checked rows have to be in a cohort this viewer may edit for
+ "Remove from cohort…" to mean anything. Same bar, same selection, opposite
+ direction — the ellipsis says a picker follows, because unlike cohort mode
+ the target is not implied by where you are standing. */}
+ {!cohortMode && removableLists.length > 0 && (
+ setSelRemoveOpen((open) => !open)}
+ aria-expanded={selRemoveOpen}
+ aria-haspopup="dialog"
+ >
+ Remove from cohort…
+
+ )}
+
+ Clear
+
+
+ )}
+
+ {/* Owner item 8 — the cap is VISIBLE, beside the control that lifts it. The toolbar
+ count above still states the full matched count; this bar says what is painted.
+ Grid + list only: calendar/kanban project the FULL set with their own stated
+ bounds (a month, a stack's Show-all). */}
+ {capped && (displayMode === "grid" || displayMode === "list") && (
+
+
+ Showing first {shownRecords.toLocaleString()} of{" "}
+ {recordCount.toLocaleString()}
+
+ setDisplayCap((c) => c + DISPLAY_STEP)}
+ >
+ See more
+
+ setDisplayCap(visibleRows.length)}
+ >
+ Show all
+
+
+ )}
+
+
+ {menuField && columnMenu && (
+
+ * t.key !== scope)` and said so honestly: "a legal relation in Airtable and a confusing
+ * one to explain in a one-line picker … left out until somebody asks". It was a taste
+ * call, not a constraint, and nothing below the picker ever needed it — `_clean_link`
+ * only requires a `ut_*` key, and `sync_reciprocal_link` puts the backlink on the target
+ * whichever table that is. "This lead is a duplicate of that lead" and "this record's
+ * parent" are the ordinary cases, and both were unbuildable.
+ *
+ * ⚠ THE UNFILTERED LIST ALSO REPAIRS THE ROLLUP EDITOR, which is the half a one-line
+ * change hides: it resolves the fold's source columns with
+ * `linkTargets.find(t => t.key === chosenLink.link.table)`, so a rollup over a self-link
+ * would have offered an EMPTY column picker and looked like the rollup was broken.
+ * ⚠ Both `` renders take the same list — the full menu and the "+" create
+ * menu. One of the two is the half-landing this repo keeps paying for.
+ */
+ linkTargets={linkTargets}
+ rollupSourceOffer={rollupSourceOffer}
+ locked={menuField.key === lockedKey}
+ schemaLocked={isSchemaLocked(menuField)}
+ viewer={viewer}
+ sortedDir={menuSortedDir}
+ isFiltered={menuIsFiltered}
+ isGrouped={config.groupBy === menuField.key}
+ groupable={isGroupableField(menuField, lockedKey)}
+ // Item 12 (R8) — the kanban's "Add option" opens THIS menu straight into its Edit-field
+ // pane. Absent for every other opening, which is the action list as before.
+ initialPane={columnMenu.pane}
+ onClose={() => setColumnMenu(null)}
+ onNote={(note) => saveField({ ...menuField, note })}
+ onHide={() => {
+ setColumnVisible(menuField.key, false);
+ setColumnMenu(null);
+ }}
+ onCreate={createField}
+ onChangeField={(newKey) => changeField(menuField.key, newKey)}
+ onCreateAndSwap={(label, type, options, extra) =>
+ createAndSwapField(menuField.key, label, type, options, extra)
+ }
+ onProfileFlag={isUserTable ? setProfileFlag : undefined}
+ onFieldConfig={isUserTable ? (patch) => setFieldConfig(menuField.key, patch) : undefined}
+ onRetype={
+ // Item 2 — a CREATED field retypes IN PLACE from the pane.
+ //
+ // ⭐⭐ WAVE-29 T27 (owner item 14): the predicate is `isUserSchemaField`, not the raw
+ // `custom` flag. `user_tables._clean_field` never stamps `custom`, so EVERY column on
+ // a `ut_*` database failed the old test — the Field-type picker vanished, the pane was
+ // left showing only the "Change field" swap picker (a different feature entirely,
+ // holding the autofocus), and the owner reported the innocent control as broken.
+ isUserSchemaField(menuField, isUserTable)
+ ? (type, options, extra) =>
+ retypeField(menuField.key, type, options, extra)
+ : undefined
+ }
+ onRename={
+ // Item 5 — rename is a def edit on created strata.
+ //
+ // ⛔ T27: withheld by the SAME flag, which is why the Name box went disabled saying
+ // *"A source field keeps its name and type from the data source"* about a column the
+ // user created themselves. A `ut_*` definition field renames through the DEFINITION
+ // door; `saveField` would fork it into a private overlay copy of itself (the reason
+ // `onFormula` below already branches this way).
+ !isUserSchemaField(menuField, isUserTable)
+ ? undefined
+ : menuField.custom
+ ? (label) => saveField({ ...menuField, label })
+ : (label) => setFieldConfig(menuField.key, { label })
+ }
+ onFormula={
+ // Owner item 8 — edit a created FORMULA field's source in place; the optional
+ // label rides the SAME upsert (see ColumnMenu's one-upsert rule).
+ //
+ // ⭐⭐ 2026-08-10 — TWO STRATA, TWO DOORS, and the gate is `custom` rather than the
+ // type. A formula column on a `ut_*` database now lives in the SHARED definition, and
+ // definition fields carry no `custom` flag (`_clean_field` never stamps one) — so the
+ // old single condition made the expression editor DISAPPEAR for exactly the columns
+ // this change creates: creatable and then permanently frozen, which is a worse state
+ // than the one it replaces. `setFieldConfig` PATCHes the definition; `saveField`
+ // writes this user's overlay, and sending a definition field down that path would
+ // fork the column into a private second copy of itself.
+ menuField.type !== "formula"
+ ? undefined
+ : isUserTable && !menuField.custom
+ ? (formula, label) =>
+ setFieldConfig(menuField.key, {
+ formula,
+ ...(label ? { label } : {}),
+ })
+ : menuField.custom
+ ? (formula, label) =>
+ saveField({
+ ...menuField,
+ formula,
+ ...(label ? { label } : {}),
+ })
+ : undefined
+ }
+ onPeriod={
+ menuField.measure
+ ? (w) => changeMeasurePeriod(menuField.key, w)
+ : undefined
+ }
+ onDelete={
+ menuField.custom && menuField.key !== lockedKey
+ ? () => {
+ deleteField(menuField.key);
+ setColumnMenu(null);
+ }
+ : /**
+ * ⭐⭐ 2026-08-10 — the DEFINITION stratum's own delete.
+ *
+ * ⛔ NARROWED TO THE THREE KINDS THAT HOLD NO DATA OF THEIR OWN, which is
+ * `preset_editable`'s argument reused rather than a new one: a link, a rollup and
+ * a formula are all QUESTIONS asked of other cells, so removing one destroys no
+ * measurement and can be undone by asking it again. Every other definition column
+ * on a `ut_*` database holds real values, and offering a type-blind Delete over
+ * those is a destructive widening this change has no mandate for (booked as
+ * D-114). It is also exactly the three kinds `createField` now routes to the
+ * definition — so what a user can make here, a user can unmake.
+ * ⚠ `schemaLocked` still gates the button inside ColumnMenu, so a pre-set link
+ * never shows it; a pre-set ROLLUP does, by the same owner ruling that made it
+ * editable, and the confirm says it will come back.
+ */
+ isUserTable &&
+ !menuField.custom &&
+ menuField.key !== lockedKey &&
+ (menuField.type === "link" || menuField.type === "rollup" ||
+ menuField.type === "formula")
+ ? () => {
+ deleteDefinitionField(menuField.key);
+ setColumnMenu(null);
+ }
+ : undefined
+ }
+ onDuplicate={
+ // Creatable strata only — a base Odoo field offers no Duplicate (contract).
+ menuField.custom ? () => duplicateField(menuField) : undefined
+ }
+ onPermissions={
+ // Creatable strata only; ColumnMenu further gates on the viewer (creator/admin).
+ menuField.custom
+ ? (edit) => saveField({ ...menuField, permissions: { edit } })
+ : undefined
+ }
+ onFormat={(format) => saveField({ ...menuField, format })}
+ /**
+ * ⭐ WAVE-29 T33 (owner item 17) — the column summary, through whichever door owns this
+ * column's definition. Same two-strata split as `onRename` above: a `ut_*` definition
+ * field PATCHes the definition (`_clean_field` stores `agg`), everything else rides the
+ * ordinary overlay upsert. ⚠ `undefined` CLEARS it, and the clear has to travel — a
+ * PATCH that merely omits `agg` keeps the stored one, which is the stickiness the money
+ * columns depend on.
+ */
+ onAggregate={(agg) =>
+ isUserTable && !menuField.custom
+ ? setFieldConfig(menuField.key, { agg: agg ?? "" })
+ : saveField({ ...menuField, agg })
+ }
+ onSort={(dir) =>
+ updateConfig({ ...config, sorts: [{ colId: menuField.key, dir }] })
+ }
+ onClearSort={() =>
+ updateConfig({
+ ...config,
+ sorts: config.sorts.filter((s) => s.colId !== menuField.key),
+ })
+ }
+ onFilterBy={() =>
+ // Item 6c — a MEASURE-carrying column seeds the equivalent MEASURE condition
+ // (same measure key + window, value empty); every other filterable column seeds
+ // an ordinary column condition. Same signal, one consumer (the Toolbar).
+ setFilterSeed((prev) => ({
+ n: (prev?.n ?? 0) + 1,
+ ...(menuField.measure
+ ? { measure: { key: menuField.measure.key, window: menuField.measure.window } }
+ : { key: menuField.key }),
+ }))
+ }
+ pinnedTo={menuPinnedTo}
+ onPinTo={
+ // Item 11c — pinning the identity column is the same as unpinning; skip it.
+ menuVisIndex > 0 && !menuPinnedTo
+ ? () => pinFieldTo(menuField.key)
+ : undefined
+ }
+ onUnpin={frozenN > 1 ? unpinFields : undefined}
+ scopeChoice={scopeChoice}
+ onClearFilter={() =>
+ updateConfig({
+ ...config,
+ filters: dropFieldFromTree(config.filters, menuField.key, measureCols),
+ })
+ }
+ onGroupByField={() => updateConfig({ ...config, groupBy: menuField.key })}
+ onClearGroup={() => updateConfig({ ...config, groupBy: null })}
+ userOptions={payload?.userOptions ?? []}
+ measures={measures}
+ />
+ )}
+
+ {/* Wave-6 item 4 — the header "+": the SAME ColumnMenu, opened straight into its
+ create form in insert-at-end mode. Anchored on the button; the nominal field is the
+ locked identity column (only its position matters — the create form never shows it).
+ The action/pane handlers below are unreachable in this mode (create-only; Cancel
+ closes), supplied inert to satisfy the surface. */}
+ {plusMenu && fieldByKey.get(lockedKey) && (
+ setPlusMenu(null)}
+ onNote={() => undefined}
+ onHide={() => undefined}
+ onCreate={createField}
+ onChangeField={() => undefined}
+ onCreateAndSwap={() => undefined}
+ onSort={() => undefined}
+ onClearSort={() => undefined}
+ onFilterBy={() => undefined}
+ onClearFilter={() => undefined}
+ onGroupByField={() => undefined}
+ onClearGroup={() => undefined}
+ scopeChoice={scopeChoice}
+ userOptions={payload?.userOptions ?? []}
+ measures={measures}
+ // ⛔⛔ 2026-08-09 — THE SECOND HALF OF THE SAME DEFECT. The sibling instance above has
+ // passed these since the relational types shipped; this one never did, so a rollup
+ // created through "+" had no databases to resolve its link against and no source offer
+ // to switch modes with — the Column picker held its placeholder and nothing else.
+ // ⚠ TWO THINGS WERE BOTH WRONG (the fetch was gated on `columnMenu` and these props
+ // were absent), so fixing either alone would still have shipped an empty picker and
+ // read as "the fix did not work" — [[defects-that-mask-each-other]].
+ // ⚠ 2026-08-10 — THE SELF-EXCLUSION IS GONE FROM BOTH SITES (D-113). The line this
+ // replaces said the two call sites "must not disagree about that", and it was right for
+ // a reason it did not anticipate: the exclusion was removed from the sibling first and
+ // this copy kept it, so for one edit the "+" menu and the column menu offered DIFFERENT
+ // databases — caught by grepping the prop rather than by the build, which is happy
+ // either way. See the sibling's note for why self-links are offered now.
+ linkTargets={linkTargets}
+ rollupSourceOffer={rollupSourceOffer}
+ />
+ )}
+
+ {/* ⭐ WAVE 21 item 11 (R10 / contract C5) — "Select records from a list".
+ ⚠ `inView` is `visibleRows`, NOT `displayRows`. `displayRows` is a capped slice
+ (`DISPLAY_PAGE`) of what the view matched; matching over it would report every
+ record past the cap as "not found" — a lie that grows with the table. The whole
+ pool goes in beside it so a value the FILTER hides is reported as filtered, not
+ as missing. */}
+ {/* ⭐ WAVE-29 T25 — the import dialog, mounted as a conditional sibling exactly like
+ `SelectFromFile` above it. `scope` is the `ut_*` table key the import door is per. */}
+ {importOpen && (
+ {
+ signal(TOAST_EVENT,
+ `${count.toLocaleString()} record${count === 1 ? "" : "s"} imported.`);
+ signal(ROWS_STALE_EVENT);
+ }}
+ onClose={() => setImportOpen(false)}
+ />
+ )}
+
+ {selectFromFile && (
+ (vr.kind === "data" ? [vr.record] : []))}
+ wholeTable={computedRows}
+ viewName={activeView?.name ?? "this view"}
+ onSelect={(pids) => selectPids(pids, "replace")}
+ onClose={() => setSelectFromFile(false)}
+ />
+ )}
+
+ {/* The selection bar's cohort picker: existing cohorts, or a new one by name. Mirrors the
+ view menu's Add-to-list surface so the two ways of building a cohort read identically. */}
+ {selAddOpen && selAddRef.current && selectedPids.size > 0 && (
+ setSelAddOpen(false)}
+ dataKind="selection-add-to-list"
+ >
+ Add to locked view
+
+ Adds the {selectedPids.size.toLocaleString()} checked customer
+ {selectedPids.size === 1 ? "" : "s"}. The cohort stays fixed as the data changes.
+
+ {lists.map((l) => (
+ addSelectionToList(l.id, l.name)}
+ >
+ {l.name}
+
+ ))}
+
+
Lock records into a new view
+
setSelListName(event.target.value)}
+ onKeyDown={(event) => {
+ if (event.key === "Enter" && selListName.trim())
+ addSelectionToList("", selListName.trim());
+ }}
+ />
+
+ addSelectionToList("", selListName.trim())}
+ >
+ Create and add
+
+
+
+
+ )}
+
+ {/* ⭐ Owner item 9 — the remove picker. Every row states how many of the CHECKED rows it
+ would take out ("3 of 4"), because the selection and the cohort are two different sets
+ and the difference is the whole reason this needs a picker rather than a button. */}
+ {selRemoveOpen && selRemoveRef.current && removableLists.length > 0 && (
+ setSelRemoveOpen(false)}
+ dataKind="selection-remove-from-list"
+ >
+ Remove from locked view
+
+ Only the locked views holding one of the {selectedPids.size.toLocaleString()} checked
+ customer{selectedPids.size === 1 ? "" : "s"} are listed. The records themselves are
+ not touched.
+
+ {removableLists.map((l) => (
+ removeSelectionFromList(l.id)}
+ >
+ {l.name}
+
+ {l.hits.toLocaleString()} of {selectedPids.size.toLocaleString()}
+
+
+ ))}
+
+ )}
+
+ {/* Cohort mode's "+ Add customers" picker (item 2c): search the POOL, tick many, confirm
+ once. Candidates exclude current members; picks ACCUMULATE across searches, so the
+ confirm button carries the running total. The 50-row render cap is stated beside the
+ list — nothing silently truncated. */}
+ {cohortMode && addCustOpen && addCustRef.current && activeCohortId && (
+ setAddCustOpen(false)}
+ dataKind="cohort-add-customers"
+ >
+ Add customers
+
+ Search your customer pool and tick who to add to{" "}
+ {activeCohort?.name ?? "this locked view"} .
+
+ setAddCustQuery(e.target.value)}
+ />
+
+ {addCandidates.slice(0, 50).map((c) => (
+
+
+ setAddCustPicked((prev) => {
+ const next = new Set(prev);
+ if (next.has(c.pid)) next.delete(c.pid);
+ else next.add(c.pid);
+ return next;
+ })
+ }
+ />
+ {c.name}
+
+ ))}
+ {addCandidates.length === 0 && (
+
+ {addCustQuery.trim()
+ ? "No customer outside this locked view matches your search."
+ : "Every customer in your pool is already in this locked view."}
+
+ )}
+
+ {addCandidates.length > 50 && (
+
+ Showing the first 50 of {addCandidates.length.toLocaleString()} matches — keep
+ typing to narrow.
+
+ )}
+
+
+ {addCustPicked.size === 0
+ ? "Add customers"
+ : `Add ${addCustPicked.size.toLocaleString()} customer${
+ addCustPicked.size === 1 ? "" : "s"
+ }`}
+
+ setAddCustOpen(false)}
+ >
+ Cancel
+
+
+
+ )}
+
+ {/* ⭐ Wave-23 C7 — the document viewer. `onSave` is OMITTED, not disabled, when the reader
+ may not edit: an absent handler is what makes the raw tab a `` instead of a
+ textarea, so there is no editor to be refused by. */}
+ {jsonField && jsonAt && (
+ patchAndRecord(jsonAt.pid, { [jsonField.key]: next }, "a document")
+ : undefined
+ }
+ onClose={closeJson}
+ />
+ )}
+ {linkField?.type === "link" && linkField.link?.table.startsWith("ut_") && linkAt && (
+ patchAndRecord(
+ linkAt.pid,
+ { [linkField.key]: ids.join(",") },
+ "linked records"
+ )}
+ onClose={closeLink}
+ />
+ )}
+ {pickerField && picker && (
+ {
+ const fwd = e.key === "ArrowDown" || e.key === "ArrowRight";
+ const back = e.key === "ArrowUp" || e.key === "ArrowLeft";
+ if (!fwd && !back) return;
+ const items = Array.from(
+ e.currentTarget.querySelectorAll(
+ ".cg-pick-row, .cg-star-btn"
+ )
+ );
+ if (!items.length) return;
+ e.preventDefault();
+ const at = items.indexOf(document.activeElement as HTMLButtonElement);
+ const next = fwd
+ ? (at + 1) % items.length
+ : (at - 1 + items.length) % items.length;
+ items[next]?.focus();
+ }}
+ >
+ {/* Wave-5 item 11 — a RATING cell's picker is the star row itself. */}
+ {pickerField.type === "rating" && (
+
+ {Array.from({ length: ratingMax(pickerField) }, (_, i) => i + 1).map(
+ (n) => (
+ {
+ patchAndRecord(picker.pid, { [pickerField.key]: String(n) }, "a rating");
+ closePicker();
+ }}
+ >
+ = n} />
+
+ )
+ )}
+
+ )}
+ {pickerField.type !== "rating" && pickerChoices.length === 0 && (
+
+ {pickerField.type === "user"
+ ? "No assignable people were supplied for this workspace."
+ : "This field has no choices yet. Add them from the column menu."}
+
+ )}
+
+ {pickerChoices.map((choice) => {
+ const tint = optionTint(pickerField, choice);
+ const on = pickerMulti
+ ? pickerParts.includes(choice)
+ : pickerValue === choice;
+ return (
+
{
+ if (pickerMulti) {
+ // Toggle membership; each toggle persists (the event log absorbs the
+ // burst), and the picker STAYS OPEN so a set is built in one visit.
+ const next = on
+ ? pickerParts.filter((p) => p !== choice)
+ : [...pickerParts, choice];
+ patchAndRecord(picker.pid, {
+ [pickerField.key]: next.join(","),
+ }, "a choice");
+ return;
+ }
+ patchAndRecord(picker.pid, { [pickerField.key]: choice }, "a choice");
+ closePicker();
+ }}
+ >
+ {/* Wave-14 item 11 / R6 — a PICKER shows the avatar AND the name. The grid
+ cell shows the face alone (you already know who your people are); this is
+ where you find out which face is whose, so dropping the name here would
+ make the feature unusable the first time two colleagues share initials.
+ ⚠ Inline styles, no new class rules: `index.css` is PANEL's fence this
+ wave. The class name is a QA hook only and styles nothing. */}
+ {pickerField.type === "user" && (
+
+ {/* ⚠ The initials are ALWAYS rendered and the photo is laid OVER them, so
+ a corrupt data URL hides itself and the fallback is already underneath.
+ C-AVATAR's rule is "never a broken-image glyph, anywhere" — the canvas
+ path honours it via `img.onerror`, and this is the DOM path's version
+ of the same promise. It cannot fire until HOST serves a photo, which is
+ exactly why it is easy to ship without. */}
+ {avatarInitials(choice)}
+ {userAvatars?.[choice] && (
+ {
+ e.currentTarget.style.visibility = "hidden";
+ }}
+ style={{
+ position: "absolute",
+ inset: 0,
+ width: "100%",
+ height: "100%",
+ objectFit: "cover",
+ }}
+ />
+ )}
+
+ )}
+
+ {choice}
+
+
+ );
+ })}
+
+ {pickerMulti && (
+
+
+ Done
+
+
+ )}
+ {/* Clearing must be possible, and it must write "" rather than delete the key —
+ the overlay patch contract is a value per field, and a missing key would leave
+ the previous value standing on the next render. */}
+ {pickerValue !== "" && (
+ {
+ patchAndRecord(picker.pid, { [pickerField.key]: "" }, "a choice");
+ closePicker();
+ }}
+ >
+ Clear
+
+ )}
+
+ )}
+
+ {!embedded && detailRecord && detailPid !== null && (
+ go(-1)}
+ onNext={() => go(1)}
+ onClose={() => setDetailPid(null)}
+ // Item 5 / C-LAYOUT — see `recordLayout` above. Both optional on RECORD's side, so
+ // this compiles whether or not their reorder UI has landed yet.
+ recordLayout={recordLayout}
+ onRecordLayout={onRecordLayout}
+ // Item 13 / C-EMBED — the host names WHICH question Insights asks. Without this the
+ // tab deliberately does not render (RECORD's no-safe-default rule: a wrong scope
+ // succeeds with plausible wrong numbers, so absence must mean absent, not guessed).
+ scope={scope}
+ docs={payload?.docs?.[String(detailPid)] ?? []}
+ docPayload={payload?.docPayload}
+ // C5 — documents ride the ordinary event log. The host answers a fetch
+ // on the NEXT render via `payload.docPayload`; Documents.tsx matches it
+ // to its own pending request before touching it.
+ onDocAdd={
+ payload?.docs
+ ? (file) =>
+ emitHostEvent({
+ id: eventId("docadd"),
+ type: "doc_add",
+ pid: detailPid,
+ docId: `doc_${eventId("d").slice(2, 14).replace(/[^a-z0-9]/gi, "")}`,
+ ...file,
+ })
+ : undefined
+ }
+ onDocFetch={
+ payload?.docs
+ ? (docId) =>
+ emitHostEvent({
+ id: eventId("docget"),
+ type: "doc_fetch",
+ pid: detailPid,
+ docId,
+ })
+ : undefined
+ }
+ onDocDelete={
+ payload?.docs
+ ? (docId) =>
+ emitHostEvent({
+ id: eventId("docdel"),
+ type: "doc_delete",
+ pid: detailPid,
+ docId,
+ })
+ : undefined
+ }
+ onNotesChange={(key, value) =>
+ setOverlayEdits((current) => ({
+ ...current,
+ [detailPid]: { ...current[detailPid], [key]: value },
+ }))
+ }
+ onNotesCommit={(key, value) => patchAndRecord(detailPid, { [key]: value })}
+ viewer={viewer}
+ // Wave-14 close-out stitch (HOST): TSREC's two light-switch props, unwired when
+ // the GRID session ended — item 9's picker vocabulary + item 11's photos. Same
+ // values the grid's own surfaces already read (the wave-13 sixth-prop precedent).
+ userOptions={payload?.userOptions ?? []}
+ userAvatars={userAvatars}
+ />
+ )}
+
+ );
+}
+
+export default memo(CustomerGrid);
diff --git a/web/src/customer-grid/FormInterface.css b/web/src/customer-grid/FormInterface.css
new file mode 100644
index 0000000000000000000000000000000000000000..d716625a38e3ffdcc0c5bba7dd22119ede362b0c
--- /dev/null
+++ b/web/src/customer-grid/FormInterface.css
@@ -0,0 +1,269 @@
+/* ---------------------------------------------------------------------------
+ customer-grid / FormInterface.css
+ ⭐ WAVE-29 R7 — the FORM builder's own stylesheet.
+
+ Its own file rather than `index.css` for two reasons, and only one of them is this wave's
+ fence: a mode's surface that ships with its component can be read, moved or retired in one
+ place (`RecordDetail.css` and `ImportDialog.css` set the precedent), and `index.css` is
+ already the app's largest single file.
+
+ Every colour, radius and type size is a `--lp-*` token. Nothing here invents a hue: the token
+ parity gate compares `index.css` against `theme.ts`, and a raw hex would be invisible to it.
+ --------------------------------------------------------------------------- */
+
+.cg-form-build {
+ display: flex;
+ flex-wrap: wrap;
+ gap: 24px;
+ align-items: flex-start;
+ overflow: auto;
+ padding: 20px 24px 32px;
+ height: 100%;
+ background: var(--lp-surface);
+ color: var(--lp-ink);
+ font-size: var(--lp-fs-xs);
+}
+
+.cg-form-pane {
+ flex: 1 1 320px;
+ min-width: 300px;
+ max-width: 560px;
+}
+
+.cg-form-h {
+ margin: 0 0 12px;
+ font-size: var(--lp-fs-lg);
+ font-weight: 600;
+}
+
+.cg-form-sub {
+ margin: 22px 0 8px;
+ font-size: var(--lp-fs-2xs);
+ font-weight: 600;
+ letter-spacing: 0.02em;
+ color: var(--lp-muted);
+}
+
+/* The first `.cg-form-sub` in a pane sits under no heading of its own. */
+.cg-form-pane > .cg-form-sub:first-child {
+ margin-top: 0;
+}
+
+.cg-form-note {
+ margin: 6px 0;
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-2xs);
+ line-height: 1.5;
+}
+
+/* An honest warning, not an alarm: it is a state the reader can fix, so it is tinted rather
+ than shouted. Red is reserved for a refusal that has already cost something. */
+.cg-form-warn {
+ margin: 8px 0;
+ padding: 8px 10px;
+ border: 1px solid var(--lp-red);
+ border-radius: var(--lp-r-sm);
+ background: var(--lp-red-tint);
+ color: var(--lp-red-deep);
+ font-size: var(--lp-fs-2xs);
+ line-height: 1.5;
+}
+
+.cg-form-empty {
+ display: grid;
+ place-items: center;
+ height: 100%;
+ padding: 40px;
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-xs);
+ text-align: center;
+}
+
+/* --- the spec fields ------------------------------------------------------------------- */
+
+.cg-form-row {
+ display: block;
+ margin: 0 0 12px;
+}
+
+.cg-form-row > span {
+ display: block;
+ margin-bottom: 4px;
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-3xs);
+ font-weight: 600;
+}
+
+.cg-form-row input[type="text"],
+.cg-form-row textarea,
+.cg-form-addmail input {
+ width: 100%;
+ padding: 7px 9px;
+ border: 1px solid var(--lp-line);
+ border-radius: var(--lp-r-sm);
+ background: var(--lp-surface);
+ color: var(--lp-ink);
+ font-family: inherit;
+ font-size: var(--lp-fs-xs);
+ resize: vertical;
+}
+
+.cg-form-row input:disabled,
+.cg-form-row textarea:disabled {
+ background: var(--lp-wash);
+ color: var(--lp-muted);
+}
+
+/* --- the question list ----------------------------------------------------------------- */
+
+.cg-form-fields {
+ margin: 0;
+ padding: 0;
+ list-style: none;
+ border: 1px solid var(--lp-line);
+ border-radius: var(--lp-r-md);
+ overflow: hidden;
+}
+
+.cg-form-field {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 10px;
+ padding: 7px 10px;
+ border-bottom: 1px solid var(--lp-line);
+}
+
+.cg-form-field:last-child {
+ border-bottom: 0;
+}
+
+.cg-form-field.on {
+ background: var(--lp-blue-tint);
+}
+
+.cg-form-pick {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ min-width: 0;
+ cursor: pointer;
+}
+
+.cg-form-label {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.cg-form-type {
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-3xs);
+}
+
+.cg-form-req {
+ display: flex;
+ align-items: center;
+ gap: 5px;
+ flex: none;
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-3xs);
+ cursor: pointer;
+}
+
+/* --- who can submit -------------------------------------------------------------------- */
+
+.cg-form-radio {
+ display: flex;
+ align-items: flex-start;
+ gap: 9px;
+ margin-bottom: 10px;
+ cursor: pointer;
+}
+
+.cg-form-radio strong {
+ display: block;
+ font-weight: 600;
+}
+
+.cg-form-radio em {
+ display: block;
+ margin-top: 2px;
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-3xs);
+ font-style: normal;
+ line-height: 1.5;
+}
+
+.cg-form-emails ul {
+ margin: 0 0 8px;
+ padding: 0;
+ list-style: none;
+}
+
+.cg-form-emails li {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 8px;
+ padding: 5px 0;
+ border-bottom: 1px solid var(--lp-line);
+}
+
+.cg-form-emails li:last-child {
+ border-bottom: 0;
+}
+
+.cg-form-addmail {
+ display: flex;
+ gap: 8px;
+}
+
+/* --- the share link -------------------------------------------------------------------- */
+
+.cg-form-link {
+ padding: 10px 12px;
+ border: 1px solid var(--lp-line);
+ border-radius: var(--lp-r-md);
+ background: var(--lp-surface-2);
+}
+
+.cg-form-link code {
+ display: block;
+ margin-bottom: 8px;
+ overflow-wrap: anywhere;
+ color: var(--lp-blue-deep);
+ font-size: var(--lp-fs-2xs);
+}
+
+.cg-form-actions {
+ display: flex;
+ flex-wrap: wrap;
+ gap: 8px;
+}
+
+.cg-form-build button {
+ padding: 5px 11px;
+ border: 1px solid var(--lp-line);
+ border-radius: var(--lp-r-pill);
+ background: var(--lp-surface);
+ color: var(--lp-ink);
+ font-family: inherit;
+ font-size: var(--lp-fs-2xs);
+ cursor: pointer;
+}
+
+.cg-form-build button:hover:not(:disabled) {
+ border-color: var(--lp-blue-solid);
+ color: var(--lp-blue-deep);
+}
+
+.cg-form-build button:disabled {
+ color: var(--lp-muted);
+ cursor: default;
+}
+
+.cg-form-emails button {
+ padding: 2px 8px;
+ font-size: var(--lp-fs-3xs);
+}
diff --git a/web/src/customer-grid/FormInterface.tsx b/web/src/customer-grid/FormInterface.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..4c6456c6235becebf9e02ee57bd367fec116c438
--- /dev/null
+++ b/web/src/customer-grid/FormInterface.tsx
@@ -0,0 +1,504 @@
+// ---------------------------------------------------------------------------
+// customer-grid / FormInterface.tsx
+// ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3 / C4) — THE FORM INTERFACE.
+//
+// The owner's ask, verbatim: "a user fills in information; the form has a shareable link that
+// accepts submissions either from authorized people (by email) or from the public. Google Forms
+// is the reference."
+//
+// This file is the BUILDER — the in-app surface a `form`-kind view renders. The page a stranger
+// fills in is `FormPublic.tsx` and it already shipped in wave 23, along with the whole public
+// door (`api/routes_forms.py`). What had never existed was any way to CREATE the thing they
+// serve: `form` was not a legal display mode and `_clean_display` dropped `display.form`, so the
+// key the public door reads could not be written by anything. That is D-90, and it is why this
+// component is mostly plumbing rather than a new idea — the door was built first and the handle
+// never fitted.
+//
+// THE TWO HALVES, and they are stored in two different places on purpose:
+// * the SPEC (questions, title, who may submit) lives on the view, at `config.display.form`,
+// written through the host's ordinary view-save path like every other mode's configuration;
+// * the TOKEN lives in a server-owned index a browser cannot write, because a share token that
+// rides the wire is a token a browser can CHOOSE, and the public resolver answers with the
+// first tenant that matches. See `routes_forms.py`'s note for the whole argument.
+// ---------------------------------------------------------------------------
+
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { API_V1, CREDENTIALS } from "../apiContract";
+import type { Field, FieldType, FormSpec } from "./types";
+import { TYPE_LABELS } from "./iconShapes";
+import "./FormInterface.css";
+
+/**
+ * The field types a form may COLLECT. **Mirrors `platform/aios_grid.py::FORM_FIELD_TYPES` and
+ * `verify_forms.py` compares the two files name-for-name** — a type offered here and refused
+ * there is a question that silently vanishes from the published form, which is the failure a
+ * one-sided list produces every time.
+ *
+ * An ALLOW-LIST, not a list of exclusions: a type missing here is a question the builder cannot
+ * ask yet, while a type wrongly present is a public door writing values its column cannot mean.
+ * The two mistakes do not cost the same, so the list fails closed.
+ */
+export const FORM_FIELD_TYPES: ReadonlySet = new Set([
+ "text", "select", "multiselect", "int", "currency", "pct", "date",
+ "checkbox", "phone", "email", "url", "rating",
+]);
+
+/**
+ * The stored bag at `view.config.display.form` is `FormSpec`, and it is DECLARED IN `./types`
+ * (imported above), not here. ⛔ Declaring it in this file made a type-only circular import —
+ * `types.ts` needs it for `DisplaySpec.form` while this file needs `Field` from `types.ts` — which
+ * vite erased and `tsc --ignoreConfig` could not, killing `verify_optimism` and
+ * `verify_live_workspace` with TS6142. Re-exported here so this file still names the contract it
+ * edits; the single declaration stays in the type module. ⛔ No `token` — see the header.
+ */
+export type { FormSpec } from "./types";
+
+export interface FormInterfaceProps {
+ /** The database this form collects into (`topicForScope(scope)`), e.g. `ut_leads`. */
+ topic: string;
+ /** The view holding the spec. `null` while no saved view is active — a form has nowhere to live. */
+ viewId: string | null;
+ /** Every column of the table; this component decides which can be questions. */
+ fields: Field[];
+ /**
+ * The stored spec, or `null` when this view has no form yet.
+ * ⚠ REQUIRED and NULLABLE, never optional: the host passes `displaySpec?.form ?? null`, and the
+ * `?? null` is the point — an optional prop degrades to "the feature does not exist", which is
+ * indistinguishable from never having been built.
+ */
+ spec: FormSpec | null;
+ /** Writes the spec back into `config.display.form`. `null` clears it. */
+ onSpec: (next: FormSpec | null) => void;
+ /** May this viewer change the form's configuration (the view-config grant)? */
+ canEdit: boolean;
+ /** Why not, in one sentence, when `canEdit` is false. `null` when it is true. */
+ readOnlyReason: string | null;
+ /** Does this database accept new records at all? A locked one can never be a form's target. */
+ canCollect: boolean;
+}
+
+interface LinkState {
+ token: string | null;
+ url: string | null;
+ stored: { fields: number; access: string; title: string } | null;
+}
+
+const EMPTY_LINK: LinkState = { token: null, url: null, stored: null };
+
+/** The share URL the page shows. The server returns an origin-qualified one when it knows the
+ * public base; otherwise it returns a path, and only the browser knows where it is. */
+function absolute(url: string): string {
+ return url.startsWith("http") ? url : `${window.location.origin}${url}`;
+}
+
+export function FormInterface({
+ topic,
+ viewId,
+ fields,
+ spec,
+ onSpec,
+ canEdit,
+ readOnlyReason,
+ canCollect,
+}: FormInterfaceProps) {
+ const [link, setLink] = useState(EMPTY_LINK);
+ const [busy, setBusy] = useState(false);
+ const [problem, setProblem] = useState("");
+ const [copied, setCopied] = useState(false);
+ const [emailDraft, setEmailDraft] = useState("");
+
+ /**
+ * The columns that can be questions.
+ *
+ * ⚠ The server enforces this too, and both walls are needed rather than one: a column can
+ * BECOME computed (a formula, a rollup, a metric binding) long after it was added to a live
+ * form, and this component is not running when that happens. Here it is a courtesy that keeps
+ * the builder honest; `routes_forms._public_form` is the wall.
+ */
+ const askable = useMemo(
+ () =>
+ fields.filter(
+ (f) =>
+ FORM_FIELD_TYPES.has(f.type) &&
+ f.source !== "odoo" &&
+ !f.automation &&
+ !f.rollup &&
+ !f.metric
+ ),
+ [fields]
+ );
+ const byKey = useMemo(() => new Map(fields.map((f) => [f.key, f])), [fields]);
+
+ const chosen = spec?.fields ?? [];
+ const required = useMemo(() => new Set(spec?.required ?? []), [spec]);
+ const emails = spec?.emails ?? [];
+ const restricted = spec?.access === "emails";
+
+ /**
+ * ⭐ THE DEAD-STATE DETECTOR, and it earns its four lines.
+ *
+ * The spec reaches this panel through the host's `cleanDisplay`, which builds its output key by
+ * key — so a host that does not carry `display.form` hands back `null` forever. The panel would
+ * then look exactly like a form nobody has built yet: every keystroke accepted, nothing stored,
+ * no error anywhere. That is [W-07]'s failure verbatim — "the feature does not exist" and "the
+ * feature was never wired" are indistinguishable from the outside, and four consecutive waves
+ * lost a hand-off to it.
+ *
+ * One write plus one render answers it: if this panel has SENT a spec and the next render still
+ * has none, the carry is missing. It cannot false-positive on a slow save — `onSpec` is
+ * synchronous host state, not a request.
+ */
+ const [wrote, setWrote] = useState(false);
+ const carryBroken = wrote && spec === null;
+
+ const patch = useCallback(
+ (next: Partial) => {
+ if (!canEdit) return;
+ setWrote(true);
+ onSpec({ ...(spec ?? {}), ...next });
+ },
+ [canEdit, onSpec, spec]
+ );
+
+ // ---- the share link, read from the server (never minted on read) -------------------------
+ const url = `${API_V1}/form-link?topic=${encodeURIComponent(topic)}&view=${encodeURIComponent(
+ viewId ?? ""
+ )}`;
+
+ useEffect(() => {
+ if (!viewId) {
+ setLink(EMPTY_LINK);
+ return;
+ }
+ let live = true;
+ fetch(url, { credentials: CREDENTIALS })
+ .then((r) => (r.ok ? r.json() : EMPTY_LINK))
+ .then((body: LinkState) => {
+ if (live) setLink(body ?? EMPTY_LINK);
+ })
+ .catch(() => {
+ if (live) setLink(EMPTY_LINK);
+ });
+ return () => {
+ live = false;
+ };
+ }, [url, viewId]);
+
+ const call = useCallback(
+ async (method: "POST" | "DELETE", body?: unknown) => {
+ setBusy(true);
+ setProblem("");
+ try {
+ const response = await fetch(method === "POST" ? `${API_V1}/form-link` : url, {
+ method,
+ credentials: CREDENTIALS,
+ ...(body
+ ? { headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }
+ : {}),
+ });
+ const payload = await response.json().catch(() => null);
+ if (!response.ok) {
+ setProblem(
+ (payload as { error?: { message?: string } } | null)?.error?.message ??
+ `The server answered ${response.status}.`
+ );
+ return;
+ }
+ setLink(
+ method === "DELETE"
+ ? { ...EMPTY_LINK, stored: link.stored }
+ : { ...(payload as LinkState), stored: link.stored }
+ );
+ } finally {
+ setBusy(false);
+ }
+ },
+ [link.stored, url]
+ );
+
+ const copy = useCallback(() => {
+ if (!link.url) return;
+ void navigator.clipboard?.writeText(absolute(link.url));
+ setCopied(true);
+ window.setTimeout(() => setCopied(false), 1600);
+ }, [link.url]);
+
+ // ---- the states that are not a form -------------------------------------------------------
+ if (!viewId) {
+ return (
+
+ A form belongs to a saved view. Save this view first, then build the form on it.
+
+ );
+ }
+ if (!canCollect) {
+ // ⚠ NOT `readOnlyReason`, and the two must not be merged: this is a fact about the DATABASE
+ // (a locked one accepts no new records, so a form could only ever collect refusals), while
+ // `readOnlyReason` is about the VIEWER's grant.
+ return (
+
+ This database does not accept new records, so a form has nowhere to put an answer.
+
+ );
+ }
+
+ const diverged = spec === null && (link.stored?.fields ?? 0) > 0;
+
+ return (
+
+
+
Form
+ {!canEdit && readOnlyReason &&
{readOnlyReason}
}
+ {/* ⚠ The divergence banner. The `spec` prop comes from the host's `cleanDisplay`, and if
+ that ever stops carrying `display.form` the panel would render an empty form over a
+ stored one — and the next save would erase real questions. Saying so beats losing
+ them quietly. */}
+ {diverged && (
+
+ The server holds {link.stored?.fields} question
+ {link.stored?.fields === 1 ? "" : "s"} for this view that this page is not carrying.
+ Reload before editing — saving now would replace them.
+
+ )}
+ {/* ⚠ The banner above needs a STORED spec to compare against, so it can only speak once a
+ form has been saved successfully at least once. This one covers the state before
+ that: nothing has ever been stored, so there is nothing to diverge from. */}
+ {carryBroken && (
+
+ Nothing is being saved: this view gave back no form after the last change. Reload the
+ page — if it repeats, this build cannot store forms and an administrator should be
+ told, because everything typed here is being discarded.
+
+ )}
+
+
+ Title
+ patch({ title: e.target.value })}
+ />
+
+
+ Description
+
+
+ Submit button
+ patch({ submitLabel: e.target.value })}
+ />
+
+
+
Questions
+ {askable.length === 0 ? (
+
+ No column on this database can be a form question yet. A form can ask for text,
+ numbers, dates, choices and yes/no — it cannot ask for a value the app computes.
+
+ ) : (
+
+ )}
+
+
+
+
Who can submit
+
+ patch({ access: "public" })}
+ />
+
+ Anyone with the link
+ No sign-in. Rate-limited, and capped per day.
+
+
+
+ patch({ access: "emails" })}
+ />
+
+ Only the people I list
+ {/* ⚠ The honest boundary, in the UI and not only in the docstring: this asks the
+ submitter to name themselves and refuses an address that is not on the list. It
+ does not PROVE the address is theirs. */}
+
+ They enter their email address and it must match the list. This identifies them; it
+ does not verify them — anyone with the link who knows a listed address can submit.
+
+
+
+
+ {restricted && (
+
+
+ {emails.map((e) => (
+
+ {e}
+ {canEdit && (
+ patch({ emails: emails.filter((x) => x !== e) })}
+ aria-label={`Remove ${e}`}
+ >
+ Remove
+
+ )}
+
+ ))}
+ {emails.length === 0 && (
+
+ Nobody is listed yet, so nobody can submit this form.
+
+ )}
+
+ {canEdit && (
+
+ )}
+
+ )}
+
+
Share link
+ {problem &&
{problem}
}
+ {link.url ? (
+
+
{absolute(link.url)}
+
+
+ {copied ? "Copied" : "Copy link"}
+
+ {canEdit && (
+ <>
+ void call("POST", { topic, view: viewId, regenerate: true })}
+ title="Mints a new link. The old one stops working immediately."
+ >
+ Regenerate
+
+ void call("DELETE")}>
+ Turn off
+
+ >
+ )}
+
+
+ ) : (
+
+
+ This form has no link yet, so nobody outside the app can reach it.
+
+ {canEdit && (
+
void call("POST", { topic, view: viewId })}
+ >
+ Create link
+
+ )}
+ {chosen.length === 0 && (
+
Pick at least one question first.
+ )}
+
+ )}
+ {chosen.length > 0 && (
+
+ Answers land as new records in this database
+ {chosen.length === 1 ? ", one column" : `, ${chosen.length} columns`}:{" "}
+ {chosen.map((k) => byKey.get(k)?.label ?? k).join(", ")}.
+
+ )}
+
+
+ );
+}
+
+export default FormInterface;
diff --git a/web/src/customer-grid/ImportDialog.css b/web/src/customer-grid/ImportDialog.css
new file mode 100644
index 0000000000000000000000000000000000000000..77555a1118bfd4175f72b4a015eb51cc5db01dcb
--- /dev/null
+++ b/web/src/customer-grid/ImportDialog.css
@@ -0,0 +1,154 @@
+/* customer-grid / ImportDialog.css — WAVE-29 T25 (owner item 6).
+ Tokens only (`--lp-*`): this file may not invent a colour, and `index.css` is another
+ session's fence, so the dialog's own styles live beside its component. */
+
+.cg-import-scrim {
+ position: fixed;
+ inset: 0;
+ z-index: 60;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ background: rgba(32, 36, 51, 0.28);
+ padding: 24px;
+}
+
+.cg-import {
+ display: flex;
+ flex-direction: column;
+ width: min(560px, 100%);
+ max-height: min(80vh, 720px);
+ background: var(--lp-bg);
+ border: 1px solid var(--lp-line);
+ border-radius: var(--lp-r-lg);
+ box-shadow: 0 12px 32px rgba(32, 36, 51, 0.18);
+ overflow: hidden;
+}
+
+.cg-import-head {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 12px;
+ padding: 14px 16px;
+ border-bottom: 1px solid var(--lp-line);
+}
+
+.cg-import-head h2 {
+ margin: 0;
+ font-size: var(--lp-fs-md);
+ font-weight: 600;
+ color: var(--lp-ink);
+}
+
+.cg-import-x {
+ border: 0;
+ background: none;
+ color: var(--lp-muted);
+ font-size: var(--lp-fs-lg);
+ line-height: 1;
+ cursor: pointer;
+ padding: 2px 6px;
+}
+
+.cg-import-x:hover {
+ color: var(--lp-ink);
+}
+
+.cg-import-body {
+ padding: 14px 16px;
+ overflow: auto;
+}
+
+.cg-import-lede {
+ margin: 0 0 8px;
+ font-size: var(--lp-fs-sm);
+ color: var(--lp-ink);
+}
+
+.cg-import-note {
+ margin: 0 0 12px;
+ font-size: var(--lp-fs-xs);
+ color: var(--lp-muted);
+}
+
+.cg-import-file {
+ font-size: var(--lp-fs-sm);
+}
+
+/* The mapping list: the spreadsheet's column on the left, this database's field on the right,
+ so the eye reads it as "this becomes that" rather than as a form. */
+.cg-import-map {
+ display: flex;
+ flex-direction: column;
+ gap: 6px;
+}
+
+.cg-import-row {
+ display: grid;
+ grid-template-columns: 1fr 200px;
+ align-items: center;
+ gap: 10px;
+}
+
+.cg-import-col {
+ font-size: var(--lp-fs-sm);
+ color: var(--lp-ink);
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+}
+
+.cg-import-row select {
+ font-family: inherit;
+ font-size: var(--lp-fs-xs);
+ padding: 4px 6px;
+ border: 1px solid var(--lp-line);
+ border-radius: var(--lp-r-sm);
+ background: var(--lp-bg);
+ color: var(--lp-ink);
+}
+
+/* A refusal is not an error state to be styled red-alert; it is a list of things to fix, so it
+ takes the yellow tint the rest of the product uses for "look at this before continuing". */
+.cg-import-problems {
+ margin-top: 12px;
+ padding: 10px 12px;
+ border: 1px solid var(--lp-yellow);
+ border-radius: var(--lp-r-sm);
+ background: var(--lp-yellow-tint);
+ color: var(--lp-yellow-deep);
+ font-size: var(--lp-fs-xs);
+}
+
+.cg-import-problems ul {
+ margin: 6px 0;
+ padding-left: 18px;
+}
+
+.cg-import-problems p {
+ margin: 6px 0 0;
+}
+
+.cg-import-error {
+ margin: 0;
+ padding: 10px 16px;
+ background: var(--lp-red-tint);
+ color: var(--lp-red-deep);
+ font-size: var(--lp-fs-xs);
+}
+
+.cg-import-foot {
+ display: flex;
+ align-items: center;
+ justify-content: flex-end;
+ gap: 8px;
+ padding: 12px 16px;
+ border-top: 1px solid var(--lp-line);
+}
+
+.cg-import-count {
+ margin-right: auto;
+ font-size: var(--lp-fs-xs);
+ color: var(--lp-muted);
+}
diff --git a/web/src/customer-grid/ImportDialog.tsx b/web/src/customer-grid/ImportDialog.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..d6c1d1d6842c7e40520bdbdb816cf0871da84cd7
--- /dev/null
+++ b/web/src/customer-grid/ImportDialog.tsx
@@ -0,0 +1,325 @@
+// ---------------------------------------------------------------------------
+// customer-grid / ImportDialog.tsx
+//
+// ⭐⭐ WAVE-29 T25 (owner item 6) — IMPORT A SPREADSHEET INTO A USER DATABASE.
+//
+// Three steps in one window, because each answers the question the last one raised:
+// 1. pick a file -> the server parses it and says which columns it found
+// 2. map its columns -> to fields, or to nothing; unmapped is the default
+// 3. check + import -> every value coerced FIRST, the problems named BEFORE anything writes
+//
+// ⛔ APPEND-ONLY IN V1, AND THE DIALOG SAYS SO. Nothing is matched, nothing is overwritten — every
+// row becomes a new record. The only bulk-merge primitive in the product is upsert-BY-KEY shaped,
+// and choosing a key on the user's behalf is how an import quietly rewrites rows somebody edited.
+// The sentence is on screen rather than in a doc, because that is where the decision is made.
+//
+// ⛔ REUSE, NOT A SECOND COERCER. `clipboard.coerceClipboardValue` is THE client answer to "can
+// this text be this field's value" (it is what a paste already goes through), and
+// `POST /api/v1/uploads/tabular` is the parser that already exists — 5 MB / 20,000 rows, xlsx AND
+// csv, and it never writes a row. A second copy of either is a second set of rules for one
+// question ([[one-evaluator-per-question]]).
+//
+// ⚠ THE SERVER IS THE WALL, not this dialog. It refuses computed columns BY NAME and refuses the
+// whole batch if it would pass the row cap. What happens here is the part a person needs before
+// they press the button: which column goes where, and what will not survive the trip.
+// ---------------------------------------------------------------------------
+
+import { useMemo, useRef, useState } from "react";
+// ⚠ From `apiContract`, which is where these two are DECLARED — not re-exported from
+// `apiBridge`, which does not export them and is another session's fence this wave.
+import { API_V1, CREDENTIALS } from "../apiContract";
+import { coerceClipboardValue } from "./clipboard";
+import { choiceOptions, isImportTarget } from "./types";
+import type { Field } from "./types";
+import "./ImportDialog.css";
+
+/** The one header a JSON POST needs. `apiBridge` keeps its own module-local copy; this is a
+ * literal rather than an import because that constant is not exported and that file is not
+ * this session's to change. */
+const JSON_HEADERS = { "Content-Type": "application/json" };
+
+/** The upload door's answer (`routes_uploads.tabular`). */
+interface Parsed {
+ columns: string[];
+ rows: number;
+ values: string[][];
+ truncated?: boolean;
+}
+
+export interface ImportDialogProps {
+ /** The target database's fields — the mapping targets, before this file filters them. */
+ fields: Field[];
+ /** `ut_*` table key; the import door is per-table. */
+ scope: string;
+ /** How the database calls itself, for the sentence that says where rows are going. */
+ tableName: string;
+ /** Fired after a successful import so the host can re-read its rows. */
+ onImported: (count: number) => void;
+ onClose: () => void;
+}
+
+const NONE = "";
+
+export default function ImportDialog({
+ fields,
+ scope,
+ tableName,
+ onImported,
+ onClose,
+}: ImportDialogProps): React.ReactElement {
+ const [parsed, setParsed] = useState(null);
+ const [mapping, setMapping] = useState>({});
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState("");
+ const fileRef = useRef(null);
+
+ const targets = useMemo(() => fields.filter(isImportTarget), [fields]);
+
+ /**
+ * The parse. `FormData` and NO `Content-Type` header: the browser has to set the multipart
+ * boundary itself, and spelling the header by hand is the classic way to make a working upload
+ * 400 with a message about a missing part.
+ */
+ async function pick(file: File): Promise {
+ setBusy(true);
+ setError("");
+ try {
+ const form = new FormData();
+ form.append("file", file);
+ const res = await fetch(`${API_V1}/uploads/tabular`, {
+ method: "POST",
+ credentials: CREDENTIALS,
+ body: form,
+ });
+ const body = (await res.json().catch(() => null)) as
+ | (Parsed & { detail?: { error?: { message?: string } } })
+ | null;
+ if (!res.ok || !body) {
+ setError(
+ body?.detail?.error?.message ??
+ `That file could not be read (the server answered ${res.status}).`
+ );
+ return;
+ }
+ setParsed(body);
+ // ⭐ AUTO-MAP BY NAME, case- and space-insensitively, because the overwhelmingly common
+ // case is a sheet exported from this very grid. Every guess is VISIBLE in a picker the
+ // user can change — a silent auto-map is only dangerous when it is also invisible.
+ const byName = new Map(
+ targets.map((f) => [f.label.trim().toLowerCase(), f.key] as const)
+ );
+ const guess: Record = {};
+ (body.columns ?? []).forEach((name, i) => {
+ const hit = byName.get(String(name).trim().toLowerCase());
+ if (hit) guess[i] = hit;
+ });
+ setMapping(guess);
+ } catch {
+ setError("That file could not be read.");
+ } finally {
+ setBusy(false);
+ }
+ }
+
+ /**
+ * Coerce every mapped cell and collect the refusals. Runs over the WHOLE parsed file, not the
+ * preview: the point of this step is that a bad value on row 900 is reported BEFORE anything is
+ * written, rather than discovered as a half-imported file.
+ */
+ const plan = useMemo(() => {
+ const problems: { row: number; column: string; value: string; field: string }[] = [];
+ const rows: Record[] = [];
+ if (!parsed) return { rows, problems };
+ const mapped = Object.entries(mapping).filter(([, key]) => key !== NONE);
+ for (let r = 0; r < (parsed.values?.length ?? 0); r += 1) {
+ const out: Record = {};
+ let blank = true;
+ for (const [idxText, key] of mapped) {
+ const field = targets.find((f) => f.key === key);
+ if (!field) continue;
+ const raw = String(parsed.values[r]?.[Number(idxText)] ?? "");
+ if (raw.trim() === "") continue;
+ blank = false;
+ const value = coerceClipboardValue(field, raw, choiceOptions(field));
+ if (value === null) {
+ problems.push({
+ row: r + 2, // +1 for the header row, +1 because humans count from 1
+ column: parsed.columns[Number(idxText)] ?? `Column ${Number(idxText) + 1}`,
+ value: raw.slice(0, 40),
+ field: field.label,
+ });
+ continue;
+ }
+ out[key] = String(value);
+ }
+ if (!blank) rows.push(out);
+ }
+ return { rows, problems };
+ }, [parsed, mapping, targets]);
+
+ async function run(): Promise {
+ setBusy(true);
+ setError("");
+ try {
+ const res = await fetch(
+ `${API_V1}/tables/${encodeURIComponent(scope)}/rows/import`,
+ {
+ method: "POST",
+ credentials: CREDENTIALS,
+ headers: JSON_HEADERS,
+ body: JSON.stringify({ rows: plan.rows }),
+ }
+ );
+ const body = (await res.json().catch(() => null)) as
+ | { imported?: number; detail?: { error?: { message?: string } } }
+ | null;
+ if (!res.ok) {
+ // The server's sentence beats anything invented here: it names the column, or the cap.
+ setError(body?.detail?.error?.message ?? `The server answered ${res.status}.`);
+ return;
+ }
+ onImported(body?.imported ?? plan.rows.length);
+ onClose();
+ } catch {
+ setError("Nothing was imported — the request did not complete.");
+ } finally {
+ setBusy(false);
+ }
+ }
+
+ const mappedCount = Object.values(mapping).filter((k) => k !== NONE).length;
+
+ return (
+
+
event.stopPropagation()}
+ >
+
+
Import records
+
+ ×
+
+
+
+ {!parsed ? (
+
+
+ Add records to {tableName} from a spreadsheet. Excel
+ (.xlsx) or .csv, up to 5 MB.
+
+
+ Every row becomes a new record . Nothing already in this database is
+ matched or overwritten.
+
+
{
+ const file = event.currentTarget.files?.[0];
+ if (file) void pick(file);
+ }}
+ />
+
+ ) : (
+
+
+ {parsed.rows.toLocaleString()} row{parsed.rows === 1 ? "" : "s"} read
+ {parsed.truncated ? " (the first 20,000 — the rest were not read)" : ""}. Choose
+ where each column goes.
+
+
+ {parsed.columns.map((name, i) => (
+
+
+ {name || `Column ${i + 1}`}
+
+ {
+ const next = event.target.value;
+ setMapping((cur) => ({ ...cur, [i]: next }));
+ }}
+ >
+ Don’t import
+ {targets.map((f) => (
+
+ {f.label}
+
+ ))}
+
+
+ ))}
+
+ {plan.problems.length > 0 && (
+
+
+ {plan.problems.length.toLocaleString()} value
+ {plan.problems.length === 1 ? "" : "s"} cannot go in the column
+ {plan.problems.length === 1 ? "" : "s"} chosen for them
+
+
+ {plan.problems.slice(0, 5).map((p, i) => (
+
+ Row {p.row}, {p.column}: {p.value ? `“${p.value}”` : "(blank)"} is not a
+ valid {p.field}
+
+ ))}
+
+ {plan.problems.length > 5 && (
+
…and {(plan.problems.length - 5).toLocaleString()} more.
+ )}
+
+ Fix them in the file, or point that column somewhere else — nothing has been
+ imported yet.
+
+
+ )}
+
+ )}
+
+ {error && (
+
+ {error}
+
+ )}
+
+
+ {parsed && (
+
+ {mappedCount === 0
+ ? "No columns chosen yet"
+ : `${plan.rows.length.toLocaleString()} record${
+ plan.rows.length === 1 ? "" : "s"
+ } from ${mappedCount} column${mappedCount === 1 ? "" : "s"}`}
+
+ )}
+
+ Cancel
+
+ {parsed && (
+ 0}
+ onClick={() => void run()}
+ >
+ {busy ? "Importing…" : `Import ${plan.rows.length.toLocaleString()} record${
+ plan.rows.length === 1 ? "" : "s"
+ }`}
+
+ )}
+
+
+
+ );
+}
diff --git a/web/src/customer-grid/MapView.tsx b/web/src/customer-grid/MapView.tsx
index 73d493603857203d1aba67ade047584bea736b72..30d5940ec4fd7349573edbd202374b299a799e89 100644
--- a/web/src/customer-grid/MapView.tsx
+++ b/web/src/customer-grid/MapView.tsx
@@ -41,6 +41,7 @@ import {
BASEMAP_DETAIL_K,
bubbleRadius,
cardBox,
+ clientToUser,
dashPattern,
fitView,
fromScreen,
@@ -53,7 +54,9 @@ import {
haversineKm,
isPlottable,
normRect,
+ pathBounds,
planRoute,
+ pointInPolygon,
project,
toScreen,
unproject,
@@ -154,7 +157,19 @@ export function MapView({
const [roundTrip, setRoundTrip] = useState(false);
const [routeStartPid, setRouteStartPid] = useState(null);
const [view, setView] = useState(null);
- const [drag, setDrag] = useState<{ x0: number; y0: number; x1: number; y1: number } | null>(null);
+ /**
+ * The selection gesture in flight. Two shapes, never a mode for SELECTING
+ * itself: shift-drag always selects, and `lassoTool` only changes what it
+ * draws. That distinction is why the "no mode toggle" note over `onPointerDown`
+ * still holds — the thing people would never find behind a mode is selection,
+ * and it is still on the bare gesture.
+ */
+ const [drag, setDrag] = useState<
+ | { kind: "rect"; x0: number; y0: number; x1: number; y1: number }
+ | { kind: "lasso"; pts: Pt[] }
+ | null
+ >(null);
+ const [lassoTool, setLassoTool] = useState(false);
/**
* ⚠ The svg element lives in STATE, not in a ref, and that is load-bearing.
*
@@ -328,15 +343,20 @@ export function MapView({
[sizeField, sizeRange]
);
- /** Client coords -> the SVG's own user-space coords (the viewBox scales). */
+ /**
+ * Client coords -> the svg's own user-space coords.
+ *
+ * ⚠ The maths lives in `clientToUser` (mapProjection) rather than here, and
+ * that placement is the fix's other half. Wave 8 wrote the conversion inline
+ * in this .tsx — where the gate's negative controls, every one of which
+ * mutates the compiled `mapProjection.js`, could not reach it. So the box-
+ * select legs passed on coordinates handed to them ALREADY in user space, and
+ * a broken conversion shipped green for four waves. See `clientToUser`.
+ */
const localPoint = useCallback(
(clientX: number, clientY: number): Pt => {
if (!svgEl) return { x: 0, y: 0 };
- const r = svgEl.getBoundingClientRect();
- return {
- x: ((clientX - r.left) / r.width) * VIEW_W,
- y: ((clientY - r.top) / r.height) * VIEW_H,
- };
+ return clientToUser(clientX, clientY, svgEl.getBoundingClientRect(), VIEW_W, VIEW_H);
},
[svgEl]
);
@@ -417,10 +437,11 @@ export function MapView({
// Shift (or Ctrl/Cmd) turns the drag into a SELECTION rectangle; a plain
// drag pans. Both gestures are on the same button because a map that
// needs a mode toggle to select is a map people never select on.
- if (e.shiftKey || e.ctrlKey || e.metaKey) setDrag({ x0: x, y0: y, x1: x, y1: y });
+ if (e.shiftKey || e.ctrlKey || e.metaKey)
+ setDrag(lassoTool ? { kind: "lasso", pts: [{ x, y }] } : { kind: "rect", x0: x, y0: y, x1: x, y1: y });
else panRef.current = { x, y, tx: view.tx, ty: view.ty };
},
- [view, localPoint]
+ [view, localPoint, lassoTool]
);
const onPointerMove = useCallback(
@@ -428,7 +449,18 @@ export function MapView({
const { x, y } = localPoint(e.clientX, e.clientY);
if (drag) {
movedRef.current = true;
- setDrag((d) => (d ? { ...d, x1: x, y1: y } : d));
+ setDrag((d) => {
+ if (!d) return d;
+ if (d.kind === "rect") return { ...d, x1: x, y1: y };
+ // `pointermove` fires far faster than a loop needs vertices, so a slow
+ // hand tracing 200 px would otherwise build a thousand-point polygon
+ // that every pin is then tested against on every frame. Drop a sample
+ // that has not travelled ~2 px — the viewBox is fixed relative to the
+ // screen, so this threshold means the same thing at every zoom.
+ const last = d.pts[d.pts.length - 1];
+ if (Math.abs(x - last.x) + Math.abs(y - last.y) < 2) return d;
+ return { kind: "lasso", pts: [...d.pts, { x, y }] };
+ });
return;
}
const pan = panRef.current;
@@ -443,14 +475,24 @@ export function MapView({
const onPointerUp = useCallback(
(e: ReactPointerEvent) => {
if (drag && view) {
- const r = normRect(drag.x0, drag.y0, drag.x1, drag.y1);
- // A rectangle smaller than a few px is a mis-click, not a selection —
+ // A gesture smaller than a few px is a mis-click, not a selection —
// clearing the user's set on a stray shift-click would be its own bug.
- if (r.x1 - r.x0 > 3 && r.y1 - r.y0 > 3) {
+ // ONE rule for both shapes, measured on the lasso's own bounds; a loop
+ // also needs three points before it is a polygon at all.
+ const b =
+ drag.kind === "rect"
+ ? normRect(drag.x0, drag.y0, drag.x1, drag.y1)
+ : pathBounds(drag.pts);
+ const drawn = b.x1 - b.x0 > 3 && b.y1 - b.y0 > 3 && (drag.kind === "rect" || drag.pts.length >= 3);
+ if (drawn) {
const hits: number[] = [];
for (const p of points) {
const s = toScreen(p.p, view);
- if (s.x >= r.x0 && s.x <= r.x1 && s.y >= r.y0 && s.y <= r.y1) hits.push(p.pid);
+ const held =
+ drag.kind === "rect"
+ ? s.x >= b.x0 && s.x <= b.x1 && s.y >= b.y0 && s.y <= b.y1
+ : pointInPolygon(s, drag.pts);
+ if (held) hits.push(p.pid);
}
onSelectPids(hits, e.altKey ? "add" : "replace");
}
@@ -474,7 +516,8 @@ export function MapView({
}
const hovered = hoverPid != null ? points.find((p) => p.pid === hoverPid) : undefined;
- const rect = drag ? normRect(drag.x0, drag.y0, drag.x1, drag.y1) : null;
+ const rect = drag?.kind === "rect" ? normRect(drag.x0, drag.y0, drag.x1, drag.y1) : null;
+ const lassoPts = drag?.kind === "lasso" && drag.pts.length > 1 ? drag.pts : null;
const tf = `translate(${view.tx.toFixed(2)} ${view.ty.toFixed(2)}) scale(${view.k.toFixed(6)})`;
// Strokes live in the transformed group, so they are pre-divided by the zoom.
// ⚠ This is the map's ONE stroke mechanism — see the note over `hairline` in
@@ -630,8 +673,13 @@ export function MapView({
)}
)}
+ {/* Every gesture the map has, named. `alt` (add to the selection rather
+ than replace it, onPointerUp) shipped in wave 8 and was disclosed
+ NOWHERE, so "select these as well" was a feature only the source
+ knew about. It needs shift too — alt alone still pans. */}
- Scroll to zoom · drag to pan · shift-drag to select
+ Scroll to zoom · drag to pan · {lassoTool ? "shift-drag to lasso" : "shift-drag to select"}{" "}
+ · shift-alt-drag to extend the selection
)}
+ {/* The loop paints as a POLYGON, so the shape on screen is the shape
+ the hit test uses — an open would draw a mouth the
+ selection does not have. Same class as the rectangle: one marquee
+ look, and no new stylesheet rule (index.css is not this fence). */}
+ {lassoPts && (
+ `${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" ")}
+ />
+ )}
{/* --- I18 — the camera controls. Zoom in / zoom out / fit to data, the
@@ -850,6 +908,35 @@ export function MapView({
+ {/* The selection SHAPE. Not a mode for selecting — shift-drag selects
+ either way — so the note over `onPointerDown` still stands. The
+ icon shows the shape you will get, which is the only "on" state
+ available: `.cg-map-ctl-b` has no pressed style and index.css
+ belongs to another session this wave, so a pressed look would have
+ been a control that cannot show its own state. */}
+ setLassoTool((v) => !v)}
+ >
+ {lassoTool ? (
+
+
+
+
+ ) : (
+
+
+
+ )}
+
{/* The street-level hand-off, aimed at whatever is on screen right
now. This is what makes the zoom cap honest rather than merely
restrictive: the map stops where its geometry stops, and points at
diff --git a/web/src/customer-grid/RecordDetail.tsx b/web/src/customer-grid/RecordDetail.tsx
index 4f35b49175950ccd68c0719b16453564c668fd4f..3f845f775c7025176327e35af0e40fbc564d12bf 100644
--- a/web/src/customer-grid/RecordDetail.tsx
+++ b/web/src/customer-grid/RecordDetail.tsx
@@ -47,7 +47,21 @@ import {
toggleMulti,
userInitials,
} from "./recordFields";
-import { pickTint } from "./choiceColors";
+/**
+ * ⭐⭐ WAVE-29 T24 (owner item 13, and D-121's class) — the record panel READS THE OPTION COLOURS.
+ *
+ * The canvas has honoured them since wave 14 (`cells.optionTint`) and so has the kanban card
+ * (`viewModes.tsx`), while this panel painted every chosen chip the same fixed blue and every
+ * single-select as an unstyled native `` — so the SAME record showed a colour-coded
+ * status in the grid and a colourless one the moment somebody opened it. That is D-121 exactly
+ * (the record panel ignoring a field's format while the canvas honours it), one property along.
+ *
+ * ⚠ `optionTint` is THE resolver and it is not called twice: it already answers "colour coding is
+ * switched off" with `undefined`, in which case no inline style is set and the stylesheet's
+ * neutral chip is what paints. `optionBorderColor` is the same darkened companion the canvas
+ * draws its outline with, so a chip here and a bubble there cannot drift apart.
+ */
+import { optionBorderColor, optionTint, pickTint } from "./choiceColors";
// Wave-19 R7 — ONE asset resolver and ONE upload path, shared with the catalog. A second URL
// builder here is how the modal and the printed page would disagree about a row's picture.
import { assetUrl, uploadRecordImage } from "./catalogData";
@@ -741,10 +755,27 @@ export default function RecordDetail({
{field.type === "user" && current !== "" && (
)}
+ {/* T24: the control wears the SELECTED option's colour. An `` cannot be
+ styled portably (the same platform limit this block already records for avatars),
+ so the colour goes on the box — which is where the reader is looking anyway, and
+ matches what the canvas paints for the same cell. `user` columns are excluded: a
+ person is not a choice with a palette, and `optionTint` would hash a name into
+ one. */}
{
+ const tint = optionTint(field, current);
+ return tint
+ ? { background: tint.bg, color: tint.fg,
+ borderColor: optionBorderColor(tint.bg) }
+ : undefined;
+ })()
+ : undefined
+ }
onChange={(event) => onNotesCommit(field.key, event.target.value)}
>
@@ -780,11 +811,24 @@ export default function RecordDetail({
{entries.map((entry) => {
const on = held.includes(entry.value);
+ // T24: the HELD chips wear the option's own colour; an unheld one stays neutral,
+ // because tinting every choice would make "which of these is on" a judgement
+ // about saturation rather than a state you can read.
+ const tint = on ? optionTint(field, entry.value) : undefined;
return (
diff --git a/web/src/customer-grid/SwipeView.css b/web/src/customer-grid/SwipeView.css
new file mode 100644
index 0000000000000000000000000000000000000000..42a8ea52e8970a015ee8aab1fb579e79ac1a2830
--- /dev/null
+++ b/web/src/customer-grid/SwipeView.css
@@ -0,0 +1,25 @@
+
+/* ⭐⭐ WAVE-29 T26 (owner R9) — the card renders the RECORD: fields, attachments, comments.
+ The card was a title plus three cells and could stay any height it liked; carrying a whole
+ record it needs a ceiling and a scroll of its own, or the deck's own controls (Later, the two
+ decision buttons) leave the viewport on a record with twenty columns. */
+.cg-swipe-card {
+ max-height: min(62vh, 560px);
+ overflow-y: auto;
+}
+
+/* Both sections are INSIDE the card, which is a `role="button"`. They stop their own events, so
+ the separation has to read visually too — otherwise a reviewer cannot tell what will open the
+ record and what will not. */
+.cg-swipe-docs,
+.cg-swipe-comments {
+ margin-top: 10px;
+ padding-top: 8px;
+ border-top: 1px solid var(--lp-line);
+ cursor: default;
+}
+
+/* The comments box carries its own composer; the card's own padding would double it. */
+.cg-swipe-comments {
+ text-align: left;
+}
diff --git a/web/src/customer-grid/SwipeView.tsx b/web/src/customer-grid/SwipeView.tsx
index a7ef20a0126c4279f0c465f101615cf31e8ae4b2..73be833b2e5761aa8afd61e5b847a639fac3d17a 100644
--- a/web/src/customer-grid/SwipeView.tsx
+++ b/web/src/customer-grid/SwipeView.tsx
@@ -1,476 +1,562 @@
-// ---------------------------------------------------------------------------
-// customer-grid / SwipeView.tsx
-// ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the SWIPE deck.
-//
-// One record at a time, decided left or right into two options of ONE
-// single-select. R2 fixes the shape and it is narrow on purpose:
-// · SINGLE-SELECT ONLY, one field, two of its options. No checkbox binding.
-// · the deck holds only the records whose bound field is EMPTY — a record
-// that already has a value has been decided, and re-asking is not triage.
-// · a swipe writes through the NORMAL cell door (`onSwipe`, which the host
-// maps to the same `patchAndRecord` a kanban card move uses), so undo,
-// echo-suppression and the permission wall all hold without being
-// re-implemented here.
-//
-// Honesty rules carried from the other modes:
-// · nothing is silently hidden — the deck states what is left, and the
-// empty state names the bound field rather than saying "all done".
-// · a lost binding is SHOWN, never repaired by guessing. `_clean_display`
-// validates that `fieldKey` names a field the table HAS and stops there;
-// it cannot know the field is still a select, or that the two options are
-// still in its vocabulary (its docstring draws that line explicitly). So
-// those three losses land here, and each one says what happened and
-// re-opens the picker — the `viewModes.tsx:190-194` rule, which exists
-// because the wave-7 trap was a picker silently falling back to its first
-// option.
-// ---------------------------------------------------------------------------
-
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
-import { choiceOptions } from "./types";
-import type { DisplaySpec, Field, Row } from "./types";
-import { formatDisplay } from "./cells";
-import { actionHref } from "./display";
-import { optionTint } from "./choiceColors";
-import { ModeIcon } from "./icons";
-
-export type SwipeSpec = NonNullable;
-
-/** The three ways a stored binding can rot that the host cannot see (see the header note). */
-type BindingFault =
- | { kind: "unbound" }
- | { kind: "field-gone"; fieldKey: string }
- | { kind: "not-select"; field: Field }
- | { kind: "option-gone"; field: Field; missing: string[] };
-
-function readBinding(spec: SwipeSpec | undefined, fieldByKey: Map):
- { ok: true; field: Field; spec: SwipeSpec } | { ok: false; fault: BindingFault } {
- if (!spec) return { ok: false, fault: { kind: "unbound" } };
- const field = fieldByKey.get(spec.fieldKey);
- if (!field) return { ok: false, fault: { kind: "field-gone", fieldKey: spec.fieldKey } };
- // R2 — SINGLE-select only. `multiselect` is deliberately not accepted: its cell holds a SET,
- // so "write this option" would mean append-or-replace, and a triage gesture that sometimes
- // adds and sometimes overwrites is two gestures wearing one button.
- if (field.type !== "select") return { ok: false, fault: { kind: "not-select", field } };
- const options = choiceOptions(field);
- const has = new Set(options.map((o) => o.toLowerCase()));
- const missing = [spec.leftOption, spec.rightOption].filter((o) => !has.has(o.toLowerCase()));
- if (missing.length) return { ok: false, fault: { kind: "option-gone", field, missing } };
- return { ok: true, field, spec };
-}
-
-/** The bound cell is EMPTY — the one predicate that decides deck membership (R2). */
-const isUndecided = (row: Row, key: string): boolean =>
- String(row[key] ?? "").trim() === "";
-
-export function SwipeView({
- rows,
- fields,
- fieldByKey,
- spec,
- cardKeys,
- titleKey,
- canWrite,
- readOnlyReason,
- canBind,
- onSpec,
- onSwipe,
- onOpen,
-}: {
- /** DISTINCT data rows from the full pipeline, overlay edits layered — the kanban's contract.
- * The deck re-derives from these, so a written record leaves it as soon as the optimistic
- * patch lands; there is no local cursor to drift out of step with the data. */
- rows: Row[];
- /** Every field, for the picker's field list. */
- fields: Field[];
- fieldByKey: Map;
- /** The stored binding, or undefined for a deck nobody has configured yet. */
- spec: SwipeSpec | undefined;
- /** The handful of visible fields the card lists under its title. */
- cardKeys: string[];
- titleKey: string;
- /** May this viewer write the BOUND field's value? (the kanban's `canMove`) */
- canWrite: boolean;
- /** Stated when the deck cannot be decided — permissions, or a computed column. */
- readOnlyReason: string | null;
- /** May this viewer change what the deck is bound TO? A VIEW-config act, so it is the
- * view-editing permission and NOT `canWrite` — conflating the two is
- * [[schema-role-is-not-a-value-wall]]. REQUIRED, because a picker that silently does
- * nothing is worse than no picker ([[wrong-parent-not-broken-control]]). */
- canBind: boolean;
- /** `undefined` DELETES the binding — absent is the honest unconfigured state, and storing a
- * half-binding is what `cleanDisplay` drops on both engines (the `kanbanClamp` law). */
- onSpec: (next: SwipeSpec | undefined) => void;
- onSwipe: (pid: number, value: string) => void;
- onOpen: (pid: number) => void;
-}) {
- // "Later" — session-only, never stored. A deck with no way past a record you cannot decide
- // blocks on that record forever, so this is navigation WITHIN the deck rather than a third
- // gesture: it writes nothing, survives no reload, and the count is disclosed in the empty
- // state rather than quietly shrinking the deck (rule 8b).
- const [later, setLater] = useState>(new Set());
- const deckRef = useRef(null);
-
- const binding = useMemo(() => readBinding(spec, fieldByKey), [spec, fieldByKey]);
- const boundKey = binding.ok ? binding.field.key : null;
-
- const undecided = useMemo(
- () => (boundKey ? rows.filter((r) => isUndecided(r, boundKey)) : []),
- [rows, boundKey]
- );
- const deck = useMemo(() => undecided.filter((r) => !later.has(r.pid)), [undecided, later]);
- const card = deck[0];
-
- // A record decided elsewhere (the grid, another user's echo) must not stay "later" forever —
- // otherwise the disclosed skip count drifts away from what is actually on the deck.
- useEffect(() => {
- setLater((prev) => {
- if (prev.size === 0) return prev;
- const live = new Set(undecided.map((r) => r.pid));
- const next = new Set([...prev].filter((pid) => live.has(pid)));
- return next.size === prev.size ? prev : next;
- });
- }, [undecided]);
-
- const decide = useCallback(
- (value: string) => {
- if (!card || !canWrite) return;
- onSwipe(card.pid, value);
- },
- [card, canWrite, onSwipe]
- );
-
- // ← / → decide, and they are the SAME two actions the buttons are rather than a second code
- // path: the whole point of the mode is that one hand can clear a queue.
- useEffect(() => {
- if (!binding.ok || !canWrite || !card) return;
- const onKey = (e: KeyboardEvent) => {
- if (e.key !== "ArrowLeft" && e.key !== "ArrowRight") return;
- const el = document.activeElement as HTMLElement | null;
- // Never steal an arrow key from a text field or a native control the user is inside.
- if (el && (el.tagName === "INPUT" || el.tagName === "SELECT" || el.tagName === "TEXTAREA"))
- return;
- if (!deckRef.current?.contains(el ?? null) && el !== document.body) return;
- e.preventDefault();
- decide(e.key === "ArrowLeft" ? binding.spec.leftOption : binding.spec.rightOption);
- };
- window.addEventListener("keydown", onKey);
- return () => window.removeEventListener("keydown", onKey);
- }, [binding, canWrite, card, decide]);
-
- if (!binding.ok) {
- return (
-
-
-
- );
- }
-
- const { field } = binding;
- const leftTint = optionTint(field, binding.spec.leftOption);
- const rightTint = optionTint(field, binding.spec.rightOption);
-
- return (
-
-
-
-
- Sorting into {field.label}
-
-
- {deck.length.toLocaleString()} {deck.length === 1 ? "record" : "records"} to sort
-
- {canBind && (
- onSpec(undefined)}
- title="Choose a different field or different options for this deck"
- >
- Change
-
- )}
-
- {readOnlyReason &&
{readOnlyReason}
}
- {card ? (
-
-
decide(binding.spec.leftOption)}
- title={`Set ${field.label} to ${binding.spec.leftOption} (left arrow key)`}
- >
-
-
-
-
-
- {binding.spec.leftOption}
-
-
onOpen(card.pid)}
- onKeyDown={(e) => {
- if (e.key !== "Enter" && e.key !== " ") return;
- e.preventDefault();
- onOpen(card.pid);
- }}
- >
- {String(card[titleKey] ?? "")}
-
- {cardKeys.map((k) => {
- const f = fieldByKey.get(k);
- if (!f) return null;
- const text = formatDisplay(f, card[k]);
- if (!text) return null;
- // A url cell is a LINK here too (wave-26 item 11), through `display.ts`'s one
- // scheme guard — and every gesture that reaches it must be stopped from also
- // reaching the card, which is a button (the kanban card's note, same trap).
- const href = actionHref(f, card[k]);
- return (
-
- );
- })}
-
- {canWrite && (
- {
- e.stopPropagation();
- setLater((prev) => new Set(prev).add(card.pid));
- }}
- >
- Decide later
-
- )}
-
-
decide(binding.spec.rightOption)}
- title={`Set ${field.label} to ${binding.spec.rightOption} (right arrow key)`}
- >
- {binding.spec.rightOption}
-
-
-
-
-
-
-
- ) : (
- // The empty state NAMES the bound field (C3), because "nothing to sort" is ambiguous
- // between "the deck is finished" and "the filter hid everything", and the two want
- // different next actions. `later` is disclosed rather than quietly subtracted.
-
- {later.size > 0 ? (
- <>
-
- {later.size.toLocaleString()}{" "}
- {later.size === 1 ? "record is" : "records are"} set aside for later. Nothing else
- in this view is missing a {field.label} .
-
-
setLater(new Set())}>
- Bring them back
-
- >
- ) : (
-
- Every record in this view already has a {field.label} . Widen the
- view's filters to sort more.
-
- )}
-
- )}
-
- );
-}
-
-/**
- * The binding picker: field + two options. It is also the surface every LOST binding lands on,
- * with the loss stated above it — one place that answers "what is this deck for", rather than an
- * error screen beside a separate setup screen.
- */
-function SwipeBinder({
- fault,
- fields,
- fieldByKey,
- spec,
- canBind,
- onSpec,
-}: {
- fault: BindingFault;
- fields: Field[];
- fieldByKey: Map;
- spec: SwipeSpec | undefined;
- canBind: boolean;
- onSpec: (next: SwipeSpec | undefined) => void;
-}) {
- const selects = useMemo(
- () => fields.filter((f) => f.type === "select" && choiceOptions(f).length >= 2),
- [fields]
- );
- const [fieldKey, setFieldKey] = useState(() => {
- if (spec && fieldByKey.get(spec.fieldKey)?.type === "select") return spec.fieldKey;
- return selects[0]?.key ?? "";
- });
- const chosen = fieldByKey.get(fieldKey);
- const options = useMemo(() => (chosen ? choiceOptions(chosen) : []), [chosen]);
- const [left, setLeft] = useState("");
- const [right, setRight] = useState("");
-
- // The two option selects follow the FIELD. Kept in an effect rather than derived so the user's
- // pick survives re-renders, and reset whenever the field changes — carrying an option from the
- // previous field would offer a value the new field cannot hold.
- useEffect(() => {
- setLeft(options[0] ?? "");
- setRight(options.find((o) => o !== options[0]) ?? "");
- }, [options]);
-
- const message = ((): string => {
- switch (fault.kind) {
- case "field-gone":
- return "The field this deck sorted into has been deleted. Pick another one.";
- case "not-select":
- return `“${fault.field.label}” is no longer a single-select, so it has no options to`
- + " sort into. Pick another field.";
- case "option-gone":
- return `${fault.missing.map((m) => `“${m}”`).join(" and ")} ${
- fault.missing.length === 1 ? "is" : "are"
- } no longer ${fault.missing.length === 1 ? "an option" : "options"} on “${
- fault.field.label
- }”. Pick the sides again.`;
- default:
- return "Sort records one at a time into two options of a single-select field.";
- }
- })();
-
- if (!canBind) {
- return (
-
-
{message}
-
Its creator or an admin can set this view up.
-
- );
- }
- if (selects.length === 0) {
- // Honest and specific: the mode is not broken, the TABLE has nothing to bind to. Naming the
- // requirement is what turns a dead end into a next action.
- return (
-
-
- A swipe deck sorts into a single-select field with at least two options. This database
- does not have one yet — add a single-select column, then come back.
-
-
- );
- }
- const ready = !!chosen && !!left && !!right && left.toLowerCase() !== right.toLowerCase();
- return (
-
-
{message}
-
- Sort into
- setFieldKey(e.target.value)}
- >
- {/* Every option carries an explicit `value`: a whose value names nothing
- renders its FIRST option and reports a choice the user never made
- ([[cg-condition-builder-items]]). */}
- {selects.map((f) => (
-
- {f.label}
-
- ))}
-
-
-
-
- Swipe left
- setLeft(e.target.value)}>
- {options.map((o) => (
-
- {o}
-
- ))}
-
-
-
- Swipe right
- setRight(e.target.value)}>
- {options.map((o) => (
-
- {o}
-
- ))}
-
-
-
- {!ready && left && right && (
- // Stated, not silently refused: both engines DROP a binding whose sides are the same
- // option, so a Save that looked like it worked would simply not persist.
-
- The two sides must be different options — otherwise both gestures do the same thing.
-
- )}
-
- chosen && onSpec({ fieldKey: chosen.key, leftOption: left, rightOption: right })
- }
- >
- Start sorting
-
-
- );
-}
-
-export default SwipeView;
+// ---------------------------------------------------------------------------
+// customer-grid / SwipeView.tsx
+// ⭐ WAVE-27 item 8 (owner ruling R2, contract C3) — the SWIPE deck.
+//
+// One record at a time, decided left or right into two options of ONE
+// single-select. R2 fixes the shape and it is narrow on purpose:
+// · SINGLE-SELECT ONLY, one field, two of its options. No checkbox binding.
+// · the deck holds only the records whose bound field is EMPTY — a record
+// that already has a value has been decided, and re-asking is not triage.
+// · a swipe writes through the NORMAL cell door (`onSwipe`, which the host
+// maps to the same `patchAndRecord` a kanban card move uses), so undo,
+// echo-suppression and the permission wall all hold without being
+// re-implemented here.
+//
+// Honesty rules carried from the other modes:
+// · nothing is silently hidden — the deck states what is left, and the
+// empty state names the bound field rather than saying "all done".
+// · a lost binding is SHOWN, never repaired by guessing. `_clean_display`
+// validates that `fieldKey` names a field the table HAS and stops there;
+// it cannot know the field is still a select, or that the two options are
+// still in its vocabulary (its docstring draws that line explicitly). So
+// those three losses land here, and each one says what happened and
+// re-opens the picker — the `viewModes.tsx:190-194` rule, which exists
+// because the wave-7 trap was a picker silently falling back to its first
+// option.
+// ---------------------------------------------------------------------------
+
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { choiceOptions } from "./types";
+import type { CustomerDoc, DisplaySpec, Field, Row, Viewer } from "./types";
+// ⚠ `SurfaceScope` is `apiBridge`'s (a TYPE-only import, so nothing of that module is
+// loaded here) — the same import `RecordComments` takes for the same prop.
+import type { SurfaceScope } from "./apiBridge";
+import RecordComments from "./RecordComments";
+import { Documents } from "./Documents";
+import { formatDisplay } from "./cells";
+import { actionHref } from "./display";
+import { optionTint } from "./choiceColors";
+import { ModeIcon } from "./icons";
+
+export type SwipeSpec = NonNullable;
+
+/** The three ways a stored binding can rot that the host cannot see (see the header note). */
+type BindingFault =
+ | { kind: "unbound" }
+ | { kind: "field-gone"; fieldKey: string }
+ | { kind: "not-select"; field: Field }
+ | { kind: "option-gone"; field: Field; missing: string[] };
+
+function readBinding(spec: SwipeSpec | undefined, fieldByKey: Map):
+ { ok: true; field: Field; spec: SwipeSpec } | { ok: false; fault: BindingFault } {
+ if (!spec) return { ok: false, fault: { kind: "unbound" } };
+ const field = fieldByKey.get(spec.fieldKey);
+ if (!field) return { ok: false, fault: { kind: "field-gone", fieldKey: spec.fieldKey } };
+ // R2 — SINGLE-select only. `multiselect` is deliberately not accepted: its cell holds a SET,
+ // so "write this option" would mean append-or-replace, and a triage gesture that sometimes
+ // adds and sometimes overwrites is two gestures wearing one button.
+ if (field.type !== "select") return { ok: false, fault: { kind: "not-select", field } };
+ const options = choiceOptions(field);
+ const has = new Set(options.map((o) => o.toLowerCase()));
+ const missing = [spec.leftOption, spec.rightOption].filter((o) => !has.has(o.toLowerCase()));
+ if (missing.length) return { ok: false, fault: { kind: "option-gone", field, missing } };
+ return { ok: true, field, spec };
+}
+
+/** The bound cell is EMPTY — the one predicate that decides deck membership (R2). */
+const isUndecided = (row: Row, key: string): boolean =>
+ String(row[key] ?? "").trim() === "";
+
+export function SwipeView({
+ rows,
+ fields,
+ fieldByKey,
+ spec,
+ cardKeys,
+ detailKeys,
+ scope,
+ viewer,
+ docs,
+ docPayload,
+ onDocAdd,
+ onDocFetch,
+ onDocDelete,
+ titleKey,
+ canWrite,
+ readOnlyReason,
+ canBind,
+ onSpec,
+ onSwipe,
+ onOpen,
+}: {
+ /** DISTINCT data rows from the full pipeline, overlay edits layered — the kanban's contract.
+ * The deck re-derives from these, so a written record leaves it as soon as the optimistic
+ * patch lands; there is no local cursor to drift out of step with the data. */
+ rows: Row[];
+ /** Every field, for the picker's field list. */
+ fields: Field[];
+ fieldByKey: Map;
+ /** The stored binding, or undefined for a deck nobody has configured yet. */
+ spec: SwipeSpec | undefined;
+ /** The handful of visible fields the card lists under its title. */
+ cardKeys: string[];
+ titleKey: string;
+ /** May this viewer write the BOUND field's value? (the kanban's `canMove`) */
+ canWrite: boolean;
+ /** Stated when the deck cannot be decided — permissions, or a computed column. */
+ readOnlyReason: string | null;
+ /** May this viewer change what the deck is bound TO? A VIEW-config act, so it is the
+ * view-editing permission and NOT `canWrite` — conflating the two is
+ * [[schema-role-is-not-a-value-wall]]. REQUIRED, because a picker that silently does
+ * nothing is worse than no picker ([[wrong-parent-not-broken-control]]). */
+ canBind: boolean;
+ /** `undefined` DELETES the binding — absent is the honest unconfigured state, and storing a
+ * half-binding is what `cleanDisplay` drops on both engines (the `kanbanClamp` law). */
+ onSpec: (next: SwipeSpec | undefined) => void;
+ onSwipe: (pid: number, value: string) => void;
+ onOpen: (pid: number) => void;
+ /**
+ * ⭐⭐ WAVE-29 T26 (owner R9) — WHAT MAKES THE CARD A RECORD RATHER THAN A SUMMARY.
+ *
+ * R9: *"the swipe CARD itself renders the record detail inline — fields, comments and
+ * attachments — so a reviewer decides without leaving the deck."* Everything below is that,
+ * and every one of them is OPTIONAL for one reason: this deck runs on hosts that supply
+ * different amounts. A card must render with none of them rather than throw — the standalone
+ * embed has no `scope`, and a deployment with no document storage serves no handlers.
+ */
+ /** Every field the VIEW shows, in its order — not the three-key card summary. */
+ detailKeys?: string[];
+ /** The surface these records live on. Comments are keyed by it; absent ⇒ no comments section,
+ * which is the pre-existing precondition `RecordDetail` already carries. */
+ scope?: SurfaceScope;
+ viewer?: Viewer;
+ /** This record's attachments, and the plumbing. Handlers absent ⇒ the section is not rendered
+ * at all, exactly as `RecordDetail` decides it (a dead upload control is a promise the app
+ * cannot keep). */
+ /** Keyed by pid, because a deck shows many records — `RecordDetail` takes ONE record's list
+ * and that shape cannot serve a deck. */
+ docs?: Record;
+ docPayload?: { pid: number; docId: string; name: string; mime: string; data_b64: string };
+ /** ⚠ EVERY HANDLER TAKES THE PID, unlike `RecordDetail`'s, whose host closes over the ONE
+ * open record. A deck paints many records at once, so a handler bound to a single pid would
+ * attach every reviewer's upload to whichever card happened to be open. */
+ onDocAdd?: (
+ pid: number,
+ file: { name: string; mime: string; size: number; data_b64: string }
+ ) => void;
+ onDocFetch?: (pid: number, docId: string) => void;
+ onDocDelete?: (pid: number, docId: string) => void;
+}) {
+ // "Later" — session-only, never stored. A deck with no way past a record you cannot decide
+ // blocks on that record forever, so this is navigation WITHIN the deck rather than a third
+ // gesture: it writes nothing, survives no reload, and the count is disclosed in the empty
+ // state rather than quietly shrinking the deck (rule 8b).
+ const [later, setLater] = useState>(new Set());
+ const deckRef = useRef(null);
+
+ const binding = useMemo(() => readBinding(spec, fieldByKey), [spec, fieldByKey]);
+ const boundKey = binding.ok ? binding.field.key : null;
+
+ const undecided = useMemo(
+ () => (boundKey ? rows.filter((r) => isUndecided(r, boundKey)) : []),
+ [rows, boundKey]
+ );
+ const deck = useMemo(() => undecided.filter((r) => !later.has(r.pid)), [undecided, later]);
+ const card = deck[0];
+
+ // A record decided elsewhere (the grid, another user's echo) must not stay "later" forever —
+ // otherwise the disclosed skip count drifts away from what is actually on the deck.
+ useEffect(() => {
+ setLater((prev) => {
+ if (prev.size === 0) return prev;
+ const live = new Set(undecided.map((r) => r.pid));
+ const next = new Set([...prev].filter((pid) => live.has(pid)));
+ return next.size === prev.size ? prev : next;
+ });
+ }, [undecided]);
+
+ const decide = useCallback(
+ (value: string) => {
+ if (!card || !canWrite) return;
+ onSwipe(card.pid, value);
+ },
+ [card, canWrite, onSwipe]
+ );
+
+ // ← / → decide, and they are the SAME two actions the buttons are rather than a second code
+ // path: the whole point of the mode is that one hand can clear a queue.
+ useEffect(() => {
+ if (!binding.ok || !canWrite || !card) return;
+ const onKey = (e: KeyboardEvent) => {
+ if (e.key !== "ArrowLeft" && e.key !== "ArrowRight") return;
+ const el = document.activeElement as HTMLElement | null;
+ // Never steal an arrow key from a text field or a native control the user is inside.
+ if (el && (el.tagName === "INPUT" || el.tagName === "SELECT" || el.tagName === "TEXTAREA"))
+ return;
+ if (!deckRef.current?.contains(el ?? null) && el !== document.body) return;
+ e.preventDefault();
+ decide(e.key === "ArrowLeft" ? binding.spec.leftOption : binding.spec.rightOption);
+ };
+ window.addEventListener("keydown", onKey);
+ return () => window.removeEventListener("keydown", onKey);
+ }, [binding, canWrite, card, decide]);
+
+ if (!binding.ok) {
+ return (
+
+
+
+ );
+ }
+
+ const { field } = binding;
+ const leftTint = optionTint(field, binding.spec.leftOption);
+ const rightTint = optionTint(field, binding.spec.rightOption);
+
+ return (
+
+
+
+
+ Sorting into {field.label}
+
+
+ {deck.length.toLocaleString()} {deck.length === 1 ? "record" : "records"} to sort
+
+ {canBind && (
+ onSpec(undefined)}
+ title="Choose a different field or different options for this deck"
+ >
+ Change
+
+ )}
+
+ {readOnlyReason &&
{readOnlyReason}
}
+ {card ? (
+
+
decide(binding.spec.leftOption)}
+ title={`Set ${field.label} to ${binding.spec.leftOption} (left arrow key)`}
+ >
+
+
+
+
+
+ {binding.spec.leftOption}
+
+
onOpen(card.pid)}
+ onKeyDown={(e) => {
+ if (e.key !== "Enter" && e.key !== " ") return;
+ e.preventDefault();
+ onOpen(card.pid);
+ }}
+ >
+ {String(card[titleKey] ?? "")}
+
+ {/* ⭐ WAVE-29 T26 (R9) — the RECORD's visible fields, not the three-key summary the
+ kanban card lends. `detailKeys` falls back to `cardKeys` so a host that has not
+ been widened yet renders exactly what it rendered before. */}
+ {(detailKeys && detailKeys.length ? detailKeys : cardKeys).map((k) => {
+ const f = fieldByKey.get(k);
+ if (!f) return null;
+ const text = formatDisplay(f, card[k]);
+ if (!text) return null;
+ // A url cell is a LINK here too (wave-26 item 11), through `display.ts`'s one
+ // scheme guard — and every gesture that reaches it must be stopped from also
+ // reaching the card, which is a button (the kanban card's note, same trap).
+ const href = actionHref(f, card[k]);
+ return (
+
+ );
+ })}
+
+ {/* ⭐ R9's other two thirds. Both stop their own events: the card is a `role="button"`
+ that opens the record, and a click on a comment box or an upload control must not
+ also open the modal it exists to make unnecessary (the kanban card's link trap,
+ one surface over). */}
+ {onDocAdd && onDocFetch && onDocDelete && (
+ e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ role="presentation"
+ >
+ onDocAdd(card.pid, file)}
+ onFetch={(docId) => onDocFetch(card.pid, docId)}
+ onDelete={(docId) => onDocDelete(card.pid, docId)}
+ />
+
+ )}
+ {scope != null && (
+ e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ role="presentation"
+ >
+ {/* ⚠ KEYED ON scope+pid, the same key `RecordDetail` uses: without it React
+ reuses the mounted instance across a card change and paints one record's
+ comments under another's name until the fetch returns. */}
+
+
+ )}
+ {canWrite && (
+ {
+ e.stopPropagation();
+ setLater((prev) => new Set(prev).add(card.pid));
+ }}
+ >
+ Decide later
+
+ )}
+
+
decide(binding.spec.rightOption)}
+ title={`Set ${field.label} to ${binding.spec.rightOption} (right arrow key)`}
+ >
+ {binding.spec.rightOption}
+
+
+
+
+
+
+
+ ) : (
+ // The empty state NAMES the bound field (C3), because "nothing to sort" is ambiguous
+ // between "the deck is finished" and "the filter hid everything", and the two want
+ // different next actions. `later` is disclosed rather than quietly subtracted.
+
+ {later.size > 0 ? (
+ <>
+
+ {later.size.toLocaleString()}{" "}
+ {later.size === 1 ? "record is" : "records are"} set aside for later. Nothing else
+ in this view is missing a {field.label} .
+
+
setLater(new Set())}>
+ Bring them back
+
+ >
+ ) : (
+
+ Every record in this view already has a {field.label} . Widen the
+ view's filters to sort more.
+
+ )}
+
+ )}
+
+ );
+}
+
+/**
+ * The binding picker: field + two options. It is also the surface every LOST binding lands on,
+ * with the loss stated above it — one place that answers "what is this deck for", rather than an
+ * error screen beside a separate setup screen.
+ */
+function SwipeBinder({
+ fault,
+ fields,
+ fieldByKey,
+ spec,
+ canBind,
+ onSpec,
+}: {
+ fault: BindingFault;
+ fields: Field[];
+ fieldByKey: Map;
+ spec: SwipeSpec | undefined;
+ canBind: boolean;
+ onSpec: (next: SwipeSpec | undefined) => void;
+}) {
+ const selects = useMemo(
+ () => fields.filter((f) => f.type === "select" && choiceOptions(f).length >= 2),
+ [fields]
+ );
+ const [fieldKey, setFieldKey] = useState(() => {
+ if (spec && fieldByKey.get(spec.fieldKey)?.type === "select") return spec.fieldKey;
+ return selects[0]?.key ?? "";
+ });
+ const chosen = fieldByKey.get(fieldKey);
+ const options = useMemo(() => (chosen ? choiceOptions(chosen) : []), [chosen]);
+ const [left, setLeft] = useState("");
+ const [right, setRight] = useState("");
+
+ // The two option selects follow the FIELD. Kept in an effect rather than derived so the user's
+ // pick survives re-renders, and reset whenever the field changes — carrying an option from the
+ // previous field would offer a value the new field cannot hold.
+ useEffect(() => {
+ setLeft(options[0] ?? "");
+ setRight(options.find((o) => o !== options[0]) ?? "");
+ }, [options]);
+
+ const message = ((): string => {
+ switch (fault.kind) {
+ case "field-gone":
+ return "The field this deck sorted into has been deleted. Pick another one.";
+ case "not-select":
+ return `“${fault.field.label}” is no longer a single-select, so it has no options to`
+ + " sort into. Pick another field.";
+ case "option-gone":
+ return `${fault.missing.map((m) => `“${m}”`).join(" and ")} ${
+ fault.missing.length === 1 ? "is" : "are"
+ } no longer ${fault.missing.length === 1 ? "an option" : "options"} on “${
+ fault.field.label
+ }”. Pick the sides again.`;
+ default:
+ return "Sort records one at a time into two options of a single-select field.";
+ }
+ })();
+
+ if (!canBind) {
+ return (
+
+
{message}
+
Its creator or an admin can set this view up.
+
+ );
+ }
+ if (selects.length === 0) {
+ // Honest and specific: the mode is not broken, the TABLE has nothing to bind to. Naming the
+ // requirement is what turns a dead end into a next action.
+ return (
+
+
+ A swipe deck sorts into a single-select field with at least two options. This database
+ does not have one yet — add a single-select column, then come back.
+
+
+ );
+ }
+ const ready = !!chosen && !!left && !!right && left.toLowerCase() !== right.toLowerCase();
+ return (
+
+
{message}
+
+ Sort into
+ setFieldKey(e.target.value)}
+ >
+ {/* Every option carries an explicit `value`: a whose value names nothing
+ renders its FIRST option and reports a choice the user never made
+ ([[cg-condition-builder-items]]). */}
+ {selects.map((f) => (
+
+ {f.label}
+
+ ))}
+
+
+
+
+ Swipe left
+ setLeft(e.target.value)}>
+ {options.map((o) => (
+
+ {o}
+
+ ))}
+
+
+
+ Swipe right
+ setRight(e.target.value)}>
+ {options.map((o) => (
+
+ {o}
+
+ ))}
+
+
+
+ {!ready && left && right && (
+ // Stated, not silently refused: both engines DROP a binding whose sides are the same
+ // option, so a Save that looked like it worked would simply not persist.
+
+ The two sides must be different options — otherwise both gestures do the same thing.
+
+ )}
+
+ chosen && onSpec({ fieldKey: chosen.key, leftOption: left, rightOption: right })
+ }
+ >
+ Start sorting
+
+
+ );
+}
+
+export default SwipeView;
diff --git a/web/src/customer-grid/ViewSidebar.tsx b/web/src/customer-grid/ViewSidebar.tsx
index 0638995e468d240cc164fe58011976cd9fe86f67..1755176b3a733bb972270a9a27aa645509e41ebf 100644
--- a/web/src/customer-grid/ViewSidebar.tsx
+++ b/web/src/customer-grid/ViewSidebar.tsx
@@ -1,2006 +1,2069 @@
-import { useState } from "react";
-import type { KeyboardEvent as ReactKeyboardEvent } from "react";
-import { AnchoredOverlay } from "./OverlaySurface";
-import { EXPORT_FORMATS, EXPORT_LABELS } from "./export";
-import type { ExportFormat } from "./export";
-import type {
- DisplayMode,
- FolderIcon,
- GridFolder,
- SavedView,
- ViewEditMode,
- ViewPermissions,
- Viewer,
-} from "./types";
-import {
- DEFAULT_FOLDER_SHAPE,
- DEFAULT_FOLDER_TONE,
- FOLDER_SHAPES,
- FOLDER_TONES,
- MAX_VIEW_USERS,
- VIEW_EDIT_BLURBS,
- VIEW_EDIT_LABELS,
- VIEW_EDIT_MODES,
- cleanFolderIcon,
- cleanViewPermissions,
- isModeFrozen,
- isUndeletableView,
- mayEditView,
- mayToggleViewLock,
- viewDisplayMode,
-} from "./types";
-import {
- CREATABLE_MODES,
- FOLDER_SHAPE_LABELS,
- FOLDER_TONE_LABELS,
- MODE_LABELS,
- MODE_TONE,
-} from "./iconShapes";
-import { FOLDER_TONE_PAINT } from "./iconShapes";
-import { FolderMark, LockMark, MenuLabel, ModeIcon, ToneModeIcon } from "./icons";
-import { SHARED_FOLDER_ID, groupByFolder, reorderFolderIds } from "./folders";
-
-/* ─────────────────────────────────────────────────────────────────────────────────────────
- ⭐ WAVE 27 · OWNER ITEM 1 — a MARK on each "Who can edit" row.
-
- The three choices used to be three radio buttons and three sentences, which is a wall of
- prose at the exact moment a user is deciding who else can change their work. A mark is the
- part you recognise on the second visit, and Airtable's own share step leads every row with
- one.
-
- ⛔ WHY THE PATHS LIVE HERE AND NOT IN `icons.tsx`, stated so a later wave does not "tidy" it
- back: `icons.tsx` renders `MODE_SHAPES` from `iconShapes.ts`, which the glide CANVAS also
- paints — geometry with two painters belongs there. These three have exactly ONE call site,
- thirty lines below, and no canvas draws them. `LockMark` and `EyeOffIcon` already carry the
- same argument at their definitions ("paths are inline… no header ever draws this one").
- ⚠ The wave-27 ownership map puts `iconShapes.ts` in session D's fence and leaves `icons.tsx`
- in NOBODY's; a one-call-site component in a file this session owns has zero collision risk,
- and the ask to settle `icons.tsx`'s owner is in the mailbox (C-3).
-
- The tone follows the STRENGTH of the grant — neutral for the private default, blue for the
- open one, green for the deliberate pick — and the paint comes from the shared pastel table,
- never a literal, so these can never be the one place in the product with its own colours.
- Outline-only, the register the owner set in 2026-07-29: "it doesn't have a full color, only
- the linings". */
-const PERM_TONE: Record = {
- personal: "neutral",
- collaborative: "blue",
- users: "green",
-};
-
-/** head + shoulders, at `x`. One drawing, three uses — a second hand-drawn person is how two
- * marks in one panel end up different heights. */
-const person = (x: number) =>
- `M${x} 7.4a2 2 0 1 0 0-4 2 2 0 0 0 0 4Z` +
- `M${x - 3.2} 12.6c0-1.9 1.4-3.1 3.2-3.1s3.2 1.2 3.2 3.1`;
-
-const PERM_MARK: Record = {
- // One person: only you.
- personal: [person(8)],
- // Two, side by side and overlapping: anyone who can see the table.
- collaborative: [person(5.9), person(10.6)],
- // One person and a tick: the people you picked.
- users: [person(6), "M10.6 9.4l1.5 1.6 2.6-3"],
-};
-
-function PermMark({ mode }: { mode: ViewEditMode }) {
- const paint = FOLDER_TONE_PAINT[PERM_TONE[mode]];
- return (
-
- {PERM_MARK[mode].map((d, i) => (
-
- ))}
-
- );
-}
-
-/* WAVE 20 item 18 (C-SHARE) — the system folder every view shared WITH you lands in until
- you file it somewhere of your own. It is the only folder id this component recognises by
- name, and as of wave 21 (item 9) it is DECLARED IN `folders.ts` beside the synthesis that
- produces it — a component-local copy of a contract id is a copy that can drift from the
- rule, and the rule is the half a gate can run. */
-
-interface ViewSidebarProps {
- views: SavedView[];
- activeViewId: string;
- saveState: "saved" | "saving";
- onSelect: (id: string) => void;
- /** I14 + I17 — the flyout names the display mode and the prompt names who may edit, so
- * creation carries both. `permissions` is always explicit (C4: absent on create means
- * personal, so an omission would silently contradict the form). */
- onCreate: (name: string, mode: DisplayMode, permissions: ViewPermissions) => void;
- onRename: (id: string, name: string) => void;
- /** Persist a view's description. "" is a real value — it clears it. */
- onNote: (id: string, note: string) => void;
- onDuplicate: (id: string) => void;
- onDelete: (id: string) => void;
- /** Cohorts this user can add to (owner item 6). Empty is fine — the menu then only offers
- * "New cohort…", which is how the first cohort gets created. */
- lists?: { id: string; name: string }[];
- /** Add the customers THIS view currently matches to a cohort. `cohortId` empty = create one
- * called `name`. The view need not be the active one — the caller re-runs the engine for it. */
- onAddToList?: (viewId: string, cohortId: string, name: string) => void;
- /**
- * ⭐ WAVE 21 item 11 (ruling R10) — open "Select from file…" for the ACTIVE view.
- *
- * ⛔ NO ARGUMENTS, and that is the contract rather than an omission. The dialog matches
- * over the rows the GRID holds, which are the active view's; passing a view id here
- * would invite a caller to match one view's list against another's rows and report the
- * difference as missing records. The menu switches views first (see its own note) so
- * this can stay a verb with one subject.
- */
- onSelectFromFile?: () => void;
- /**
- * Item 12 (contract C-LOCK) — lock a view to a cohort, or `null` to unlock.
- *
- * ⚠ This menu is the ONLY emitter of `cohortLock` in the client. Optional so the rail still
- * compiles and renders wherever the host has not accepted the key — the same posture every
- * other optional action here takes, and the reason a half-deployed wave degrades to "the
- * action is absent" rather than to "the action silently does nothing".
- */
- onCohortLock?: (viewId: string, cohortId: string | null) => void;
- /**
- * The TENANT'S today (wave-2 item 4) — the payload's `today` string VERBATIM, seeding the
- * default cohort name ` · `. Never the browser clock: a viewer a day
- * ahead of the tenant would stamp a cohort with a date the host's own data denies
- * ([[date-window-vocabulary]]). Absent = the name seeds without a date.
- */
- today?: string;
- /**
- * Wave-7 item W2 (contract C2) — export THIS view's current matches (filters + sorts,
- * visible fields in display order), generated client-side. Absent on tables where the
- * client does not hold the rows (server-windowed), so the entry never lies.
- */
- onExport?: (viewId: string, format: ExportFormat) => void;
- /**
- * Wave-8 I11c (contract C4) — folders. All of it is OPTIONAL: a caller that
- * passes no `folders` and no handlers renders exactly the pre-wave-8 flat rail,
- * which is what keeps the cohort page (and any future rail) from having to opt
- * into a feature it does not want.
- */
- folders?: GridFolder[];
- /** The folder a view belongs to RIGHT NOW — resolved by the caller through
- * `resolveFolderId`, so a pending drag and a dangling ref are already handled
- * before the rail sees them. */
- folderIdOf?: (viewId: string) => string | null;
- /** C5 (I15) — `icon` absent means the caller should send no icon at all (the folder wears
- * the default), NOT that it should clear one. */
- onFolderCreate?: (name: string, icon?: FolderIcon) => void;
- onFolderRename?: (folderId: string, name: string) => void;
- onFolderDelete?: (folderId: string) => void;
- onFolderDuplicate?: (folderId: string) => void;
- onItemMove?: (viewId: string, folderId: string | null) => void;
- /**
- * WAVE 20 item 19 (C-FOLDER-REORDER) — the folders, in the order the user just dragged
- * them into. The FULL list every time, never a delta: a partial order cannot say where an
- * unnamed folder went, and the host has to be able to stamp `order` on all of them.
- *
- * Optional like every other folder handler here: a caller that does not pass it gets a
- * rail with no folder drag at all, rather than a grip that lifts a folder and drops it
- * back — the posture this whole component takes toward an action nobody can honour.
- */
- onFolderReorder?: (order: string[]) => void;
- /**
- * ⭐ WAVE 27 · OWNER ITEM 5 (contract C7) — the rail's new VIEW order, whole.
- *
- * ⛔ REQUIRED, AND THE OPTIONAL VERSION OF THIS EXACT PROP IS WHY. `onFolderReorder` above
- * shipped optional in wave 20: its absence disabled the entire folder drag SILENTLY, every
- * gate stayed green, and folders simply would not move (the note at its call site in
- * `CustomerGrid` records it). A required prop fails `tsc` at the unmounted call site, which
- * is the only version of this rule a person cannot forget.
- */
- onViewReorder: (order: string[]) => void;
- /**
- * ⭐ WAVE 27 · OWNER ITEM 21 / RULING R14 — `viewId -> how many records match it RIGHT NOW`,
- * for the views that carry an alert. A view with no alert is ABSENT from this map, and that
- * absence is the whole display rule ("alert-on = badge-on").
- *
- * ⛔ REQUIRED, and `{}` is the honest empty. An optional prop would make "no alerts on this
- * table" and "the caller forgot to compute them" the same state on screen — which is the
- * shape wave 20's optional `onFolderReorder` shipped in, and it went unnoticed for a wave.
- * ⚠ ABSENT vs ZERO are DIFFERENT and both are real: absent = no alert, so no badge; `0` = an
- * alert on a view that currently matches nothing, which is exactly the case a user wants to
- * see. `!== undefined` rather than a truthiness test, for that reason.
- */
- alertCounts: Record;
- /**
- * C4 as AMENDED 2026-07-28 — the folder-level bulk "Add to cohort". The caller
- * owns the arithmetic (it holds the engine); the rail owns the confirm.
- * `preview` must return the DEDUPED union plus anything it could not compute,
- * because the amendment is explicit that a partial union is never silent.
- */
- folderAddPreview?: (folderId: string) => {
- pids: number[];
- counted: number;
- skipped: { name: string; why: string }[];
- };
- onFolderAddToList?: (folderId: string, cohortId: string, name: string) => void;
- /**
- * Wave-9 contract C3 (I12) — who is looking, so the lock entry appears only for the view's
- * creator or an admin. Absent = the entry never appears (fail-closed), which is also what
- * every caller that has not wired `onToggleLock` gets.
- */
- viewer?: Viewer;
- /** C3 — freeze/unfreeze this view's DISPLAY MODE. The host re-checks the actor. */
- onToggleLock?: (viewId: string, locked: boolean) => void;
- /* wave17 R1 / C-LOCKV — `cohorts` and `onSelectCohort` are GONE from this component's
- surface. They carried the transient-lock selection for the retired Cohorts section; a
- locked view is now an ordinary member of `views` and is selected by `onSelect` like any
- other, which is the whole point of R1. */
- /** I17 (C4) — the tenant's real account list, for the "Specific users" picker. The HOST
- * re-validates every name against core/users.py and drops what it cannot resolve, so this
- * list is a convenience, never the authority. */
- userOptions?: string[];
-}
-
-export default function ViewSidebar({
- views,
- activeViewId,
- saveState,
- onSelect,
- onCreate,
- onRename,
- onNote,
- onDuplicate,
- onDelete,
- lists = [],
- onAddToList,
- onSelectFromFile,
- onCohortLock,
- today,
- onExport,
- folders,
- folderIdOf,
- onFolderCreate,
- onFolderRename,
- onFolderDelete,
- onFolderDuplicate,
- onItemMove,
- onFolderReorder,
- onViewReorder,
- alertCounts,
- folderAddPreview,
- onFolderAddToList,
- viewer,
- onToggleLock,
- userOptions = [],
-}: ViewSidebarProps) {
- /**
- * I14 — creating a view now always knows WHICH KIND of view is being created, because the
- * flyout is the only door in. `null` = not creating; a mode = the name step for that mode.
- */
- const [creating, setCreating] = useState(null);
- /** I14 — the "+ Create new…" flyout's anchor. */
- const [createMenu, setCreateMenu] = useState(null);
- const [name, setName] = useState("");
- /** I17 (C4) — the create prompt's who-can-edit draft. Seeded PERSONAL, matching the host's
- * create-side default, so the form and the store agree before the user touches anything. */
- const [permEdit, setPermEdit] = useState("personal");
- const [permUsers, setPermUsers] = useState([]);
- const [menu, setMenu] = useState<{
- viewId: string;
- anchor: HTMLButtonElement;
- } | null>(null);
- const [renamingId, setRenamingId] = useState(null);
- const [noteFor, setNoteFor] = useState<{
- viewId: string;
- anchor: HTMLButtonElement;
- } | null>(null);
- const [noteDraft, setNoteDraft] = useState("");
- const [addFor, setAddFor] = useState<{
- viewId: string;
- anchor: HTMLButtonElement;
- } | null>(null);
- /** Item 12 (C-LOCK) — which view's cohort lock the picker is open for. */
- const [lockFor, setLockFor] = useState<{
- viewId: string;
- anchor: HTMLButtonElement;
- } | null>(null);
- const [newListName, setNewListName] = useState("");
- /** W2 — the Export format pane, anchored where the "…" menu was. */
- const [exportFor, setExportFor] = useState<{
- viewId: string;
- anchor: HTMLButtonElement;
- } | null>(null);
- const exportView = exportFor
- ? views.find((view) => view.id === exportFor.viewId)
- : undefined;
- const addView = addFor ? views.find((view) => view.id === addFor.viewId) : undefined;
- const menuView = menu ? views.find((view) => view.id === menu.viewId) : undefined;
- /** C4 — may this viewer change the menu view's IDENTITY? Courtesy only; the host is the
- * wall. C3's lock toggle ANDs with this rather than replacing it: the two contracts
- * compose (C4 says who may edit, C3 says who may freeze the mode). */
- const canEditMenuView = !!menuView && mayEditView(menuView, viewer);
- // I11c local UI state. Collapse is deliberately LOCAL (C4: "minimize state
- // local") — which folders you have open is a per-person, per-session habit,
- // not a property of the shared workspace.
- const [collapsed, setCollapsed] = useState>(new Set());
- const [folderMenu, setFolderMenu] = useState<{ id: string; anchor: HTMLElement } | null>(null);
- const [renamingFolder, setRenamingFolder] = useState(null);
- const [creatingFolder, setCreatingFolder] = useState(false);
- /**
- * I15 — the folder-create draft. The name became CONTROLLED with an explicit Create button
- * for a reason that is not cosmetic: the old input committed on BLUR, and clicking an icon
- * swatch blurs the input — so an icon picker beside a commit-on-blur field would create the
- * folder the moment you tried to colour it.
- */
- const [folderName, setFolderName] = useState("");
- const [folderIcon, setFolderIcon] = useState({
- shape: DEFAULT_FOLDER_SHAPE,
- tone: DEFAULT_FOLDER_TONE,
- });
- const [confirmDelete, setConfirmDelete] = useState(null);
- const [addTarget, setAddTarget] = useState(null);
- /** The element the bulk-add pane hangs off. Anchoring to document.body
- * put it at the bottom of the DOCUMENT rather than beside the folder —
- * painted, and effectively unreachable ([[ui-invisible-to-assertions]]).
- * Captured from the folder menu before that menu is dismissed. */
- const [addAnchor, setAddAnchor] = useState(null);
- const [addName, setAddName] = useState("");
- const [dropTarget, setDropTarget] = useState(null);
- // Owner item 9 (2026-07-31) — "Find a view". Local and never persisted: a search is a
- // moment, not a setting. Filtering runs BEFORE grouping so a folder with no matches
- // simply drops out of the rail while the query is live.
- const [viewQuery, setViewQuery] = useState("");
- // Owner item 11 — this rail is the SECOND navigation bar: it folds to the same slim strip
- // the shell nav does, remembered per browser under its own key.
- const [railShut, setRailShut] = useState(() => {
- try {
- return localStorage.getItem("aios-views-rail") === "1";
- } catch {
- return false;
- }
- });
- const toggleRail = () =>
- setRailShut((v) => {
- const next = !v;
- try {
- localStorage.setItem("aios-views-rail", next ? "1" : "0");
- } catch {
- // storage can be blocked; the fold still works for the session
- }
- return next;
- });
- const viewNeedle = viewQuery.trim().toLowerCase();
- const shownViews = viewNeedle
- ? views.filter((v) => (v.name || "").toLowerCase().includes(viewNeedle))
- : views;
- // ── WAVE 20 item 19 (C-FOLDER-REORDER) — dragging a FOLDER to reorder the rail ──────
- //
- // ⚠ A PRIVATE dataTransfer MIME, and it is not decoration. The VIEW drag two blocks down
- // carries `text/plain`, and the folder groups already accept that drop ("file this view
- // into me"). Sharing one channel would make every folder drag look like a view drop to
- // the handler that fires first — so the two drags are told apart by TYPE, the same
- // discipline the shell nav's own drag uses, and a drop of this type anywhere
- // text-editable types nothing.
- const FOLD_DRAG_TYPE = "application/x-loopable-fold";
- /** The folder being dragged, and the group its insertion line is currently drawn above. */
- const [foldDrag, setFoldDrag] = useState(null);
- const [foldOver, setFoldOver] = useState(null);
- const foldsReorderable = !!folders && !!onFolderReorder;
- /**
- * Land `draggedId` before `beforeId` (null = last) and emit the new order. The arithmetic
- * lives in `folders.ts` so a gate can run it under node — the handler around it is DOM.
- *
- * ⚠ The order is computed from the `folders` PROP, never from the rendered groups: a live
- * "Find a view" query hides folders with no matches, and an order derived from what is on
- * screen would silently drop the hidden ones out of the sequence.
- */
- const reorderFolders = (draggedId: string, beforeId: string | null) => {
- const next = reorderFolderIds((folders ?? []).map((f) => f.id), draggedId, beforeId);
- // `null` is "nothing to say" — an unknown id, or a drop that changed nothing.
- if (next) onFolderReorder?.(next);
- };
- // ── ⭐ WAVE 27 · OWNER ITEM 5 (contract C7) — dragging a VIEW to reorder the rail ────────
- //
- // The owner asked for both lists to be draggable top-to-bottom; folders already were
- // (wave 20 item 19) and views could only be FILED INTO a folder, never moved past each
- // other. This is that drag, built on the same three pieces so the two gestures cannot drift:
- // a private MIME, the shared `reorderFolderIds` arithmetic, and a whole-order emit.
- //
- // ⛔ THE ARITHMETIC IS REUSED, NOT COPIED. `reorderFolderIds` is id-generic — it takes a list
- // of ids, a dragged id and a "before" id, and every clause in it (dropped-on-itself, the
- // not-found fallback, the no-op return) is about SEQUENCES rather than about folders. It is
- // already exercised by `verify_folders.py`, and a second copy here would be a second place
- // for the dropped-on-itself bug that function's own comment records ([[one-evaluator-per-question]]).
- const VIEW_DRAG_TYPE = "application/x-loopable-view";
- const [viewDrag, setViewDrag] = useState(null);
- const [viewOver, setViewOver] = useState(null);
- /**
- * Land `draggedId` before `beforeId` and emit the FULL order.
- *
- * ⚠ Computed from the `views` PROP, never from the rendered rows — a live "Find a view"
- * query hides non-matching rows, and an order derived from what is on screen would silently
- * drop every hidden view out of the sequence. (The folder half carries the identical note;
- * it is the same trap and this rail has both filters on it.)
- */
- const reorderViews = (draggedId: string, beforeId: string | null) => {
- const next = reorderFolderIds(views.map((v) => v.id), draggedId, beforeId);
- if (next) onViewReorder(next); // `null` = an unknown id, or a drop that changed nothing
- };
- /**
- * One pair of handlers per VIEW ROW, listening for the view MIME only.
- *
- * ⛔ `stopPropagation` IS LOAD-BEARING. A view row sits inside its folder group, whose own
- * `onDrop` reads `text/plain` and files the view into that folder. Both types are on this
- * drag, so without this a reorder inside a folder would ALSO re-file the view — usually into
- * the folder it is already in, i.e. invisible, until the day it is not.
- * ⚠ WHAT THIS DELIBERATELY DOES NOT DO: change the view's folder. Dropping a view onto a row
- * in another folder reorders it and leaves it where it lives; filing stays a drop on the
- * folder itself. Two gestures, two meanings, rather than one gesture guessing.
- */
- const viewDropHandlers = (viewId: string) => ({
- onDragOver: (e: React.DragEvent) => {
- if (!e.dataTransfer.types.includes(VIEW_DRAG_TYPE)) return;
- e.preventDefault();
- e.stopPropagation();
- e.dataTransfer.dropEffect = "move";
- setViewOver(viewId);
- },
- onDragLeave: () => setViewOver(null),
- onDrop: (e: React.DragEvent) => {
- if (!e.dataTransfer.types.includes(VIEW_DRAG_TYPE)) return;
- const dragged = e.dataTransfer.getData(VIEW_DRAG_TYPE);
- setViewOver(null);
- setViewDrag(null);
- if (!dragged) return;
- e.preventDefault();
- e.stopPropagation();
- reorderViews(dragged, viewId);
- },
- });
- const viewsReorderable = views.length > 1;
- const foldersOn = !!folders && !!folderIdOf && !!onItemMove;
- const groups = groupByFolder(
- shownViews,
- foldersOn ? (folders as GridFolder[]) : [],
- (v) => (folderIdOf ? folderIdOf(v.id) : null),
- /**
- * ⭐ WAVE 21 item 9 (R12 / contract C1) — THE CONSUMER of `shared`.
- *
- * The wire marks each granted view `shared: true` (C1); this is where that mark becomes
- * a place on screen. Passed unconditionally — NOT gated on `foldersOn` — because the
- * folder machinery is an opt-in feature of the views rail and "who shared this with me"
- * is not: a surface with folders switched off must still be able to show you somebody
- * else's view under a heading that says whose it is.
- */
- (v) => v.shared === true
- );
- const folderMenuF = folderMenu ? folders?.find((f) => f.id === folderMenu.id) : undefined;
- const addPreview = addTarget && folderAddPreview ? folderAddPreview(addTarget) : null;
- /**
- * ONE pair of drop handlers per group, serving TWO drags — a view being filed into this
- * folder (`text/plain`) and, since wave 20 item 19, a folder being dropped in front of
- * this one (`FOLD_DRAG_TYPE`). They must be one pair rather than two spreads: a second
- * `onDragOver`/`onDrop` on the same element replaces the first, silently, and the drag
- * that lost would simply stop working.
- *
- * The type is read BEFORE anything else, so a folder drag never paints the "file a view
- * in here" fill and a view drag never draws the insertion rule.
- */
- const dropHandlers = (folderId: string | null) =>
- foldersOn || foldsReorderable
- ? {
- onDragOver: (e: React.DragEvent) => {
- if (e.dataTransfer.types.includes(FOLD_DRAG_TYPE)) {
- if (!foldsReorderable) return;
- e.preventDefault();
- e.dataTransfer.dropEffect = "move";
- setFoldOver(folderId ?? "__root__");
- return;
- }
- if (!foldersOn) return;
- e.preventDefault();
- setDropTarget(folderId ?? "__root__");
- },
- onDragLeave: () => {
- setDropTarget(null);
- setFoldOver(null);
- },
- onDrop: (e: React.DragEvent) => {
- if (e.dataTransfer.types.includes(FOLD_DRAG_TYPE)) {
- const dragged = e.dataTransfer.getData(FOLD_DRAG_TYPE);
- setFoldOver(null);
- setFoldDrag(null);
- if (!foldsReorderable || !dragged) return;
- e.preventDefault();
- e.stopPropagation();
- reorderFolders(dragged, folderId);
- return;
- }
- if (!foldersOn) return;
- e.preventDefault();
- setDropTarget(null);
- const id = e.dataTransfer.getData("text/plain");
- if (id) onItemMove?.(id, folderId);
- },
- }
- : {};
- const noteView = noteFor
- ? views.find((view) => view.id === noteFor.viewId)
- : undefined;
-
- const create = () => {
- const value = name.trim();
- if (!value || !creating) return;
- // C4: permissions are sent EXPLICITLY on create. Omitting them means 'personal'
- // host-side, so a user who picked "Collaborative" and got a personal view would have no
- // way to tell — the form said one thing and the store did the other.
- // Cleaned through the same rules the host applies, so an empty "Specific users" grant
- // collapses to 'personal' HERE too and the confirmation the user sees is the truth.
- onCreate(value, creating, cleanViewPermissions(
- { edit: permEdit, users: permUsers }, "personal", userOptions));
- setName("");
- setPermEdit("personal");
- setPermUsers([]);
- setCreating(null);
- // WAVE 20 item 20 — the two steps share ONE panel, so the anchor outlives step 1 and
- // has to be released here. Left set, the flyout would spring back to the type chooser
- // the instant the view was created.
- setCreateMenu(null);
- };
-
- /** Item 20 — abandon the whole flyout, from either step. Escape, an outside click and
- * Cancel all mean the same thing ("I am not creating anything"), so they call one
- * function rather than three combinations of setters — and abandoning drops the DRAFT
- * too, or the next "+ Create new…" would hand back a half-filled form from a decision
- * the user already walked away from. */
- const closeCreate = () => {
- setCreating(null);
- setCreateMenu(null);
- setName("");
- setPermEdit("personal");
- setPermUsers([]);
- };
-
- const createFolder = () => {
- const value = folderName.trim();
- if (!value) return;
- // cleanFolderIcon returns undefined for the all-defaults pair, so a folder the user
- // never restyled stays byte-identical to a pre-wave-9 one (the no-churn rule).
- onFolderCreate?.(value, cleanFolderIcon(folderIcon));
- setFolderName("");
- setFolderIcon({ shape: DEFAULT_FOLDER_SHAPE, tone: DEFAULT_FOLDER_TONE });
- setCreatingFolder(false);
- };
-
- /**
- * WAVE 20 item 20 — the view door KEEPS the anchor, because the name/permissions step
- * now renders in the SAME anchored box the type chooser used. The old comment ("close
- * the flyout first, or the menu sits on top of the input") described the bug the owner
- * reported from the other side: the two steps of one action were drawn in two different
- * places, one beside the rail and one inside it, so step 2 read as an unrelated form.
- * One panel, two contents, one geometry.
- *
- * The FOLDER door still closes it: that path is a single step and has no second panel
- * to disagree with. Moving it too would be a change the ruling does not ask for.
- */
- const startView = (mode: DisplayMode) => {
- setCreatingFolder(false);
- setCreating(mode);
- };
- const startFolder = () => {
- setCreateMenu(null);
- setCreating(null);
- setCreatingFolder(true);
- };
-
- /**
- * WAVE 20 items 18/23/26 (R10, C-SHARE) — open the access editor for a view or a
- * folder.
- *
- * ⚠ A WINDOW EVENT, NOT A PROP, and not for convenience. The dialog is the SHELL's
- * (one editor for views, folders and databases — R10 asks for one vocabulary), and
- * this tree is host-neutral: `customer-grid/**` must not import a shell. The frame
- * already listens on this channel for toasts and staleness; this is one more note on
- * it. The literal is spelled once here and once in `shell/shareModel.ts`, which is
- * where it becomes a shared constant when S3 publishes it in `apiContract.ts`
- * (amendment A-S4-4).
- *
- * ⛔ It degrades to NOTHING when no shell is listening (the Streamlit-era bare grid):
- * a dispatched event nobody hears opens no dialog and throws no error — which is the
- * right failure for a frame-level surface reached from a rail.
- */
- const openShare = (kind: "view" | "folder", id: string, label: string) => {
- window.dispatchEvent(
- new CustomEvent("aios:share-open", { detail: { kind, id, label } })
- );
- };
-
- const onMenuKeyDown = (event: ReactKeyboardEvent) => {
- if (!["ArrowDown", "ArrowUp", "Home", "End"].includes(event.key)) return;
- const items = [
- ...event.currentTarget.querySelectorAll('[role="menuitem"]'),
- ];
- if (!items.length) return;
- event.preventDefault();
- const current = items.indexOf(document.activeElement as HTMLButtonElement);
- const next =
- event.key === "Home"
- ? 0
- : event.key === "End"
- ? items.length - 1
- : event.key === "ArrowDown"
- ? (current + 1) % items.length
- : (current - 1 + items.length) % items.length;
- items[next]?.focus();
- };
-
- return (
-