diff --git "a/api/automation_engine.py" "b/api/automation_engine.py" --- "a/api/automation_engine.py" +++ "b/api/automation_engine.py" @@ -1,5501 +1,6082 @@ -"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. - -THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant -store and a RUN that lives in this process. The definition is durable, small and rarely written; -the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file -is built to avoid, and it has two halves: - - * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the - store outlives the process that wrote it, so a container restart mid-run leaves an automation - that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives - in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written - exactly ONCE per run, at the end. - * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s - floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape - by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes - its whole result first and commits it in a single `update` callback. - -WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not -deployed, so its extraction core is copied into this file (the wave doc's instruction), ported -httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with -it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: -the skill version takes a URL from a developer on a CLI; this takes one from a request body. -""" -from __future__ import annotations - -import datetime as _dt -import hmac -import ipaddress -import json -import os -import re -import socket -import threading -import time -from urllib.parse import urljoin, urlparse - -import requests - -try: # the parser the extraction half needs - from bs4 import BeautifulSoup -except Exception: # pragma: no cover — deploy lag; see mailbox ② - BeautifulSoup = None - -# --------------------------------------------------------------------------------------------- -# THE BUCKET (C4-AUTO) -# --------------------------------------------------------------------------------------------- - -#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. -STORE_KEY = "automations" -#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through -#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for -#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the -#: session-D mailbox as a finding against A's file rather than fixed from here. -UT_STORE_KEY = "user_tables" -UT_PREFIX = "ut_" - -MAX_AUTOMATIONS = 40 -MAX_RUNS = 20 # trimmed history per automation -MAX_NAME = 80 -MAX_UT_ROWS = 5000 # mirrors core.user_tables.MAX_ROWS -MAX_UT_TABLES = 40 - -#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. -#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the -#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull -#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's -#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour -#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still -#: report success. A time series that stops after five days without saying so is worse than one -#: that was never built. -#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE -#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: -#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day -#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts -#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had -#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the -#: silent version would have flatlined a chart inside a fortnight with a green dot over it. -#: -#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises -#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, -#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 -#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a -#: smaller number. -#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) -MAX_UT_IG_ROWS = 200_000 -IG_TABLE_PREFIX = "ut_ig_" - -#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was -#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — -#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY -#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would -#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket -#: serialisation cost, purely by an accident of naming. Naming the append tables is the version -#: that stays true when the next `ut_ig_*` table is not one. -APPEND_TABLES = frozenset({"ut_ig_snapshots", "ut_ig_post_snapshots"}) - -KINDS = ("scrape_db", "field_instagram", "discover_instagram") -EXTRACTS = ("table", "jsonld") -STATES = ("idle", "running", "ok", "error", "partial") -#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). -#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a -#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). -TIERS = ("anonymous", "brightdata") -#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap -#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown -#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation -#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps -#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. -TIER_ALIASES = {"hiker": "brightdata"} - - -def clean_tier(raw): - """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" - t = str(raw or "").strip().lower() - t = TIER_ALIASES.get(t, t) - return t if t in TIERS else "" - - -def row_cap(table_key): - """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. - - An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything - else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. - """ - return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS - -#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI -#: shows and the vocabulary the parser accepts cannot drift. -CRON_PRESETS = [ - {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, - {"cron": "0 * * * *", "label": "Hourly"}, - {"cron": "0 6 * * *", "label": "Daily at 06:00"}, - {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, - {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, -] - -UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" - - -def _now(): - return _dt.datetime.now() - - -def _stamp(dt=None): - return (dt or _now()).strftime("%Y-%m-%d %H:%M") - - -def _iso(dt=None): - """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). - - ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables - and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A - naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC - minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is - not expressible at all. With the offset the same string is an instant, and relative rendering - becomes possible without migrating a single stored row. - - ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in - this module is against a naive `_now()`, and mixing aware and naive datetimes raises - `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. - """ - dt = dt or _now() - if dt.tzinfo is None: - dt = dt.astimezone() # a naive stamp from this process IS local time - return dt.isoformat(timespec="seconds") - - -def _parse_iso(s): - """A stamp → a NAIVE LOCAL datetime, with or without an offset. - - Both shapes exist in the store simultaneously and always will: every run committed before - D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new - shape would silently return None for them — and `is_due` reads None as "never ran", which - would re-fire every schedule once. Reading both is what makes the change additive. - """ - raw = str(s or "").strip().replace("Z", "+00:00") - dt = None - try: - dt = _dt.datetime.fromisoformat(raw) - except Exception: # noqa: BLE001 - try: - dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") - except Exception: # noqa: BLE001 - return None - return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt - - -# --------------------------------------------------------------------------------------------- -# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher -# --------------------------------------------------------------------------------------------- -# The skill version guards the URL a developer typed. This one guards a URL that arrived in a -# request body, which changes the threat model in two ways the original does not cover: -# -# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` -# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` -# sails straight past a guard that only ever saw the first URL. The negative control ("a -# localhost URL is refused") would still pass while the rail was wide open. `fetch` below -# therefore takes the hops MANUALLY and re-guards every one. -# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the -# host and checks EVERY answer, not just the literal. -# -# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the -# record between the guard and the socket is not stopped by it. Closing that needs a pinned -# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather -# than implied away — the rail refuses the realistic cases and says what it does not cover. - -class Refused(ValueError): - """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" - - -def _ip_public(ip): - return not (ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_reserved or ip.is_unspecified or ip.is_multicast) - - -def guard(url, allowed=()): - """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers - checked too, plus the optional `--allowed` domain rail scrape.py carries.""" - p = urlparse(url or "") - if p.scheme not in ("http", "https"): - raise Refused(f"only http(s) allowed, not {p.scheme!r}") - host = (p.hostname or "").strip() - if not host: - raise Refused("no host in the URL") - low = host.lower() - if (low in ("localhost", "localhost.localdomain") - or low.endswith(".local") or low.endswith(".internal") - or low.endswith(".localhost")): - raise Refused(f"refused non-public host {host!r} (public web only)") - try: # a bare-IP host is decided on the literal - ip = ipaddress.ip_address(low) - except ValueError: - ip = None - if ip is not None and not _ip_public(ip): - raise Refused(f"refused non-public IP {host} (public web only)") - if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): - raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") - if ip is None: - try: - infos = socket.getaddrinfo(host, None) - except Exception as e: - raise Refused(f"could not resolve {host!r}: {type(e).__name__}") - for info in infos: - addr = info[4][0] - try: - resolved = ipaddress.ip_address(addr.split("%")[0]) - except ValueError: - continue - if not _ip_public(resolved): - raise Refused( - f"refused {host!r}: it resolves to the non-public address {resolved}") - return True - - -def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): - """GET `url`, taking redirects BY HAND so every hop passes `guard`. - - Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, - not a silent truncation — an endless redirect is indistinguishable from an attempt to walk - the fetcher somewhere it was told not to go. - """ - current = url - hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} - hdrs.update(headers or {}) - for _hop in range(max_hops + 1): - guard(current, allowed) - r = requests.get(current, timeout=timeout, headers=hdrs, - allow_redirects=False, stream=True) - if r.status_code in (301, 302, 303, 307, 308): - loc = r.headers.get("Location") - r.close() - if not loc: - raise Refused(f"redirect {r.status_code} with no Location header") - current = urljoin(current, loc) - continue - body = r.raw.read(max_kb * 1024, decode_content=True) or b"" - status, final = r.status_code, str(r.url) - r.close() - return status, final, body - raise Refused(f"more than {max_hops} redirects — refusing to follow further") - - -def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): - """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. - - ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by - hand because a redirected GET is ordinary. A redirected POST is not: re-sending a - credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail - exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever - re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. - (The vendor calls below are the only POSTs this module makes, and they carry the API key.) - """ - guard(url) - hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} - hdrs.update(headers or {}) - r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, - data=json.dumps(body if body is not None else {}), stream=True) - try: - if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, " - f"never re-sent to a location the server picked") - return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") - finally: - r.close() - - -# --------------------------------------------------------------------------------------------- -# EXTRACTION — vendored from scrape.py, unchanged in behaviour -# --------------------------------------------------------------------------------------------- - -def _soup(body): - if BeautifulSoup is None: - raise RuntimeError( - "beautifulsoup4 is not installed in this environment — the automation engine's " - "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " - "aios-web/requirements.txt).") - try: - return BeautifulSoup(body, "lxml") - except Exception: - return BeautifulSoup(body, "html.parser") - - -def tables(soup): - """Every HTML table as rows. Dict rows when the first row looks like a header.""" - out = [] - for t in soup.find_all("table"): - rows = [] - for tr in t.find_all("tr"): - cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] - if cells: - rows.append(cells) - if not rows: - continue - head, body = rows[0], rows[1:] - if body and len(head) == len(body[0]) and all(head): - out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) - else: - out.append(rows) - return out - - -def jsonld(soup): - out = [] - for s in soup.find_all("script", attrs={"type": "application/ld+json"}): - try: - out.append(json.loads(s.string or s.get_text())) - except Exception: - pass - return out - - -def meta(soup): - m = {} - if soup.title and soup.title.string: - m["title"] = soup.title.string.strip() - for tag in soup.find_all("meta"): - k = tag.get("name") or tag.get("property") - v = tag.get("content") - if k and v and (k in ("description", "keywords", "author") - or k.startswith(("og:", "twitter:"))): - m[k] = v.strip() - return m - - -def preview(url, extract="table", table_index=0, allowed=()): - """The field-map preview the editor calls BEFORE anything is created: what columns does this - page actually offer, and what do the first rows look like? Never writes.""" - status, final, body = fetch(url, allowed=allowed) - if not (200 <= status < 300): - return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status} — it may be bot-gated or gone", - "columns": [], "sample": [], "rowCount": 0} - soup = _soup(body) - rows = [] - if extract == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 - picked = found[idx] if found else [] - rows = [r for r in picked if isinstance(r, dict)] - cols, seen = [], set() - for r in rows[:50]: - for k in r: - if k not in seen: - seen.add(k) - cols.append(k) - return {"ok": True, "status": status, "url": final, "columns": cols, - "sample": rows[:8], "rowCount": len(rows), - "tableCount": len(tables(soup)) if extract != "jsonld" else 0, - "title": (meta(soup) or {}).get("title", "")} - - -def _scalar(v): - if isinstance(v, (str, int, float)) and not isinstance(v, bool): - return str(v) - if isinstance(v, bool): - return "1" if v else "" - if isinstance(v, dict): - return str(v.get("name") or v.get("@id") or "") - if isinstance(v, list): - return ", ".join(_scalar(x) for x in v[:8]) - return "" - - -# --------------------------------------------------------------------------------------------- -# CRON — a 5-field parser and a "what was the last fire time" walk -# --------------------------------------------------------------------------------------------- -# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward -# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute -# between the last run and now?"* — and answering it that way is what makes a missed tick -# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and -# not never. - -_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] - - -def _parse_field(spec, lo, hi): - out = set() - for part in str(spec).split(","): - part = part.strip() - if not part: - raise ValueError("empty cron field part") - step = 1 - if "/" in part: - part, _, s = part.partition("/") - step = int(s) - if step < 1: - raise ValueError("cron step must be >= 1") - if part in ("*", "?"): - a, b = lo, hi - elif "-" in part.lstrip("-"): - a_s, _, b_s = part.partition("-") - a, b = int(a_s), int(b_s) - else: - a = b = int(part) - if a < lo or b > hi or a > b: - raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") - out.update(range(a, b + 1, step)) - return out - - -def parse_cron(expr): - """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. - - Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at - WRITE time, because a cron string nobody can evaluate is an automation that silently never - runs and reports no error anywhere. - """ - parts = str(expr or "").split() - if len(parts) != 5: - raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") - sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] - dom_r = parts[2].strip() not in ("*", "?") - dow_r = parts[4].strip() not in ("*", "?") - return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), - dom_r, dow_r) - - -def _day_matches(day, doms, months, dows, dom_r, dow_r): - if day.month not in months: - return False - # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the - # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes - # a schedule that looks right fire almost never. - dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 - if dom_r and dow_r: - return day.day in doms or dow in dows - if dom_r: - return day.day in doms - if dow_r: - return dow in dows - return True - - -def prev_fire(expr, now=None, lookback_days=400): - """The most recent scheduled minute at or before `now`, or None inside the lookback. - - Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, - and only a matching day needs its hours/minutes searched. - """ - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - now = (now or _now()).replace(second=0, microsecond=0) - for back in range(lookback_days + 1): - day = (now - _dt.timedelta(days=back)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = back == 0 - for h in sorted(hours, reverse=True): - if same_day and h > now.hour: - continue - for m in sorted(minutes, reverse=True): - if same_day and h == now.hour and m > now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -def is_due(defn, now=None): - """Should the scheduler run this automation right now? - - Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the - moment the schedule was enabled, else creation) and at or before now. Anchoring on - `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at - 14:00" from running immediately and looking like a bug. - """ - if not isinstance(defn, dict): - return False - sched = defn.get("schedule") or {} - if not sched.get("enabled"): - return False - try: - fire = prev_fire(sched.get("cron"), now=now) - except ValueError: - return False - if fire is None: - return False - since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) - or _parse_iso(sched.get("enabledAt")) - or _parse_iso(defn.get("created"))) - return since is None or fire > since - - -def next_fire(expr, now=None, lookahead_days=400): - """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" - try: - minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) - except ValueError: - return None - now = (now or _now()).replace(second=0, microsecond=0) - for ahead in range(lookahead_days + 1): - day = (now + _dt.timedelta(days=ahead)).date() - if not _day_matches(day, doms, months, dows, dom_r, dow_r): - continue - same_day = ahead == 0 - for h in sorted(hours): - if same_day and h < now.hour: - continue - for m in sorted(minutes): - if same_day and h == now.hour and m <= now.minute: - continue - return _dt.datetime(day.year, day.month, day.day, h, m) - return None - - -# --------------------------------------------------------------------------------------------- -# THE UPSERT — pure, so the arithmetic is testable without a store or a network -# --------------------------------------------------------------------------------------------- - -def upsert_rows(existing, incoming, key_field, cap=None): - """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. - - THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped - appearing on the source page has not necessarily stopped existing — the page changed its - filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream - hiccup into silent data loss, and the row may be carrying user-typed overlay values in - columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. - - Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. - - ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on - entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to - call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` - that same loop is hundreds of millions of dict copies and the automation simply never - finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) - - ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different - facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the - DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — - a property of the SYSTEM, and never benign. One number for both meant a table hitting its - ceiling was indistinguishable from a page with a few blank cells, which is how a time series - stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. - """ - cap = MAX_UT_ROWS if cap is None else int(cap) - rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} - counts = {"inserted": 0, "updated": 0, "unchanged": 0, - "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} - by_key, dupe_ids = {}, set() - for rid, row in rows.items(): - kv = str(row.get(key_field, "") or "").strip() - if not kv: - continue - if kv in by_key: - dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left - continue - by_key[kv] = rid - next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 - seen_keys, incoming_dupes = set(), 0 - for src in incoming or []: - kv = str((src or {}).get(key_field, "") or "").strip() - if not kv: - counts["skipped"] += 1 # no key -> cannot be upserted; never guessed - continue - if kv in seen_keys: - incoming_dupes += 1 # the SOURCE listed it twice; first wins - continue - seen_keys.add(kv) - rid = by_key.get(kv) - if rid is None: - if len(rows) >= cap: - counts["capped"] += 1 # LOUD: the runner turns this into a partial run - continue - rid = str(next_id) - next_id += 1 - rows[rid] = dict(src) - by_key[kv] = rid - counts["inserted"] += 1 - continue - before = rows[rid] - changed = {k: v for k, v in src.items() if str(before.get(k, "")) != str(v)} - if changed: - before.update(src) - counts["updated"] += 1 - else: - counts["unchanged"] += 1 - counts["duplicates"] = incoming_dupes + len(dupe_ids) - counts["orphans"] = sum( - 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) - return rows, counts - - -# --------------------------------------------------------------------------------------------- -# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) -# --------------------------------------------------------------------------------------------- - -def _ut_slug(label): - s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") - return (s or "table")[:40] - - -def ut_all(rt): - try: - return dict(rt.get(UT_STORE_KEY) or {}) - except Exception: - return {} - - -def disable_for_table(rt, table_key, note="target database deleted"): - """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. - - The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still - shows what existed and why it stopped — silently deleting a user's automation because its - target died would read as data loss. Returns the ids it touched.""" - key = str(table_key or "") - touched = [] - - def _up(cur): - for aid, d in (cur or {}).items(): - if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: - sch = d.get("schedule") - if not isinstance(sch, dict): - sch = d["schedule"] = {} - sch["enabled"] = False - d["statusNote"] = note - touched.append(str(aid)) - return cur - - if key: - rt.update(STORE_KEY, _up, flush="sync") - return touched - - -def ut_get(rt, key): - return ut_all(rt).get(str(key)) - - -def bind_unbound_fields(rt): - """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` - are bound to the definition that already writes them — matched on the (targetTable, - fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no - definition references is DISABLED with the reason on it rather than deleted or guessed: - the column was already dead (nothing runs a column no flow names), and now it says so. - Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" - by_binding = {} - for aid, d in all_definitions(rt).items(): - cfg = d.get("config") or {} - if cfg.get("fieldKey") and cfg.get("targetTable"): - by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) - plan = {} - for tk, t in ut_all(rt).items(): - for f in (t.get("fields") or []): - a = f.get("automation") - # An already-DISABLED bag is a decision this migration made on a previous pass — - # re-planning it every call would turn the once-per-process sweep into a write - # per read. - if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ - and not a.get("disabled"): - plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) - if not plan: - return 0, 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for (tk, fk), aid in plan.items(): - for f in ((cur.get(tk) or {}).get("fields") or []): - if f.get("key") == fk and isinstance(f.get("automation"), dict): - if aid: - f["automation"]["flowId"] = aid - else: - f["automation"]["disabled"] = True - f["automation"]["statusNote"] = ( - "not bound to any flow — create an automation for this column " - "or delete it") - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - bound = sum(1 for v in plan.values() if v) - return bound, len(plan) - bound - - -def ut_key_for(label, key=None): - """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which - must not create anything) resolves the same key by construction rather than by copying the - derivation and drifting from it.""" - k = str(key or (UT_PREFIX + _ut_slug(label))) - return k if k.startswith(UT_PREFIX) else UT_PREFIX + k - - -#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same -#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging -#: `core` into the API's boot path — and the two are held in step by a gate check. -MACHINE_OWNERS = ("automation", "scheduler") - - -def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag=""): - """Create the table if it is missing; return its key. Idempotent — a re-run of an automation - that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) - and an existing table with that key is adopted, not duplicated. - - Fields are MERGED, never replaced: a user who added a column to an automation's table keeps - it, and a new source column joins on the next run. - - `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped - `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their - provenance. Fields already on the table keep whatever tag they have (first flow wins; - shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not - ownership). - - ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever - or WHATEVER ran the automation, so the same table belonged to a person if its first run was - manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` - admits only the creator or an admin, so **whether you could open your own Instagram - snapshots depended on a race you never saw**. The owner is now the automation's creator, and - a table already stamped with a machine name is ADOPTED the next time a run knows a human - one. Adoption is a repair, not a widening: the automation's creator is the person who asked - for the table in the first place. - """ - key = ut_key_for(label, key) - wanted = [({**f, "automation": {"flowId": str(flow_tag)}} - if flow_tag and not f.get("automation") else dict(f)) - for f in (fields or [])] - created = _iso() - human = username if username and username not in MACHINE_OWNERS else "" - - # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store - # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo - # budget, an idempotent helper that always writes is the same defect as a per-row insert. - have = ut_get(rt, key) - if (have is not None - and not [f for f in wanted - if f.get("key") not in {g.get("key") for g in (have.get("fields") or [])}] - and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): - return key - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - if len(cur) >= MAX_UT_TABLES: - return cur - cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", - "createdBy": username, "created": created, - "fields": wanted, "rows": {}} - return cur - have = {f.get("key") for f in (t.get("fields") or [])} - for f in wanted: - if f.get("key") not in have: - t.setdefault("fields", []).append(f) - have.add(f.get("key")) - # ADOPTION: a machine name is not an owner. It never overwrites a human one. - if human and (t.get("createdBy") or "") in MACHINE_OWNERS: - t["createdBy"] = human - # An automation's table SAYS an automation owns it — the nav badge reads from this. - t["source"] = t.get("source") or "Automation" - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return key - - -def ut_write_rows(rt, key, rows): - """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t["rows"] = rows - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): - """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is not None: - t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value - for k, rws in (extra_rows or {}).items(): - tt = cur.get(k) - if tt is not None: - tt["rows"] = rws - return cur - rt.update(UT_STORE_KEY, _up, flush="sync") - - -def field_def(text_key, label, ftype="text"): - return {"key": text_key, "label": label, "type": ftype, "source": "overlay"} - - -# --------------------------------------------------------------------------------------------- -# DEFINITIONS — validation and the bucket's read/write half -# --------------------------------------------------------------------------------------------- - -def _s(v, n=200): - return str(v if v is not None else "")[:n] - - -def clean_config(kind, raw, previous=None): - """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. - - ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail - rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not - edit `tier` / `noFallback` / `dryRun` — those are node SWITCHES. So a plain "Save" from a - panel that never knew about them would silently reset the paid rung to `anonymous` and turn - off the dry run: a save that quietly changes what the automation costs. Absent ⇒ keep; - present ⇒ take it, including `false`. - """ - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - - def flag(name): - return bool(raw[name]) if name in raw else bool(prev.get(name)) - if kind == "scrape_db": - url = _s(raw.get("url"), 2000).strip() - if not url: - return None, "a source URL is required" - try: - guard(url) - except Refused as e: - return None, str(e) - extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" - fmap = {} - for k, v in list((raw.get("fieldMap") or {}).items())[:60]: - tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") - if tk: - fmap[_s(k, 120)] = tk[:60] - if not fmap: - return None, "map at least one source column to a field" - key_field = _s(raw.get("keyField"), 60).strip() - if key_field not in fmap.values(): - return None, "the key field must be one of the mapped fields" - target = _s(raw.get("targetTable"), 60).strip() - if target and not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - lanes, lerr = clean_lanes(raw.get("lanes") if "lanes" in raw else prev.get("lanes")) - if lerr: - return None, lerr - return {"url": url, "extract": extract, - "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), - "fieldMap": fmap, "keyField": key_field, - "targetTable": target, - "dryRun": flag("dryRun"), - "lanes": lanes, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None - if kind == "field_instagram": - table = _s(raw.get("targetTable"), 60).strip() - if not table.startswith(UT_PREFIX): - return None, "an Instagram automation runs against a blank database (ut_*)" - fkey = _s(raw.get("fieldKey"), 80).strip() - if not fkey: - return None, "pick the automation column this run writes into" - # ⚠ UNKNOWN TIERS FALL BACK TO `anonymous`, NOT to the paid rung. A typo must never be - # the thing that starts spending money. An ABSENT tier keeps the one already stored (see - # the `flag` note above), and a stored `hiker` is carried FORWARD to the new vendor - # rather than silently demoted — see `TIER_ALIASES`. - tier = clean_tier(raw["tier"]) if "tier" in raw else "" - if not tier and "tier" not in raw: - tier = clean_tier(prev.get("tier")) - lanes, lerr = clean_lanes(raw.get("lanes") if "lanes" in raw else prev.get("lanes")) - if lerr: - return None, lerr - return {"targetTable": table, "fieldKey": fkey, - "urlField": _s(raw.get("urlField"), 80).strip(), - "tier": tier or "anonymous", - "noFallback": flag("noFallback"), - # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked - # for: the Profiles row carries post IDENTITY for free but NO engagement - # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. - "postMetrics": flag("postMetrics"), - "dryRun": flag("dryRun"), - "lanes": lanes, - "maxPosts": max(1, min(int(raw.get("maxPosts") or 24), 200))}, None - if kind == "discover_instagram": - limit = int(raw.get("recordsLimit") or 0) - if limit < 1: - return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the " - "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") - if limit > BD_MAX_RECORDS: - return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - preds, perr = clean_predicates(raw.get("predicates")) - if perr: - return None, perr - if raw.get("lanes") if "lanes" in raw else prev.get("lanes"): - # R6: discovery's one path is the human gate — Review, then Tracked. A lane after - # the gate would be a second router over a set a person is already routing. - return None, ("a discovery automation routes through Review — its path is the " - "human gate; lanes belong on scrape and column automations") - target = _s(raw.get("targetTable"), 60).strip() or DISCOVER_TABLE - if not target.startswith(UT_PREFIX): - target = UT_PREFIX + target - return {"recordsLimit": limit, "predicates": preds, - "operator": "or" if str(raw.get("operator") or "").lower() == "or" else "and", - "targetTable": target, - "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "IG candidates", - "dryRun": flag("dryRun")}, None - return None, f"unknown automation kind {kind!r}" - - -def clean_schedule(raw, previous=None): - """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` - measures from — without it, enabling a daily job fires it immediately).""" - raw = raw if isinstance(raw, dict) else {} - cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" - parse_cron(cron) # raises -> the route answers 400 - enabled = bool(raw.get("enabled")) - prev = previous or {} - out = {"cron": cron, "enabled": enabled} - if enabled: - out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() - return out - - -# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── -# The owner's words: "we can make this automation a loop, so the ending should always be defined, -# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, -# or the user have to click a button to reset, or after a certain amount of time it can -# automatically reset to first cycle." -# -# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition -# that predates this field must not start moving records on its own the day the code ships. A -# loop is a thing somebody turns on. -ENDING_MODES = ("terminal", "auto_reset", "manual_reset", "timed_reset") -MAX_RESET_HOURS = 720 # 30 days; past that "it resets eventually" is a lie -#: The per-record cycle counter's cell suffix, beside `stage_` and `stage__at`. -CYCLES_SUFFIX = "_cycles" - - -def cycles_field_key(auto_id): - return stage_field_key(auto_id) + CYCLES_SUFFIX - - -def clean_ending(raw, previous=None): - """Validate `{mode, hours?}`. Returns `(ending, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - mode = _s(raw.get("mode") or prev.get("mode"), 20).strip() or "terminal" - if mode not in ENDING_MODES: - return None, f"{mode!r} is not one of: " + ", ".join(ENDING_MODES) - out = {"mode": mode} - if mode == "timed_reset": - try: - hours = int(raw.get("hours") if raw.get("hours") is not None else prev.get("hours") - or 24) - except (TypeError, ValueError): - return None, "the reset delay must be a whole number of hours" - if hours < 1 or hours > MAX_RESET_HOURS: - return None, f"the reset delay is between 1 and {MAX_RESET_HOURS} hours" - out["hours"] = hours - return out, None - - -def clean_flow(raw, previous=None): - """Validate `{actions, ending}` — the builder's half of a definition. `(flow, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions")) - if err: - return None, err - ending, err = clean_ending(raw.get("ending") if "ending" in raw else prev.get("ending"), - prev.get("ending")) - if err: - return None, err - return {"actions": actions, "ending": ending}, None - - -def clean_definition(raw, previous=None, username=""): - """Whole-definition validation. Returns `(defn, error)`.""" - raw = raw if isinstance(raw, dict) else {} - prev = previous or {} - kind = _s(raw.get("kind") or prev.get("kind"), 40) - if kind not in KINDS: - return None, f"unknown automation kind {kind!r}" - name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) - if not name: - return None, "name the automation" - cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") - config, err = clean_config(kind, cfg_raw, prev.get("config")) - if err: - return None, err - try: - schedule = clean_schedule( - raw.get("schedule") if isinstance(raw.get("schedule"), dict) - else prev.get("schedule"), prev.get("schedule")) - except ValueError as e: - return None, str(e) - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr - flow, ferr = clean_flow(raw.get("flow") if "flow" in raw else prev.get("flow"), - prev.get("flow")) - if ferr: - return None, ferr - return { - "id": _s(prev.get("id") or raw.get("id"), 40), - "name": name, "kind": kind, "config": config, "schedule": schedule, - "trigger": trigger, - # WAVE 23 · C4/C5 — the BUILDER's half: the ordered actions a record walks and what - # happens when it reaches the end. Absent on every definition written before this wave, - # which is exactly why `clean_flow` defaults it to "no actions, terminal ending": the - # code shipping must not put existing automations into a loop. - "flow": flow, - "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, - "runs": list(prev.get("runs") or [])[:MAX_RUNS], - # C3-A2(5): the review-decision audit survives a Save the way `state` does — an - # ordinary patch must not amnesty every recorded approve/decline. - "reviews": list(prev.get("reviews") or [])[:100], - # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 - # minutes to build, so a run persists its id here and the NEXT run collects it. It is - # carried through `patch` untouched because a user pressing Save must not silently orphan - # a set the vendor is already building (and already counting against the funds gate). - "state": dict(prev.get("state") or {}), - "created": prev.get("created") or _iso(), - "createdBy": prev.get("createdBy") or username, - }, None - - -def set_state(rt, auto_id, patch_state): - """Merge into ONE automation's engine state. A key set to None is removed. - - Its own tiny store write rather than a field on `_commit_run`, because the handoff must - survive a run that ends in `error` — a snapshot the vendor is building does not stop existing - because the run that started it failed afterwards. - """ - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - st = dict(d.get("state") or {}) - for k, v in (patch_state or {}).items(): - st.pop(k, None) if v is None else st.__setitem__(k, v) - d["state"] = st - return cur - - rt.update(STORE_KEY, _up, flush="sync") - - -def all_definitions(rt): - try: - return dict(rt.get(STORE_KEY) or {}) - except Exception: - return {} - - -def _new_id(existing): - n = 1 - while f"auto_{n}" in existing: - n += 1 - return f"auto_{n}" - - -#: C2's three answers to "which database does this automation work on?" — the FIRST question the -#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). -TARGET_MODES = ("existing", "new", "automated") - - -def resolve_target(rt, raw, username=""): - """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. - - - `existing` — bind a database that is already there. - - `new` — mint a blank one now, so the automation has somewhere to write before its first - run instead of conjuring a table the person never agreed to. - - `automated` — leave it to the runner: a scraping automation MINTS its target on first run - (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. - Nothing is created here, deliberately — an empty table created up front for a scrape that - never runs is litter nobody can explain. - """ - if not isinstance(raw, dict) or not raw: - return {}, None # no target block = the pre-wizard shape - mode = _s(raw.get("mode"), 20).strip() or "existing" - if mode not in TARGET_MODES: - return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) - if mode == "automated": - label = " ".join(_s(raw.get("label"), 60).split()) - return ({"targetLabel": label} if label else {}), None - if mode == "existing": - key = _s(raw.get("table"), 60).strip() - if not key: - return None, "choose the database this automation works on" - t = ut_get(rt, key) - if t is None: - return None, f"{key!r} is not a database in this workspace" - return {"targetTable": key, "targetLabel": t.get("label") or key}, None - label = " ".join(_s(raw.get("label"), 60).split()) - if not label: - return None, "name the new database" - import core.user_tables as _ut_new - key = _ut_new.create(label, username or "automation", st=rt) - if not key: - return None, ("the database could not be created — this workspace may be at its table " - "limit") - return {"targetTable": key, "targetLabel": label}, None - - -def create(rt, raw, username=""): - existing = all_definitions(rt) - if len(existing) >= MAX_AUTOMATIONS: - return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" - raw = dict(raw or {}) - # C2 runs BEFORE validation so a `new` target's freshly-minted key is part of the config - # `clean_definition` sees — one write, not a create-then-patch the client has to sequence. - patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) - if terr: - return None, terr - if patch_cfg: - raw["config"] = {**(raw.get("config") or {}), **patch_cfg} - defn, err = clean_definition(raw, None, username) - if err: - return None, err - defn["id"] = _new_id(existing) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - rt.update(STORE_KEY, _up, flush="sync") - _stage_field_after_write(rt, defn, username) - if defn.get("trigger"): - _seed_event_state(rt, defn) - return defn, None - - -def patch(rt, auto_id, raw, username=""): - existing = all_definitions(rt) - prev = existing.get(str(auto_id)) - if prev is None: - return None, "no such automation" - merged = dict(prev) - # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a - # key missing here is accepted by the route, validated by `clean_definition`, and then - # DISCARDED — the client shows a saved flow that the store never received, and nothing goes - # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only - # ordering that never has a window where the bug exists. - for k in ("name", "config", "schedule", "kind", "trigger", "flow"): - if k in (raw or {}): - merged[k] = raw[k] - defn, err = clean_definition(merged, prev, username) - if err: - return None, err - defn["id"] = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur[defn["id"]] = defn - return cur - - rt.update(STORE_KEY, _up, flush="sync") - _stage_field_after_write(rt, defn, username) - # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching - # DISARMED — enabling never fires for records already in the state. - if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): - _seed_event_state(rt, defn) - return defn, None - - -def _stage_field_after_write(rt, defn, username): - """C2's 'at automation create (or first bind)': when the target database already exists, - the stage field appears/refreshes with the definition write — a patch that adds a lane - grows the select's vocabulary in the same breath. A table that does not exist yet gets its - field from the first RUN instead (the runner ensures after `ut_ensure`).""" - tgt = str((defn.get("config") or {}).get("targetTable") or "") - if tgt and ut_get(rt, tgt) is not None: - ensure_stage_field(rt, tgt, defn, username) - - -def remove(rt, auto_id): - aid = str(auto_id) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur.pop(aid, None) - return cur - rt.update(STORE_KEY, _up, flush="sync") - # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned - # silently: the column keeps its values and its config, and says why it stopped. Scanned - # first so a delete with no tagged fields costs no store commit. - hit = any(str((f.get("automation") or {}).get("flowId") or "") == aid - for t in ut_all(rt).values() for f in (t.get("fields") or [])) - if not hit: - return - - def _disable(cur): - cur = cur if isinstance(cur, dict) else {} - for t in cur.values(): - for f in (t.get("fields") or []): - a = f.get("automation") - if isinstance(a, dict) and str(a.get("flowId") or "") == aid: - a["disabled"] = True - a["statusNote"] = f"its automation was deleted {_stamp()}" - return cur - rt.update(UT_STORE_KEY, _disable, flush="sync") - - -# --------------------------------------------------------------------------------------------- -# RUN STATE — process memory only (see the module header for why this may never be persisted) -# --------------------------------------------------------------------------------------------- - -_RUN_LOCK = threading.RLock() -_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} - - -def running(tenant, auto_id): - with _RUN_LOCK: - return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None - - -def _claim(tenant, auto_id): - with _RUN_LOCK: - if (tenant, str(auto_id)) in _RUNNING: - return False - _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} - return True - - -def _step(tenant, auto_id, text): - with _RUN_LOCK: - cur = _RUNNING.get((tenant, str(auto_id))) - if cur is not None: - cur["step"] = text - - -def _release(tenant, auto_id): - with _RUN_LOCK: - _RUNNING.pop((tenant, str(auto_id)), None) - - -def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None): - """THE ONE store write a run performs on the `automations` bucket. - - `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here - because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to - invent a node status when this map is absent (every run stored before W19-C has no `steps`, - and painting those nodes green from the run's overall state would be a fabrication). - """ - entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), - "counts": {k: int(v) for k, v in (counts or {}).items() - if isinstance(v, (int, float))}, - "steps": {str(k): str(v) for k, v in (steps or {}).items()}, - "affected": list(affected or [])[:200]} - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is None: - return cur - d["status"] = {"state": state, "lastRunAt": entry["ts"], - "lastSummary": entry["summary"]} - d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] - # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with - # the reason on it — a dead credential must not burn quota (or a paid vendor's records) - # 96 times a day while a green toggle sits over a buried error log. Errors only: - # `partial` is the honest-progress state and pausing on it would punish honesty. - runs = d.get("runs") or [] - if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( - not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): - sch = d.get("schedule") - if isinstance(sch, dict) and sch.get("enabled"): - sch["enabled"] = False - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " - f"failures — fix the cause, then re-enable: " - f"{entry['summary'][:120]}") - return cur - - rt.update(STORE_KEY, _up, flush="sync") - return entry - - -# --------------------------------------------------------------------------------------------- -# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN -# --------------------------------------------------------------------------------------------- -# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and -# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) -# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: -# -# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, -# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the -# supplier takes the scraping risk; it is never an Instagram credential and this module must -# never be given one. -# -# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). -# -# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress -# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been -# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and -# the cell says which. A pull that quietly wrote zero posts and reported success would be worse -# than no automation at all: the table would look maintained and be empty. -# -# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, -# so a run history answers "what still works anonymously?" from data rather than from memory. - -PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) -IG_APP_ID = "936619743392459" # the public web client id the browser itself sends -IG_MAX_RETRIES = 2 - -#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's -#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number -#: is a NOW value, so the series starts the day capture starts and a field we do not capture -#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows -#: has exactly the same cost as delaying capture — for that column — and is justified by exactly -#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to -#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. -#: ⛔ THE ONLY EXCLUSIONS ARE RULINGS, NOT TASTE: contact PII (D-24) and third-party comment text -#: (R11), both enforced in `BD_NEVER_MAP`. -#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose -#: almost none of them, which is what the `source` column is for. -SNAPSHOT_FIELDS = [ - field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), - field_def("pulled_at", "Pulled at"), field_def("followers", "Followers"), - field_def("following", "Following"), field_def("posts_count", "Posts"), - field_def("full_name", "Name"), field_def("bio", "Bio"), - field_def("verified", "Verified"), field_def("source", "Read via"), - field_def("approx", "Counts are approximate"), - # The link in bio — the paid rung's field, and commercially the most useful single string an - # influencer row can carry (it is the shop/affiliate destination). - field_def("external_url", "Link in bio", "url"), - # --- the rest of the vendor's Profiles schema, minus the PII and the comment text. - field_def("ig_id", "Instagram id"), - field_def("profile_url", "Profile", "url"), - field_def("avg_engagement", "Avg engagement"), - field_def("category", "Category"), - field_def("business_category", "Business category"), - field_def("is_business", "Business account"), - field_def("is_professional", "Professional account"), - field_def("is_private", "Private"), - field_def("highlights_count", "Highlights"), - field_def("bio_hashtags", "Bio hashtags"), - field_def("post_hashtags", "Post hashtags"), - field_def("pronouns", "Pronouns"), -] -POST_FIELDS = [ - field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), - field_def("posted_at", "Posted at"), field_def("type", "Type"), - field_def("caption", "Caption"), field_def("url", "URL", "url"), - # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous - # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). - field_def("paid_partnership", "Paid partnership"), - field_def("partner", "Partner brand"), - field_def("hashtags", "Hashtags"), - field_def("alt_text", "Alt text"), -] -POST_SNAPSHOT_FIELDS = [ - field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), - field_def("pulled_at", "Pulled at"), field_def("likes", "Likes"), - field_def("comments", "Comments"), - # Plays/views move independently of likes on video, so it is its own series rather than a - # thing to derive. Paid rung only; blank means not read. - field_def("views", "Views"), -] - - -def ig_handle(url): - """The handle out of a profile URL (or a bare handle). '' when it is not one.""" - raw = str(url or "").strip() - if not raw: - return "" - if raw.startswith("@"): - return re.sub(r"[^A-Za-z0-9._]", "", raw[1:])[:40] - if "instagram.com" not in raw: - return re.sub(r"[^A-Za-z0-9._]", "", raw)[:40] if "/" not in raw else "" - path = urlparse(raw if "//" in raw else "https://" + raw).path.strip("/") - first = (path.split("/") or [""])[0] - if first in ("p", "reel", "reels", "explore", "accounts", "stories", ""): - return "" - return re.sub(r"[^A-Za-z0-9._]", "", first)[:40] - - -def _ig_get(url, headers=None, timeout=20.0): - """One anonymous GET with 429 backoff. Returns `(status, body_bytes, note)`.""" - delay = PACE_SECONDS - for attempt in range(IG_MAX_RETRIES + 1): - try: - status, _final, body = fetch(url, timeout=timeout, headers=headers, max_kb=4096) - except Refused: - raise - except Exception as e: # noqa: BLE001 - return 0, b"", f"{type(e).__name__}: {str(e)[:120]}" - if status == 429 and attempt < IG_MAX_RETRIES: - time.sleep(delay) - delay *= 2 # exponential, never a tight retry - continue - return status, body, "" - return 429, b"", "rate limited after retries" - - -def _ig_int(v): - try: - return int(v) - except Exception: - return None - - -def _profile_from_web_api(node, handle): - return { - "username": node.get("username") or handle, - "full_name": node.get("full_name") or "", - "bio": node.get("biography") or "", - "followers": _ig_int((node.get("edge_followed_by") or {}).get("count")), - "following": _ig_int((node.get("edge_follow") or {}).get("count")), - "posts_count": _ig_int((node.get("edge_owner_to_timeline_media") or {}).get("count")), - "verified": "1" if node.get("is_verified") else "", - } - - -def _posts_from_edges(edges): - out = [] - for e in edges or []: - n = (e or {}).get("node") or {} - code = n.get("shortcode") - if not code: - continue - caption = "" - for ce in ((n.get("edge_media_to_caption") or {}).get("edges") or []): - caption = ((ce or {}).get("node") or {}).get("text") or "" - break - ts = n.get("taken_at_timestamp") - out.append({ - "shortcode": code, - "posted_at": (_dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M") - if isinstance(ts, (int, float)) else ""), - "type": ("video" if n.get("is_video") else - "carousel" if n.get("__typename") == "GraphSidecar" else "image"), - "caption": caption, - "url": f"https://www.instagram.com/p/{code}/", - "likes": _ig_int((n.get("edge_liked_by") or n.get("edge_media_preview_like") - or {}).get("count")), - "comments": _ig_int((n.get("edge_media_to_comment") or {}).get("count")), - }) - return out - - -_OG_COUNTS = re.compile( - r"([\d.,KMkm]+)\s+Followers?,\s*([\d.,KMkm]+)\s+Following,\s*([\d.,KMkm]+)\s+Posts?") - - -def _loose_count(txt): - """'1.2M' / '12,345' -> int. IG's og:description abbreviates, so the snapshot is honest - about being approximate rather than pretending to a precision it does not have.""" - t = str(txt or "").strip().replace(",", "") - mult = 1 - if t[-1:].lower() == "k": - mult, t = 1_000, t[:-1] - elif t[-1:].lower() == "m": - mult, t = 1_000_000, t[:-1] - try: - return int(float(t) * mult) - except Exception: - return None - - -# --------------------------------------------------------------------------------------------- -# BRIGHT DATA — THE PAID RUNG (owner ruling R1 / wave-20 D-21, contract C4) -# --------------------------------------------------------------------------------------------- -# WHY A VENDOR AT ALL, stated once so nobody re-opens it: the anonymous ladder above is honest but -# it is ALSO geographically fenced — Instagram blocks datacenter egress, so from the HF Space or a -# Lambda the ladder degrades to `blocked` and the whole automation becomes a thing that only works -# on the owner's desk. The vendor takes the egress risk, returns EXACT counts (not -# og:description's "204K") and reaches post-level data the anonymous surface has closed. -# -# ⚠ WHY THIS VENDOR AND NOT THE PREVIOUS ONE. HikerAPI was crypto-only with no legal entity — -# no invoice, no counterparty, nothing that can go on a subprocessor list at the EU/enterprise -# gates. Bright Data is nameable, litigated Meta and won on logged-off public scraping, and runs -# KYC. That is a POSTURE purchase, not a feature one; the price is ~2.5× and it is the right -# trade. Full comparison: `.claude/wiki/research/instagram-capture.md` §2a. -# -# ⛔ THE FOUR RAILS TRANSFER UNCHANGED (contract C4 — they were written vendor-agnostic): -# 1. **FAIL-CLOSED on the key.** No `AIOS_BRIGHTDATA_KEY` ⇒ this rung refuses with an honest -# blocked ATTEMPT and the anonymous ladder still runs. Never a crash, never a silent skip. -# 2. **The key is never logged, never in a URL, never in an exception.** It travels in the -# `Authorization: Bearer` header only, and every error below quotes a STATUS, never the -# request. A secret that reaches a log line has leaked to everyone who can read logs. -# 3. **The base URL is guarded like any other.** `AIOS_BRIGHTDATA_BASE` is env-provided, so it -# goes through the same SSRF rail — an env var pointed at the metadata endpoint would -# otherwise be a credential-bearing request to it. -# 4. **A rung failure is a RUNG failure.** `bd_call` swallows its own `Refused`/transport -# errors, because `run_now` turns a `Refused` into a whole-run error — and a mistyped base -# URL must not kill a run whose anonymous rung would have answered. -# 5. **Exact ⇒ NO `approx` TAG.** That tag is the anonymous rung's rounding disclosure. -# -# ⚠ THE KEY IS READ LAZILY, INSIDE THE CALL. A module-level snapshot would be taken at import, -# which makes the "key absent ⇒ blocked, not crash" gate untestable (nothing could flip it between -# checks) and would also mean a deploy that sets the secret after import silently has no key. -# -# ⛔ TWO SNAPSHOT NAMESPACES THAT 404 EACH OTHER (measured; the trap that costs an hour): -# the SCRAPER `/datasets/v3/trigger` + `/v3/scrape` -> `sd_…` read at `/datasets/v3/…` -# the CORPUS `/datasets/filter` (⚠ NO `/v3/`) -> `snap_…` read at `/datasets/…` -# Cross them and you get a flat "404 Snapshot does not exist" about a snapshot that is alive. - -#: Pinned from live probes against `api.brightdata.com` (2026-08-04 and 2026-08-05) — see -#: `.claude/wiki/research/instagram-capture.md` §2c/§2d for the recipe and its provenance tags. -#: They live in ONE named block so a vendor rename is a one-line fix rather than a hunt. -BD_BASE_DEFAULT = "https://api.brightdata.com" -BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records -BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields -BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id -BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"} -BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /?format=json -> the rows -BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ — the CORPUS query (discovery) -BD_PATH_FILTER_SNAPSHOT = "/datasets/snapshot" # / -> status -#: A sync scrape MEASURED at 15–23 s for one URL and ~40 s for two, so the timeout is generous; -#: it is a ceiling against a hung socket, not a latency budget. -BD_TIMEOUT = 300.0 -#: The READ ceiling for one vendor answer, in KB. `fetch` truncates SILENTLY at this size, so it -#: is load-bearing arithmetic rather than a round number: MEASURED `file_size` was 175,617 bytes -#: for 5 corpus rows ⇒ **~35 KB per profile row**, and `BD_MAX_RECORDS` is set from it with -#: headroom (500 × 35 KB ≈ 17 MB against 32 MB). A truncated body is REPORTED as truncated in -#: `bd_call` — never allowed to surface as "that is not JSON". -BD_MAX_KB = 32768 -#: How long a run waits on a scrape batch the vendor DEFERRED (see `_bd_deferral`). Much shorter -#: than the corpus filter's budget because a scrape snapshot is minutes, not tens of minutes — -#: and unlike the corpus path there is no handoff to a later run, so this is the whole wait. -BD_SCRAPE_WAIT = float(os.environ.get("AIOS_BD_SCRAPE_WAIT") or 180) -BD_SCRAPE_POLL = float(os.environ.get("AIOS_BD_SCRAPE_POLL") or 10) -#: ⚠ SPEC, NOT MEASURED — off the pricing page, never returned by the API (the funds gate fires -#: BEFORE a price is quoted, and `price: 0` means "not priced", not "free"). Every surface that -#: shows a number derived from this MUST say it is an estimate. `/customer/balance` answers 403 -#: for our token, so there is no way to check a balance or a spend from here either. -BD_RECORD_PRICE_SPEC = 0.0025 - - -def bd_key(): - """The key, or ''. Read fresh every call — see the module note above on why.""" - return (os.environ.get("AIOS_BRIGHTDATA_KEY") or "").strip() - - -def bd_base(): - return (os.environ.get("AIOS_BRIGHTDATA_BASE") or BD_BASE_DEFAULT).strip().rstrip("/") - - -def bd_ready(): - """Is the paid rung configured at all? The UI reads this to say so honestly.""" - return bool(bd_key()) - - -def bd_call(path, params=None, body=None): - """One authenticated vendor call. GET when `body` is None, POST otherwise. - - Returns `(payload, note)`; a non-empty `note` means it did NOT answer, and the note is safe to - show a user and safe to write to a log. - - Never raises. Every refusal shape — no key, a guarded base, a 402, a 429, a transport error, - unparseable JSON — comes back as a note so the caller can record the attempt and drop to the - next rung. - """ - key = bd_key() - if not key: - # ⛔ THE FAIL-CLOSED PATH (C4). Named precisely, because "blocked" with no reason sends - # somebody to read Instagram's status page instead of setting an env var. - return None, "AIOS_BRIGHTDATA_KEY is not configured — the paid rung is closed" - qs = "&".join(f"{k}={requests.utils.quote(str(v))}" - for k, v in (params or {}).items() if v not in (None, "")) - url = f"{bd_base()}{path}" + (f"?{qs}" if qs else "") - hdrs = {"Authorization": f"Bearer {key}", "Accept": "application/json"} - try: - if body is None: - status, _final, raw = fetch(url, timeout=BD_TIMEOUT, max_kb=BD_MAX_KB, headers=hdrs) - else: - status, raw = fetch_json(url, body, timeout=BD_TIMEOUT, max_kb=BD_MAX_KB, - headers=hdrs) - except Refused as e: - # The BASE was refused by the SSRF rail — a configuration fault, not a vendor outage. - return None, f"the Bright Data base URL was refused by the URL rail: {e}" - except Exception as e: # noqa: BLE001 - # ⚠ `type(e).__name__` only. A requests exception's str() can carry the full URL, and the - # Authorization header is one refactor away from being a query param; this line must not - # be what leaks it. - return None, f"Bright Data did not answer ({type(e).__name__})" - if status == 402: - return None, "Bright Data answered 402 — the account is out of credit" - if status in (401, 403): - return None, f"Bright Data refused the key ({status})" - if status == 429: - # ⚠ MEASURED during the discovery probe: `too_many_parallel_jobs`. The filter API has a - # concurrency cap and the probe's most important test died on it — hence every caller - # below is SERIAL by construction. - return None, "Bright Data answered 429 — too many parallel jobs; the calls are serialised" - if not (200 <= status < 300): - return None, f"Bright Data answered {status}" - # ⛔ A BODY THAT REACHED THE READ CEILING IS A TRUNCATION, AND IT MUST SAY SO. `fetch` reads - # at most `max_kb` and returns what it got — silently. A truncated JSON document then fails - # `json.loads`, falls through to the NDJSON branch, and comes back as "answered with - # something that is not JSON": a sentence that sends somebody to read the vendor's schema - # looking for a fault that is OUR byte ceiling — after a result set that has already been - # paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow. - if len(raw) >= BD_MAX_KB * 1024: - return None, (f"Bright Data's answer exceeded this client's {BD_MAX_KB // 1024} MB read " - f"ceiling and was TRUNCATED — ask for fewer records; nothing was parsed " - f"from a partial document") - text = raw.decode("utf-8", "replace").strip() - if not text: - return None, f"Bright Data answered {status} with an empty body" - try: - return json.loads(text), "" - except Exception: # noqa: BLE001 - pass - # NDJSON is the other shape this API uses for row sets. Tried SECOND, so a genuine JSON error - # is not silently reinterpreted as a one-line NDJSON document. - rows = [] - for line in text.splitlines(): - line = line.strip() - if not line: - continue - try: - rows.append(json.loads(line)) - except Exception: # noqa: BLE001 - return None, "Bright Data answered with something that is not JSON" - return (rows, "") if rows else (None, "Bright Data answered with something that is not JSON") - - -def _first(node, *names, default=None): - """The first of `names` present on `node` with a real value. - - ⚠ THE TOLERANCE IS THE POINT, not laziness. The vendor's `components/schemas` was not readable - on 2026-08-04 (recorded as RECALLED in the digest), so every media field name below is a - hypothesis. Reading through a candidate list means a rename costs one more candidate instead - of a rung that returns nothing and cannot say why. **A field that matches nothing returns - `default` (None), never 0** — an invented zero is a measurement, and this module's whole - honest-status contract exists to stop those. - """ - if not isinstance(node, dict): - return default - for n in names: - v = node.get(n) - if v not in (None, ""): - return v - return default - - -#: ⛔ FIELDS THAT ARE NEVER MAPPED, WHATEVER THE VENDOR SENDS. Two separate reasons, one list, -#: because the enforcement is the same: a field never collected needs no erasure path (D-24). -#: PII (the vendor's own `pii` flag): contact details are a different ORDER of privacy exposure -#: than a follower count, and `email_address` + `phone_number` change the lawful-basis analysis -#: materially. `business_email`'s own description says it "no longer exists without login!". -#: R11 (owner ruling, wave 20): NO Instagram comment-text capture. ⚠ This is not merely "do not -#: buy the Comments dataset" — MEASURED 2026-08-05, a POSTS row arrives with `latest_comments` -#: as an array of 15 comment texts WITH their authors, whether we asked or not. The ruling is -#: enforced HERE, at the map, because that is the only place it can be. -BD_NEVER_MAP = frozenset({ - "email_address", "phone_number", "business_email", "profile_image_link", - "latest_comments", "comment_user", "post_user", -}) - -#: The vendor's content vocabulary → ours. MEASURED values: profiles' `posts[].content_type` is -#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`. -_BD_TYPES = {"image": "image", "photo": "image", "video": "video", "reel": "video", - "reels": "video", "clips": "video", "igtv": "video", "carousel": "carousel", - "carousel_container": "carousel", "sidecar": "carousel", "posts": "image"} - - -def _bd_type(*raw): - """The first recognisable content type among `raw`, defaulting to `image`.""" - for v in raw: - hit = _BD_TYPES.get(str(v or "").strip().lower()) - if hit: - return hit - return "image" - - -def ig_shortcode(url): - """The shortcode out of a post/reel permalink. '' when it is not one. - - The `ut_ig_posts` identity key. The vendor gives a post URL (sometimes without its trailing - slash) and a numeric `pk`; the shortcode is what both datasets agree on and what a human can - paste into a browser, so it stays the key. - """ - raw = str(url or "").strip() - if not raw: - return "" - path = urlparse(raw if "//" in raw else "https://" + raw).path.strip("/").split("/") - if len(path) >= 2 and path[0] in ("p", "reel", "reels", "tv"): - return re.sub(r"[^A-Za-z0-9_-]", "", path[1])[:40] - return "" - - -def _bd_first_url(v): - """A link-in-bio value → one string. - - ⚠ MEASURED 2026-08-05: `external_url` is an **ARRAY** on the real row - (`["http://linktr.ee/utopianevents/"]`), not the string the previous vendor returned. A dict - (`external_url_title`) is also in the family. Read through the shape rather than assuming one. - """ - if isinstance(v, list): - v = next((x for x in v if x), "") - if isinstance(v, dict): - v = _first(v, "url", "link", default="") - return str(v or "") - - -def _bd_posts_count(node): - """`posts_count`, or None — **and a vendor ZERO IS DISCARDED.** - - ⛔ MEASURED 2026-08-05: the Profiles dataset returned `posts_count: 0` for BOTH probe accounts - — `inayma` (203,800 followers, 12 posts returned in the same row) and `utopian_events` (20,924 - followers, 12 posts). The POSTS dataset carries the true value for the same two accounts - (1,215 and 751), so the zero is not the account being empty; it is a measurement failure - wearing a number. Writing it would be the invented measurement this module's honest-status - contract exists to forbid — and a `_first`-style candidate list cannot help, because the - vendor DID answer, with the wrong answer. - - ⚠ THE ASYMMETRY IS DELIBERATE AND IT COSTS SOMETHING: an account that genuinely has zero posts - also reads blank. "Not read" is honest about an unknown; "0" is a claim, and on the only - evidence we have that claim is wrong 2 times out of 2. - ⛔ `len(posts)` IS NOT A SUBSTITUTE. The profile row carries the TOP 12 — a cap, not a count. - """ - v = _ig_int(_first(node, "posts_count", "media_count", "post_count")) - return None if v == 0 else v - - -def _bd_flag(node, *names): - """A vendor boolean → `'1'` / `''` / None. **None when the key is ABSENT or null**, which is - the distinction that matters: `is_business_account: false` is a measurement and `''` records - it; a missing key is not, and must not arrive as "false".""" - for n in names: - v = node.get(n) if isinstance(node, dict) else None - if v is None: - continue - return "1" if v is True or str(v).strip().lower() in ("true", "1", "yes") else "" - return None - - -def _bd_list(node, *names): - """An array field → a comma-joined string, or None when it is absent/empty.""" - for n in names: - v = node.get(n) if isinstance(node, dict) else None - if isinstance(v, list) and v: - return ", ".join(str(x) for x in v if x)[:400] - return None - - -def _bd_profile(node, handle): - """A Profiles row → our snapshot shape. Every name below is MEASURED off a real row - (2026-08-05, 35 keys); the candidates guard renames, not guesses. - - ⚠ Several of these were **null on both probe rows** (`avg_engagement`, `category_name`, - `business_category_name`, `is_professional_account`, `bio_hashtags`, `post_hashtags`). They - are mapped ANYWAY — see the note on `SNAPSHOT_FIELDS`: n=2 is not evidence enough to drop a - column permanently, and history is unbuyable, so a field omitted today is lost for today. - ⚠ They ARE populated on the CORPUS (`/datasets/filter`) rows, which is a different path with - a different population; that is a reason to capture them, not a licence to promise a - scrape-side feature before D-25 measures the fill rate. - """ - out = { - "username": str(_first(node, "account", "username", default=handle) or handle), - "full_name": str(_first(node, "full_name", "profile_name", default="") or ""), - "bio": str(_first(node, "biography", "bio", default="") or ""), - "followers": _ig_int(_first(node, "followers", "follower_count")), - "following": _ig_int(_first(node, "following", "following_count")), - "posts_count": _bd_posts_count(node), - "verified": "1" if _first(node, "is_verified", default=False) else "", - "external_url": _bd_first_url(_first(node, "external_url", "external_urls", - "external_url_title")), - # ⛔ NO `approx` KEY. These counts are exact, and carrying the anonymous rung's rounding - # tag onto them would be a lie in the other direction (R1: "drop the approx tag when - # exact"). Its ABSENCE is what tells the snapshot row it is trustworthy. - "ig_id": str(_first(node, "id", "fbid", default="") or ""), - "profile_url": str(_first(node, "profile_url", "url", default="") or ""), - "avg_engagement": _first(node, "avg_engagement"), - "category": str(_first(node, "category_name", default="") or ""), - "business_category": str(_first(node, "business_category_name", default="") or ""), - "is_business": _bd_flag(node, "is_business_account"), - "is_professional": _bd_flag(node, "is_professional_account"), - "is_private": _bd_flag(node, "is_private"), - "highlights_count": _ig_int(_first(node, "highlights_count")), - "bio_hashtags": _bd_list(node, "bio_hashtags"), - "post_hashtags": _bd_list(node, "post_hashtags"), - "pronouns": str(_first(node, "pronouns", default="") or ""), - } - # ⛔ THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP` - # field, and this asserts it stays that way after the next edit — a map is exactly the kind - # of code somebody extends in a hurry. - return {k: v for k, v in out.items() if k not in BD_NEVER_MAP} - - -def _bd_post_identity(p): - """One entry of a Profiles row's `posts[]` → our post shape. None when it has no shortcode. - - ⚠ IDENTITY ONLY — AND THAT IS THE FINDING THIS WHOLE RUNG IS SHAPED AROUND. MEASURED - 2026-08-05 across 24 posts on two accounts: `caption`, `content_type`, `id`, `image_url`, - `url` are populated 24/24, and **`likes`, `comments` and `datetime` are `None` 24/24.** So the - profile scrape can fill `ut_ig_posts` and CANNOT fill `ut_ig_post_snapshots` — the engagement - time series §3's append law exists for needs the POSTS dataset, one record per post. That is - why `postMetrics` is a separate, opt-in, ~13×-cost rung rather than a free by-product. - ⚠ `is_pinned` was `True` on all 24 — a constant, not a measurement. Not mapped. - ⚠ Blank-valued keys are OMITTED rather than written, so a later engagement pull that DOES know - `posted_at` fills it in instead of being overwritten by this rung's silence. - """ - code = ig_shortcode(p.get("url")) if isinstance(p, dict) else "" - if not code: - return None - out = {"shortcode": code, "url": f"https://www.instagram.com/p/{code}/", - "type": _bd_type(p.get("content_type"), p.get("type"))} - caption = _first(p, "caption", "description", default="") - if isinstance(caption, dict): - caption = _first(caption, "text", default="") - if caption: - out["caption"] = str(caption) - return out - - -def _bd_post_metrics(row): - """One POSTS-dataset row → the engagement half. None when it has no shortcode. - - MEASURED 2026-08-05 on three real post rows: `likes` (308 / 3,736), `num_comments` (37 / 134) - and `date_posted` (`2026-05-04T17:51:58.000Z`) all populated. - ⚠ NO VIEW COUNT CAME BACK — `video_play_count` / `video_view_count` are declared in the schema - and the KEY WAS ABSENT from every row, including a Reel. So `views` reads None and the column - stays blank, which is the honest answer; it is not folded into likes and it is not zeroed. - ⚠ The candidates below are all COUNTS OF VIEWS in different vendor typings (`video_view_count` - is declared `text`, the others `number`). A candidate list may span types only when the - MEANING is identical — never the way `like_and_view_counts_disabled`, a boolean, would have - turned "likes are hidden" into "1 like". - ⛔ `latest_comments` IS PRESENT ON THESE ROWS AND IS NEVER READ (R11 / `BD_NEVER_MAP`). - """ - if not isinstance(row, dict): - return None - code = str(_first(row, "shortcode", "content_id", default="") or "") or \ - ig_shortcode(row.get("url")) - if not code: - return None - posted = str(_first(row, "date_posted", "taken_at", default="") or "") - partner = row.get("partnership_details") - out = { - "shortcode": code, - "url": str(_first(row, "url", default=f"https://www.instagram.com/p/{code}/")), - "influencer_key": str(_first(row, "user_posted", default="") or ""), - "posted_at": posted.replace("T", " ")[:16], - "type": _bd_type(row.get("content_type"), row.get("product_type")), - "caption": str(_first(row, "description", "caption", default="") or ""), - "likes": _ig_int(_first(row, "likes", "like_count")), - "comments": _ig_int(_first(row, "num_comments", "comment_count", "comments")), - "views": _ig_int(_first(row, "video_play_count", "video_view_count", - "engagement_score_view", "play_count")), - # Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) — and - # it is the one field here that answers a commercial question the counts cannot. - "paid_partnership": _bd_flag(row, "is_paid_partnership"), - "partner": (str(_first(partner, "username", "profile_id", default="") or "") - if isinstance(partner, dict) else ""), - "hashtags": _bd_list(row, "hashtags"), - "alt_text": str(_first(row, "alt_text", default="") or ""), - } - # ⛔ SAME RAIL AS THE PROFILE MAP — and it matters MORE here, because a Posts row genuinely - # arrives carrying `latest_comments`: fifteen third-party comment texts with their authors, - # sent whether we asked or not (R11). - return {k: v for k, v in out.items() if k not in BD_NEVER_MAP} - - -def _bd_rows(payload): - """A vendor answer → the list of row dicts inside it, whatever envelope it arrived in. - - MEASURED shapes: a bare list (`/v3/scrape`), NDJSON (already listified by `bd_call`), and - `{"data": [...]}` / `{"results": [...]}` on the snapshot reads. An unreadable envelope yields - NOTHING rather than a guess. - """ - if isinstance(payload, list): - return [x for x in payload if isinstance(x, dict)] - if isinstance(payload, dict): - for k in ("data", "results", "records", "items"): - v = payload.get(k) - if isinstance(v, list): - return [x for x in v if isinstance(x, dict)] - return [payload] if payload else [] - return [] - - -def _bd_deferral(payload): - """The snapshot id when the 'sync' endpoint DEFERRED, else ''. - - ⛔ MEASURED 2026-08-05, AND IT IS THE TRAP OF THIS WHOLE RUNG. `/v3/scrape` is documented and - named as the synchronous call, and for a small batch it is — two profile URLs came back - inline. Give it more work and it answers **200** with no rows and a note instead: - - {"snapshot_id": "sd_…", "message": "Your request is still in progress and cannot be - retrieved in this call. Use the provided Snapshot ID to track progress…"} - - Read naively that is a 200 with a body, so a parser looking for "did it error?" sails past - it, `_bd_rows` wraps the envelope as ONE unusable row, and the caller reports that the vendor - returned nothing — **while a snapshot we have already been billed for sits on the account, - finishing, and is never collected.** That is the D-26 failure shape (paid work abandoned) - arriving through the success path. - ⚠ It is a TIME threshold, not a row count: 2 URLs answered inline, 6 and 10 deferred. - """ - if isinstance(payload, dict) and payload.get("snapshot_id") and not payload.get("account"): - return str(payload["snapshot_id"]) - return "" - - -def bd_scrape(dataset_id, urls, wait=None): - """Scrape a batch. `(rows, note)`; a note means it did not answer. - - Sends the SYNC call and, when the vendor defers it (see `_bd_deferral`), **collects the - snapshot it handed back** rather than discarding it. One call carries many URLs, which is - what keeps the per-profile pacing floor from turning a 20-profile run into an hour. - """ - payload, note = bd_call(BD_PATH_SCRAPE, {"dataset_id": dataset_id}, - body=[{"url": u} for u in urls if u]) - if note: - return [], note - sid = _bd_deferral(payload) - if not sid: - return _bd_rows(payload), "" - - # --- THE DEFERRED PATH. Poll the scraper namespace (`sd_…` at `/datasets/v3/…` — crossing it - # with the corpus namespace is a flat 404 about a snapshot that is alive). - budget = BD_SCRAPE_WAIT if wait is None else float(wait) - waited = 0.0 - while True: - got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) - if not gnote: - rows = _bd_rows(got) - # The status document is itself JSON, so "did it deliver?" is decided by whether - # what came back looks like ROWS — never by the HTTP code. - if rows and not _bd_deferral(got) and not ( - len(rows) == 1 and str(rows[0].get("status") or "") in - ("running", "building", "collecting")): - return rows, "" - if waited >= budget: - break - time.sleep(BD_SCRAPE_POLL) - waited += BD_SCRAPE_POLL - # ⛔ NAME THE SNAPSHOT. It has been paid for and it is still finishing; a note that said only - # "no rows" would throw away both the data and the way to get it. - return [], (f"Bright Data deferred this batch to snapshot {sid} and it was not ready within " - f"{int(budget)}s — the records are collected, not lost; ask for a smaller batch " - f"or collect {sid} from the control panel") - - -def pull_profile_bd(url, max_posts=24, post_metrics=False, log=print): - """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides. - - Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner - treats every rung identically. `blocked` here means the VENDOR rung did not answer — the - caller decides whether to drop to the anonymous ladder. - - `post_metrics=True` adds the second, PAID-PER-POST call against the Posts dataset, which is - the ONLY source of likes/comments (see `_bd_post_identity`). It is off by default because it - multiplies the bill by roughly the post count. - """ - handle = ig_handle(url) - if not handle: - return {"state": "error", "note": f"{url!r} is not an Instagram profile URL", - "profile": {}, "posts": [], "via": ""} - rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"]) - if note: - return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", - "note": note} - node = rows[0] if rows else {} - profile = _bd_profile(node, handle) - if profile.get("followers") is None and profile.get("following") is None: - # It answered 200 with something we could not read. Say THAT — not "0 followers". - # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and - # is discarded, so a row where it is the only "readable" field is a row we cannot read. - return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", - "note": "Bright Data answered, but no follower/following counts were readable in " - "it (the field names may have moved — see instagram-capture.md)"} - posts = [] - for p in (node.get("posts") or [])[:max_posts]: - got = _bd_post_identity(p) - if got: - posts.append(got) - - if post_metrics and posts: - time.sleep(min(PACE_SECONDS, 1.0)) # the vendor is paid, but it is still someone's API - rows2, note2 = bd_scrape(BD_DS_POSTS, [p["url"] for p in posts]) - if note2: - # The engagement half refusing does NOT lose the identity half — the profile and the - # post rows still land, and the run says which part was not readable. - note = f"post metrics unavailable: {note2}" - else: - by_code = {} - for r in rows2: - got = _bd_post_metrics(r) - if got: - by_code[got["shortcode"]] = got - for p in posts: - extra = by_code.get(p["shortcode"]) - if extra: - # The metrics row is RICHER (it knows posted_at); merge it over the identity - # row, dropping keys it could not read so a blank never overwrites a value. - p.update({k: v for k, v in extra.items() if v not in (None, "")}) - if not by_code: - note = "post metrics were requested but the Posts dataset returned no rows" - - # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften - # because we are paying: a run that wrote a follower count and no posts must not paint green - # over a posts table that did not grow. - return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts, - "via": "brightdata", - "note": note or ("" if posts else "profile read; Bright Data returned no posts")} - - -# --------------------------------------------------------------------------------------------- -# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) -# --------------------------------------------------------------------------------------------- -# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's -# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns -# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND -# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower -# Georgia/Florida floral-design company — Fisch Floral's actual market. -# -# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: -# -# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` -# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is -# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the -# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. -# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, -# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, -# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still -# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a -# snapshot that is not ready when it looks. -# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery -# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` -# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is -# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the -# next run collects it. A `partial` that says "still building, the next run collects it" is -# the honest report of exactly what happened. -# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and -# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into -# the tracked profile set would silently multiply the enrichment bill AND pollute the -# snapshot series with rows nobody chose. The `tracked` column is NEVER written here. -# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important -# test died on it. -# -# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE -# the API returns a number, `price: 0` means "not priced" rather than "free", and -# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to -# report. Every estimate below is derived from the PRICING PAGE and says so. - -#: The table a discovery run writes. Upsert by handle, so re-finding a profile is free. -DISCOVER_TABLE = "ut_ig_candidates" -#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at -#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed -#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real -#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. -BD_MAX_RECORDS = 500 -#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). -BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) -BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) - -#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error -#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. -BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", - "array_includes", "not_array_includes", "is_null", "is_not_null") -#: Operators that take NO value (everything else requires one). -BD_NULLARY_OPS = ("is_null", "is_not_null") - -#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** -#: by the API and is therefore absent rather than offered-and-broken). -#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY -#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier -#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the -#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by -#: accident; adding them back is a decision, not a typo. -BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", - "category_name", "business_category_name", "is_business_account", - "is_professional_account", "is_verified", "account", "full_name", - "external_url", "bio_hashtags", "post_hashtags", "profile_url", - "related_accounts", "profile_name", "id", "fbid", "highlights_count") -#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: -#: these are the fields seen carrying values on real CORPUS rows. `category_name` and -#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter -#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". -BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") - -#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. -#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still -#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not -#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a -#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the -#: scanner still has to walk, so they refine a search and cannot BE one. -BD_NARROWING_FIELDS = frozenset({ - "biography", "account", "full_name", "profile_name", "category_name", - "business_category_name", "external_url", "bio_hashtags", "post_hashtags", - "related_accounts", "id", "fbid", "profile_url"}) -#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the -#: whole corpus, which is breadth wearing a condition's clothes. -BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) -#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed -#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% -#: populated: below it a filter misses more corpus than it matches, and the surface should -#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, -#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never -#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. -#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding -#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. -BD_POPULATED_FIELDS = frozenset({ - "account", "followers", "following", "posts_count", "avg_engagement", "biography", - "external_url", "is_business_account", "is_professional_account", "is_verified", - "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) -BD_MIN_NARROWING = 1 - - -def predicate_narrows(p): - """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" - return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS - and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) - - -def narrowing_refusal(preds): - """C4's server law: the sentence when a predicate set has ZERO narrowing predicates, else - ''. One function so create/patch (via `clean_predicates`) and RUN (legacy stored configs - predate the law) refuse in the same words.""" - if any(predicate_narrows(p) for p in preds or []): - return "" - return ("add at least one CONTENT condition (biography includes …, account =, a hashtag) — " - "number and yes/no filters alone make the vendor scan the whole 620-million-row " - "corpus, and breadth is what costs: the measured `followers > 10000` search was " - "still building 50 minutes in") - - -def filter_meta(): - """C4's per-field flags for the Find surface: `[{name, populated, narrowing}]` over the - same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, R3).""" - return [{"name": n, "populated": n in BD_POPULATED_FIELDS, - "narrowing": n in BD_NARROWING_FIELDS} for n in BD_FILTER_FIELDS] - -CANDIDATE_FIELDS = [ - field_def("handle", "Handle"), field_def("profile_url", "Profile", "url"), - field_def("full_name", "Name"), field_def("followers", "Followers"), - field_def("following", "Following"), field_def("avg_engagement", "Avg engagement"), - field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), - field_def("verified", "Verified"), field_def("category", "Category"), - field_def("found_count", "Times found"), field_def("first_found", "First found"), - field_def("last_found", "Last found"), - # C6 (wave 22): the candidate pool is PER-USER — the upsert key is the COMPOUND - # (handle, created_by), so two people discovering the same profile each get their own row, - # their own found_count and their own review card. Stamped by the RUNNER (the automation's - # creator), never by `_candidate_row`. - field_def("created_by", "Found by"), - # ⛔ WRITTEN BY A HUMAN, NEVER BY THIS ENGINE (R7). It is declared here so the column exists - # and can be ticked; `_candidate_row` never emits the key, and `upsert_rows` only writes the - # keys it is given — so a re-find can never un-tick somebody's decision either. - field_def("tracked", "Tracked", "checkbox"), -] - - -def clean_predicates(raw): - """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. - - Refuses rather than coerces: a predicate naming a field the API rejects comes back as a - 400 with the field named, because the alternative — dropping it — turns "find me verified - accounts in Georgia" into "find me any account" and bills for the difference. - """ - out = [] - for p in (raw or [])[:12]: - if not isinstance(p, dict): - continue - name = _s(p.get("name") or p.get("field"), 60).strip() - op = _s(p.get("operator") or p.get("op"), 20).strip() - if name not in BD_FILTER_FIELDS: - return None, (f"{name or 'that field'!r} cannot be used to search — the searchable " - f"fields are: {', '.join(BD_FILTER_FIELDS)}") - if op not in BD_FILTER_OPS: - return None, f"{op or 'that comparison'!r} is not one of: {', '.join(BD_FILTER_OPS)}" - entry = {"name": name, "operator": op} - if op not in BD_NULLARY_OPS: - v = p.get("value") - if v is None or (isinstance(v, str) and not v.strip()): - return None, f"give a value to compare {name!r} against" - entry["value"] = v.strip() if isinstance(v, str) else v - out.append(entry) - if not out: - return None, ("add at least one condition — an empty filter matches the whole 620-million" - "-record corpus, which is the query the vendor refuses") - # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the - # measured hang this guards against. Same sentence at create/patch and at run. - guard_err = narrowing_refusal(out) - if guard_err: - return None, guard_err - return out, None - - -def discover_estimate(records_limit): - """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the - section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" - n = max(0, int(records_limit or 0)) - return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), - "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", - "note": "estimated from Bright Data's published dataset rate — the API never returns " - "a price before a run, and this account's token cannot read a balance"} - - -def bd_filter_start(predicates, operator="and", records_limit=5, dataset_id=BD_DS_PROFILES): - """Start ONE corpus query. Returns `(snapshot_id, note)`. - - ⚠ ALWAYS THE COMPOUND `{operator, filters:[…]}` SHAPE, even for a single condition. A bare - `{name, operator, value}` is a plausible-looking second shape that has never been sent from - here, and a one-element `filters` list is a trivial extension of the three-element body that - HAS been measured end-to-end. One shape, and it is the tested one. - """ - body = {"dataset_id": dataset_id, "records_limit": int(records_limit), - "filter": {"operator": operator, "filters": list(predicates)}} - payload, note = bd_call(BD_PATH_FILTER, body=body) - if note: - return "", note - sid = str((payload or {}).get("snapshot_id") or "") if isinstance(payload, dict) else "" - if not sid: - return "", "Bright Data accepted the search but named no snapshot to collect it from" - return sid, "" - - -def bd_filter_status(snapshot_id): - """`(status, size, note)` for a corpus snapshot. `status` ∈ building | ready | failed | ''.""" - payload, note = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snapshot_id}") - if note: - return "", 0, note - d = payload if isinstance(payload, dict) else {} - status = str(d.get("status") or "") - err = str(d.get("error") or "") - if status == "failed" and err: - # NOT_ENOUGH_FUNDS is the one everybody will hit; say what it means, not just its name. - extra = (" — the matched set is too large for this account's balance; narrow the " - "conditions or lower the limit" if "FUNDS" in err.upper() else "") - return status, 0, f"the search failed: {err}{extra}" - return status, int(d.get("dataset_size") or 0), "" - - -def bd_filter_rows(snapshot_id): - """The rows of a READY corpus snapshot. `(rows, note)`. - - ⚠ `/download` answered *"Snapshot is building. Try again in a few minutes"* while the status - endpoint ALREADY said `ready` — delivery lags readiness by minutes. So a download that does - not parse as rows is reported as not-yet-collectable rather than as an error. - """ - payload, note = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snapshot_id}/download", - {"format": "json"}) - if note: - return [], note - rows = _bd_rows(payload) - if not rows: - return [], "the search is ready but its rows are not downloadable yet — the vendor's " \ - "delivery lags its own `ready` status by minutes" - return rows, "" - - -def _candidate_row(row, stamp): - """One corpus row → a `ut_ig_candidates` row. None when it has no handle. - - ⛔ NEVER EMITS `tracked` OR `found_count`. `tracked` is the human's decision (R7) and - `found_count` is arithmetic the runner does against what is already stored — writing either - from here would overwrite a person's choice or reset a counter to 1 on every re-find. - """ - if not isinstance(row, dict): - return None - handle = str(_first(row, "account", "username", default="") or "").strip() - if not handle: - return None - return { - "handle": handle, - "profile_url": str(_first(row, "profile_url", "url", - default=f"https://www.instagram.com/{handle}/")), - "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), - "followers": _s(_ig_int(_first(row, "followers"))), - "following": _s(_ig_int(_first(row, "following"))), - # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is - # the path where it carries values (measured on all five of the 2026-08-04 result set). - "avg_engagement": _s(_first(row, "avg_engagement", default="")), - "bio": _s(_first(row, "biography", "bio", default=""), 500), - "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), - "verified": "1" if _first(row, "is_verified", default=False) else "", - "category": _s(_first(row, "category_name", "business_category_name", default="")), - "last_found": stamp, - } - - -def pull_profile(url, max_posts=24, log=print, tier="anonymous", fallback=True, - post_metrics=False): - """Everything readable about one public profile, cheapest-honest rung last. - - Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error. - `partial` means the identity was read but the media was not — which is the outcome the - anonymous surface most often gives, and saying so is the point. - - **RUNG 0 IS THE PAID ONE (R1's hybrid).** With `tier="brightdata"` the vendor is tried FIRST - and, when it answers, this returns exact counts with no `approx` tag. When it does not answer - the attempt is RECORDED and the anonymous ladder runs underneath it — that is what - "fail-closed on the key, and the anonymous rung still works" means in C4. - - `fallback=False` turns that safety net off on purpose: it means *"exact numbers or nothing"*. - An influencer table where some rows are exact and some are `og:description` roundings, with - no way to tell them apart at a glance, is a table you cannot do arithmetic on — so a user who - needs the guarantee can have it, and gets an honest `blocked` instead of a quiet downgrade. - (The `source`/`approx` columns tell them apart anyway; this makes it impossible to forget.) - """ - handle = ig_handle(url) - if not handle: - return {"state": "error", "note": f"{url!r} is not an Instagram profile URL", - "profile": {}, "posts": [], "via": ""} - attempts = [] - - if clean_tier(tier) == "brightdata": - paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics, log=log) - if paid["state"] in ("ok", "partial"): - return paid - attempts.append(f"brightdata:{_s(paid.get('note'), 90)}") - if not fallback: - return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", - "note": "the paid rung did not answer and the anonymous fallback is turned " - f"off for this automation ({paid.get('note') or 'blocked'})"} - time.sleep(PACE_SECONDS) - - # RUNG 1 — the web client's own profile endpoint. Public, no cookie; the one that used to - # carry both the profile AND the first page of media in a single answer. - api = f"https://www.instagram.com/api/v1/users/web_profile_info/?username={handle}" - status, body, note = _ig_get(api, headers={ - "X-IG-App-ID": IG_APP_ID, "Accept": "*/*", - "Referer": f"https://www.instagram.com/{handle}/"}) - attempts.append(f"web_profile_info:{status or note}") - if 200 <= status < 300 and body: - try: - node = ((json.loads(body.decode("utf-8", "replace")) or {}) - .get("data") or {}).get("user") or {} - except Exception: - node = {} - if node: - profile = _profile_from_web_api(node, handle) - media = node.get("edge_owner_to_timeline_media") or {} - posts = _posts_from_edges(media.get("edges"))[:max_posts] - page = media.get("page_info") or {} - if page.get("has_next_page") and page.get("end_cursor") and len(posts) < max_posts: - more, pnote = _paginate(node.get("id"), page.get("end_cursor"), - max_posts - len(posts), log) - posts.extend(more) - if pnote: - attempts.append(pnote) - state = "ok" if posts else "partial" - return {"state": state, "profile": profile, "posts": posts, - "via": "web_profile_info", - "note": "" if posts else "profile read; media edges empty (" - + ", ".join(attempts) + ")"} - - time.sleep(PACE_SECONDS) - - # RUNG 2 — the profile HTML. Its og:description carries the three counts even when every - # JSON surface refuses, so it is the honest floor: identity without media. - status, body, note = _ig_get(f"https://www.instagram.com/{handle}/") - attempts.append(f"html:{status or note}") - if 200 <= status < 300 and body: - text = body.decode("utf-8", "replace") - if "/accounts/login" in text[:4000] and "og:description" not in text: - return {"state": "blocked", "profile": {}, "posts": [], "via": "html", - "note": "Instagram served the login wall (" + ", ".join(attempts) + ")"} - desc = "" - m = re.search(r']+property="og:description"[^>]+content="([^"]*)"', text) - if m: - desc = m.group(1) - counts = _OG_COUNTS.search(desc or "") - title = "" - mt = re.search(r']+property="og:title"[^>]+content="([^"]*)"', text) - if mt: - title = mt.group(1) - if counts: - profile = { - "username": handle, "full_name": title.split("(")[0].strip(), - # ⚠ NO BIO. `og:description` on a profile page is - # "204K Followers, 5,245 Following, 750 Posts - See Instagram photos and videos - # from Inayma (@inayma)" — the tail is Instagram's own boilerplate, not the - # person's biography. Storing it would fill a Bio column with the same sentence - # for every influencer, which reads as data and is not. Measured 2026-08-04. - "bio": "", - "followers": _loose_count(counts.group(1)), - "following": _loose_count(counts.group(2)), - "posts_count": _loose_count(counts.group(3)), - "verified": "", - # The page abbreviates ("204K"), so the number is the page's rounding, not a - # count we read. Recorded on the row rather than left for someone to discover - # when 204,000 fails to reconcile with anything. - "approx": "1", - } - return {"state": "partial", "profile": profile, "posts": [], "via": "og:description", - "note": "counts are the page's own abbreviations; media is not anonymously " - "readable (" + ", ".join(attempts) + ")"} - if status in (401, 403, 429) or status == 0: - return {"state": "blocked", "profile": {}, "posts": [], "via": "", - "note": f"Instagram refused the anonymous read ({', '.join(attempts)})"} - return {"state": "error", "profile": {}, "posts": [], "via": "", - "note": f"nothing anonymously readable ({', '.join(attempts)})"} - - -#: The historical anonymous media query. Kept because trying it and RECORDING the refusal is -#: what makes "as deep as anonymously accessible" (R7) a measurement instead of an assumption. -IG_MEDIA_QUERY_HASH = "e769aa130647d2354c40ea6a439bfc08" - - -def _paginate(user_id, cursor, want, log=print): - """Walk further back through the media edges while the anonymous surface allows it.""" - out, note = [], "" - while user_id and cursor and len(out) < want: - time.sleep(PACE_SECONDS) - variables = json.dumps({"id": str(user_id), "first": min(50, want - len(out)), - "after": cursor}, separators=(",", ":")) - url = (f"https://www.instagram.com/graphql/query/?query_hash={IG_MEDIA_QUERY_HASH}" - f"&variables={requests.utils.quote(variables)}") - status, body, err = _ig_get(url, headers={"X-IG-App-ID": IG_APP_ID}) - if not (200 <= status < 300) or not body: - note = f"graphql:{status or err}" - break - try: - media = (((json.loads(body.decode("utf-8", "replace")) or {}).get("data") or {}) - .get("user") or {}).get("edge_owner_to_timeline_media") or {} - except Exception: - note = "graphql:unparseable" - break - edges = media.get("edges") or [] - if not edges: - note = "graphql:empty" - break - out.extend(_posts_from_edges(edges)) - page = media.get("page_info") or {} - if not page.get("has_next_page"): - break - cursor = page.get("end_cursor") - return out[:want], note - - -# --------------------------------------------------------------------------------------------- -# RUNNERS -# --------------------------------------------------------------------------------------------- - -def cap_note(capped, missing=()): - """The sentence(s) a cap breach adds to a run summary. - - `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. - `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not - be created at all. - - ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the - ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that - does not say WHERE is only marginally better than a silent one. - - ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused - SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the - runners' `if tgt is not None` write guard then skipped that bucket, and the run reported - success over a database that does not exist. Exactly the silent-stop failure the row cap - already had, one level up — and harder to spot, because the table is not there to look at. - """ - out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} " - f"from this run were NOT written" for k, n in (capped or []) if n] - out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its " - f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" - for k in (missing or [])] - return "; ".join(out) - - -def ut_missing(rt, *keys): - """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" - have = ut_all(rt) - return [k for k in keys if k and k not in have] - - -def run_scrape_db(rt, defn, username="automation", log=print): - """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" - cfg = defn.get("config") or {} - fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") - dry = bool(cfg.get("dryRun")) - # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from - # the rollup — inferring is exactly how a green dot ended up over an empty table (see the - # rollup comment in `run_field_instagram`). - steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} - status, final, body = fetch(cfg["url"]) - if not (200 <= status < 300): - steps["fetch"] = "error" - return ("error", f"{cfg['url']} answered {status} — nothing was written", - {"status": status}, [], steps) - steps["fetch"] = "ok" - soup = _soup(body) - if cfg.get("extract") == "jsonld": - blocks = jsonld(soup) - flat = [] - for b in blocks: - if isinstance(b, list): - flat.extend([x for x in b if isinstance(x, dict)]) - elif isinstance(b, dict): - items = b.get("itemListElement") - flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) - else [b]) - raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] - else: - found = tables(soup) - idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 - raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] - if not raw_rows: - steps["extract"] = "error" - return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], - steps) - steps["extract"] = "ok" - - incoming = [] - for r in raw_rows: - mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} - if any(v for v in mapped.values()): - incoming.append(mapped) - - fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) - for src, tgt in fmap.items()] - # the key column leads, so the table opens on the identity it is upserted by - fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) - label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" - if dry: - # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, - # which would create the table: a dry run that leaves a new empty database behind is not - # a dry run. - table_key = ut_key_for(label, cfg.get("targetTable") or None) - else: - table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) - t = ut_get(rt, table_key) or {} - rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, - cap=row_cap(table_key)) - touched = [rid for rid, row in rows.items() - if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] - # R5 — the engine moves each record down its winning lane, over the row's FINAL values. - # Blank ⇒ no lane matched: the record shows in the derived Write bucket, never a guess. - lanes = cfg.get("lanes") or [] - if lanes: - skey = stage_field_key(defn.get("id")) - stamp = _iso() - counts["routed"] = 0 - for rid in touched: - row = rows.get(rid) or {} - label = route_record(lanes, row) - if label: - counts["routed"] += 1 - if str(row.get(skey) or "") != label: - row[skey] = label - row[skey + "_at"] = stamp if label else "" - if not dry: - ut_write_rows(rt, table_key, rows) - if lanes: - ensure_stage_field(rt, table_key, defn, username) - counts["source_rows"] = len(incoming) - affected = touched[:200] - summary = (f"{counts['inserted']} new, {counts['updated']} updated, " - f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " - f"(kept)") - if lanes: - summary += f", {counts['routed']}/{len(touched)} routed to lanes" - capped = [(table_key, counts.get("capped", 0))] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - note = cap_note(capped, missing) - if note: - summary += f" — {note}" - steps["write"] = "partial" - if dry: - summary = f"DRY RUN (the Write step is turned off) — would have written: {summary}" - steps["write"] = "skipped" - elif not note: - steps["write"] = "ok" - state = "partial" if (counts.get("capped") or counts.get("skipped") or missing - or dry) else "ok" - log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -def run_field_instagram(rt, defn, username="automation", log=print): - """Automation #2 (R7 + R1's hybrid): for every row of a blank database that carries a profile - URL, pull the public profile — the PAID rung first when the tier says so, the anonymous - ladder underneath it — write a status string into the automation column, and append a - timestamped row to each of the three IG tables.""" - cfg = defn.get("config") or {} - table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") - tier = clean_tier(cfg.get("tier")) or "anonymous" - paid = tier == "brightdata" - fallback = not cfg.get("noFallback") - post_metrics = bool(cfg.get("postMetrics")) - dry = bool(cfg.get("dryRun")) - steps = {"trigger": "ok", "source": "idle", "capture": "idle", - "capture_paid": "idle" if paid else "skipped", - "capture_anon": "idle" if (not paid or fallback) else "skipped", - "capture_metrics": "idle" if post_metrics else "skipped", - "write": "idle"} - t = ut_get(rt, table_key) - if not t: - steps["source"] = "error" - return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) - url_field = cfg.get("urlField") or _auto_url_field(t, fkey) - if not url_field: - steps["source"] = "error" - return ("error", "the automation column has no URL field bound to it", {}, [], steps) - steps["source"] = "ok" - rows = dict(t.get("rows") or {}) - # the relational tables the pull lands in (R7): every row timestamped for time-range filters - if dry: - # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) - snap_key, post_key, ps_key = ("ut_ig_snapshots", "ut_ig_posts", "ut_ig_post_snapshots") - else: - snap_key = ut_ensure(rt, "IG snapshots", SNAPSHOT_FIELDS, username, - key="ut_ig_snapshots", flow_tag=str(defn.get("id") or "")) - post_key = ut_ensure(rt, "IG posts", POST_FIELDS, username, key="ut_ig_posts", - flow_tag=str(defn.get("id") or "")) - ps_key = ut_ensure(rt, "IG post snapshots", POST_SNAPSHOT_FIELDS, username, - key="ut_ig_post_snapshots", flow_tag=str(defn.get("id") or "")) - missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key) - snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) - posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) - psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) - - counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, - "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, - "missing_tables": len(missing)} - cells, affected, notes = {}, [], [] - # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` - # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. - # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation - # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) - in_snaps, in_posts, in_psnaps = [], [], [] - paid_tried = paid_answered = anon_tried = anon_answered = False - targets = [(rid, str(r.get(url_field, "") or "").strip()) - for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] - for i, (rid, url) in enumerate(targets): - if i: - time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) - counts["profiles"] += 1 - res = pull_profile(url, max_posts=cfg.get("maxPosts") or 24, log=log, - tier=tier, fallback=fallback, post_metrics=post_metrics) - pulled = _iso() - via = res.get("via") or "" - read_ok = res["state"] in ("ok", "partial") - # WHICH RUNG ANSWERED — tracked per profile so the canvas's rung dots are a measurement. - if paid: - paid_tried = True - if via == "brightdata" and read_ok: - paid_answered = True - counts["paid"] += 1 - elif fallback: - anon_tried = True - anon_answered = anon_answered or read_ok - else: - anon_tried = True - anon_answered = anon_answered or read_ok - if read_ok: - counts["ok" if res["state"] == "ok" else "partial"] += 1 - prof = res["profile"] - # ⚠ A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written - # every time and a blank is unambiguous: it means THIS PULL did not read it. That is - # the opposite of the identity rows below, which are upserted and therefore omit - # blanks so a cheap rung cannot erase an expensive one's value. - snap_row = { - "snapshot_key": f"{prof['username']}@{pulled}", - "influencer_key": prof["username"], "pulled_at": pulled, - "source": _s(res.get("via")), "approx": _s(prof.get("approx")), - "bio": _s(prof.get("bio"), 500), - "external_url": _s(prof.get("external_url"), 300), - } - for fd in SNAPSHOT_FIELDS: - k = fd["key"] - if k not in snap_row: - snap_row[k] = _s(prof.get(k), 400) - in_snaps.append(snap_row) - for p in res.get("posts") or []: - # ⚠ A BLANK IS OMITTED, NOT WRITTEN. `posted_at` / `caption` / `type` are - # readable on some rungs and not others — the Bright Data PROFILE row carries - # post identity with `datetime: None` (measured 24/24), while the engagement rung - # knows `date_posted`. `upsert_rows` writes exactly the keys it is handed, so - # sending "" would let a cheap run ERASE what an expensive one learned. - ident = {"shortcode": p["shortcode"], "influencer_key": prof["username"], - "url": _s(p.get("url"), 300)} - for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), - ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), - ("alt_text", 400)): - if p.get(k): - ident[k] = _s(p.get(k), n) - in_posts.append(ident) - # ⛔ AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The - # post series is the append table that FILLS (maxPosts rows per profile per - # pull), and with the paid engagement rung OFF every one of those rows would - # carry three blanks — pure noise, eating a 200k ceiling and drawing a chart of - # nothing. A row in a time series should mean "this was true then"; a row that - # means "nobody looked" belongs nowhere. - metrics = {k: p.get(k) for k in ("likes", "comments", "views")} - if any(v is not None for v in metrics.values()): - in_psnaps.append({ - "post_snapshot_key": f"{p['shortcode']}@{pulled}", - "shortcode": p["shortcode"], "pulled_at": pulled, - "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), - "views": _s(metrics["views"])}) - counts["metrics"] += 1 - counts["posts"] += len(res.get("posts") or []) - detail = f"{prof.get('followers') or '?'} followers" - if prof.get("approx"): - detail += " (approx)" - if res.get("posts"): - detail += f", {len(res['posts'])} posts" - elif res["state"] == "partial": - detail += ", posts not readable on the rung that answered" - cells[rid] = f"{res['state']} · {_stamp()} · {detail}" - elif res["state"] == "blocked": - counts["blocked"] += 1 - cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" - notes.append(res.get("note") or "blocked") - else: - counts["error"] += 1 - cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" - notes.append(res.get("note") or "error") - affected.append(rid) - - # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. - snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) - posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) - psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) - counts["new_posts"] = c_post["inserted"] - capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), - (ps_key, c_ps["capped"])] - counts["capped"] = sum(n for _k, n in capped) - - # R5 — route each processed record down its winning lane, judged over the row's FINAL - # values (the fresh status cell included, so a lane on the cell itself is answerable). - lanes = cfg.get("lanes") or [] - stage_writes = {} - if lanes: - skey = stage_field_key(defn.get("id")) - stamp = _iso() - counts["routed"] = 0 - for rid, _url in targets: - merged = dict(rows.get(rid) or {}) - if rid in cells: - merged[fkey] = cells[rid] - label = route_record(lanes, merged) - if label: - counts["routed"] += 1 - if str((rows.get(rid) or {}).get(skey) or "") != label: - stage_writes[rid] = {skey: label, skey + "_at": stamp if label else ""} - - if not dry: - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for rid, val in cells.items(): - tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val - for rid, vals in stage_writes.items(): - tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals) - for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)): - tgt = cur.get(k) - if tgt is not None: - tgt["rows"] = rws - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables - if lanes: - ensure_stage_field(rt, table_key, defn, username) - - # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: - # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), - # `error` a LOUD partial — a pooled history with silent holes is worse than none. - master_note = "" - if not dry and (in_snaps or in_posts or in_psnaps): - import ig_master - m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), - in_snaps, in_posts, in_psnaps) - if m_status == "error": - master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \ - f"is complete and the next run re-appends" - counts["master_failed"] = 1 - elif m_status == "ok": - counts["master"] = len(in_snaps) + len(in_psnaps) - - # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while - # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail - # showed a green dot over a table with no posts in it. A summary that disagrees with the - # cells it summarises is the failure this whole module's honest-status rule exists to - # prevent, so `partial` now propagates. Measured 2026-08-04. - read = counts["ok"] + counts["partial"] - if not read: - state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" - elif counts["blocked"] or counts["error"] or counts["partial"]: - state = "partial" - else: - state = "ok" - # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) - if not targets: - cap_state = "idle" - elif not read: - cap_state = "error" if counts["error"] else "blocked" - else: - cap_state = "partial" if (counts["blocked"] or counts["error"] - or counts["partial"]) else "ok" - steps["capture"] = cap_state - steps["capture_paid"] = ("ok" if paid_answered - else "blocked" if paid_tried else "skipped") - steps["capture_anon"] = ("ok" if anon_answered - else "blocked" if anon_tried else "skipped") - # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually - # appended. "It was switched on" is not the same fact as "it answered", and painting the - # second from the first is the fabrication `node_status` refuses to make. - # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A - # run that found no posts to enrich did not have a rung refuse it — nothing was requested — - # so that reads `skipped`, the same word an off switch earns. - steps["capture_metrics"] = ("ok" if counts["metrics"] - else "blocked" if (post_metrics and counts["posts"]) - else "skipped") - - summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " - f"({counts['new_posts']} new)") - if lanes: - summary += f", {counts['routed']}/{len(targets)} routed to lanes" - if counts["paid"]: - summary += f", {counts['paid']} exact via Bright Data" - if counts["metrics"]: - summary += f", {counts['metrics']} post engagement snapshots" - elif post_metrics and counts["posts"]: - # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there - # was no engagement", which is a claim about Instagram rather than about our run. - summary += ", no post engagement was readable" - if counts["partial"]: - summary += f", {counts['partial']} profile-only (posts not readable)" - if counts["blocked"]: - summary += f", {counts['blocked']} blocked" - note = cap_note(capped, missing) - if note: - summary += f" — {note}" - state = "partial" if state != "error" else state - steps["write"] = "partial" - elif not dry: - steps["write"] = "ok" - if master_note: - summary += f" — {master_note}" - state = "partial" if state != "error" else state - steps["write"] = "partial" - if dry: - summary = f"DRY RUN (the Write step is turned off) — would have written: {summary}" - steps["write"] = "skipped" - state = "partial" if state != "error" else state - if notes: - summary += f" — {notes[0][:120]}" - if not dry: - # C7: the fresh master rows are exactly what the table's metric cells summarise. - try: - compute_metric_cells(rt, table_key) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - return (state, summary, counts, affected, steps) - - -def _auto_url_field(table, fkey): - """The URL field an automation column is bound to: its own `automation.urlField`, else the - first url-typed column on the table. Never guessed silently — the caller reports which.""" - for f in table.get("fields") or []: - if f.get("key") == fkey: - bound = ((f.get("automation") or {}).get("urlField") or "").strip() - if bound: - return bound - for f in table.get("fields") or []: - if f.get("type") == "url": - return f.get("key") - return "" - - -def run_discover_instagram(rt, defn, username="automation", log=print): - """Automation #3 (owner ruling R7 / D-23): FIND handles nobody here has typed in. - - Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus - query and hands its snapshot id to the next run, or COLLECTS one a previous run started. - Every outcome is a sentence about what actually happened; none of them is a green zero. - """ - cfg = defn.get("config") or {} - dry = bool(cfg.get("dryRun")) - limit = int(cfg.get("recordsLimit") or 5) - preds = list(cfg.get("predicates") or []) - auto_id = str(defn.get("id") or "") - steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} - counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, - "missing_tables": 0} - - if not bd_ready(): - # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is - # the vendor's — so this is where the run stops, saying which env var is missing. - steps["find"] = "blocked" - return ("partial", "AIOS_BRIGHTDATA_KEY is not configured — profile search is closed " - "(nothing was searched and nothing was billed)", counts, [], steps) - - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - if not pending: - # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and - # the vendor bills for breadth whether the filter was saved yesterday or last month. - guard_err = narrowing_refusal(preds) - if guard_err: - steps["find"] = "blocked" - return ("error", f"the search was not started — {guard_err}", counts, [], steps) - if dry: - est = discover_estimate(limit) - steps["find"] = steps["write"] = "skipped" - return ("partial", - f"DRY RUN (the Write step is turned off) — would search for up to {limit} " - f"profile{'' if limit == 1 else 's'} matching " - f"{_predicate_sentence(preds, cfg.get('operator'))}. Estimated cost " - f"${est['usd']} (ESTIMATE — {est['note']})", counts, [], steps) - sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit) - if note: - steps["find"] = "blocked" - return ("error", f"the search was not started — {note}", counts, [], steps) - pending = sid - # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes - # through `rt.update`, which can RAISE on an unavailable store — and that exception - # becomes an error run, so a `log()` placed after it never executes. The snapshot would - # then exist, be billed, and have its id in neither the store nor the logs. Logging first - # means the worst case is still recoverable by a human reading the Space output. - log(f"[aios-auto] discover {auto_id}: started {sid}") - # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop - # existing because this process dies thirty seconds later, and a lost id is a set we - # paid attention to and can never collect. - set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) - steps["find"] = "ok" - - waited, status, size, note = 0.0, "", 0, "" - while True: - status, size, note = bd_filter_status(pending) - if note: - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", note, counts, [], steps) - if status != "building" or waited >= BD_FILTER_WAIT: - break - time.sleep(BD_FILTER_POLL) - waited += BD_FILTER_POLL - - if status == "building": - # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 - # minutes; holding a worker thread that long on a free-tier container to poll would be - # the wrong shape. Say plainly that it is still running and who collects it. - steps["collect"] = "partial" - return ("partial", - f"the search is still running at Bright Data ({pending}) — searches take about " - f"20 minutes, and the NEXT run of this automation collects it. Nothing is lost " - f"and nothing is re-billed", counts, [], steps) - if status != "ready": - steps["collect"] = "error" - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", - counts, [], steps) - - rows, dnote = bd_filter_rows(pending) - if dnote: - # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. - steps["collect"] = "partial" - return ("partial", f"the search finished with {size} match" - f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) - set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - steps["collect"] = "ok" - - stamp = _iso() - incoming = [c for c in (_candidate_row(r, stamp) for r in rows) if c] - counts["found"] = len(incoming) - # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and - # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to - # "the vendor delivered 3". It also catches the shape where an error document comes back - # down the rows path and parses as one unusable row instead of an error. - counts["dropped"] = max(0, len(rows) - len(incoming)) - label = cfg.get("targetLabel") or "IG candidates" - table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry - else ut_ensure(rt, label, CANDIDATE_FIELDS, username, - key=cfg.get("targetTable") or DISCOVER_TABLE, - flow_tag=str(defn.get("id") or ""))) - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - # --- C6: the upsert key is the COMPOUND (handle, created_by) — per-user rows, per-user - # found_counts ("each user only works with their own data" without breaking the upsert - # law). Implemented as a synthesised key column that exists only inside this merge, so - # `upsert_rows` keeps its one-field contract and its id/cap arithmetic runs over the WHOLE - # table. A pre-wave row with no owner is ADOPTED by the first user who re-finds its handle - # (the `ut_ensure` owner-adoption move) rather than duplicated beside itself. - def _ck(handle, owner): - return f"{str(handle or '').strip()}\n{str(owner or '').strip()}" - - incoming_handles = {c["handle"] for c in incoming} - for r in existing.values(): - if not str(r.get("created_by") or "").strip() and \ - str(r.get("handle") or "").strip() in incoming_handles: - r["created_by"] = username - existing2 = {rid: {**r, "_ckey": _ck(r.get("handle"), r.get("created_by"))} - for rid, r in existing.items()} - # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding - # a profile is the signal that it keeps matching, so the counter grows instead of - # resetting. `first_found` is written only when the pair is new, so it never moves. - seen = {r["_ckey"]: r for r in existing2.values()} - skey = stage_field_key(auto_id) - for c in incoming: - c["created_by"] = username - c["_ckey"] = _ck(c["handle"], username) - prev_row = seen.get(c["_ckey"]) - if prev_row: - counts["seen_again"] += 1 - c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) - else: - counts["new"] += 1 - c["found_count"] = "1" - c["first_found"] = stamp - # C2/R6 — a NEW candidate's card starts at the human gate. Stamped only at birth: - # a re-find must never pull a card somebody has already moved back into Review, - # for the same reason it never un-ticks `tracked`. - c[skey] = "Review" - c[skey + "_at"] = stamp - merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) - merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} - for rid, r in merged.items()} - counts["capped"] = mc["capped"] - missing = [] if dry else ut_missing(rt, table_key) - counts["missing_tables"] = len(missing) - if not dry and not missing: - ut_write_rows(rt, table_key, merged) - ensure_stage_field(rt, table_key, defn, username) - steps["write"] = "ok" - elif dry: - steps["write"] = "skipped" - - affected = [rid for rid, r in merged.items() - if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] - est = discover_estimate(counts["found"]) - summary = (f"{counts['found']} profile{'' if counts['found'] == 1 else 's'} found — " - f"{counts['new']} new, {counts['seen_again']} seen before. Est. cost " - f"${est['usd']} (ESTIMATE, not a billed figure). None is tracked until somebody " - f"ticks it") - if counts["dropped"]: - summary += (f". ⚠ {counts['dropped']} row{'' if counts['dropped'] == 1 else 's'} the " - f"search returned had no handle and could not be used") - cnote = cap_note([(table_key, counts["capped"])], missing) - if cnote: - summary += f" — {cnote}" - steps["write"] = "partial" - if dry: - summary = f"DRY RUN (the Write step is turned off) — would have written: {summary}" - state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" - if not dry and not missing: - try: - compute_metric_cells(rt, table_key) # C7 — see the field runner - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") - log(f"[aios-auto] discover {auto_id} -> {table_key}: {summary}") - return (state, summary, counts, affected, steps) - - -def _predicate_sentence(preds, operator="and"): - """A predicate list as something a person can read back. Used in summaries and the canvas.""" - joiner = " or " if str(operator or "").lower() == "or" else " and " - parts = [] - for p in preds or []: - v = p.get("value") - parts.append(f"{p.get('name')} {p.get('operator')}" - + ("" if v is None else f" {v}")) - return joiner.join(parts) or "no conditions" - - -RUNNERS = {"scrape_db": run_scrape_db, "field_instagram": run_field_instagram, - "discover_instagram": run_discover_instagram} - - -# --------------------------------------------------------------------------------------------- -# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) -# --------------------------------------------------------------------------------------------- -# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → -# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness -# boolean on the wire, four client strings and two gate sections. That is what a hard-coded -# vendor costs, and TikTok (D-9) would have paid it again from scratch. -# -# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the -# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: -# -# probe "is this configured?" -> the honest readiness bit the UI shows -# capture enrich a handle we know -> `pull_profile`-shaped -# discover find handles we do not -> the corpus query -# -# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in -# later; it is the honest statement that this source has no discovery route, and a surface that -# offers one anyway is offering a button that must refuse. -SOURCES = { - "instagram": { - "key": "instagram", "label": "Instagram", "vendor": "Bright Data", - "probe": bd_ready, - "capture": pull_profile, - "discover": bd_filter_start, - "captureKind": "field_instagram", - "discoverKind": "discover_instagram", - "tiers": TIERS, - }, - # D-9 (TikTok) lands here as one more entry: a new field map and a new `capture`, with the - # canvas, the append tables, the caps and the honest-status contract already in place. -} - - -def source_status(): - """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. - - ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface - needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the - credential to a browser would put a billable secret in every user's devtools. - """ - return [{"key": s["key"], "label": s["label"], "vendor": s["vendor"], - "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} - for s in SOURCES.values()] - - -# --------------------------------------------------------------------------------------------- -# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client -# --------------------------------------------------------------------------------------------- -# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are -# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — -# a canvas that still shows a step the engine stopped running looks completely fine. So the node -# list is derived from the DEFINITION by the same module that executes it, rides on the payload -# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. -# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. -# -# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is -# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine -# walks it linearly. Nothing here should be read as a general branching runtime. - -#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be -#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw -#: away the only information the dot is there to carry. -NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") - -#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a -#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is -#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones -#: here are all REAL — each changes what the next run does: -#: schedule the trigger fires on its cron, or only by hand -#: paid the paid rung is reached for, or it is not (this is the one that spends money) -#: fallback the anonymous ladder catches a refused paid rung, or the run says `blocked` -#: instead of quietly writing an approximate number -#: postMetrics likes/comments per post are bought, or the engagement series does not grow — -#: the expensive one, because it costs a vendor record PER POST rather than per -#: profile (measured: the profile row carries post identity and no engagement) -#: write DRY RUN — read everything, compute the counts, write nothing anywhere -NODE_TOGGLES = {"trigger": "schedule", "capture_paid": "paid", - "capture_anon": "fallback", "capture_metrics": "postMetrics", - "write": "write"} - - -def _cron_label(cron): - return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") - - -def node_status(steps, nid): - """A node's dot — from the last run's MEASURED step outcome, or `idle`. - - ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely - so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, - and painting those nodes green because the run said `ok` would manufacture a measurement — the - same defect as the green dot over the empty posts table recorded in `run_field_instagram`. - No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. - """ - v = str((steps or {}).get(nid) or "") - return v if v in NODE_STATES else "idle" - - -def graph(defn): - """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" - defn = defn if isinstance(defn, dict) else {} - cfg = defn.get("config") or {} - sched = defn.get("schedule") or {} - runs = defn.get("runs") or [] - last = runs[0] if runs and isinstance(runs[0], dict) else {} - steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} - - def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): - return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, - "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), - "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} - - on = bool(sched.get("enabled")) - trg = defn.get("trigger") or {} - if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): - # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail - # read the same node, so an event-triggered flow must not read "Schedule". The membership - # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old - # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so - # the else-branch was reachable) — green, compiling, and wrong on screen. - t_on = bool(trg.get("enabled", True)) and not trg.get("paused") - sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) - watched = ", ".join(trg.get("fields") or []) or "any field" - det = {"event_field": (f"{trg.get('table', '')}" - + (f".{trg.get('field')}" if trg.get("field") else "") - + (f" — {_lane_sentence(trg.get('when'))}" - if trg.get("when") else "")), - "record_updated": f"{trg.get('table', '')} — {watched}", - "record_created": str(trg.get("table") or ""), - "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}", - "form_submitted": str(trg.get("table") or ""), - "webhook": "POST the hook URL to fire it", - "email": str(trg.get("query") or "")}.get(trg["key"], "") - if not trg.get("configured", True): - det = "Finish setting this trigger up before it can fire" - if trg.get("paused"): - det = str(defn.get("statusNote") or "paused") - nodes = [node("trigger", "trigger", sub, - "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] - else: - nodes = [node("trigger", "trigger", "Schedule", - _cron_label(sched.get("cron")) if on else "Manual only", 0, - panel="schedule", on=on, - detail="" if on else "It still runs when you press Run now")] - edges = [] - - if defn.get("kind") == "field_instagram": - paid = clean_tier(cfg.get("tier")) == "brightdata" - fb = not cfg.get("noFallback") - anon_on = (not paid) or fb - metrics_on = bool(cfg.get("postMetrics")) - max_posts = cfg.get("maxPosts") or 24 - nodes += [ - node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, - panel="source", - detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") - else "URL column: the one the field is bound to"), - node("capture", "branch", "Capture", - "Bright Data, then anonymous" if paid else "Anonymous only", 2, panel="capture", - detail=f"Up to {max_posts} posts per profile"), - node("capture_paid", "capture", "Bright Data", - "Exact counts, top posts" if paid else "Off — the paid rung is not used", - 3, row=-1, panel="capture", on=paid, - detail="" if bd_ready() else "No AIOS_BRIGHTDATA_KEY is configured"), - node("capture_anon", "capture", "Anonymous", - "Approximate counts, $0" if anon_on else "Off — exact numbers or nothing", - 3, row=1, panel="capture", on=anon_on, - detail="Local egress only — Instagram blocks datacenters"), - # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY - # and no engagement (measured), so likes/comments are a SECOND vendor call per post. - # A switch that multiplies a run's cost by the post count deserves to be visible on - # the canvas rather than buried in a config panel. - node("capture_metrics", "capture", "Post engagement", - "Likes and comments per post" if metrics_on - else "Off — no likes/comments are bought", - 4, panel="capture", on=metrics_on, - detail=(f"~{max_posts}x the records of a profile-only run" - if metrics_on else - "ut_ig_post_snapshots only grows while this is on")), - node("write", "write", "Write", - cfg.get("targetTable") or "No database", 5, panel="write", - on=not cfg.get("dryRun"), - detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), - ] - edges = [{"from": "trigger", "to": "source", "label": ""}, - {"from": "source", "to": "capture", "label": ""}, - {"from": "capture", "to": "capture_paid", "label": "paid"}, - {"from": "capture", "to": "capture_anon", "label": "fallback"}, - {"from": "capture_paid", "to": "capture_metrics", "label": ""}, - {"from": "capture_anon", "to": "capture_metrics", "label": ""}, - {"from": "capture_metrics", "to": "write", "label": ""}] - # When there is no paid rung the anonymous one is the ONLY rung, so it must not carry a - # switch that would leave the automation with nothing to run. - if not paid: - for n in nodes: - if n["id"] == "capture_anon": - n["toggle"] = "" - elif defn.get("kind") == "discover_instagram": - limit = int(cfg.get("recordsLimit") or 0) - est = discover_estimate(limit) - pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") - nodes += [ - node("find", "source", "Find profiles", - _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, - panel="find", - detail=(f"Up to {limit} profiles · est. ${est['usd']} (ESTIMATE)" - if bd_ready() - else "No AIOS_BRIGHTDATA_KEY is configured — search is closed")), - node("collect", "capture", "Collect results", - f"Building: {pending}" if pending else "Searches take about 20 minutes", 2, - panel="find", - detail="A run that finds it still building hands off to the next run"), - node("write", "write", "Write candidates", - cfg.get("targetTable") or DISCOVER_TABLE, 3, panel="write", - on=not cfg.get("dryRun"), - detail="Nothing is tracked until somebody ticks it"), - ] - edges = [{"from": "trigger", "to": "find", "label": ""}, - {"from": "find", "to": "collect", "label": ""}, - {"from": "collect", "to": "write", "label": ""}] - else: - url = str(cfg.get("url") or "") - host = (urlparse(url).hostname or url or "No URL") if url else "No URL" - which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" - else f"HTML table #{int(cfg.get('tableIndex') or 0)}") - mapped = len(cfg.get("fieldMap") or {}) - nodes += [ - node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), - node("extract", "capture", "Extract", which, 2, panel="columns", - detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " - f"{cfg.get('keyField') or '—'}"), - node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, - panel="write", on=not cfg.get("dryRun"), - detail=cfg.get("targetTable") or "a new database"), - ] - edges = [{"from": "trigger", "to": "fetch", "label": ""}, - {"from": "fetch", "to": "extract", "label": ""}, - {"from": "extract", "to": "write", "label": ""}] - return {"nodes": nodes, "edges": edges} - - -def toggle_node(rt, auto_id, node_id): - """Flip ONE node's switch. Returns `(defn, error)`. - - The node → config mapping lives on the SERVER for the reason the whole graph does: a client - that knew which field each node writes would be a second copy of that knowledge, free to - drift. The canvas just says "this node was clicked". - """ - prev = all_definitions(rt).get(str(auto_id)) - if prev is None: - return None, "no such automation" - target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) - if target is None: - return None, f"there is no {node_id!r} step in this automation" - if not target.get("toggle"): - return None, (f"the {target['title']} step cannot be turned off — it is what this " - f"automation is") - cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) - which = target["toggle"] - if which == "schedule" and (prev.get("trigger") or {}).get("key") in ( - "event_field", "record_created", "webhook", "email"): - # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping - # the cron under a node labelled "When a field changes" would be a switch that lies. - trg = dict(prev.get("trigger") or {}) - trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) - trg["paused"] = False # a deliberate flip clears an auto-pause: human wins - return patch(rt, auto_id, {"trigger": trg}) - if which == "schedule": - sched["enabled"] = not sched.get("enabled") - elif which == "paid": - cfg["tier"] = ("anonymous" if clean_tier(cfg.get("tier")) == "brightdata" - else "brightdata") - elif which == "fallback": - cfg["noFallback"] = not cfg.get("noFallback") - elif which == "postMetrics": - cfg["postMetrics"] = not cfg.get("postMetrics") - elif which == "write": - cfg["dryRun"] = not cfg.get("dryRun") - return patch(rt, auto_id, {"config": cfg, "schedule": sched}) - - -# --------------------------------------------------------------------------------------------- -# THE BOARD (wave 22, contracts C1/C2 — owner ruling R1: RECORDS AS CARDS) -# --------------------------------------------------------------------------------------------- -# The automation detail IS a kanban board: columns = the flow's stages, cards = records of the -# target database, each record's position stored in a single-select STAGE FIELD auto-created on -# that database — so the same field drives a plain Kanban view with no board code at all. -# -# TWO KINDS OF STAGE, and conflating them is the mistake this section is built around: -# -# * RUN stages (trigger/fetch/find/collect/capture/write) are MACHINERY — a record passes -# through them but can never sit in one, the way a parcel is never "at" the conveyor belt. -# They come straight off `graph()` so the board and the runner cannot drift. -# * RECORD stages hold cards. They are the REVIEW gate (R6: discovery's review IS the -# `tracked` promotion), its terminal, and the R5 IF-routing lanes. Only these are choices -# of the stage field — a select offering "Fetch page" as a place to PUT a record would be -# offering a lie. -# -# ⛔ WHO WRITES THE STAGE FIELD (C2): the ENGINE, everywhere — except a review stage, where a -# HUMAN may advance a card to a member of the review stage's `next` and nothing else. That wall -# is enforced twice, once per door: `move_card` below (the board's own POST) and -# `grid_events.overlay_patch` (the plain grid/kanban door), which reads the law OUT OF THE FIELD -# DEFINITION (`automation.humanMoves` / `automation.arrive`) because platform code must not -# import this module. The engine STATES the law; the grid ENFORCES it mechanically. -# -# R7's approval-posture vocabulary rides every stage: `strict` = holds for a human, -# `auto` = the engine advances. Today review⇒strict and everything else⇒auto BY DERIVATION — -# the vocabulary is on the wire so a later wave can flip a stage without a client change. - -STAGE_FIELD_PREFIX = "stage_" -STAGE_KINDS = ("trigger", "action", "review", "branch") -POSTURES = ("auto", "strict") -#: One board answer carries at most this many cards PER STAGE; `counts` always carries the -#: true totals and `truncated` names the stages that overflowed — a cap nobody can see is the -#: silent-[:N] defect ([[no-unverifiable-aggregates]]), so this one is disclosed on the wire. -BOARD_CARD_CAP = 200 - -#: R5 lanes — the per-record IF-routing vocabulary. Enumerated HERE (and shipped on the wire) -#: so the client can never grow its own union of operators (the wave-9 silent-drop class). -MAX_LANES = 6 -LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", - "is_empty", "is_not_empty") -LANE_NULLARY_OPS = ("is_empty", "is_not_empty") - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` -# -# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why -# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches -# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc -# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the -# same function applied one level up, so the cheaper honest version is to evaluate the leaf where -# it lies and never touch the bytes.) -# -# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An -# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and -# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. -# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an -# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that -# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine -# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. -MAX_COND_DEPTH = 3 -MAX_COND_CHILDREN = 12 -COND_GROUP_KEYS = ("all", "any") - - -def clean_cond(raw, depth=0, where=""): - """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no - condition", which is legal everywhere a condition is optional (the catch-all lane, an - unconditioned trigger, an action group that always runs).""" - if raw in (None, "", {}): - return None, None - at = f" on {where}" if where else "" - if not isinstance(raw, dict): - return None, f"the condition{at} must be an object" - conj = [k for k in COND_GROUP_KEYS if k in raw] - if len(conj) > 1: - return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the " - f"other") - if conj: - key = conj[0] - if depth + 1 > MAX_COND_DEPTH: - return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — " - f"the group{at} is deeper") - kids_raw = raw.get(key) - if not isinstance(kids_raw, list) or not kids_raw: - return None, f"the '{key}' group{at} needs at least one condition inside it" - if len(kids_raw) > MAX_COND_CHILDREN: - return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — " - f"the '{key}' group{at} has {len(kids_raw)}") - kids = [] - for child in kids_raw: - c, err = clean_cond(child, depth + 1, where) - if err: - return None, err - if c is None: - return None, (f"an empty condition sits inside the '{key}' group{at} — " - f"finish it or remove it") - kids.append(c) - return {key: kids}, None - field = _s(raw.get("field"), 80).strip() - op = _s(raw.get("op") or raw.get("operator"), 20).strip() - if not field: - return None, f"the condition{at} names no field" - if op not in LANE_OPS: - return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) - cond = {"field": field, "op": op} - if op not in LANE_NULLARY_OPS: - v = raw.get("value") - if v is None or (isinstance(v, str) and not v.strip()): - return None, f"give a value to compare {field!r} against{at}" - cond["value"] = v.strip() if isinstance(v, str) else v - return cond, None - - -def cond_fields(cond): - """Every field name a tree reads — the set a caller must have on the row image before the - answer means anything.""" - if not isinstance(cond, dict): - return set() - for key in COND_GROUP_KEYS: - if key in cond: - out = set() - for child in cond.get(key) or []: - out |= cond_fields(child) - return out - f = cond.get("field") - return {f} if f else set() -#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would -#: collide with the stage the label is a choice FOR, and the board could no longer tell a -#: routed card from a gated one. -_RESERVED_STAGE_LABELS = frozenset({"review", "tracked", "declined", "write", "profile set", - "schedule", "stage"}) - - -def stage_field_key(auto_id): - """The stage field's key on the target database. Derived, never chosen — two automations on - one table must mint two distinct fields, and a rename must never move the key.""" - return STAGE_FIELD_PREFIX + re.sub(r"[^a-z0-9_]+", "_", str(auto_id or "").lower())[:40] - - -def clean_lanes(raw): - """Validate the R5 lane list. Returns `(lanes, error)` — refuses, never coerces - (the `clean_predicates` discipline: a condition we cannot answer is refused at WRITE time - with the reason named, because dropping or bending it would route records nobody asked to - route).""" - if raw in (None, ""): - return [], None - if not isinstance(raw, list): - return None, "lanes must be a list" - if len(raw) > MAX_LANES: - return None, f"an automation may have at most {MAX_LANES} lanes" - out, seen_ids, defaults = [], set(), 0 - for entry in raw: - if not isinstance(entry, dict): - return None, "each lane must be an object with a label and an optional condition" - label = " ".join(_s(entry.get("label"), 60).split()) - if not label: - return None, "every lane needs a label" - if label.strip().lower() in _RESERVED_STAGE_LABELS: - return None, (f"{label!r} is the name of a fixed stage — pick another label for " - f"the lane") - lid = "lane_" + (re.sub(r"[^a-z0-9]+", "_", label.lower()).strip("_") or "x")[:40] - base, n = lid, 2 - while lid in seen_ids: - lid = f"{base}_{n}" - n += 1 - seen_ids.add(lid) - cond, cerr = clean_cond(entry.get("when"), where=repr(label)) - if cerr: - return None, cerr - if cond is None: - defaults += 1 - if defaults > 1: - return None, ("only one lane may have no condition — it is the catch-all, " - "and two catch-alls would race for every record") - out.append({"id": lid, "label": label, "when": cond}) - return out, None - - -def _lane_num(v): - try: - return float(str(v).replace(",", "")) - except (TypeError, ValueError): - return None - - -def lane_match(cond, row): - """Does `row` satisfy this condition TREE? None matches everything (the catch-all). - - Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 - caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping - this name rather than growing a second entry point that could disagree with it. - - ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse - as a number answers False — the record simply does not enter the lane — never "treat blank - as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank - cell is an unknown, and an unknown cannot be less than 10,000. - """ - if cond is None: - return True - if isinstance(cond, dict): - if "all" in cond: - return all(lane_match(c, row) for c in cond.get("all") or []) - if "any" in cond: - return any(lane_match(c, row) for c in cond.get("any") or []) - raw = (row or {}).get(cond.get("field")) - op = cond.get("op") - if op == "is_empty": - return raw in (None, "") - if op == "is_not_empty": - return raw not in (None, "") - want = cond.get("value") - if op in (">", ">=", "<", "<="): - a, b = _lane_num(raw), _lane_num(want) - if a is None or b is None: - return False - return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] - have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") - if op == "includes": - return want_s.lower() in have_s.lower() - if op == "not_includes": - return want_s.lower() not in have_s.lower() - a, b = _lane_num(raw), _lane_num(want) - same = (a is not None and b is not None and a == b) or have_s == want_s - return same if op == "=" else (not same) if op == "!=" else False - - -def route_record(lanes, row): - """The winning lane's LABEL for one record, or '' when nothing matches. - - First declared match wins — declaration order is the priority order, the same rule every - rule engine the owner has used applies, and the catch-all (condition None) is simply the - first lane nothing above it beat. - """ - for lane in lanes or []: - if lane_match(lane.get("when"), row): - return lane["label"] - return "" - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. -# -# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. -# A `group` holds nested actions behind a condition — Airtable's "conditional action group", -# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's -# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this -# builder is a column, not a canvas). -# -# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a -# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / -# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty -# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready -# kind with a sentence, so the faded state is a wall and not a styling choice. -MAX_ACTIONS = 20 # per automation, counting nested ones -MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow -MAX_ACTION_VALUES = 20 # cells one update/create action may write -FIND_LIMIT_MAX = 100 -ACTION_KINDS = ("group", "update_record", "create_record", "find_records", "review") -ACTION_CATALOG = [ - {"kind": "group", "label": "Conditional logic", "group": "Advanced logic", "ready": True, - "detail": "Run different actions based off a certain condition"}, - {"kind": "update_record", "label": "Update record", "group": "Actions", "ready": True, - "detail": "Write values onto the record walking the flow"}, - {"kind": "create_record", "label": "Create record", "group": "Actions", "ready": True, - "detail": "Add a row to another database"}, - {"kind": "find_records", "label": "Find records", "group": "Actions", "ready": True, - "detail": "Look rows up by condition; the run log opens them"}, - {"kind": "review", "label": "Manual review", "group": "Advanced logic", "ready": True, - "detail": "Hold the record for a person — or an AI — to decide where it goes next"}, - {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", - "ready": False, "detail": "Run the same actions on every item in a list"}, - {"kind": "send_email", "label": "Send email", "group": "Actions", "ready": False, - "detail": "Needs a send scope on the Gmail connection"}, - {"kind": "slack", "label": "Send Slack message", "group": "Actions", "ready": False, - "detail": "Needs the Slack connector"}, - {"kind": "run_script", "label": "Run script", "group": "Actions", "ready": False, - "detail": "Not built — a sandbox is its own decision"}, - {"kind": "generate_ai", "label": "Generate with AI", "group": "Actions", "ready": False, - "detail": "Review stages can already be decided by AI"}, -] -REVIEW_DECIDERS = ("user", "ai") -MAX_AI_PROMPT = 600 - - -def action_catalog(): - """The catalog as the wire carries it — a copy, because a caller that mutated the module - constant would change every later reader's answer.""" - return [dict(a) for a in ACTION_CATALOG] - - -def _action_label(act): - for row in ACTION_CATALOG: - if row["kind"] == act.get("kind"): - return row["label"] - return str(act.get("kind") or "Action") - - -def clean_actions(raw, depth=0, _seen=None, _count=None): - """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. - - Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder - survives a Save. A missing or colliding one is minted `act_`; minting on every clean would - move the selection under the person editing it. - """ - if raw in (None, ""): - return [], None - if not isinstance(raw, list): - return None, "actions must be a list" - seen = _seen if _seen is not None else set() - count = _count if _count is not None else [0] - out = [] - for entry in raw: - if not isinstance(entry, dict): - return None, "each action must be an object with a kind" - kind = _s(entry.get("kind"), 30).strip() - row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) - if row is None: - return None, (f"{kind or 'that action'!r} is not one of: " - + ", ".join(a["kind"] for a in ACTION_CATALOG)) - if not row["ready"]: - return None, (f"{row['label']!r} is on the menu but not built yet — " - f"{row['detail'][0].lower()}{row['detail'][1:]}") - count[0] += 1 - if count[0] > MAX_ACTIONS: - return None, f"an automation runs at most {MAX_ACTIONS} actions" - aid = _s(entry.get("id"), 40).strip() - if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: - n = 1 - while f"act_{n}" in seen: - n += 1 - aid = f"act_{n}" - seen.add(aid) - when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") - if cerr: - return None, cerr - cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} - cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count) - if cerr: - return None, cerr - out.append({"id": aid, "kind": kind, - "enabled": bool(entry.get("enabled", True)), - "when": when, "config": cfg}) - return out, None - - -def _clean_action_config(kind, cfg, depth, seen, count): - """One action's `config`, per kind. Returns `(config, error)`.""" - if kind == "group": - if depth + 1 > MAX_GROUP_DEPTH: - return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — " - f"past that a flow is a flowchart and belongs on the board") - cond, cerr = clean_cond(cfg.get("cond"), where="the group") - if cerr: - return None, cerr - kids, kerr = clean_actions(cfg.get("actions"), depth + 1, seen, count) - if kerr: - return None, kerr - if not kids: - return None, "a conditional group with no actions inside it does nothing" - return {"cond": cond, "actions": kids}, None - if kind in ("update_record", "create_record"): - values = cfg.get("values") - if not isinstance(values, dict) or not values: - return None, f"the {kind.replace('_', ' ')} action writes no values" - if len(values) > MAX_ACTION_VALUES: - return None, f"one action writes at most {MAX_ACTION_VALUES} cells" - clean_vals = {} - for k, v in values.items(): - key = _s(k, 80).strip() - if not key: - return None, "a value is written to a field with no name" - if v is not None and not isinstance(v, (str, int, float, bool)): - return None, (f"the value for {key!r} must be text or a number — " - f"an automation writes cells, not objects") - clean_vals[key] = _s(v, 500) if isinstance(v, str) else v - out = {"values": clean_vals} - if kind == "create_record": - table = _s(cfg.get("table"), 60).strip() - if not table: - return None, "the create record action names no database" - if not table.startswith(UT_PREFIX): - return None, (f"actions write to blank databases (ut_*) this wave — " - f"{table!r} is not one") - out["table"] = table - return out, None - if kind == "find_records": - table = _s(cfg.get("table"), 60).strip() - if not table: - return None, "the find records action names no database" - cond, cerr = clean_cond(cfg.get("cond"), where="the find") - if cerr: - return None, cerr - try: - limit = int(cfg.get("limit") or 25) - except (TypeError, ValueError): - return None, "the find records limit must be a whole number" - if limit < 1 or limit > FIND_LIMIT_MAX: - return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" - return {"table": table, "cond": cond, "limit": limit}, None - if kind == "review": - by = _s(cfg.get("decidedBy"), 10).strip() or "user" - if by not in REVIEW_DECIDERS: - return None, f"a review is decided by one of: {', '.join(REVIEW_DECIDERS)}" - label = " ".join(_s(cfg.get("label"), 60).split()) or "Review" - if label.strip().lower() in _RESERVED_STAGE_LABELS and label != "Review": - return None, f"{label!r} is the name of a fixed stage — name the review something else" - nxt = cfg.get("next") - outs = [] - if isinstance(nxt, list): - for entry in nxt[:MAX_LANES]: - lbl = " ".join(_s(entry, 60).split()) - if lbl and lbl not in outs: - outs.append(lbl) - if not outs: - outs = ["Approved", "Declined"] - prompt = _s(cfg.get("prompt"), MAX_AI_PROMPT).strip() - if by == "ai" and not prompt: - return None, ("an AI review needs a prompt — tell it what to decide, or set the " - "review back to a person") - return {"decidedBy": by, "label": label, "next": outs, "prompt": prompt}, None - return {}, None - - -def walk_actions(actions): - """Every action in the tree, depth-first, groups included. One walker, so "how many actions - does this flow have" and "which review stages exist" cannot answer differently.""" - for act in actions or []: - yield act - if act.get("kind") == "group": - for kid in walk_actions((act.get("config") or {}).get("actions")): - yield kid - - -def review_actions(defn): - """The review actions of one definition, in flow order — the stages a card can REST at.""" - flow = (defn or {}).get("flow") or {} - return [a for a in walk_actions(flow.get("actions")) if a.get("kind") == "review" - and a.get("enabled", True)] - - -def interpolate(text, row): - """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an - action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would - put template syntax in front of a customer, and a blank is the visible failure.""" - if not isinstance(text, str) or "{{" not in text: - return text - def _sub(m): - v = (row or {}).get(m.group(1).strip()) - return "" if v is None else str(v) - return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) - - -def stages_for(defn): - """One automation → `(stages, routes)` — C1's vocabulary, derived from `graph()` so the - board can never disagree with the runner about what the flow does. - - Stage extras beyond the graph node: `posture` (R7), `record` (may this stage HOLD cards), - `assignable` (may a card be PLACED here — i.e. is its label a choice of the stage field). - `record` without `assignable` marks the derived entry buckets (Profile set / Write): a card - shows there because of what is true about the row, not because anything stamped it. - """ - defn = defn if isinstance(defn, dict) else {} - cfg = defn.get("config") or {} - g = graph(defn) - kind_map = {"trigger": "trigger", "branch": "branch"} - stages, routes = [], [] - for n in g["nodes"]: - stages.append({ - "id": n["id"], "label": n["title"], - "kind": kind_map.get(n["kind"], "action"), - "col": n["col"], "row": n["row"], "enabled": n["enabled"], - "toggle": n["toggle"], "status": n["status"], "detail": n["detail"], - # C1 amendment (2026-08-05): the node's `panel` rides VERBATIM — B joins stage → - # config face on this key, never by guessing that ids match. - "panel": n.get("panel") or n["id"], - "posture": "auto", "record": False, "assignable": False, - }) - for e in g["edges"]: - routes.append({"from": e["from"], "to": e["to"], "label": e.get("label") or "", - "condition": None}) - maxcol = max((s["col"] for s in stages), default=0) - - def _mark(sid, record=True, assignable=False): - for s in stages: - if s["id"] == sid: - s["record"], s["assignable"] = record, assignable - - if defn.get("kind") == "discover_instagram": - # R6: the review gate lives at the END of discovery, and the human card-move IS the - # `tracked` promotion — R7-W20's tick, become a gesture. Nothing here auto-promotes. - # C3-A2(5): the gate has TWO exits — Approve (→ Tracked) and Decline (→ Declined, a - # TERMINAL state, never a delete: a candidate somebody rejected is a decision worth - # keeping, and deleting the row would un-decide it on the next re-find). - stages.append({"id": "review", "label": "Review", "kind": "review", - "col": maxcol + 1, "row": 0, "enabled": True, "toggle": "", - "status": "idle", "panel": "review", "posture": "strict", - "record": True, "assignable": True, - "detail": "Cards wait here for a person — moving one on IS the " - "tracked promotion"}) - stages.append({"id": "tracked", "label": "Tracked", "kind": "action", - "col": maxcol + 2, "row": 0, "enabled": True, "toggle": "", - "status": "idle", "panel": "tracked", "posture": "auto", - "record": True, "assignable": True, - "detail": "The set the Instagram column automation enriches"}) - stages.append({"id": "declined", "label": "Declined", "kind": "action", - "col": maxcol + 2, "row": 1, "enabled": True, "toggle": "", - "status": "idle", "panel": "declined", "posture": "auto", - "record": True, "assignable": True, - "detail": "Rejected candidates stay here — a kept decision, not a " - "delete"}) - routes.append({"from": "write", "to": "review", "label": "", "condition": None}) - routes.append({"from": "review", "to": "tracked", "label": "Approve", - "condition": None}) - routes.append({"from": "review", "to": "declined", "label": "Decline", - "condition": None}) - else: - entry = "source" if defn.get("kind") == "field_instagram" else "write" - _mark(entry) - _mark("write") - for i, lane in enumerate(cfg.get("lanes") or []): - cond = lane.get("when") - stages.append({"id": lane["id"], "label": lane["label"], "kind": "branch", - "col": maxcol + 1, "row": i, "enabled": True, "toggle": "", - "status": "idle", "panel": "lanes", "posture": "auto", - "record": True, "assignable": True, - "detail": _lane_sentence(cond)}) - routes.append({"from": "write", "to": lane["id"], - "label": lane["label"] if cond is None else _lane_sentence(cond), - "condition": dict(cond) if cond else None}) - - # ── WAVE 23 · C4/C6 — REVIEW ACTIONS ARE STAGES. ───────────────────────────────────────── - # A review action is the one action kind that HOLDS a record rather than doing something to - # it, so it has to exist on the board: a flow whose review lived only in the builder would - # gate records at a place the kanban cannot show, which is the whole complaint the board was - # built to answer. Its declared exits become assignable stages too — a gate with nowhere to - # go is a hole records fall into. - tail = max((s["col"] for s in stages), default=0) - for i, act in enumerate(review_actions(defn)): - acfg = act.get("config") or {} - rid = act["id"] - stages.append({"id": rid, "label": acfg.get("label") or "Review", "kind": "review", - "col": tail + 1 + (i * 2), "row": 0, "enabled": True, "toggle": "", - "status": "idle", "panel": "review_action", "posture": "strict", - "record": True, "assignable": True, "actionId": rid, - "decidedBy": acfg.get("decidedBy") or "user", - "detail": ("An AI decides where each card goes — a person can override it" - if acfg.get("decidedBy") == "ai" - else "Cards wait here for a person to move them on")}) - for j, label in enumerate(acfg.get("next") or []): - oid = f"{rid}_out_{j}" - stages.append({"id": oid, "label": label, "kind": "action", - "col": tail + 2 + (i * 2), "row": j, "enabled": True, "toggle": "", - "status": "idle", "panel": "review_action", "posture": "auto", - "record": True, "assignable": True, "actionId": rid, - "detail": f"Where {acfg.get('label') or 'Review'} sends a card"}) - routes.append({"from": rid, "to": oid, "label": label, "condition": None}) - return stages, routes - - -def terminal_stage_ids(stages, routes): - """Stages a record can REST at with nowhere further to go — the ends C5's `ending` acts on. - Derived from the routes rather than declared, so a flow cannot gain an exit without this - noticing (and a lane that stops being terminal stops being reset).""" - assignable = {s["id"] for s in stages if s.get("assignable")} - onward = {r["from"] for r in routes if r["to"] in assignable and r["to"] != r["from"]} - return [sid for sid in assignable if sid not in onward] - - -def _stage_of(stages, sid): - for s in stages: - if s["id"] == sid: - return s - return None - - -# ───────────────────────────────────────────────────────────────────────────────────────────── -# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. -# -# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits -# them once at the end (the 256-commits/hr budget every writer in this module respects). A -# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget -# on one busy flow. -# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the -# runner stamps its stage and stops walking THAT record — later actions belong to the branch a -# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. -# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the -# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write -# cannot fire an event trigger, its own or a sibling's. - -def _act_row_patch(patches, table, rid, values): - patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) - - -def _flow_table(defn): - """The database a flow's actions and ending operate on: the automation's own target, or the - table its event trigger watches when the flow has no target of its own.""" - cfg = (defn or {}).get("config") or {} - if defn.get("kind") == "discover_instagram": - return cfg.get("targetTable") or DISCOVER_TABLE - return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") - - -def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print): - """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. - - ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery - automation's terminal stages (`tracked`/`declined`) come from `stages_for`, not from actions, - so an early return on an empty action list would have silently disabled the owner's loop - feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in - review; the tests all happened to pass a non-empty action list, which is why it was green. - """ - flow = (defn or {}).get("flow") or {} - actions = flow.get("actions") or [] - counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0, "review": 0, - "aiDecided": 0, "cycles": 0} - if not table_key: - return counts - table = str(table_key) - rows = dict((ut_get(rt, table) or {}).get("rows") or {}) - if not rows: - return counts - stages, routes = stages_for(defn) - skey = stage_field_key(defn.get("id")) - stamp = _iso() - patches, creates, ai_budget = {}, {}, [AI_DECISIONS_PER_RUN] - - def _walk(acts, row, rid, depth=0): - """Returns True when the record was SUSPENDED (a review holds it).""" - for act in acts: - if not act.get("enabled", True): - continue - if not lane_match(act.get("when"), row): - continue - kind = act.get("kind") - cfg = act.get("config") or {} - counts["actionsRun"] += 1 - if kind == "group": - if not lane_match(cfg.get("cond"), row): - continue - if _walk(cfg.get("actions") or [], row, rid, depth + 1): - return True - elif kind == "update_record": - vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} - row.update(vals) # later actions see the write, as they must - _act_row_patch(patches, table, rid, vals) - counts["updated"] += 1 - elif kind == "create_record": - target = str(cfg.get("table") or "") - vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} - creates.setdefault(target, []).append(vals) - counts["created"] += 1 - elif kind == "find_records": - found = find_records(rt, cfg.get("table"), cfg.get("cond"), - int(cfg.get("limit") or 25)) - counts["found"] += len(found) - elif kind == "review": - label = cfg.get("label") or "Review" - at = str(row.get(skey) or "").strip() - if at == label: - return True # already waiting here — do not re-stamp - if at and at in (cfg.get("next") or []): - # ⛔ ALREADY DECIDED — the card went through this gate on an earlier run, so - # the flow continues past it. Without this the next run would stamp a - # finished card back to the review gate and hold it there again, forever: - # a card could never leave a flow that runs on a schedule. Found by this - # wave's own terminal-ending negative control, which is the argument for - # writing the NC in the same change as the feature. - continue - decided = "" - if cfg.get("decidedBy") == "ai" and ai_budget[0] > 0: - ai_budget[0] -= 1 - decided = ai_decide(rt, defn, act, row, rid, log=log) - if decided: - counts["aiDecided"] += 1 - _act_row_patch(patches, table, rid, - {skey: decided, skey + "_at": stamp}) - row[skey] = decided - continue # the AI moved it on — keep walking - _act_row_patch(patches, table, rid, {skey: label, skey + "_at": stamp}) - row[skey] = label - counts["review"] += 1 - return True - return False - - held = [] - if actions: - for rid in list(row_ids or [])[:FLOOD_LIMIT]: - row = dict(rows.get(str(rid)) or {}) - if not row: - continue - if _walk(actions, row, str(rid)): - held.append(str(rid)) - counts["cycles"] = _apply_ending(rt, defn, table, rows, patches, stages, routes, stamp) - _commit_action_writes(rt, table, patches, creates, username, log) - if held: - notify_review(rt, defn, held, log=log) - return counts - - -#: C6 cap: AI decisions ONE run may make. A flow that suddenly matches 500 records must not turn -#: into 500 paid calls before anybody notices — the rest simply wait for a human, which is the -#: behaviour the feature degrades to everywhere else too. -AI_DECISIONS_PER_RUN = 25 - - -def ai_decide(rt, defn, act, row, row_id="", log=print): - """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" - to leave the card for a person. - - ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed - answer, or a label the review does not offer all return "" — and "" means the card sits at - the review gate exactly as it would with no AI at all. The feature can be broken, absent or - wrong and the worst outcome is a human doing the work. - """ - cfg = act.get("config") or {} - options = list(cfg.get("next") or []) - if not options: - return "" - try: - import ai_review - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") - return "" - fields = [f.get("key") for f in - (ut_get(rt, ((defn.get("config") or {}).get("targetTable") - or (defn.get("trigger") or {}).get("table") or "")) or {} - ).get("fields") or []] - choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, - row=row, fields=[f for f in fields if f], - label=cfg.get("label") or "Review") - if not choice: - if meta.get("problem"): - log(f"[aios-auto] ai review declined to answer: {meta['problem']}") - return "" - review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", - choice, meta.get("provider") or "ai", by="ai", - note=meta.get("reason") or "", model=meta.get("model") or "") - return choice - - -def notify_review(rt, defn, row_ids, log=print): - """C6: one notification per RUN naming the count — never one per record. - - A discovery pass that gates 300 candidates would otherwise put 300 rows in somebody's inbox - and the inbox would stop being read, which costs more than the notification gains. The board - is where the individual cards live; the bell says "there is work", once. - """ - if not row_ids: - return - owner = str((defn or {}).get("createdBy") or "").strip() - if not owner: - return - n = len(row_ids) - try: - import core.alerts as alerts - alerts.notify(owner, - f"{defn.get('name') or 'An automation'} — {n} " - f"record{'' if n == 1 else 's'} waiting for review", - topic="automation", key=str(defn.get("id") or ""), st=rt) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] review notification failed: {type(e).__name__}: {e}") - - -def find_records(rt, table_key, cond, limit=25): - """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts - the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" - rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} - out = [] - for rid, row in rows.items(): - if lane_match(cond, row or {}): - out.append(str(rid)) - if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): - break - return out - - -def _apply_ending(rt, defn, table, rows, patches, stages, routes, stamp): - """C5: send terminal cards back round, per the ending mode. Returns cycles started. - - A RESET CLEARS THE STAMP rather than writing a start stage, so the card falls back to the - DERIVED bucket `_card_stage` would give a record that had never been stamped at all — which - is what "back to the first cycle" means, and it needs no opinion about which stage is first. - """ - ending = ((defn or {}).get("flow") or {}).get("ending") or {} - mode = ending.get("mode") or "terminal" - if mode in ("terminal", "manual_reset"): - return 0 - terminals = set(terminal_stage_ids(stages, routes)) - if not terminals: - return 0 - labels = {s["label"] for s in stages if s["id"] in terminals} - skey = stage_field_key(defn.get("id")) - cutoff = None - if mode == "timed_reset": - cutoff = _now() - _dt.timedelta(hours=int(ending.get("hours") or 24)) - started = 0 - for rid, row in rows.items(): - merged = {**(row or {}), **((patches.get(table) or {}).get(str(rid)) or {})} - if str(merged.get(skey) or "").strip() not in labels: - continue - if cutoff is not None: - at = _parse_iso(str(merged.get(skey + "_at") or "")) - if at is None or at > cutoff: - continue - n = (_ig_int(merged.get(cycles_field_key(defn.get("id")))) or 0) + 1 - _act_row_patch(patches, table, rid, - {skey: "", skey + "_at": stamp, - cycles_field_key(defn.get("id")): n}) - started += 1 - return started - - -def _commit_action_writes(rt, table, patches, creates, username, log): - """The ONE write. Row patches merge into the target's rows; creates append to their own - tables through the ordinary cap-respecting path.""" - if not patches and not creates: - return - for tkey, rowpatch in (patches or {}).items(): - cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) - for rid, vals in rowpatch.items(): - cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} - ut_write_rows(rt, tkey, cur) - for tkey, new_rows in (creates or {}).items(): - t = ut_get(rt, tkey) - if t is None: - log(f"[aios-auto] create_record: {tkey} no longer exists — {len(new_rows)} skipped") - continue - cur = dict(t.get("rows") or {}) - nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 - cap = row_cap(tkey) - for vals in new_rows: - if len(cur) >= cap: - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap") - break - cur[str(nxt)] = dict(vals) - nxt += 1 - ut_write_rows(rt, tkey, cur) - - -def _lane_sentence(cond, top=True): - """One condition tree → the sentence the board and the field definition carry. - - C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a - lane label never claims a flat comparison the tree does not make. The server composes it for - the same reason it composes every other stage `detail` — a client paraphrase of a structure - the engine evaluates is a second implementation of the same sentence, free to drift from it. - """ - if cond is None: - return "Everything else" if top else "" - if isinstance(cond, dict): - for key, joiner in (("all", " and "), ("any", " or ")): - if key in cond: - parts = [_lane_sentence(c, False) for c in cond.get(key) or []] - parts = [p for p in parts if p] - if not parts: - return "" - inner = joiner.join(parts) - return inner if top or len(parts) == 1 else f"({inner})" - v = cond.get("value") - return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") - - -def _stage_choices(stages): - """The stage field's select vocabulary: the ASSIGNABLE stage labels, in board order.""" - return [s["label"] for s in sorted(stages, key=lambda s: (s["col"], s["row"])) - if s.get("assignable")] - - -def ensure_stage_field(rt, table_key, defn, username="automation"): - """C2: the stage field on the target database — created/refreshed idempotently, skipped - entirely when the flow has nothing a record can be PLACED in (a select with an empty - vocabulary is a column that can hold no value; it appears when the first lane or review - stage does). - - The field carries the WHOLE law the grid door needs (`automation.humanMoves` — from-label → - admissible to-labels — and `automation.arrive` — to-label → extra cell writes, which is how - the kanban drag Review→Tracked also sets `tracked`), because `grid_events` may not import - this module: the engine states the law where the enforcer can read it. - """ - key = str(table_key or "") - if not key or ut_get(rt, key) is None: - return "" - stages, _routes = stages_for(defn) - choices = _stage_choices(stages) - if not choices: - return "" - fkey = stage_field_key(defn.get("id")) - review_next = {} - for s in stages: - if s["kind"] != "review": - continue - nxt = [r["to"] for r in _routes if r["from"] == s["id"]] - labels = [t["label"] for t in stages if t["id"] in nxt and t.get("assignable")] - if labels: - review_next[s["label"]] = labels - auto = {"flowId": str(defn.get("id") or ""), "stageField": True, - "humanMoves": review_next} - if defn.get("kind") == "discover_instagram": - # Leaving Tracked is a human decision too (an admin demoting a mistake), so the flag - # follows the CARD in both directions — the engine itself still never writes it (R7). - auto["arrive"] = {"Tracked": {"tracked": "1"}, "Review": {"tracked": ""}, - "Declined": {"tracked": ""}} - field = {"key": fkey, "label": f"Stage — {defn.get('name') or defn.get('id')}"[:80], - "type": "select", "source": "overlay", "options": choices, "automation": auto} - # ⭐ WAVE 23 (C5, D-35's minimum) — THE CYCLE COLUMN, created beside the stage field when - # the flow can loop. The owner asked to SEE how many times a record has been round, and - # `_apply_ending` was writing the cell without anything ever declaring the column: the value - # would sit in the row, correct and invisible, because a ut grid renders the DEFINITION's - # fields. A machine-written cell with no field definition is the shape of a number nobody - # can find. Tagged `automation` so it greys out and refuses human writes like every other - # machine column (item 4 / R9). - cyc_key = cycles_field_key(defn.get("id")) - loops = (((defn.get("flow") or {}).get("ending") or {}).get("mode") or "terminal") != \ - "terminal" - cyc_field = {"key": cyc_key, "label": "Cycles"[:80], "type": "int", "source": "overlay", - "automation": {"flowId": str(defn.get("id") or ""), "cyclesField": True}} - - have, have_cyc = None, False - for f in (ut_get(rt, key) or {}).get("fields") or []: - if f.get("key") == fkey: - have = f - if f.get("key") == cyc_key: - have_cyc = True - if have is not None and have.get("options") == choices and \ - (have.get("automation") or {}) == auto and have_cyc == loops: - return fkey # nothing would change — save the commit - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - return cur - for f in t.setdefault("fields", []): - if f.get("key") == fkey: - f["options"] = choices - f["automation"] = auto - f["type"] = "select" - break - else: - t["fields"].append(field) - if loops and not any(f.get("key") == cyc_key for f in t["fields"]): - t["fields"].append(cyc_field) - # ⚠ A flow switched BACK to `terminal` keeps its cycle column rather than dropping it: - # the counts already in those cells are a record of work that really happened, and a - # schema change must not be a way to lose data nobody asked to delete. - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return fkey - - -def _rid_num(rid): - return int(rid) if str(rid).isdigit() else 10 ** 9 - - -def _card_stage(defn, stages, row, skey): - """Which stage holds this record. The STAMPED label wins; a blank or unrecognised value - falls to the DERIVED bucket — what is true about the row, not what anything wrote: - discovery splits on the human `tracked` decision, an Instagram column on whether its cell - has ever been written, and a scrape lands where its rows are born (Write).""" - val = str((row or {}).get(skey) or "").strip() - if val: - for s in stages: - if s.get("assignable") and s["label"] == val: - return s["id"] - kind = defn.get("kind") - if kind == "discover_instagram": - return "tracked" if str((row or {}).get("tracked") or "").strip() in ("1", "true", "True") \ - else "review" - if kind == "field_instagram": - fkey = (defn.get("config") or {}).get("fieldKey") or "" - return "write" if str((row or {}).get(fkey) or "").strip() else "source" - return "write" - - -def _card_next(stages, routes, row, sid): - """Where THIS card may go from `sid` — every outgoing route whose condition the record - satisfies (declaration order kept, null = catch-all), targets limited to assignable stages. - Evaluated per record so R5's winning path is decided on the SERVER, once.""" - assignable = {s["id"] for s in stages if s.get("assignable")} - out = [] - for r in routes: - if r["from"] != sid or r["to"] not in assignable or r["to"] == sid: - continue - cond = r.get("condition") - if cond is not None and not lane_match(cond, row): - continue - if r["to"] not in out: - out.append(r["to"]) - return out - - -def _card_title(defn, row): - kind = defn.get("kind") - cfg = defn.get("config") or {} - if kind == "discover_instagram": - return str(row.get("handle") or row.get("full_name") or "?") - if kind == "scrape_db": - return str(row.get(cfg.get("keyField") or "") or "?") - url_f = cfg.get("urlField") or "" - return str(row.get("name") or row.get("handle") or row.get(url_f) or "?") - - -def _card_sub(defn, row): - kind = defn.get("kind") - if kind == "discover_instagram": - f = str(row.get("followers") or "").strip() - nm = str(row.get("full_name") or "").strip() - return f"{nm} · {f} followers" if (nm and f) else (nm or (f and f + " followers") or "") - if kind == "field_instagram": - return _s(row.get((defn.get("config") or {}).get("fieldKey") or ""), 90) - fmap = (defn.get("config") or {}).get("fieldMap") or {} - keys = [k for k in fmap.values() if k != (defn.get("config") or {}).get("keyField")] - return _s(row.get(keys[0]) if keys else "", 90) - - -def board(rt, defn, admin=False): - """C1's whole payload: `{stages, routes, cards, stageField, counts, truncated}`. - - `counts` carries the TRUE per-stage totals for every record stage (zeros included); - `cards` is capped per stage at `BOARD_CARD_CAP` and any stage that overflowed is NAMED in - `truncated` — the cap is disclosed, never silent. Cards ride in row-id order so the board - is stable across polls. - """ - stages, routes = stages_for(defn) - cfg = defn.get("config") or {} - table_key = str(cfg.get("targetTable") or "") - skey = stage_field_key(defn.get("id")) - t = ut_get(rt, table_key) - counts = {s["id"]: 0 for s in stages if s.get("record")} - # C1 amendment (2026-08-05): an automation with NO BOUND STAGE FIELD answers - # `stageField: null, cards: []` — the stages still ride (they come from the definition), - # and an empty board is a state, never an error. Bound = the field exists on the table. - bound = any(f.get("key") == skey for f in ((t or {}).get("fields") or [])) - if t is None or not bound: - return {"stages": stages, "routes": routes, "cards": [], "stageField": None, - "counts": counts, "truncated": []} - rows = t.get("rows") or {} - cards, truncated, per = [], [], {} - for rid in sorted(rows, key=_rid_num): - row = rows.get(rid) or {} - sid = _card_stage(defn, stages, row, skey) - counts[sid] = counts.get(sid, 0) + 1 - if per.get(sid, 0) >= BOARD_CARD_CAP: - if sid not in truncated: - truncated.append(sid) - continue - per[sid] = per.get(sid, 0) + 1 - stage = next((s for s in stages if s["id"] == sid), None) - nxt = _card_next(stages, routes, row, sid) - cards.append({ - "rowId": str(rid), "stageId": sid, - "title": _s(_card_title(defn, row), 80), - "sub": _s(_card_sub(defn, row), 90), - "movedAt": str(row.get(skey + "_at") or ""), - "canAdvance": bool(stage and stage["kind"] == "review" and nxt), - "next": nxt, - }) - return {"stages": stages, "routes": routes, "cards": cards, - "stageField": {"table": table_key, "key": skey}, - "counts": counts, "truncated": truncated} - - -def move_card(rt, auto_id, row_id, to, username="", admin=False): - """One card move, SERVER-decided (C1): a review-stage card advances only to a member of its - per-record `next`; every other move is an admin repair onto an assignable stage. Returns - `(fresh_board, None)` or `(None, (status, code, sentence))` — the route maps the tuple. - - The write lands in the DEFINITION rows (the shared truth the engine and every viewer read), - stamped with `movedAt`; on a discovery board arriving at Tracked sets `tracked` and leaving - it clears it — the human decision travelling with the card (R6/R7).""" - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return None, (404, "unknown_automation", "no automation with that id") - stages, routes = stages_for(defn) - cfg = defn.get("config") or {} - table_key = str(cfg.get("targetTable") or "") - t = ut_get(rt, table_key) - if t is None: - return None, (404, "no_target", f"{table_key or 'the target database'} does not exist " - f"yet — run the automation once") - skey = stage_field_key(defn.get("id")) - if not any(f.get("key") == skey for f in (t.get("fields") or [])): - return None, (400, "unbound", - "this automation has no stage field on its database yet — save or run " - "it once to bind the board") - row = (t.get("rows") or {}).get(str(row_id)) - if row is None: - return None, (404, "unknown_row", "that record is not in the target database") - target = next((s for s in stages if s["id"] == str(to or "")), None) - if target is None or not target.get("assignable"): - return None, (400, "bad_stage", - "cards can only be placed in a review, lane or terminal stage — the " - "run stages are machinery a record passes through") - cur_id = _card_stage(defn, stages, row, skey) - cur = next((s for s in stages if s["id"] == cur_id), None) - from_review = cur is not None and cur["kind"] == "review" - if from_review: - allowed = _card_next(stages, routes, row, cur_id) - if target["id"] not in allowed and not admin: - return None, (400, "not_next", - "a review card advances only to its listed next stage") - elif not admin: - return None, (403, "admin_only", - "only a review card can be advanced by hand — moving anything else is " - "an admin repair") - values = {skey: target["label"], skey + "_at": _iso()} - if defn.get("kind") == "discover_instagram": - values["tracked"] = "1" if target["id"] == "tracked" else "" - - def _up(cur_b): - cur_b = cur_b if isinstance(cur_b, dict) else {} - tt = cur_b.get(table_key) - if tt is not None: - tt.setdefault("rows", {}).setdefault(str(row_id), {}).update(values) - return cur_b - - rt.update(UT_STORE_KEY, _up, flush="sync") - if from_review: - review_audit(rt, defn.get("id"), row_id, cur["label"], target["label"], username) - return board(rt, defn, admin=admin), None - - -#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can -#: grow a definition without limit is a serialisation cost wearing a compliance hat. -MAX_REVIEWS = 100 - - -def review_audit(rt, auto_id, row_id, from_label, to_label, username, - by="user", note="", model=""): - """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to - the definition (newest first, bounded) by BOTH doors: `move_card` here, and the grid door - in `core.grid_events`, which writes the same shape mechanically off the field's `flowId`. - - ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that - made it and the one-line reason it gave. One audit log, not two — a reader asking "who - decided this card" must not have to know there are two places to look, and the moment an - AI decision is invisible beside a human one the log stops being an audit. - """ - entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), - "from": _s(from_label, 60), "to": _s(to_label, 60), - "by": "ai" if by == "ai" else "user"} - if note: - entry["note"] = _s(note, 300) - if model: - entry["model"] = _s(model, 60) - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] - return cur - - rt.update(STORE_KEY, _up, flush="sync") - - -# --------------------------------------------------------------------------------------------- -# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) -# --------------------------------------------------------------------------------------------- -# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — -# the value reflects every pull the platform has, not just this tenant's). Computed by the -# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files -# no session owns this wave), human-write-refused at the grid door, and every number drills to -# the exact snapshot rows behind it. -# -# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no -# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero -# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not -# readable") — the `_bd_posts_count` law, one layer up. - -METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") -METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") -#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each -#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; -#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). -PROFILE_MEASURES = ("followers", "avg_engagement") -METRIC_AGGS = ("avg", "sum", "latest") - - -def metric_value(series, measure, window, agg="", today=None): - """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. - - `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference - day; post-count windows count back from the newest post. `drill_rows` are the exact rows - the number came from, so the route can honour [[no-unverifiable-aggregates]] without - recomputing differently.""" - series = series or {} - today = today or _now() - if measure in PROFILE_MEASURES: - rows = [r for r in series.get("snapshots") or [] - if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] - if window in ("last_7d", "last_30d"): - days = 7 if window == "last_7d" else 30 - floor = today - _dt.timedelta(days=days) - rows = [r for r in rows - if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] - if not rows: - return None, [] - if window == "latest" or agg == "latest": - picked = [rows[-1]] - else: - picked = rows - vals = [_lane_num(r.get(measure)) for r in picked] - vals = [v for v in vals if v is not None] - if not vals: - return None, [] - out = vals[-1] if (window == "latest" or agg == "latest") else \ - (sum(vals) if agg == "sum" else sum(vals) / len(vals)) - if measure == "avg_engagement": - # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs - # transform-pct scar) — scaled exactly once, here. - return f"{out * 100:.2f}", picked - return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked - # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. - posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] - posts.sort(key=lambda p: str(p.get("posted_at"))) - if window == "latest": - picked_posts = posts[-1:] - elif window == "last_3_posts": - picked_posts = posts[-3:] - else: - days = 7 if window == "last_7d" else 30 - floor = today - _dt.timedelta(days=days) - picked_posts = [p for p in posts - if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) - or _dt.datetime.min) >= floor] - vals, drill = [], [] - for p in picked_posts: - snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] - for snap in reversed(snaps): - raw = snap.get(measure) - v = _lane_num(raw) - if v is not None and str(raw).strip() != "": - vals.append(v) - drill.append(snap) - break - if not vals: - return None, [] - out = sum(vals) if (agg or "sum") == "sum" else \ - (vals[-1] if agg == "latest" else sum(vals) / len(vals)) - return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill - - -def metric_fields_of_table(t): - return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] - - -def _table_handle(row, url_field): - return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) - or "").strip().lower() - - -def compute_metric_cells(rt, table_key, today=None): - """Recompute every metric cell on ONE table from the master series. One coalesced write, - only when something actually changed (the flush-ceiling law); zero reads when the table - has no metric fields or the master is off. Returns the number of rows touched.""" - t = ut_get(rt, table_key) - mfields = metric_fields_of_table(t) - if not mfields: - return 0 - import ig_master - if not ig_master.configured(): - return 0 - url_field = next((f.get("key") for f in (t.get("fields") or []) - if f.get("type") == "url"), "") - rows = t.get("rows") or {} - handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} - series = ig_master.series_for({h for h in handles.values() if h}) - changes = {} - for rid, row in rows.items(): - s = series.get(handles.get(rid) or "") - for f in mfields: - bag = f["metric"] - val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), - bag.get("agg") or "", today=today) - if s else (None, [])) - want = "" if val is None else str(val) - if str((row or {}).get(f["key"], "")) != want: - changes.setdefault(str(rid), {})[f["key"]] = want - if not changes: - return 0 - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - tt = cur.get(table_key) - if tt is not None: - for rid, vals in changes.items(): - tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return len(changes) - - -def refresh_metrics(rt, today=None, log=print): - """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window - metric can never go staler than one tick while a scheduler exists. Cheap by construction — - a table without metric fields costs a dict scan and nothing else.""" - touched = 0 - for tk, t in ut_all(rt).items(): - if metric_fields_of_table(t): - try: - touched += compute_metric_cells(rt, tk, today=today) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") - return touched - - -def purge_subject(rt, handle): - """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the - tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half - non-optional: a purge that missed the pooled copy would not be erasure). Returns - `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is - now ABSENT, which is the one aggregate whose drill is emptiness.""" - subject = str(handle or "").strip().lstrip("@").lower() - if not subject: - return {} - counts = {} - tables = ut_all(rt) - post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} - codes = {str(r.get("shortcode") or "") for r in post_rows.values() - if str((r or {}).get("influencer_key") or "").strip().lower() == subject} - - keeps = { - "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") - or "").strip().lower() != subject, - "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") - or "").strip().lower() != subject, - "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, - DISCOVER_TABLE: lambda r: str((r or {}).get("handle") - or "").strip().lower() != subject, - } - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - for tk, keep in keeps.items(): - t = cur.get(tk) - if t is None: - continue - rows = t.get("rows") or {} - nxt = {rid: r for rid, r in rows.items() if keep(r)} - counts[tk] = len(rows) - len(nxt) - t["rows"] = nxt - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - import ig_master - for bucket, n in (ig_master.purge_handle(subject) or {}).items(): - counts[f"master:{bucket}"] = n - return counts - - -# --------------------------------------------------------------------------------------------- -# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) -# --------------------------------------------------------------------------------------------- -# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | -# webhook | email. The first two are what always existed (Run now; cron via the tick). The four -# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: -# -# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, -# not for being in it. Implemented as per-record ARMED state over successive evaluations -# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms -# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing -# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the -# disarmed set with everything currently matching, so already-matching records do not fire -# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). -# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only -# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers -# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never -# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by -# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as -# a statusNote) catches whatever construction did not foresee. -# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds -# instead of running — a partial run entry names the count and the deliberate way through -# (Run now). C4's discovery guard is this rule's special case. -# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record -# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); -# event_field fires every transition. - -# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── -# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that -# renames the same events is a second vocabulary to learn for no gain. -# -# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". -# Renaming the key would have orphaned every stored trigger in production for a caption; the key -# is the contract with the store, the label is the contract with the reader, and they are allowed -# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the -# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 -# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. -# -# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show -# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a -# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop -# class wearing a friendlier face; here the two lists are separate on purpose and the refusal -# names the state rather than pretending the key is unknown. -TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", - "enters_view", "webhook", "email", "form_submitted") -TRIGGER_PLANNED = ("button_clicked", "comment_added") -TRIGGER_LABELS = { - "manual": "Manual", - "schedule": "At a scheduled time", - "event_field": "When a record matches conditions", - "record_updated": "When a record is updated", - "record_created": "When a record is created", - "enters_view": "When a record enters a view", - "webhook": "When a webhook is received", - "email": "When an email arrives", - "form_submitted": "When a form is submitted", - "button_clicked": "When a button is clicked", - "comment_added": "When a comment is added", -} -#: Triggers that watch a database and therefore need one named before they can fire. -TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", - "form_submitted") -#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so -#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. -TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") -MAX_WATCH_FIELDS = 12 -#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, -#: and so would a deployment that prefers immediacy over coalescing. -EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) -FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses -FIRE_WINDOW_SECONDS = 300 -FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on -EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation -EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest -CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota - -EMAIL_FIELDS = [ - field_def("email_id", "Email id"), field_def("email_from", "From"), - field_def("email_subject", "Subject"), field_def("email_date", "Date"), - field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), -] - - -def clean_trigger(raw, previous=None): - """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and - means what it always meant: manual + whatever `schedule` says. - - ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry - with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT - rather than refused — the picker writes `{key}` first and the table/field after, the - wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the - wire so the surface says "finish setting this up" instead of snapping back to Manual. It - cannot fire while incomplete (the hooks match on the table it does not name), which is the - fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a - condition on a field the trigger does not watch) are still refused with the sentence — - incomplete is a state, wrong is not. - """ - if raw in (None, "", {}): - return (dict(previous) if isinstance(previous, dict) and previous else None), None - if not isinstance(raw, dict): - return None, "the trigger must be an object" - prev = previous if isinstance(previous, dict) else {} - key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), - 30).strip() - if key in TRIGGER_PLANNED: - # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence - # says WHY rather than "unknown trigger", because the picker legitimately showed it. - return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — " - f"it renders so you can see it is coming, and it cannot be saved") - if key not in TRIGGER_KEYS: - return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) - if key in ("manual", "schedule"): - # Not stored: `schedule` already owns the cron and Run now always works. Storing a - # no-op trigger would be a second copy of that fact, free to disagree with it. - return None, None - out = {"key": key, - "enabled": bool(raw["enabled"]) if "enabled" in raw else - bool(prev.get("enabled", True)), - "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} - if key in TRIGGER_TABLE_KEYS: - table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() - if table and not table.startswith(UT_PREFIX): - return None, ("event triggers watch blank databases (ut_*) this wave — " - f"{table!r} is not one") - out["table"] = table - if key == "event_field": - # The watched field is OPTIONAL now (C3-v2): named ⇒ "when THIS field changes and the - # record matches"; unnamed ⇒ Airtable's plain "when a record matches conditions", - # evaluated on any write to the table. The condition is a C4 TREE over the WHOLE row — - # wave 22 could only test the written value, so it refused a condition on any other - # field; `_settle_eval` now reads the row itself, which is what lifted that restriction. - out["field"] = _s(raw.get("field") if "field" in raw else prev.get("field"), 80).strip() - cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), - where="the trigger") - if cerr: - return None, cerr - out["when"] = cond - if key == "record_updated": - # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty - # is the WIDER reading and it is the default there too, so it stays the default here. - watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") - if watch_raw in (None, ""): - watch = [] - elif not isinstance(watch_raw, list): - return None, "the watched-field list must be a list of field keys" - else: - watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] - if len(watch) > MAX_WATCH_FIELDS: - return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " - f"fields — leave the list empty to watch every field") - out["fields"] = watch - cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), - where="the trigger") - if cerr: - return None, cerr - out["when"] = cond - if key == "enters_view": - out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), - 80).strip() - if key == "form_submitted": - # Blank = any form on that database. Naming one narrows to it, which is what a table - # carrying an intake form AND a correction form needs. - out["formToken"] = _s(raw.get("formToken") if "formToken" in raw - else prev.get("formToken"), 64).strip() - if key == "webhook": - # The token is MINTED here, once, and survives every later patch — rotating it on - # every Save would silently break the external caller the URL was given to. - out["token"] = _s(prev.get("token"), 64) or _secrets_token() - if key == "email": - out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), - 200).strip() or "in:inbox is:unread" - out["configured"] = _trigger_configured(out) - return out, None - - -def _trigger_configured(trg): - """Is this trigger complete enough to fire? One reader, because "configured" is asserted in - three places (the wire, the graph node, the hooks) and three copies of a boolean is how a - surface says "ready" about a trigger the engine skips.""" - key = str((trg or {}).get("key") or "") - if key in TRIGGER_TABLE_KEYS and not trg.get("table"): - return False - if key == "event_field": - # Neither a watched field nor a condition = "fire on anything, ever" — which is not a - # trigger, it is a description of the table. Refuse to call that configured. - return bool(trg.get("field") or trg.get("when")) - if key == "enters_view": - return bool(trg.get("viewId")) - return True - - -def _secrets_token(): - import secrets as _sec - return _sec.token_urlsafe(24) - - -# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the -# process would keep punishing an automation for a storm that ended with the restart. -_FIRES = {} -_FIRES_LOCK = threading.Lock() - - -def _breaker_trips(tenant, auto_id, now=None): - now = now if now is not None else time.time() - key = (tenant, str(auto_id)) - with _FIRES_LOCK: - log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] - log.append(now) - _FIRES[key] = log - return len(log) > FIRE_LIMIT - - -def _pause_trigger(rt, auto_id, note): - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - d = cur.get(str(auto_id)) - if d is not None: - trg = d.get("trigger") - if isinstance(trg, dict): - trg["paused"] = True - d["statusNote"] = _s(note, 200) - return cur - rt.update(STORE_KEY, _up, flush="sync") - - -def trigger_fire(rt, tenant, auto_id, log=print): - """One trigger firing — breaker first, then the ordinary async run. False when it did not - start (breaker, or already running — both are answers, not errors).""" - if _breaker_trips(tenant, auto_id): - note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " - f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's " - f"subject in a loop") - _pause_trigger(rt, auto_id, note) - log(f"[aios-auto] breaker: {auto_id} {note}") - return False - return run_async(rt, tenant, auto_id, username="automation", log=log) - - -# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE -# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). -_SETTLE = {} -_SETTLE_LOCK = threading.Lock() - - -def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): - """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. - - ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave - dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the - reason is worth stating because it is invisible from this file: an ordinary ut cell typed at - the grid door lands in the editor's PER-USER OVERLAY stratum - (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` - definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for - the common case the value that just changed exists ONLY in this event, and a definition-row - read sees the pre-write value — the trigger would evaluate stale and never fire. - `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree - read the record's other columns) with the written cells on top. - """ - key = (tenant, str(auto_id)) - cell = {str(field): after} if field else {} - if EVENT_SETTLE_SECONDS <= 0: - with _SETTLE_LOCK: - buf = _SETTLE.setdefault(key, {"rows": {}}) - buf["rows"].setdefault(str(row_id), {}).update(cell) - _settle_eval(rt, tenant, auto_id, log=log) - return - with _SETTLE_LOCK: - buf = _SETTLE.setdefault(key, {"rows": {}}) - buf["rows"].setdefault(str(row_id), {}).update(cell) - timer = buf.get("timer") - if timer is not None: - timer.cancel() # the burst continues — push the window out - timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, - args=(rt, tenant, auto_id), kwargs={"log": log}) - timer.daemon = True - buf["timer"] = timer - timer.start() - - -def view_filter(rt, table_key, view_id): - """`(tree, fields, problem)` for one saved view on a user table — the substrate the - `enters_view` trigger tests membership against (C3-v2 / owner R2). - - Personal strata first (`find_view`), then the shared bucket, because a view somebody shared - is exactly the kind an automation gets pointed at. A view that has been deleted answers a - PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one - would turn "when a record enters Overdue" into "on every write", which is the widening this - module refuses everywhere else. - """ - key = str(table_key or "") - vid = str(view_id or "").strip() - if not key or not vid: - return None, [], "the trigger names no view" - try: - import core.table_store as table_store - tops = table_store.make(f"{key}_table_workspace", st=rt) - found = tops.find_view(vid) - view = (found[1] if found else None) or tops.shared_view(vid) - except Exception as e: # noqa: BLE001 - return None, [], f"the view could not be read ({type(e).__name__})" - if not isinstance(view, dict): - return None, [], f"view {vid!r} no longer exists on {key}" - cfg = view.get("config") or {} - tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} - return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" - - -def _row_gate(rt, defn, trg): - """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. - - None means the trigger has NO match gate — the write itself is the event (wave 22's "the - field changed", and `record_updated` over any field). A problem means the gate cannot be - built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that - passes. - """ - key = trg.get("key") - if key == "enters_view": - tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) - if problem: - return None, problem - import harness.filter_eval as filter_eval - return (lambda row: filter_eval.matches(tree, row, fields)), "" - when = trg.get("when") - if when: - return (lambda row: lane_match(when, row)), "" - return None, "" - - -def _settle_eval(rt, tenant, auto_id, log=print): - """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" - with _SETTLE_LOCK: - buf = _SETTLE.pop((tenant, str(auto_id)), None) - written = dict((buf or {}).get("rows") or {}) - touched = list(written) - if not touched: - return - d = all_definitions(rt).get(str(auto_id)) - trg = (d or {}).get("trigger") or {} - if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ - or trg.get("paused") or not trg.get("enabled", True) \ - or not trg.get("configured", True): - return - gate, problem = _row_gate(rt, d, trg) - if problem: - # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken - # automation, not a quiet no-op — the note is what the surface shows instead of "On". - if (d.get("statusNote") or "") != problem: - _pause_trigger(rt, auto_id, problem) - log(f"[aios-auto] trigger gate: {auto_id} {problem}") - return - fired = [] - if gate is not None: - rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} - disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) - nxt = set(disarmed) - for rid in touched: - # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 - # tree can read the record's other columns, the just-written cells on top because - # for an ordinary ut column this event is the ONLY place that value exists yet. - if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), - **(written.get(rid) or {})}): - if rid not in disarmed: - fired.append(rid) # false→true THIS evaluation: the edge - nxt.add(rid) - else: - nxt.discard(rid) # left the state — re-armed (Airtable's rule) - if nxt != disarmed: - set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) - if not fired: - return - else: - fired = list(touched) # "the field changed" — the burst is the edge - if len(fired) > FLOOD_LIMIT: - _commit_run(rt, auto_id, "partial", - f"the trigger matched {len(fired)} records in one evaluation — more than " - f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " - f"process them deliberately", {"held": len(fired)}, True) - return - trigger_fire(rt, tenant, auto_id, log=log) - - -def _seed_event_state(rt, defn): - """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so - turning the trigger on fires nothing — the first fire needs a real false→true transition. - Evaluated over the definition rows (the machine-written truth this trigger class watches). - - Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation - reads. Two implementations of "does this row match" is how a seed disagrees with the edge it - is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. - """ - trg = (defn or {}).get("trigger") or {} - if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ - or not trg.get("configured"): - return - gate, problem = _row_gate(rt, defn, trg) - if gate is None or problem: - return # no match gate ⇒ nothing to arm; a broken gate seeds nothing - rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} - matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) - set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) - - -def grid_hook(evt): - """THE listener the human write doors emit into (registered onto - `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may - import both sides). Never raises into a write path; a broken trigger listener must not - break typing into a cell.""" - try: - st = evt.get("st") - if st is None: - return - tenant = str(getattr(st, "key", "") or "royal-imports") - table = str(evt.get("table") or "") - kind = str(evt.get("type") or "") - defs = all_definitions(st) - field = str(evt.get("field") or "") - rid = str(evt.get("rowId") or "") - for aid, d in defs.items(): - trg = (d or {}).get("trigger") or {} - key = str(trg.get("key") or "") - if trg.get("paused") or not trg.get("enabled", True) \ - or not trg.get("configured", True) \ - or key not in TRIGGER_ROW_KEYS \ - or str(trg.get("table") or "") != table: - continue - if kind == "record_created" and key == "record_created": - hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 - ridn = int(rid) if rid.isdigit() else None - if ridn is None or ridn <= hw: - continue # once per record EVER (A2(4)) — undo-proof - set_state(st, aid, {"rcHighwater": ridn}) - trigger_fire(st, tenant, aid) - elif kind == "event_field" and key == "event_field" and ( - not trg.get("field") or str(trg.get("field")) == field): - # A NAMED field narrows to writes on it (wave 22's shape); an unnamed one is - # Airtable's plain condition trigger and watches every column. - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - elif kind == "event_field" and key == "record_updated" and ( - not (trg.get("fields") or []) or field in (trg.get("fields") or [])): - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - elif key == "enters_view": - # BOTH kinds feed it: a row can enter a view by being edited into its filter or - # by being CREATED already inside it. Listening only to edits would silently miss - # every new record — the half of the definition a reader assumes is covered. - _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) - except Exception as e: # noqa: BLE001 - print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") - - -def form_fired(rt, table_key, row_id, values=None, form_token=""): - """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). - - One submitted form row → every `form_submitted` automation watching that database fires. - Returns the list of automation ids that started, so the door can log what it set off (and so - the gate can assert it, rather than asserting a side effect nobody can see). - - ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, - so it fires triggers exactly like typing into a cell does. The loop-prevention law is not - weakened by that — the engine's own writers still never reach here (only `routes_forms` calls - it), so an automation cannot create a form row and re-fire itself. - - `values` is accepted and unused today: the row is already written when this is called, and - the flow reads it from the table. It stays in the signature because the caller HAS it and a - later refire policy ("only when field X was submitted") needs it — a parameter added later - would mean changing D's call site in a wave that does not own it. - """ - started, table = [], str(table_key or "") - if not table: - return started - tenant = str(getattr(rt, "key", "") or "royal-imports") - for aid, d in all_definitions(rt).items(): - trg = (d or {}).get("trigger") or {} - if trg.get("key") != "form_submitted" or trg.get("paused") \ - or not trg.get("enabled", True) or not trg.get("configured", True): - continue - if str(trg.get("table") or "") != table: - continue - want = str(trg.get("formToken") or "") - if want and not hmac.compare_digest(want, str(form_token or "")): - continue # this automation watches a DIFFERENT form on that table - if trigger_fire(rt, tenant, aid): - started.append(aid) - return started - - -def hook_fire(rt, tenant, auto_id, token): - """The webhook trigger's decision, separated from FastAPI so the gate can drive it. - Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, - 409 already running, 200 started.""" - import hmac as _hmac - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return 404, {"error": "unknown_automation"} - trg = defn.get("trigger") or {} - want = str(trg.get("token") or "") - if trg.get("key") != "webhook" or not want: - return 403, {"error": "no_webhook", "message": - "this automation has no webhook trigger"} - if trg.get("paused") or not trg.get("enabled", True): - return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} - if not _hmac.compare_digest(want, str(token or "")): - return 403, {"error": "bad_token", "message": "that token is not valid"} - started = trigger_fire(rt, tenant, auto_id) - return 200, {"started": bool(started), "at": _iso()} - - -def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): - """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a - row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the - gate drives this without a network; production leaves them None. - - Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per - tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most - `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. - """ - trg = defn.get("trigger") or {} - if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): - return None - import oauth_connect - creator = str(defn.get("createdBy") or "").strip() or "admin" - token, err = oauth_connect.google_creds(rt, creator) - if err: - if (defn.get("statusNote") or "") != err: - def _note(cur): - cur = cur if isinstance(cur, dict) else {} - dd = cur.get(str(auto_id)) - if dd is not None: - dd["statusNote"] = _s(err, 200) - return cur - rt.update(STORE_KEY, _note, flush="sync") - return None - lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) - reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) - ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) - if lerr: - return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}", - {}, True) - seen = set((defn.get("state") or {}).get("emailSeen") or []) - fresh = [m for m in ids if m not in seen] - if not fresh: - return None # nothing new is not a run — no history spam - if len(fresh) > FLOOD_LIMIT: - return _commit_run(rt, auto_id, "partial", - f"{len(fresh)} new emails matched in one poll — more than the " - f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " - f"the query, or press Run now after adjusting it", - {"held": len(fresh)}, True) - fresh = fresh[:EMAIL_MAX_PER_POLL] - rows_in, notes = [], [] - for mid in fresh: - row, rerr = reader(mid) - if row: - rows_in.append(row) - elif rerr: - notes.append(rerr) - table_key = (defn.get("config") or {}).get("targetTable") or "" - if not table_key: - return _commit_run(rt, auto_id, "error", - "the email trigger has nowhere to write — the automation names no " - "target database", {}, False) - ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", - EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) - skey = stage_field_key(auto_id) - stamp = _iso() - for r in rows_in: - r[skey + "_at"] = stamp - existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) - merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) - ut_write_rows(rt, table_key, merged) - new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] - set_state(rt, auto_id, {"emailSeen": new_seen}) - summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — " - f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") - if notes: - summary += f" — {notes[0][:100]}" - entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, - True, affected=list(merged)[:200]) - trigger_fire(rt, tenant, auto_id, log=log) - return entry - - -def compose_sentence(defn): - """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the - definition so it cannot lie about what runs.""" - cfg = defn.get("config") or {} - trg = defn.get("trigger") or {} - sched = defn.get("schedule") or {} - kind = defn.get("kind") - if trg.get("key") == "event_field": - head = f"When {trg.get('field')} changes on {trg.get('table')}" - elif trg.get("key") == "record_created": - head = f"When a record is created in {trg.get('table')}" - elif trg.get("key") == "webhook": - head = "When the webhook is called" - elif trg.get("key") == "email": - head = f"When an email matches {trg.get('query')}" - elif sched.get("enabled"): - head = f"{_cron_label(sched.get('cron'))}" - else: - head = "When you press Run now" - if kind == "scrape_db": - host = urlparse(str(cfg.get("url") or "")).hostname or "the page" - body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" - elif kind == "field_instagram": - body = (f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" - + (", exact counts first" if clean_tier(cfg.get("tier")) == "brightdata" - else ", anonymous only")) - else: - body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into " - f"{cfg.get('targetTable') or DISCOVER_TABLE}") - lanes = cfg.get("lanes") or [] - tail = f", then route each record across {len(lanes)} lanes" if lanes else "" - return f"{head}, {body}{tail}." - - -def run_now(rt, tenant, auto_id, username="automation", log=print): - """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it - directly. Returns the run entry, or None when it was already running (the 409).""" - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return None - if not _claim(tenant, auto_id): - return None - # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever - # happened to press Run — and above all not to the scheduler, which is not a person and - # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by - # whether a human or a cron got to the first run first. - owner = str(defn.get("createdBy") or "").strip() - if owner and username in MACHINE_OWNERS: - username = owner - try: - _step(tenant, auto_id, "running") - runner = RUNNERS.get(defn.get("kind")) - if runner is None: - return _commit_run(rt, auto_id, "error", - f"unknown automation kind {defn.get('kind')!r}", {}, False) - try: - state, summary, counts, affected, steps = runner(rt, defn, username=username, - log=log) - except Refused as e: - return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") - return _commit_run(rt, auto_id, "error", - f"{type(e).__name__}: {str(e)[:200]}", {}, False) - # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is - # committed, over the records this run actually touched. ONE call site rather than three - # inside the runners: every kind gets actions and endings for free, and a fourth runner - # cannot forget to opt in. - # - # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, - # wired to the wire and covered by twelve gate checks that all called `apply_actions` - # DIRECTLY — so the whole feature was green and unreachable. A person would have built a - # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. - # - # A failing action must not fail the RUN: the machine steps already wrote their rows and - # reporting that as an error would misdescribe what happened. It degrades to `partial` - # with the reason in the summary — the cap_note discipline. - try: - a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], - username=username, log=log) - counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") - state = "partial" if state != "error" else state - summary = f"{summary} — the actions did not finish ({type(e).__name__})" - return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, - steps) - finally: - _release(tenant, auto_id) - - -def run_async(rt, tenant, auto_id, username="automation", log=print): - """Start a run on a background thread. False when one is already in flight (→ 409).""" - if running(tenant, auto_id): - return False - th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log), - daemon=True, name=f"automation-{tenant}-{auto_id}") - th.start() - return True - - -# --------------------------------------------------------------------------------------------- -# THE TICK + the in-process scheduler -# --------------------------------------------------------------------------------------------- - -def due_ids(rt, now=None): - """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" - return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now)) - - -def tick(rt, tenant, now=None, log=print): - """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the - ids started. The email polls are bounded and fail-quiet per automation — one broken - mailbox connection must not stop the tenant's schedules.""" - started = [] - for aid in due_ids(rt, now): - if run_async(rt, tenant, aid, username="scheduler", log=log): - started.append(aid) - for aid, d in all_definitions(rt).items(): - if (d.get("trigger") or {}).get("key") == "email": - try: - if email_poll(rt, tenant, aid, d, log=log) is not None: - started.append(aid) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") - # C7's amendment: date-window metrics advance with the tick, so `today` is never staler - # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan. - try: - refresh_metrics(rt, log=log) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}") - if started: - log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") - return started - - -def tick_all(now=None, log=print): - """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop - the others' schedules.""" - from harness import runtime as _rt - out = {} - for slug in _rt.known_tenants(): - try: - out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}") - return out - - -#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer -#: would be a busy loop against a vocabulary that cannot express it. -TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) -_SCHEDULER = [None] - - -def scheduler_loop(log=print): - """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. - - Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and - `verify_seam` import this module through `main.py`, run against fake stores in seconds and - exit. A loop that ticked on entry would fire inside them. - """ - while True: - time.sleep(TICK_SECONDS) - try: - tick_all(log=log) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") - - -def start_scheduler(log=print): - """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** - - ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave - brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 - seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires - two or three times inside a gate that imports `main.py` — and while a tick over a fake store - finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and - mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread - that can move a gate's subject is a flaky suite waiting to happen. - - So it takes the convention `main.py:353` already established for exactly this hazard — - `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup - event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the - external EventBridge tick (R5) does not depend on it either way, since that POSTs the - endpoint rather than riding this thread. - """ - if os.environ.get("AIOS_AUTOMATIONS") != "1": - return False - if _SCHEDULER[0] is not None: - return False - th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, - daemon=True, name="automation-scheduler") - _SCHEDULER[0] = th - th.start() - return True +"""automation_engine.py — wave-18 item 5 (contract C4-AUTO): the automation RUNTIME. + +THE SHAPE, and why it is this shape. An automation is a DEFINITION that lives in the tenant +store and a RUN that lives in this process. The definition is durable, small and rarely written; +the run is hot, chatty and worthless five minutes later. Conflating them is the failure this file +is built to avoid, and it has two halves: + + * **Run state is NEVER persisted while it is running.** A `state: 'running'` written to the + store outlives the process that wrote it, so a container restart mid-run leaves an automation + that can never run again — the 409 sees a `running` nothing will ever clear. Hot state lives + in `_RUNNING` (a process dict, cleared by definition on restart) and the store is written + exactly ONCE per run, at the end. + * **One coalesced store update per run per bucket.** `core/store.py` coalesces on a 20 s + floor per key against a 256-commits/hour repo budget, so a per-ROW write is the wrong shape + by two orders of magnitude — the S&P demo alone is ~500 rows. Every runner below computes + its whole result first and commits it in a single `update` callback. + +WHAT IS VENDORED HERE AND WHY. `.claude/skills/browse/scrape.py` is not on `sys.path` and is not +deployed, so its extraction core is copied into this file (the wave doc's instruction), ported +httpx → requests (the only HTTP dependency this API already carries). The SSRF rail comes with +it, HARDENED rather than merely preserved — see `guard`/`fetch`. That hardening is the point: +the skill version takes a URL from a developer on a CLI; this takes one from a request body. +""" +from __future__ import annotations + +import datetime as _dt +import hmac +import ipaddress +import json +import os +import re +import socket +import threading +import time +from urllib.parse import urljoin, urlparse + +import requests + +try: # the parser the extraction half needs + from bs4 import BeautifulSoup +except Exception: # pragma: no cover — deploy lag; see mailbox ② + BeautifulSoup = None + +# --------------------------------------------------------------------------------------------- +# THE BUCKET (C4-AUTO) +# --------------------------------------------------------------------------------------------- + +#: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. +STORE_KEY = "automations" +#: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through +#: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for +#: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the +#: session-D mailbox as a finding against A's file rather than fixed from here. +UT_STORE_KEY = "user_tables" +UT_PREFIX = "ut_" + +MAX_AUTOMATIONS = 40 +MAX_RUNS = 20 # trimmed history per automation +MAX_NAME = 80 +MAX_UT_ROWS = 5000 # mirrors core.user_tables.MAX_ROWS +MAX_UT_TABLES = 40 + +#: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. +#: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the +#: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull +#: INSERTS a timestamped row (see `SNAPSHOT_FIELDS` and the compound keys below), so at R1's +#: 1k-profiles-daily the snapshot table crosses 5000 rows on **day five** — and the old behaviour +#: was a SILENT `skipped++`, i.e. the table would quietly stop growing and the run would still +#: report success. A time series that stops after five days without saying so is worse than one +#: that was never built. +#: 200k is the owner-directed ceiling (R1's "~200k"). ⚠ IT BUYS VERY DIFFERENT HORIZONS FOR THE +#: TWO APPEND TABLES, and the difference is worth knowing before anyone plans around it: +#: ut_ig_snapshots 1 row per profile per pull -> ~200 days at 1k profiles/day +#: ut_ig_post_snapshots maxPosts rows per pull -> ~8 days at 1k profiles x 24 posts +#: So the post series is the one that fills, and it fills FAST. That is exactly why the breach had +#: to become LOUD (a distinct `capped` count -> a `partial` run naming the table): at this rate the +#: silent version would have flatlined a chart inside a fortnight with a green dot over it. +#: +#: ⚠ MEASURED COST AT THE CEILING (2026-08-04, this box): a full `ut_ig_post_snapshots` serialises +#: to **35.8 MB** of JSON (~1.4 s). `UT_STORE_KEY` is ONE bucket for ALL of a tenant's user tables, +#: so that cost is paid by every unrelated automation write in the tenant too. Booked for the B-3 +#: Postgres tripwires (R2 clause b) rather than papered over — the fix is a row store, not a +#: smaller number. +#: (W19-C; the GENERIC loud-breach for every other table stays booked as DEBT D-11.) +MAX_UT_IG_ROWS = 200_000 +IG_TABLE_PREFIX = "ut_ig_" + +#: ⚠ THE RAISED CEILING BELONGS TO THE **APPEND** TABLES, NOT TO A NAME PREFIX (wave 20). It was +#: keyed off `ut_ig_`, which was exactly right while every `ut_ig_*` table was an append table — +#: and stopped being right the moment discovery added `ut_ig_candidates`, which is UPSERTED BY +#: HANDLE and is bounded by how many accounts exist rather than by how often we look. It would +#: have inherited a 200,000-row ceiling, and with it the measured 35.8 MB single-bucket +#: serialisation cost, purely by an accident of naming. Naming the append tables is the version +#: that stays true when the next `ut_ig_*` table is not one. +APPEND_TABLES = frozenset({"ut_ig_snapshots", "ut_ig_post_snapshots"}) + +#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. +#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger +#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram +#: column, an Instagram search — and they survive on the automations that already use them +#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). +#: +#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: +#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind +#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was +#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole +#: description: three machine nodes it does not have, and a one-line summary about searching +#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. +KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram") +#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores +#: when the body names none, which after R6 is every create the client makes. +DEFAULT_KIND = "plain" +#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — +#: the ruling retires the door, not the two automations behind it (see `create`). +#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. +RETIRED_KINDS = ("scrape_db", "field_instagram") +KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database", + "field_instagram": "Instagram profile column", + "discover_instagram": "Find Instagram profiles"} +EXTRACTS = ("table", "jsonld") +STATES = ("idle", "running", "ok", "error", "partial") +#: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). +#: `anonymous` = the $0 ladder only. `brightdata` = the paid rung FIRST, then the ladder as a +#: fallback (unless the fallback is switched off — see `graph`'s `fallback` toggle). +TIERS = ("anonymous", "brightdata") +#: ⚠ STORED CONFIGS SAY `hiker`, AND THEY MEAN "THE PAID RUNG" (wave-20 D-21). The vendor swap +#: must not silently answer that request with the free ladder: `clean_config` refuses an unknown +#: tier by falling back to `anonymous`, so without this alias every existing Instagram automation +#: would quietly stop reaching for exact counts and nothing would say so. Mapping FORWARD keeps +#: the user's expressed intent (they turned the paid step ON) at a cost of ~$0.0015 a profile. +TIER_ALIASES = {"hiker": "brightdata"} + + +def clean_tier(raw): + """A stored/posted tier → a tier this engine runs, or '' when it is neither.""" + t = str(raw or "").strip().lower() + t = TIER_ALIASES.get(t, t) + return t if t in TIERS else "" + + +def row_cap(table_key): + """The row ceiling for ONE table. Per-table rather than global — see `MAX_UT_IG_ROWS`. + + An APPEND table (one row per subject per pull, forever) gets the raised ceiling; everything + else — including the `ut_ig_` table that is an UPSERT — keeps the list-table one. + """ + return MAX_UT_IG_ROWS if str(table_key or "") in APPEND_TABLES else MAX_UT_ROWS + +#: Schedule presets the editor offers. Kept here (not in the client) so the vocabulary the UI +#: shows and the vocabulary the parser accepts cannot drift. +CRON_PRESETS = [ + {"cron": "*/15 * * * *", "label": "Every 15 minutes"}, + {"cron": "0 * * * *", "label": "Hourly"}, + {"cron": "0 6 * * *", "label": "Daily at 06:00"}, + {"cron": "0 6 * * 1", "label": "Weekly (Monday 06:00)"}, + {"cron": "0 6 1 * *", "label": "Monthly (1st, 06:00)"}, +] + +UA = "Mozilla/5.0 (compatible; AIOS-automation/1.0; +https://aios.local/automation)" + + +def _now(): + return _dt.datetime.now() + + +def _stamp(dt=None): + return (dt or _now()).strftime("%Y-%m-%d %H:%M") + + +def _iso(dt=None): + """An ISO stamp **WITH its UTC offset** — `2026-08-05T14:03:11+07:00` (DEBT D-18). + + ⚠ THE OFFSET IS NOT COSMETIC. These stamps are the time axis of the `ut_ig_*` append tables + and they are rendered by a browser, which can only subtract from an instant it can LOCATE. A + naive `2026-08-05T14:03:11` is read as the *reader's* local time, so a container running UTC + minted cells a Jakarta browser would place seven hours in the future — and "2 minutes ago" is + not expressible at all. With the offset the same string is an instant, and relative rendering + becomes possible without migrating a single stored row. + + ⚠ `_parse_iso` reads it BACK as naive local ON PURPOSE. Every cron / `is_due` comparison in + this module is against a naive `_now()`, and mixing aware and naive datetimes raises + `TypeError` — so the offset rides on the WIRE and never enters the arithmetic. + """ + dt = dt or _now() + if dt.tzinfo is None: + dt = dt.astimezone() # a naive stamp from this process IS local time + return dt.isoformat(timespec="seconds") + + +def _parse_iso(s): + """A stamp → a NAIVE LOCAL datetime, with or without an offset. + + Both shapes exist in the store simultaneously and always will: every run committed before + D-18 wrote a naive stamp, and nothing rewrites history. A parser that understood only the new + shape would silently return None for them — and `is_due` reads None as "never ran", which + would re-fire every schedule once. Reading both is what makes the change additive. + """ + raw = str(s or "").strip().replace("Z", "+00:00") + dt = None + try: + dt = _dt.datetime.fromisoformat(raw) + except Exception: # noqa: BLE001 + try: + dt = _dt.datetime.strptime(raw[:19], "%Y-%m-%dT%H:%M:%S") + except Exception: # noqa: BLE001 + return None + return dt.astimezone().replace(tzinfo=None) if dt.tzinfo is not None else dt + + +# --------------------------------------------------------------------------------------------- +# THE SSRF RAIL — vendored from scrape.py `guard`, then hardened for a SERVER-SIDE fetcher +# --------------------------------------------------------------------------------------------- +# The skill version guards the URL a developer typed. This one guards a URL that arrived in a +# request body, which changes the threat model in two ways the original does not cover: +# +# 1. REDIRECTS. `httpx.Client(follow_redirects=True)` / `requests.get(allow_redirects=True)` +# never re-enter the guard, so `http://evil.example/x` → 302 → `http://169.254.169.254/…` +# sails straight past a guard that only ever saw the first URL. The negative control ("a +# localhost URL is refused") would still pass while the rail was wide open. `fetch` below +# therefore takes the hops MANUALLY and re-guards every one. +# 2. DNS. A perfectly public hostname may resolve to a private address. `guard` resolves the +# host and checks EVERY answer, not just the literal. +# +# ⚠ STATED, NOT HIDDEN: this is check-then-connect, so a DNS-rebinding attacker who flips the +# record between the guard and the socket is not stopped by it. Closing that needs a pinned +# connection to the validated IP (a custom adapter). Out of scope for v1 and recorded here rather +# than implied away — the rail refuses the realistic cases and says what it does not cover. + +class Refused(ValueError): + """The rail refused a URL. A distinct type so a refusal is never logged as a fetch error.""" + + +def _ip_public(ip): + return not (ip.is_private or ip.is_loopback or ip.is_link_local + or ip.is_reserved or ip.is_unspecified or ip.is_multicast) + + +def guard(url, allowed=()): + """PUBLIC-WEB-ONLY rail: http(s) only, no loopback/private/link-local host, DNS answers + checked too, plus the optional `--allowed` domain rail scrape.py carries.""" + p = urlparse(url or "") + if p.scheme not in ("http", "https"): + raise Refused(f"only http(s) allowed, not {p.scheme!r}") + host = (p.hostname or "").strip() + if not host: + raise Refused("no host in the URL") + low = host.lower() + if (low in ("localhost", "localhost.localdomain") + or low.endswith(".local") or low.endswith(".internal") + or low.endswith(".localhost")): + raise Refused(f"refused non-public host {host!r} (public web only)") + try: # a bare-IP host is decided on the literal + ip = ipaddress.ip_address(low) + except ValueError: + ip = None + if ip is not None and not _ip_public(ip): + raise Refused(f"refused non-public IP {host} (public web only)") + if allowed and not any(low == d.lower() or low.endswith("." + d.lower()) for d in allowed): + raise Refused(f"host {host!r} not in the allowed domains {list(allowed)}") + if ip is None: + try: + infos = socket.getaddrinfo(host, None) + except Exception as e: + raise Refused(f"could not resolve {host!r}: {type(e).__name__}") + for info in infos: + addr = info[4][0] + try: + resolved = ipaddress.ip_address(addr.split("%")[0]) + except ValueError: + continue + if not _ip_public(resolved): + raise Refused( + f"refused {host!r}: it resolves to the non-public address {resolved}") + return True + + +def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): + """GET `url`, taking redirects BY HAND so every hop passes `guard`. + + Returns `(status, final_url, body_bytes)`. A hop chain longer than `max_hops` is a refusal, + not a silent truncation — an endless redirect is indistinguishable from an attempt to walk + the fetcher somewhere it was told not to go. + """ + current = url + hdrs = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"} + hdrs.update(headers or {}) + for _hop in range(max_hops + 1): + guard(current, allowed) + r = requests.get(current, timeout=timeout, headers=hdrs, + allow_redirects=False, stream=True) + if r.status_code in (301, 302, 303, 307, 308): + loc = r.headers.get("Location") + r.close() + if not loc: + raise Refused(f"redirect {r.status_code} with no Location header") + current = urljoin(current, loc) + continue + body = r.raw.read(max_kb * 1024, decode_content=True) or b"" + status, final = r.status_code, str(r.url) + r.close() + return status, final, body + raise Refused(f"more than {max_hops} redirects — refusing to follow further") + + +def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): + """POST a JSON body through the SAME guard. Returns `(status, body_bytes)`. + + ⛔ NO REDIRECT FOLLOWING, AND THAT IS THE WHOLE DIFFERENCE FROM `fetch`. `fetch` walks hops by + hand because a redirected GET is ordinary. A redirected POST is not: re-sending a + credential-bearing body to a Location the *server* chose is precisely the hop the SSRF rail + exists to refuse, and `requests`' own `allow_redirects=True` would do it without ever + re-entering `guard`. So a 3xx here is a REFUSAL with the reason on it, never a second request. + (The vendor calls below are the only POSTs this module makes, and they carry the API key.) + """ + guard(url) + hdrs = {"User-Agent": UA, "Content-Type": "application/json", "Accept": "application/json"} + hdrs.update(headers or {}) + r = requests.post(url, timeout=timeout, headers=hdrs, allow_redirects=False, + data=json.dumps(body if body is not None else {}), stream=True) + try: + if r.status_code in (301, 302, 303, 307, 308): + raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, " + f"never re-sent to a location the server picked") + return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") + finally: + r.close() + + +# --------------------------------------------------------------------------------------------- +# EXTRACTION — vendored from scrape.py, unchanged in behaviour +# --------------------------------------------------------------------------------------------- + +def _soup(body): + if BeautifulSoup is None: + raise RuntimeError( + "beautifulsoup4 is not installed in this environment — the automation engine's " + "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " + "aios-web/requirements.txt).") + try: + return BeautifulSoup(body, "lxml") + except Exception: + return BeautifulSoup(body, "html.parser") + + +def tables(soup): + """Every HTML table as rows. Dict rows when the first row looks like a header.""" + out = [] + for t in soup.find_all("table"): + rows = [] + for tr in t.find_all("tr"): + cells = [c.get_text(" ", strip=True) for c in tr.find_all(["th", "td"])] + if cells: + rows.append(cells) + if not rows: + continue + head, body = rows[0], rows[1:] + if body and len(head) == len(body[0]) and all(head): + out.append([dict(zip(head, r)) for r in body if len(r) == len(head)]) + else: + out.append(rows) + return out + + +def jsonld(soup): + out = [] + for s in soup.find_all("script", attrs={"type": "application/ld+json"}): + try: + out.append(json.loads(s.string or s.get_text())) + except Exception: + pass + return out + + +def meta(soup): + m = {} + if soup.title and soup.title.string: + m["title"] = soup.title.string.strip() + for tag in soup.find_all("meta"): + k = tag.get("name") or tag.get("property") + v = tag.get("content") + if k and v and (k in ("description", "keywords", "author") + or k.startswith(("og:", "twitter:"))): + m[k] = v.strip() + return m + + +def preview(url, extract="table", table_index=0, allowed=()): + """The field-map preview the editor calls BEFORE anything is created: what columns does this + page actually offer, and what do the first rows look like? Never writes.""" + status, final, body = fetch(url, allowed=allowed) + if not (200 <= status < 300): + return {"ok": False, "status": status, "url": final, + "note": f"the page answered {status} — it may be bot-gated or gone", + "columns": [], "sample": [], "rowCount": 0} + soup = _soup(body) + rows = [] + if extract == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(table_index or 0), len(found) - 1)) if found else 0 + picked = found[idx] if found else [] + rows = [r for r in picked if isinstance(r, dict)] + cols, seen = [], set() + for r in rows[:50]: + for k in r: + if k not in seen: + seen.add(k) + cols.append(k) + return {"ok": True, "status": status, "url": final, "columns": cols, + "sample": rows[:8], "rowCount": len(rows), + "tableCount": len(tables(soup)) if extract != "jsonld" else 0, + "title": (meta(soup) or {}).get("title", "")} + + +def _scalar(v): + if isinstance(v, (str, int, float)) and not isinstance(v, bool): + return str(v) + if isinstance(v, bool): + return "1" if v else "" + if isinstance(v, dict): + return str(v.get("name") or v.get("@id") or "") + if isinstance(v, list): + return ", ".join(_scalar(x) for x in v[:8]) + return "" + + +# --------------------------------------------------------------------------------------------- +# CRON — a 5-field parser and a "what was the last fire time" walk +# --------------------------------------------------------------------------------------------- +# Deliberately NOT `croniter` (a dependency for ~60 lines) and deliberately NOT a forward +# scheduler. The question a tick asks is backwards-looking — *"was there a scheduled minute +# between the last run and now?"* — and answering it that way is what makes a missed tick +# self-healing: a container that was asleep for two hours runs once on wake, not eleven times and +# not never. + +_FIELD_RANGES = [(0, 59), (0, 23), (1, 31), (1, 12), (0, 6)] + + +def _parse_field(spec, lo, hi): + out = set() + for part in str(spec).split(","): + part = part.strip() + if not part: + raise ValueError("empty cron field part") + step = 1 + if "/" in part: + part, _, s = part.partition("/") + step = int(s) + if step < 1: + raise ValueError("cron step must be >= 1") + if part in ("*", "?"): + a, b = lo, hi + elif "-" in part.lstrip("-"): + a_s, _, b_s = part.partition("-") + a, b = int(a_s), int(b_s) + else: + a = b = int(part) + if a < lo or b > hi or a > b: + raise ValueError(f"cron value {part!r} out of range {lo}..{hi}") + out.update(range(a, b + 1, step)) + return out + + +def parse_cron(expr): + """`'m h dom mon dow'` → `(minutes, hours, doms, months, dows, dom_restricted, dow_restricted)`. + + Raises ValueError on anything malformed — a schedule that cannot be parsed must be refused at + WRITE time, because a cron string nobody can evaluate is an automation that silently never + runs and reports no error anywhere. + """ + parts = str(expr or "").split() + if len(parts) != 5: + raise ValueError("a cron schedule has exactly 5 fields: minute hour day month weekday") + sets = [_parse_field(p, lo, hi) for p, (lo, hi) in zip(parts, _FIELD_RANGES)] + dom_r = parts[2].strip() not in ("*", "?") + dow_r = parts[4].strip() not in ("*", "?") + return (sets[0], sets[1], sets[2], sets[3], sets[4] | ({0} if 7 in sets[4] else set()), + dom_r, dow_r) + + +def _day_matches(day, doms, months, dows, dom_r, dow_r): + if day.month not in months: + return False + # POSIX rule: with BOTH day-of-month and weekday restricted the match is the UNION, not the + # intersection. `0 6 1 * 1` means "the 1st, and every Monday" — getting this backwards makes + # a schedule that looks right fire almost never. + dow = (day.weekday() + 1) % 7 # python Mon=0 -> cron Sun=0 + if dom_r and dow_r: + return day.day in doms or dow in dows + if dom_r: + return day.day in doms + if dow_r: + return dow in dows + return True + + +def prev_fire(expr, now=None, lookback_days=400): + """The most recent scheduled minute at or before `now`, or None inside the lookback. + + Walks by DAY (≤400 iterations) rather than by minute (≥500k) — the day fields decide first, + and only a matching day needs its hours/minutes searched. + """ + minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) + now = (now or _now()).replace(second=0, microsecond=0) + for back in range(lookback_days + 1): + day = (now - _dt.timedelta(days=back)).date() + if not _day_matches(day, doms, months, dows, dom_r, dow_r): + continue + same_day = back == 0 + for h in sorted(hours, reverse=True): + if same_day and h > now.hour: + continue + for m in sorted(minutes, reverse=True): + if same_day and h == now.hour and m > now.minute: + continue + return _dt.datetime(day.year, day.month, day.day, h, m) + return None + + +def is_due(defn, now=None): + """Should the scheduler run this automation right now? + + Due when a scheduled minute exists strictly AFTER the reference point (the last run, else the + moment the schedule was enabled, else creation) and at or before now. Anchoring on + `enabledAt` rather than firing on the first tick is what stops "enable a daily 06:00 job at + 14:00" from running immediately and looking like a bug. + """ + if not isinstance(defn, dict): + return False + sched = defn.get("schedule") or {} + if not sched.get("enabled"): + return False + try: + fire = prev_fire(sched.get("cron"), now=now) + except ValueError: + return False + if fire is None: + return False + since = (_parse_iso((defn.get("status") or {}).get("lastRunAt")) + or _parse_iso(sched.get("enabledAt")) + or _parse_iso(defn.get("created"))) + return since is None or fire > since + + +def next_fire(expr, now=None, lookahead_days=400): + """The next scheduled minute strictly after `now` — display only (the rail is `is_due`).""" + try: + minutes, hours, doms, months, dows, dom_r, dow_r = parse_cron(expr) + except ValueError: + return None + now = (now or _now()).replace(second=0, microsecond=0) + for ahead in range(lookahead_days + 1): + day = (now + _dt.timedelta(days=ahead)).date() + if not _day_matches(day, doms, months, dows, dom_r, dow_r): + continue + same_day = ahead == 0 + for h in sorted(hours): + if same_day and h < now.hour: + continue + for m in sorted(minutes): + if same_day and h == now.hour and m <= now.minute: + continue + return _dt.datetime(day.year, day.month, day.day, h, m) + return None + + +# --------------------------------------------------------------------------------------------- +# THE UPSERT — pure, so the arithmetic is testable without a store or a network +# --------------------------------------------------------------------------------------------- + +def upsert_rows(existing, incoming, key_field, cap=None): + """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. + + THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped + appearing on the source page has not necessarily stopped existing — the page changed its + filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream + hiccup into silent data loss, and the row may be carrying user-typed overlay values in + columns the scrape never touches. So the run reports `orphans: N` and leaves them alone. + + Only MAPPED keys are written: a re-run never clobbers a column a user added by hand. + + ⚠ CALL THIS ONCE PER TABLE PER RUN, NOT ONCE PER ROW. It rebuilds the whole row dict on + entry, so it is O(existing) per call — fine once, quadratic in a loop. The IG runner used to + call it per POST, which was survivable only because the cap was 5000; against `MAX_UT_IG_ROWS` + that same loop is hundreds of millions of dict copies and the automation simply never + finishes. Raising a cap and batching the writer are ONE change, not two. (W19-C.) + + ⚠ `capped` IS ITS OWN COUNT, deliberately not folded into `skipped`. They are different + facts: `skipped` means "this row had no key, so it could not be upserted" — a property of the + DATA, and usually benign. `capped` means "this table is full and the run is now losing rows" — + a property of the SYSTEM, and never benign. One number for both meant a table hitting its + ceiling was indistinguishable from a page with a few blank cells, which is how a time series + stops silently. The runners turn any `capped` into a `partial` run that NAMES the table. + """ + cap = MAX_UT_ROWS if cap is None else int(cap) + rows = {str(k): dict(v or {}) for k, v in (existing or {}).items()} + counts = {"inserted": 0, "updated": 0, "unchanged": 0, + "skipped": 0, "duplicates": 0, "orphans": 0, "capped": 0} + by_key, dupe_ids = {}, set() + for rid, row in rows.items(): + kv = str(row.get(key_field, "") or "").strip() + if not kv: + continue + if kv in by_key: + dupe_ids.add(rid) # a pre-existing duplicate: first id wins, second left + continue + by_key[kv] = rid + next_id = max((int(r) for r in rows if str(r).isdigit()), default=0) + 1 + seen_keys, incoming_dupes = set(), 0 + for src in incoming or []: + kv = str((src or {}).get(key_field, "") or "").strip() + if not kv: + counts["skipped"] += 1 # no key -> cannot be upserted; never guessed + continue + if kv in seen_keys: + incoming_dupes += 1 # the SOURCE listed it twice; first wins + continue + seen_keys.add(kv) + rid = by_key.get(kv) + if rid is None: + if len(rows) >= cap: + counts["capped"] += 1 # LOUD: the runner turns this into a partial run + continue + rid = str(next_id) + next_id += 1 + rows[rid] = dict(src) + by_key[kv] = rid + counts["inserted"] += 1 + continue + before = rows[rid] + changed = {k: v for k, v in src.items() if str(before.get(k, "")) != str(v)} + if changed: + before.update(src) + counts["updated"] += 1 + else: + counts["unchanged"] += 1 + counts["duplicates"] = incoming_dupes + len(dupe_ids) + counts["orphans"] = sum( + 1 for kv, rid in by_key.items() if kv not in seen_keys and rid not in dupe_ids) + return rows, counts + + +# --------------------------------------------------------------------------------------------- +# USER TABLES — read/write through the RUNTIME (never core.user_tables' module-global key) +# --------------------------------------------------------------------------------------------- + +def _ut_slug(label): + s = re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") + return (s or "table")[:40] + + +def ut_all(rt): + try: + return dict(rt.get(UT_STORE_KEY) or {}) + except Exception: + return {} + + +def disable_for_table(rt, table_key, note="target database deleted"): + """Wave 21 (item 6a, C3): a deleted table's automations are DISABLED loudly, never deleted. + + The definition survives with `schedule.enabled = False` + a `statusNote`, so the rail still + shows what existed and why it stopped — silently deleting a user's automation because its + target died would read as data loss. Returns the ids it touched.""" + key = str(table_key or "") + touched = [] + + def _up(cur): + for aid, d in (cur or {}).items(): + if isinstance(d, dict) and (d.get("config") or {}).get("targetTable") == key: + sch = d.get("schedule") + if not isinstance(sch, dict): + sch = d["schedule"] = {} + sch["enabled"] = False + d["statusNote"] = note + touched.append(str(aid)) + return cur + + if key: + rt.update(STORE_KEY, _up, flush="sync") + return touched + + +def ut_get(rt, key): + return ut_all(rt).get(str(key)) + + +def bind_unbound_fields(rt): + """C8's migration (wave 22, stated choice): existing automation bags WITHOUT a `flowId` + are bound to the definition that already writes them — matched on the (targetTable, + fieldKey) pair the definition carries, which is the binding W21-C2 established. A bag no + definition references is DISABLED with the reason on it rather than deleted or guessed: + the column was already dead (nothing runs a column no flow names), and now it says so. + Returns `(bound, disabled)`; costs zero store commits when there is nothing to migrate.""" + by_binding = {} + for aid, d in all_definitions(rt).items(): + cfg = d.get("config") or {} + if cfg.get("fieldKey") and cfg.get("targetTable"): + by_binding[(str(cfg["targetTable"]), str(cfg["fieldKey"]))] = str(aid) + plan = {} + for tk, t in ut_all(rt).items(): + for f in (t.get("fields") or []): + a = f.get("automation") + # An already-DISABLED bag is a decision this migration made on a previous pass — + # re-planning it every call would turn the once-per-process sweep into a write + # per read. + if isinstance(a, dict) and not a.get("flowId") and not a.get("stageField") \ + and not a.get("disabled"): + plan[(tk, str(f.get("key")))] = by_binding.get((tk, str(f.get("key")))) + if not plan: + return 0, 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for (tk, fk), aid in plan.items(): + for f in ((cur.get(tk) or {}).get("fields") or []): + if f.get("key") == fk and isinstance(f.get("automation"), dict): + if aid: + f["automation"]["flowId"] = aid + else: + f["automation"]["disabled"] = True + f["automation"]["statusNote"] = ( + "not bound to any flow — create an automation for this column " + "or delete it") + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + bound = sum(1 for v in plan.values() if v) + return bound, len(plan) - bound + + +def ut_key_for(label, key=None): + """The table key a label resolves to. Factored out of `ut_ensure` so the DRY-RUN path (which + must not create anything) resolves the same key by construction rather than by copying the + derivation and drifting from it.""" + k = str(key or (UT_PREFIX + _ut_slug(label))) + return k if k.startswith(UT_PREFIX) else UT_PREFIX + k + + +#: Machine names, mirrored from `core.user_tables.MACHINE_OWNERS`. A LOCAL literal for the same +#: reason `UT_FIELD_TYPES` is one over there — this module must stay importable without dragging +#: `core` into the API's boot path — and the two are held in step by a gate check. +MACHINE_OWNERS = ("automation", "scheduler") + + +def ut_ensure(rt, label, fields, username="automation", key=None, flow_tag=""): + """Create the table if it is missing; return its key. Idempotent — a re-run of an automation + that owns a table must not spawn `ut_x_2`, so the key is DERIVED from the label (or given) + and an existing table with that key is adopted, not duplicated. + + Fields are MERGED, never replaced: a user who added a column to an automation's table keeps + it, and a new source column joins on the next run. + + `flow_tag` (wave 22, C7/item 5): every field THIS call adds is stamped + `automation: {flowId: }` — the pre-set columns an IG automation spawns carry their + provenance. Fields already on the table keep whatever tag they have (first flow wins; + shared tables like ut_ig_snapshots are fed by many flows and the tag is provenance, not + ownership). + + ⛔ AND IT STAMPS A HUMAN OWNER, WHICH IT DID NOT (wave 20, item 3). `createdBy` was whoever + or WHATEVER ran the automation, so the same table belonged to a person if its first run was + manual and to `"scheduler"` if the schedule got there first — and `user_tables.may_open` + admits only the creator or an admin, so **whether you could open your own Instagram + snapshots depended on a race you never saw**. The owner is now the automation's creator, and + a table already stamped with a machine name is ADOPTED the next time a run knows a human + one. Adoption is a repair, not a widening: the automation's creator is the person who asked + for the table in the first place. + """ + key = ut_key_for(label, key) + wanted = [({**f, "automation": {"flowId": str(flow_tag)}} + if flow_tag and not f.get("automation") else dict(f)) + for f in (fields or [])] + created = _iso() + human = username if username and username not in MACHINE_OWNERS else "" + + # ⚠ SKIP THE WRITE WHEN NOTHING WOULD CHANGE. Without this, every re-run spends a store + # commit re-writing an identical definition — against a 20 s flush floor and a 256/hr repo + # budget, an idempotent helper that always writes is the same defect as a per-row insert. + have = ut_get(rt, key) + if (have is not None + and not [f for f in wanted + if f.get("key") not in {g.get("key") for g in (have.get("fields") or [])}] + and not (human and (have.get("createdBy") or "") in MACHINE_OWNERS)): + return key + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is None: + if len(cur) >= MAX_UT_TABLES: + return cur + cur[key] = {"key": key, "label": str(label)[:60], "source": "Automation", + "createdBy": username, "created": created, + "fields": wanted, "rows": {}} + return cur + have = {f.get("key") for f in (t.get("fields") or [])} + for f in wanted: + if f.get("key") not in have: + t.setdefault("fields", []).append(f) + have.add(f.get("key")) + # ADOPTION: a machine name is not an owner. It never overwrites a human one. + if human and (t.get("createdBy") or "") in MACHINE_OWNERS: + t["createdBy"] = human + # An automation's table SAYS an automation owns it — the nav badge reads from this. + t["source"] = t.get("source") or "Automation" + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return key + + +def ut_write_rows(rt, key, rows): + """ONE store update for the WHOLE row set — the flush-ceiling rule (see the module header).""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is not None: + t["rows"] = rows + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") + + +def ut_set_cell(rt, key, row_id, field_key, value, extra_rows=None): + """Write one automation-owned cell (+ optional whole extra tables) in ONE update.""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is not None: + t.setdefault("rows", {}).setdefault(str(row_id), {})[field_key] = value + for k, rws in (extra_rows or {}).items(): + tt = cur.get(k) + if tt is not None: + tt["rows"] = rws + return cur + rt.update(UT_STORE_KEY, _up, flush="sync") + + +def field_def(text_key, label, ftype="text"): + return {"key": text_key, "label": label, "type": ftype, "source": "overlay"} + + +# --------------------------------------------------------------------------------------------- +# DEFINITIONS — validation and the bucket's read/write half +# --------------------------------------------------------------------------------------------- + +def _s(v, n=200): + return str(v if v is not None else "")[:n] + + +def clean_config(kind, raw, previous=None): + """Validate a kind's config. Returns `(config, error)`; error is a user-facing sentence. + + ⚠ THE NODE-SWITCH FLAGS FALL BACK TO `previous` WHEN THE KEY IS ABSENT, and that is a rail + rather than a nicety. `patch` replaces the whole config, and the canvas's config panels do not + edit `tier` / `noFallback` / `dryRun` — those are node SWITCHES. So a plain "Save" from a + panel that never knew about them would silently reset the paid rung to `anonymous` and turn + off the dry run: a save that quietly changes what the automation costs. Absent ⇒ keep; + present ⇒ take it, including `false`. + """ + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + + def flag(name): + return bool(raw[name]) if name in raw else bool(prev.get(name)) + if kind == "plain": + # ⭐ WAVE 24 / R6 — a plain automation has no machine step, so it has almost no config: + # the database its records walk, and the board's lane router. + # + # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on + # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- + # trigger — so requiring a second copy of that fact here would be a mandatory field with + # exactly one legal answer, and a Save that refuses until you retype what you just picked. + # There is no `dryRun`: dry-run means "run the machine step but do not write", and there + # is no machine step to run. + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + lanes, lerr = clean_lanes(raw.get("lanes") if "lanes" in raw else prev.get("lanes")) + if lerr: + return None, lerr + return {"targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip(), + "lanes": lanes}, None + if kind == "scrape_db": + url = _s(raw.get("url"), 2000).strip() + if not url: + return None, "a source URL is required" + try: + guard(url) + except Refused as e: + return None, str(e) + extract = raw.get("extract") if raw.get("extract") in EXTRACTS else "table" + fmap = {} + for k, v in list((raw.get("fieldMap") or {}).items())[:60]: + tk = re.sub(r"[^a-z0-9_]+", "_", _s(v, 60).strip().lower()).strip("_") + if tk: + fmap[_s(k, 120)] = tk[:60] + if not fmap: + return None, "map at least one source column to a field" + key_field = _s(raw.get("keyField"), 60).strip() + if key_field not in fmap.values(): + return None, "the key field must be one of the mapped fields" + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + lanes, lerr = clean_lanes(raw.get("lanes") if "lanes" in raw else prev.get("lanes")) + if lerr: + return None, lerr + return {"url": url, "extract": extract, + "tableIndex": max(0, min(int(raw.get("tableIndex") or 0), 50)), + "fieldMap": fmap, "keyField": key_field, + "targetTable": target, + "dryRun": flag("dryRun"), + "lanes": lanes, + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "Scraped table"}, None + if kind == "field_instagram": + table = _s(raw.get("targetTable"), 60).strip() + if not table.startswith(UT_PREFIX): + return None, "an Instagram automation runs against a blank database (ut_*)" + fkey = _s(raw.get("fieldKey"), 80).strip() + if not fkey: + return None, "pick the automation column this run writes into" + # ⚠ UNKNOWN TIERS FALL BACK TO `anonymous`, NOT to the paid rung. A typo must never be + # the thing that starts spending money. An ABSENT tier keeps the one already stored (see + # the `flag` note above), and a stored `hiker` is carried FORWARD to the new vendor + # rather than silently demoted — see `TIER_ALIASES`. + tier = clean_tier(raw["tier"]) if "tier" in raw else "" + if not tier and "tier" not in raw: + tier = clean_tier(prev.get("tier")) + lanes, lerr = clean_lanes(raw.get("lanes") if "lanes" in raw else prev.get("lanes")) + if lerr: + return None, lerr + return {"targetTable": table, "fieldKey": fkey, + "urlField": _s(raw.get("urlField"), 80).strip(), + "tier": tier or "anonymous", + "noFallback": flag("noFallback"), + # ⛔ THE ONE FLAG THAT MULTIPLIES THE BILL BY THE POST COUNT. Off unless asked + # for: the Profiles row carries post IDENTITY for free but NO engagement + # (measured 2026-08-05), so likes/comments cost one extra vendor record PER POST. + "postMetrics": flag("postMetrics"), + "dryRun": flag("dryRun"), + "lanes": lanes, + "maxPosts": max(1, min(int(raw.get("maxPosts") or 24), 200))}, None + if kind == "discover_instagram": + limit = int(raw.get("recordsLimit") or 0) + if limit < 1: + return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the " + "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") + if limit > BD_MAX_RECORDS: + return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" + preds, perr = clean_predicates(raw.get("predicates")) + if perr: + return None, perr + if raw.get("lanes") if "lanes" in raw else prev.get("lanes"): + # R6: discovery's one path is the human gate — Review, then Tracked. A lane after + # the gate would be a second router over a set a person is already routing. + return None, ("a discovery automation routes through Review — its path is the " + "human gate; lanes belong on scrape and column automations") + target = _s(raw.get("targetTable"), 60).strip() or DISCOVER_TABLE + if not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + return {"recordsLimit": limit, "predicates": preds, + "operator": "or" if str(raw.get("operator") or "").lower() == "or" else "and", + "targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "IG candidates", + "dryRun": flag("dryRun")}, None + return None, f"unknown automation kind {kind!r}" + + +def clean_schedule(raw, previous=None): + """Validate `{cron, enabled}` and stamp `enabledAt` on the OFF→ON edge (the anchor `is_due` + measures from — without it, enabling a daily job fires it immediately).""" + raw = raw if isinstance(raw, dict) else {} + cron = _s(raw.get("cron"), 120).strip() or "0 6 * * *" + parse_cron(cron) # raises -> the route answers 400 + enabled = bool(raw.get("enabled")) + prev = previous or {} + out = {"cron": cron, "enabled": enabled} + if enabled: + out["enabledAt"] = (prev.get("enabledAt") if prev.get("enabled") else None) or _iso() + return out + + +# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── +# The owner's words: "we can make this automation a loop, so the ending should always be defined, +# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, +# or the user have to click a button to reset, or after a certain amount of time it can +# automatically reset to first cycle." +# +# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition +# that predates this field must not start moving records on its own the day the code ships. A +# loop is a thing somebody turns on. +ENDING_MODES = ("terminal", "auto_reset", "manual_reset", "timed_reset") +MAX_RESET_HOURS = 720 # 30 days; past that "it resets eventually" is a lie +#: The per-record cycle counter's cell suffix, beside `stage_` and `stage__at`. +CYCLES_SUFFIX = "_cycles" + + +def cycles_field_key(auto_id): + return stage_field_key(auto_id) + CYCLES_SUFFIX + + +def clean_ending(raw, previous=None): + """Validate `{mode, hours?}`. Returns `(ending, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + mode = _s(raw.get("mode") or prev.get("mode"), 20).strip() or "terminal" + if mode not in ENDING_MODES: + return None, f"{mode!r} is not one of: " + ", ".join(ENDING_MODES) + out = {"mode": mode} + if mode == "timed_reset": + try: + hours = int(raw.get("hours") if raw.get("hours") is not None else prev.get("hours") + or 24) + except (TypeError, ValueError): + return None, "the reset delay must be a whole number of hours" + if hours < 1 or hours > MAX_RESET_HOURS: + return None, f"the reset delay is between 1 and {MAX_RESET_HOURS} hours" + out["hours"] = hours + return out, None + + +def clean_flow(raw, previous=None): + """Validate `{actions, ending}` — the builder's half of a definition. `(flow, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions")) + if err: + return None, err + ending, err = clean_ending(raw.get("ending") if "ending" in raw else prev.get("ending"), + prev.get("ending")) + if err: + return None, err + return {"actions": actions, "ending": ending}, None + + +#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the +#: definition carries none. 10 is the size wave 21 proved live, at $0.15. +DISCOVER_SEED_RECORDS = 10 + + +def _seed_ig_action(flow_raw, table): + """C-TRIG LAW 3 — the follow-up action `ig_profile_match` seeds, ONCE, on the transition. + + ⛔ ONCE IS THE WHOLE DESIGN. Law 3 ends "deleting it is their call, not a refusal", so this + fires only on the EDGE into the trigger and only when the flow is empty. Seeding on every + clean would make the action un-deletable — it would reappear on the next Save — which is the + difference between a helpful default and a control that will not take no for an answer. + + The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner and no + card: the wave-23 header records four kinds that shipped with illegal seeds and did exactly + that. `create_record` needs a ut_-prefixed table and at least one value, so it gets both. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + if flow.get("actions"): + return flow_raw + flow["actions"] = [{ + "id": "act_1", "kind": "create_record", "enabled": True, "when": None, + # `config.label` follows the `review` action's precedent in `_clean_action_config` — + # an action naming itself, inside the untyped config bag, so no shared TS type changes. + "config": {"table": str(table or DISCOVER_TABLE), + "label": "Create an Instagram record", + "values": {"handle": "{{handle}}"}}, + }] + return flow + + +def clean_definition(raw, previous=None, username=""): + """Whole-definition validation. Returns `(defn, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous or {} + # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing + # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second + # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys + # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload + # invalid in BOTH its trigger and its config now answers with the trigger's sentence. + trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw + else prev.get("trigger"), prev.get("trigger")) + if terr: + return None, terr + # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this + # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a + # dead end the moment the wizard was deleted and the client's create body became {name}. + kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND + if (trigger or {}).get("key") == "ig_profile_match": + # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather + # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and + # whose kind says something else is not a preference to respect, it is two halves of one + # answer disagreeing, and the trigger is the half the person actually picked. + kind = "discover_instagram" + if kind not in KINDS: + return None, f"unknown automation kind {kind!r}" + name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) + if not name: + return None, "name the automation" + cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") + if kind == "discover_instagram" and prev.get("kind") != "discover_instagram": + # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very + # first Save after picking this trigger 400s — against the stored-inert-with- + # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). + # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard + # exists because an unbounded query is the one shape the vendor refuses outright, and + # coercing a blank into a number would be exactly the silent widening it protects against. + # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until + # a person presses Run now or arms the schedule. + cfg_raw = dict(cfg_raw or {}) + if not _ig_int(cfg_raw.get("recordsLimit")): + cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS + config, err = clean_config(kind, cfg_raw, prev.get("config")) + if err: + return None, err + if trigger: + # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own + # configuration IS the config, and `clean_trigger` could not see it. + trigger["configured"] = _trigger_configured(trigger, config) + try: + schedule = clean_schedule( + raw.get("schedule") if isinstance(raw.get("schedule"), dict) + else prev.get("schedule"), prev.get("schedule")) + except ValueError as e: + return None, str(e) + # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) + flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") + if (trigger or {}).get("key") == "ig_profile_match" \ + and (prev.get("trigger") or {}).get("key") != "ig_profile_match": + flow_raw = _seed_ig_action(flow_raw, config.get("targetTable") or DISCOVER_TABLE) + flow, ferr = clean_flow(flow_raw, prev.get("flow")) + if ferr: + return None, ferr + return { + "id": _s(prev.get("id") or raw.get("id"), 40), + "name": name, "kind": kind, "config": config, "schedule": schedule, + "trigger": trigger, + # WAVE 23 · C4/C5 — the BUILDER's half: the ordered actions a record walks and what + # happens when it reaches the end. Absent on every definition written before this wave, + # which is exactly why `clean_flow` defaults it to "no actions, terminal ending": the + # code shipping must not put existing automations into a loop. + "flow": flow, + "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, + "runs": list(prev.get("runs") or [])[:MAX_RUNS], + # C3-A2(5): the review-decision audit survives a Save the way `state` does — an + # ordinary patch must not amnesty every recorded approve/decline. + "reviews": list(prev.get("reviews") or [])[:100], + # ⚠ ENGINE-OWNED CONTINUATION STATE, and it is NOT config. A discovery snapshot takes ~20 + # minutes to build, so a run persists its id here and the NEXT run collects it. It is + # carried through `patch` untouched because a user pressing Save must not silently orphan + # a set the vendor is already building (and already counting against the funds gate). + "state": dict(prev.get("state") or {}), + "created": prev.get("created") or _iso(), + "createdBy": prev.get("createdBy") or username, + }, None + + +def set_state(rt, auto_id, patch_state): + """Merge into ONE automation's engine state. A key set to None is removed. + + Its own tiny store write rather than a field on `_commit_run`, because the handoff must + survive a run that ends in `error` — a snapshot the vendor is building does not stop existing + because the run that started it failed afterwards. + """ + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is None: + return cur + st = dict(d.get("state") or {}) + for k, v in (patch_state or {}).items(): + st.pop(k, None) if v is None else st.__setitem__(k, v) + d["state"] = st + return cur + + rt.update(STORE_KEY, _up, flush="sync") + + +def all_definitions(rt): + try: + return dict(rt.get(STORE_KEY) or {}) + except Exception: + return {} + + +def _new_id(existing): + n = 1 + while f"auto_{n}" in existing: + n += 1 + return f"auto_{n}" + + +#: C2's three answers to "which database does this automation work on?" — the FIRST question the +#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). +TARGET_MODES = ("existing", "new", "automated") + + +def resolve_target(rt, raw, username=""): + """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. + + - `existing` — bind a database that is already there. + - `new` — mint a blank one now, so the automation has somewhere to write before its first + run instead of conjuring a table the person never agreed to. + - `automated` — leave it to the runner: a scraping automation MINTS its target on first run + (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. + Nothing is created here, deliberately — an empty table created up front for a scrape that + never runs is litter nobody can explain. + """ + if not isinstance(raw, dict) or not raw: + return {}, None # no target block = the pre-wizard shape + mode = _s(raw.get("mode"), 20).strip() or "existing" + if mode not in TARGET_MODES: + return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) + if mode == "automated": + label = " ".join(_s(raw.get("label"), 60).split()) + return ({"targetLabel": label} if label else {}), None + if mode == "existing": + key = _s(raw.get("table"), 60).strip() + if not key: + return None, "choose the database this automation works on" + t = ut_get(rt, key) + if t is None: + return None, f"{key!r} is not a database in this workspace" + return {"targetTable": key, "targetLabel": t.get("label") or key}, None + label = " ".join(_s(raw.get("label"), 60).split()) + if not label: + return None, "name the new database" + import core.user_tables as _ut_new + key = _ut_new.create(label, username or "automation", st=rt) + if not key: + return None, ("the database could not be created — this workspace may be at its table " + "limit") + return {"targetTable": key, "targetLabel": label}, None + + +def _unmint(rt, key): + """Delete a database this call minted moments ago, because the call is refusing (D-50). + + ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user + needs to read; a rollback that raised would replace that sentence with a 500 and lose the + reason the create failed in the first place. The worst case of a swallowed failure here is + the orphan we had before — strictly no worse, and the refusal still reaches the person. + """ + if not key: + return + try: + import core.user_tables as _ut_del + _ut_del.delete(key, st=rt) + except Exception: # noqa: BLE001 + pass + + +def create(rt, raw, username=""): + existing = all_definitions(rt) + if len(existing) >= MAX_AUTOMATIONS: + return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" + raw = dict(raw or {}) + # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation + # happens after it, so a correct refusal ("a source URL is required") used to leave an + # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: + # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form + # twice silently accumulated empty databases in their nav. + # + # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It + # does not test for `mode == "new"`; it asks "did the table this call's target resolved to + # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and + # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target + # rather than "any table that appeared", so a concurrent create in another request is never + # collateral. + before = set(ut_all(rt) or {}) + patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) + if terr: + return None, terr + minted = str((patch_cfg or {}).get("targetTable") or "") + minted = minted if minted and minted not in before else "" + if patch_cfg: + raw["config"] = {**(raw.get("config") or {}), **patch_cfg} + defn, err = clean_definition(raw, None, username) + if err: + _unmint(rt, minted) + return None, err + # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations + # that already use them and NO NEW ONE CAN BE CREATED." + # + # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live + # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing + # the kind there would make both of them unsaveable — the ruling retires the door, not the + # automations behind it. Enforced on the server rather than left to the deleted wizard: a + # wall that holds only because the client stopped asking is a convention, not a wall. + if defn["kind"] in RETIRED_KINDS: + _unmint(rt, minted) + return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " + f"longer created — the ones already using it keep working. Start a new " + f"automation and pick its trigger instead") + defn["id"] = _new_id(existing) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur[defn["id"]] = defn + return cur + + rt.update(STORE_KEY, _up, flush="sync") + _stage_field_after_write(rt, defn, username) + if defn.get("trigger"): + _seed_event_state(rt, defn) + return defn, None + + +def patch(rt, auto_id, raw, username=""): + existing = all_definitions(rt) + prev = existing.get(str(auto_id)) + if prev is None: + return None, "no such automation" + merged = dict(prev) + # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a + # key missing here is accepted by the route, validated by `clean_definition`, and then + # DISCARDED — the client shows a saved flow that the store never received, and nothing goes + # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only + # ordering that never has a window where the bug exists. + for k in ("name", "config", "schedule", "kind", "trigger", "flow"): + if k in (raw or {}): + merged[k] = raw[k] + defn, err = clean_definition(merged, prev, username) + if err: + return None, err + defn["id"] = str(auto_id) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur[defn["id"]] = defn + return cur + + rt.update(STORE_KEY, _up, flush="sync") + _stage_field_after_write(rt, defn, username) + # A2(1): a NEW or CHANGED condition trigger starts with everything currently matching + # DISARMED — enabling never fires for records already in the state. + if (defn.get("trigger") or {}) != ((prev.get("trigger")) or {}): + _seed_event_state(rt, defn) + return defn, None + + +def _stage_field_after_write(rt, defn, username): + """C2's 'at automation create (or first bind)': when the target database already exists, + the stage field appears/refreshes with the definition write — a patch that adds a lane + grows the select's vocabulary in the same breath. A table that does not exist yet gets its + field from the first RUN instead (the runner ensures after `ut_ensure`). + + ⭐ WAVE 24 — resolved through `_flow_table` rather than reading `config.targetTable` directly, + because a `plain` automation is allowed to take its table from its TRIGGER (see `clean_config` + on why the target is optional). Reading the config alone left exactly those automations + without a stage field, so their board stayed permanently empty — `board()` answers + `cards: []` until the field is bound, which looks like "no records" rather than "no field". + """ + tgt = str(_flow_table(defn) or "") + if tgt and ut_get(rt, tgt) is not None: + ensure_stage_field(rt, tgt, defn, username) + + +def remove(rt, auto_id): + aid = str(auto_id) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur.pop(aid, None) + return cur + rt.update(STORE_KEY, _up, flush="sync") + # C8 — the fields this flow tagged are DISABLED with the reason on them, never orphaned + # silently: the column keeps its values and its config, and says why it stopped. Scanned + # first so a delete with no tagged fields costs no store commit. + hit = any(str((f.get("automation") or {}).get("flowId") or "") == aid + for t in ut_all(rt).values() for f in (t.get("fields") or [])) + if not hit: + return + + def _disable(cur): + cur = cur if isinstance(cur, dict) else {} + for t in cur.values(): + for f in (t.get("fields") or []): + a = f.get("automation") + if isinstance(a, dict) and str(a.get("flowId") or "") == aid: + a["disabled"] = True + a["statusNote"] = f"its automation was deleted {_stamp()}" + return cur + rt.update(UT_STORE_KEY, _disable, flush="sync") + + +# --------------------------------------------------------------------------------------------- +# RUN STATE — process memory only (see the module header for why this may never be persisted) +# --------------------------------------------------------------------------------------------- + +_RUN_LOCK = threading.RLock() +_RUNNING = {} # (tenant, id) -> {'startedAt', 'step'} + + +def running(tenant, auto_id): + with _RUN_LOCK: + return dict(_RUNNING.get((tenant, str(auto_id))) or {}) or None + + +def _claim(tenant, auto_id): + with _RUN_LOCK: + if (tenant, str(auto_id)) in _RUNNING: + return False + _RUNNING[(tenant, str(auto_id))] = {"startedAt": _iso(), "step": "starting"} + return True + + +def _step(tenant, auto_id, text): + with _RUN_LOCK: + cur = _RUNNING.get((tenant, str(auto_id))) + if cur is not None: + cur["step"] = text + + +def _no_step(_text): + """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner + directly). A runner must never REQUIRE the slot — the slot is process state, and the runner + is the part a test is allowed to call on its own.""" + + +def _release(tenant, auto_id): + with _RUN_LOCK: + _RUNNING.pop((tenant, str(auto_id)), None) + + +def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=None): + """THE ONE store write a run performs on the `automations` bucket. + + `steps` is the per-NODE outcome map the canvas paints its dots from. It is recorded here + because it is a MEASUREMENT the runner took as it walked — see `graph`, which refuses to + invent a node status when this map is absent (every run stored before W19-C has no `steps`, + and painting those nodes green from the run's overall state would be a fabrication). + """ + entry = {"ts": _iso(), "ok": bool(ok), "summary": _s(summary, 400), + "counts": {k: int(v) for k, v in (counts or {}).items() + if isinstance(v, (int, float))}, + "steps": {str(k): str(v) for k, v in (steps or {}).items()}, + "affected": list(affected or [])[:200]} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is None: + return cur + d["status"] = {"state": state, "lastRunAt": entry["ts"], + "lastSummary": entry["summary"]} + d["runs"] = ([entry] + list(d.get("runs") or []))[:MAX_RUNS] + # Wave 22 (airtable-brief rec 6): K consecutive FAILURES auto-pause the automation with + # the reason on it — a dead credential must not burn quota (or a paid vendor's records) + # 96 times a day while a green toggle sits over a buried error log. Errors only: + # `partial` is the honest-progress state and pausing on it would punish honesty. + runs = d.get("runs") or [] + if state == "error" and len(runs) >= CONSECUTIVE_FAILURE_PAUSE and all( + not r.get("ok") for r in runs[:CONSECUTIVE_FAILURE_PAUSE]): + sch = d.get("schedule") + if isinstance(sch, dict) and sch.get("enabled"): + sch["enabled"] = False + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " + f"failures — fix the cause, then re-enable: " + f"{entry['summary'][:120]}") + return cur + + rt.update(STORE_KEY, _up, flush="sync") + return entry + + +# --------------------------------------------------------------------------------------------- +# INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN +# --------------------------------------------------------------------------------------------- +# ⚠ AMENDED W19-C, vendor swapped W20. This section read "ANONYMOUS PUBLIC ENDPOINTS ONLY" and +# that is no longer the whole truth: R1 added a paid VENDOR rung (Bright Data, further down) +# which does authenticate — to the vendor. The rail that matters is unchanged and is sharper: +# +# **NOTHING HERE EVER AUTHENTICATES TO INSTAGRAM.** No login, no password, no session cookie, +# no account to get banned, public data only. The vendor key is a key to a SUPPLIER, and the +# supplier takes the scraping risk; it is never an Instagram credential and this module must +# never be given one. +# +# The rungs below this line are the ANONYMOUS ladder ($0, no key of any kind). +# +# ⚠ HONEST STATUSES ARE THE FEATURE. Instagram rate-limits and outright blocks datacenter egress +# (the HF Space and any AWS Lambda are both in that class), and its anonymous surface has been +# progressively closed for years. So this returns a STATE — ok / partial / blocked / error — and +# the cell says which. A pull that quietly wrote zero posts and reported success would be worse +# than no automation at all: the table would look maintained and be empty. +# +# THE LADDER, tried in order with pacing between rungs. Each rung records how it did in `via`, +# so a run history answers "what still works anonymously?" from data rather than from memory. + +PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) +IG_APP_ID = "936619743392459" # the public web client id the browser itself sends +IG_MAX_RETRIES = 2 + +#: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's +#: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number +#: is a NOW value, so the series starts the day capture starts and a field we do not capture +#: TODAY is permanently lost for today. Dropping a column because it was null on two probe rows +#: has exactly the same cost as delaying capture — for that column — and is justified by exactly +#: the evidence (n=2) that was judged too thin to close D-25. Adding a column that turns out to +#: be usually-empty costs a blank cell; omitting one costs the months before somebody notices. +#: ⛔ THE ONLY EXCLUSIONS ARE RULINGS, NOT TASTE: contact PII (D-24) and third-party comment text +#: (R11), both enforced in `BD_NEVER_MAP`. +#: ⚠ A blank in any of these means NOT READ — never "they have none". The anonymous rungs expose +#: almost none of them, which is what the `source` column is for. +SNAPSHOT_FIELDS = [ + field_def("snapshot_key", "Snapshot"), field_def("influencer_key", "Influencer"), + field_def("pulled_at", "Pulled at"), field_def("followers", "Followers"), + field_def("following", "Following"), field_def("posts_count", "Posts"), + field_def("full_name", "Name"), field_def("bio", "Bio"), + field_def("verified", "Verified"), field_def("source", "Read via"), + field_def("approx", "Counts are approximate"), + # The link in bio — the paid rung's field, and commercially the most useful single string an + # influencer row can carry (it is the shop/affiliate destination). + field_def("external_url", "Link in bio", "url"), + # --- the rest of the vendor's Profiles schema, minus the PII and the comment text. + field_def("ig_id", "Instagram id"), + field_def("profile_url", "Profile", "url"), + field_def("avg_engagement", "Avg engagement"), + field_def("category", "Category"), + field_def("business_category", "Business category"), + field_def("is_business", "Business account"), + field_def("is_professional", "Professional account"), + field_def("is_private", "Private"), + field_def("highlights_count", "Highlights"), + field_def("bio_hashtags", "Bio hashtags"), + field_def("post_hashtags", "Post hashtags"), + field_def("pronouns", "Pronouns"), +] +POST_FIELDS = [ + field_def("shortcode", "Shortcode"), field_def("influencer_key", "Influencer"), + field_def("posted_at", "Posted at"), field_def("type", "Type"), + field_def("caption", "Caption"), field_def("url", "URL", "url"), + # Sponsored-post detection — §2c called it one of the fields worth having that the anonymous + # ladder cannot reach, and it came back MEASURED-populated (`True`, with the brand attached). + field_def("paid_partnership", "Paid partnership"), + field_def("partner", "Partner brand"), + field_def("hashtags", "Hashtags"), + field_def("alt_text", "Alt text"), +] +POST_SNAPSHOT_FIELDS = [ + field_def("post_snapshot_key", "Snapshot"), field_def("shortcode", "Shortcode"), + field_def("pulled_at", "Pulled at"), field_def("likes", "Likes"), + field_def("comments", "Comments"), + # Plays/views move independently of likes on video, so it is its own series rather than a + # thing to derive. Paid rung only; blank means not read. + field_def("views", "Views"), +] + + +def ig_handle(url): + """The handle out of a profile URL (or a bare handle). '' when it is not one.""" + raw = str(url or "").strip() + if not raw: + return "" + if raw.startswith("@"): + return re.sub(r"[^A-Za-z0-9._]", "", raw[1:])[:40] + if "instagram.com" not in raw: + return re.sub(r"[^A-Za-z0-9._]", "", raw)[:40] if "/" not in raw else "" + path = urlparse(raw if "//" in raw else "https://" + raw).path.strip("/") + first = (path.split("/") or [""])[0] + if first in ("p", "reel", "reels", "explore", "accounts", "stories", ""): + return "" + return re.sub(r"[^A-Za-z0-9._]", "", first)[:40] + + +def _ig_get(url, headers=None, timeout=20.0): + """One anonymous GET with 429 backoff. Returns `(status, body_bytes, note)`.""" + delay = PACE_SECONDS + for attempt in range(IG_MAX_RETRIES + 1): + try: + status, _final, body = fetch(url, timeout=timeout, headers=headers, max_kb=4096) + except Refused: + raise + except Exception as e: # noqa: BLE001 + return 0, b"", f"{type(e).__name__}: {str(e)[:120]}" + if status == 429 and attempt < IG_MAX_RETRIES: + time.sleep(delay) + delay *= 2 # exponential, never a tight retry + continue + return status, body, "" + return 429, b"", "rate limited after retries" + + +def _ig_int(v): + try: + return int(v) + except Exception: + return None + + +def _profile_from_web_api(node, handle): + return { + "username": node.get("username") or handle, + "full_name": node.get("full_name") or "", + "bio": node.get("biography") or "", + "followers": _ig_int((node.get("edge_followed_by") or {}).get("count")), + "following": _ig_int((node.get("edge_follow") or {}).get("count")), + "posts_count": _ig_int((node.get("edge_owner_to_timeline_media") or {}).get("count")), + "verified": "1" if node.get("is_verified") else "", + } + + +def _posts_from_edges(edges): + out = [] + for e in edges or []: + n = (e or {}).get("node") or {} + code = n.get("shortcode") + if not code: + continue + caption = "" + for ce in ((n.get("edge_media_to_caption") or {}).get("edges") or []): + caption = ((ce or {}).get("node") or {}).get("text") or "" + break + ts = n.get("taken_at_timestamp") + out.append({ + "shortcode": code, + "posted_at": (_dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M") + if isinstance(ts, (int, float)) else ""), + "type": ("video" if n.get("is_video") else + "carousel" if n.get("__typename") == "GraphSidecar" else "image"), + "caption": caption, + "url": f"https://www.instagram.com/p/{code}/", + "likes": _ig_int((n.get("edge_liked_by") or n.get("edge_media_preview_like") + or {}).get("count")), + "comments": _ig_int((n.get("edge_media_to_comment") or {}).get("count")), + }) + return out + + +_OG_COUNTS = re.compile( + r"([\d.,KMkm]+)\s+Followers?,\s*([\d.,KMkm]+)\s+Following,\s*([\d.,KMkm]+)\s+Posts?") + + +def _loose_count(txt): + """'1.2M' / '12,345' -> int. IG's og:description abbreviates, so the snapshot is honest + about being approximate rather than pretending to a precision it does not have.""" + t = str(txt or "").strip().replace(",", "") + mult = 1 + if t[-1:].lower() == "k": + mult, t = 1_000, t[:-1] + elif t[-1:].lower() == "m": + mult, t = 1_000_000, t[:-1] + try: + return int(float(t) * mult) + except Exception: + return None + + +# --------------------------------------------------------------------------------------------- +# BRIGHT DATA — THE PAID RUNG (owner ruling R1 / wave-20 D-21, contract C4) +# --------------------------------------------------------------------------------------------- +# WHY A VENDOR AT ALL, stated once so nobody re-opens it: the anonymous ladder above is honest but +# it is ALSO geographically fenced — Instagram blocks datacenter egress, so from the HF Space or a +# Lambda the ladder degrades to `blocked` and the whole automation becomes a thing that only works +# on the owner's desk. The vendor takes the egress risk, returns EXACT counts (not +# og:description's "204K") and reaches post-level data the anonymous surface has closed. +# +# ⚠ WHY THIS VENDOR AND NOT THE PREVIOUS ONE. HikerAPI was crypto-only with no legal entity — +# no invoice, no counterparty, nothing that can go on a subprocessor list at the EU/enterprise +# gates. Bright Data is nameable, litigated Meta and won on logged-off public scraping, and runs +# KYC. That is a POSTURE purchase, not a feature one; the price is ~2.5× and it is the right +# trade. Full comparison: `.claude/wiki/research/instagram-capture.md` §2a. +# +# ⛔ THE FOUR RAILS TRANSFER UNCHANGED (contract C4 — they were written vendor-agnostic): +# 1. **FAIL-CLOSED on the key.** No `AIOS_BRIGHTDATA_KEY` ⇒ this rung refuses with an honest +# blocked ATTEMPT and the anonymous ladder still runs. Never a crash, never a silent skip. +# 2. **The key is never logged, never in a URL, never in an exception.** It travels in the +# `Authorization: Bearer` header only, and every error below quotes a STATUS, never the +# request. A secret that reaches a log line has leaked to everyone who can read logs. +# 3. **The base URL is guarded like any other.** `AIOS_BRIGHTDATA_BASE` is env-provided, so it +# goes through the same SSRF rail — an env var pointed at the metadata endpoint would +# otherwise be a credential-bearing request to it. +# 4. **A rung failure is a RUNG failure.** `bd_call` swallows its own `Refused`/transport +# errors, because `run_now` turns a `Refused` into a whole-run error — and a mistyped base +# URL must not kill a run whose anonymous rung would have answered. +# 5. **Exact ⇒ NO `approx` TAG.** That tag is the anonymous rung's rounding disclosure. +# +# ⚠ THE KEY IS READ LAZILY, INSIDE THE CALL. A module-level snapshot would be taken at import, +# which makes the "key absent ⇒ blocked, not crash" gate untestable (nothing could flip it between +# checks) and would also mean a deploy that sets the secret after import silently has no key. +# +# ⛔ TWO SNAPSHOT NAMESPACES THAT 404 EACH OTHER (measured; the trap that costs an hour): +# the SCRAPER `/datasets/v3/trigger` + `/v3/scrape` -> `sd_…` read at `/datasets/v3/…` +# the CORPUS `/datasets/filter` (⚠ NO `/v3/`) -> `snap_…` read at `/datasets/…` +# Cross them and you get a flat "404 Snapshot does not exist" about a snapshot that is alive. + +#: Pinned from live probes against `api.brightdata.com` (2026-08-04 and 2026-08-05) — see +#: `.claude/wiki/research/instagram-capture.md` §2c/§2d for the recipe and its provenance tags. +#: They live in ONE named block so a vendor rename is a one-line fix rather than a hunt. +BD_BASE_DEFAULT = "https://api.brightdata.com" +BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records +BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields +BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id +BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"} +BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /?format=json -> the rows +BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ — the CORPUS query (discovery) +BD_PATH_FILTER_SNAPSHOT = "/datasets/snapshot" # / -> status +#: A sync scrape MEASURED at 15–23 s for one URL and ~40 s for two, so the timeout is generous; +#: it is a ceiling against a hung socket, not a latency budget. +BD_TIMEOUT = 300.0 +#: The READ ceiling for one vendor answer, in KB. `fetch` truncates SILENTLY at this size, so it +#: is load-bearing arithmetic rather than a round number: MEASURED `file_size` was 175,617 bytes +#: for 5 corpus rows ⇒ **~35 KB per profile row**, and `BD_MAX_RECORDS` is set from it with +#: headroom (500 × 35 KB ≈ 17 MB against 32 MB). A truncated body is REPORTED as truncated in +#: `bd_call` — never allowed to surface as "that is not JSON". +BD_MAX_KB = 32768 +#: How long a run waits on a scrape batch the vendor DEFERRED (see `_bd_deferral`). Much shorter +#: than the corpus filter's budget because a scrape snapshot is minutes, not tens of minutes — +#: and unlike the corpus path there is no handoff to a later run, so this is the whole wait. +BD_SCRAPE_WAIT = float(os.environ.get("AIOS_BD_SCRAPE_WAIT") or 180) +BD_SCRAPE_POLL = float(os.environ.get("AIOS_BD_SCRAPE_POLL") or 10) +#: ⚠ SPEC, NOT MEASURED — off the pricing page, never returned by the API (the funds gate fires +#: BEFORE a price is quoted, and `price: 0` means "not priced", not "free"). Every surface that +#: shows a number derived from this MUST say it is an estimate. `/customer/balance` answers 403 +#: for our token, so there is no way to check a balance or a spend from here either. +BD_RECORD_PRICE_SPEC = 0.0025 + + +def bd_key(): + """The key, or ''. Read fresh every call — see the module note above on why.""" + return (os.environ.get("AIOS_BRIGHTDATA_KEY") or "").strip() + + +def bd_base(): + return (os.environ.get("AIOS_BRIGHTDATA_BASE") or BD_BASE_DEFAULT).strip().rstrip("/") + + +def bd_ready(): + """Is the paid rung configured at all? The UI reads this to say so honestly.""" + return bool(bd_key()) + + +def bd_call(path, params=None, body=None): + """One authenticated vendor call. GET when `body` is None, POST otherwise. + + Returns `(payload, note)`; a non-empty `note` means it did NOT answer, and the note is safe to + show a user and safe to write to a log. + + Never raises. Every refusal shape — no key, a guarded base, a 402, a 429, a transport error, + unparseable JSON — comes back as a note so the caller can record the attempt and drop to the + next rung. + """ + key = bd_key() + if not key: + # ⛔ THE FAIL-CLOSED PATH (C4). Named precisely, because "blocked" with no reason sends + # somebody to read Instagram's status page instead of setting an env var. + return None, "AIOS_BRIGHTDATA_KEY is not configured — the paid rung is closed" + qs = "&".join(f"{k}={requests.utils.quote(str(v))}" + for k, v in (params or {}).items() if v not in (None, "")) + url = f"{bd_base()}{path}" + (f"?{qs}" if qs else "") + hdrs = {"Authorization": f"Bearer {key}", "Accept": "application/json"} + try: + if body is None: + status, _final, raw = fetch(url, timeout=BD_TIMEOUT, max_kb=BD_MAX_KB, headers=hdrs) + else: + status, raw = fetch_json(url, body, timeout=BD_TIMEOUT, max_kb=BD_MAX_KB, + headers=hdrs) + except Refused as e: + # The BASE was refused by the SSRF rail — a configuration fault, not a vendor outage. + return None, f"the Bright Data base URL was refused by the URL rail: {e}" + except Exception as e: # noqa: BLE001 + # ⚠ `type(e).__name__` only. A requests exception's str() can carry the full URL, and the + # Authorization header is one refactor away from being a query param; this line must not + # be what leaks it. + return None, f"Bright Data did not answer ({type(e).__name__})" + if status == 402: + return None, "Bright Data answered 402 — the account is out of credit" + if status in (401, 403): + return None, f"Bright Data refused the key ({status})" + if status == 429: + # ⚠ MEASURED during the discovery probe: `too_many_parallel_jobs`. The filter API has a + # concurrency cap and the probe's most important test died on it — hence every caller + # below is SERIAL by construction. + return None, "Bright Data answered 429 — too many parallel jobs; the calls are serialised" + if not (200 <= status < 300): + return None, f"Bright Data answered {status}" + # ⛔ A BODY THAT REACHED THE READ CEILING IS A TRUNCATION, AND IT MUST SAY SO. `fetch` reads + # at most `max_kb` and returns what it got — silently. A truncated JSON document then fails + # `json.loads`, falls through to the NDJSON branch, and comes back as "answered with + # something that is not JSON": a sentence that sends somebody to read the vendor's schema + # looking for a fault that is OUR byte ceiling — after a result set that has already been + # paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow. + if len(raw) >= BD_MAX_KB * 1024: + return None, (f"Bright Data's answer exceeded this client's {BD_MAX_KB // 1024} MB read " + f"ceiling and was TRUNCATED — ask for fewer records; nothing was parsed " + f"from a partial document") + text = raw.decode("utf-8", "replace").strip() + if not text: + return None, f"Bright Data answered {status} with an empty body" + try: + return json.loads(text), "" + except Exception: # noqa: BLE001 + pass + # NDJSON is the other shape this API uses for row sets. Tried SECOND, so a genuine JSON error + # is not silently reinterpreted as a one-line NDJSON document. + rows = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except Exception: # noqa: BLE001 + return None, "Bright Data answered with something that is not JSON" + return (rows, "") if rows else (None, "Bright Data answered with something that is not JSON") + + +def _first(node, *names, default=None): + """The first of `names` present on `node` with a real value. + + ⚠ THE TOLERANCE IS THE POINT, not laziness. The vendor's `components/schemas` was not readable + on 2026-08-04 (recorded as RECALLED in the digest), so every media field name below is a + hypothesis. Reading through a candidate list means a rename costs one more candidate instead + of a rung that returns nothing and cannot say why. **A field that matches nothing returns + `default` (None), never 0** — an invented zero is a measurement, and this module's whole + honest-status contract exists to stop those. + """ + if not isinstance(node, dict): + return default + for n in names: + v = node.get(n) + if v not in (None, ""): + return v + return default + + +#: ⛔ FIELDS THAT ARE NEVER MAPPED, WHATEVER THE VENDOR SENDS. Two separate reasons, one list, +#: because the enforcement is the same: a field never collected needs no erasure path (D-24). +#: PII (the vendor's own `pii` flag): contact details are a different ORDER of privacy exposure +#: than a follower count, and `email_address` + `phone_number` change the lawful-basis analysis +#: materially. `business_email`'s own description says it "no longer exists without login!". +#: R11 (owner ruling, wave 20): NO Instagram comment-text capture. ⚠ This is not merely "do not +#: buy the Comments dataset" — MEASURED 2026-08-05, a POSTS row arrives with `latest_comments` +#: as an array of 15 comment texts WITH their authors, whether we asked or not. The ruling is +#: enforced HERE, at the map, because that is the only place it can be. +BD_NEVER_MAP = frozenset({ + "email_address", "phone_number", "business_email", "profile_image_link", + "latest_comments", "comment_user", "post_user", +}) + +#: The vendor's content vocabulary → ours. MEASURED values: profiles' `posts[].content_type` is +#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`. +_BD_TYPES = {"image": "image", "photo": "image", "video": "video", "reel": "video", + "reels": "video", "clips": "video", "igtv": "video", "carousel": "carousel", + "carousel_container": "carousel", "sidecar": "carousel", "posts": "image"} + + +def _bd_type(*raw): + """The first recognisable content type among `raw`, defaulting to `image`.""" + for v in raw: + hit = _BD_TYPES.get(str(v or "").strip().lower()) + if hit: + return hit + return "image" + + +def ig_shortcode(url): + """The shortcode out of a post/reel permalink. '' when it is not one. + + The `ut_ig_posts` identity key. The vendor gives a post URL (sometimes without its trailing + slash) and a numeric `pk`; the shortcode is what both datasets agree on and what a human can + paste into a browser, so it stays the key. + """ + raw = str(url or "").strip() + if not raw: + return "" + path = urlparse(raw if "//" in raw else "https://" + raw).path.strip("/").split("/") + if len(path) >= 2 and path[0] in ("p", "reel", "reels", "tv"): + return re.sub(r"[^A-Za-z0-9_-]", "", path[1])[:40] + return "" + + +def _bd_first_url(v): + """A link-in-bio value → one string. + + ⚠ MEASURED 2026-08-05: `external_url` is an **ARRAY** on the real row + (`["http://linktr.ee/utopianevents/"]`), not the string the previous vendor returned. A dict + (`external_url_title`) is also in the family. Read through the shape rather than assuming one. + """ + if isinstance(v, list): + v = next((x for x in v if x), "") + if isinstance(v, dict): + v = _first(v, "url", "link", default="") + return str(v or "") + + +def _bd_posts_count(node): + """`posts_count`, or None — **and a vendor ZERO IS DISCARDED.** + + ⛔ MEASURED 2026-08-05: the Profiles dataset returned `posts_count: 0` for BOTH probe accounts + — `inayma` (203,800 followers, 12 posts returned in the same row) and `utopian_events` (20,924 + followers, 12 posts). The POSTS dataset carries the true value for the same two accounts + (1,215 and 751), so the zero is not the account being empty; it is a measurement failure + wearing a number. Writing it would be the invented measurement this module's honest-status + contract exists to forbid — and a `_first`-style candidate list cannot help, because the + vendor DID answer, with the wrong answer. + + ⚠ THE ASYMMETRY IS DELIBERATE AND IT COSTS SOMETHING: an account that genuinely has zero posts + also reads blank. "Not read" is honest about an unknown; "0" is a claim, and on the only + evidence we have that claim is wrong 2 times out of 2. + ⛔ `len(posts)` IS NOT A SUBSTITUTE. The profile row carries the TOP 12 — a cap, not a count. + """ + v = _ig_int(_first(node, "posts_count", "media_count", "post_count")) + return None if v == 0 else v + + +def _bd_flag(node, *names): + """A vendor boolean → `'1'` / `''` / None. **None when the key is ABSENT or null**, which is + the distinction that matters: `is_business_account: false` is a measurement and `''` records + it; a missing key is not, and must not arrive as "false".""" + for n in names: + v = node.get(n) if isinstance(node, dict) else None + if v is None: + continue + return "1" if v is True or str(v).strip().lower() in ("true", "1", "yes") else "" + return None + + +def _bd_list(node, *names): + """An array field → a comma-joined string, or None when it is absent/empty.""" + for n in names: + v = node.get(n) if isinstance(node, dict) else None + if isinstance(v, list) and v: + return ", ".join(str(x) for x in v if x)[:400] + return None + + +def _bd_profile(node, handle): + """A Profiles row → our snapshot shape. Every name below is MEASURED off a real row + (2026-08-05, 35 keys); the candidates guard renames, not guesses. + + ⚠ Several of these were **null on both probe rows** (`avg_engagement`, `category_name`, + `business_category_name`, `is_professional_account`, `bio_hashtags`, `post_hashtags`). They + are mapped ANYWAY — see the note on `SNAPSHOT_FIELDS`: n=2 is not evidence enough to drop a + column permanently, and history is unbuyable, so a field omitted today is lost for today. + ⚠ They ARE populated on the CORPUS (`/datasets/filter`) rows, which is a different path with + a different population; that is a reason to capture them, not a licence to promise a + scrape-side feature before D-25 measures the fill rate. + """ + out = { + "username": str(_first(node, "account", "username", default=handle) or handle), + "full_name": str(_first(node, "full_name", "profile_name", default="") or ""), + "bio": str(_first(node, "biography", "bio", default="") or ""), + "followers": _ig_int(_first(node, "followers", "follower_count")), + "following": _ig_int(_first(node, "following", "following_count")), + "posts_count": _bd_posts_count(node), + "verified": "1" if _first(node, "is_verified", default=False) else "", + "external_url": _bd_first_url(_first(node, "external_url", "external_urls", + "external_url_title")), + # ⛔ NO `approx` KEY. These counts are exact, and carrying the anonymous rung's rounding + # tag onto them would be a lie in the other direction (R1: "drop the approx tag when + # exact"). Its ABSENCE is what tells the snapshot row it is trustworthy. + "ig_id": str(_first(node, "id", "fbid", default="") or ""), + "profile_url": str(_first(node, "profile_url", "url", default="") or ""), + "avg_engagement": _first(node, "avg_engagement"), + "category": str(_first(node, "category_name", default="") or ""), + "business_category": str(_first(node, "business_category_name", default="") or ""), + "is_business": _bd_flag(node, "is_business_account"), + "is_professional": _bd_flag(node, "is_professional_account"), + "is_private": _bd_flag(node, "is_private"), + "highlights_count": _ig_int(_first(node, "highlights_count")), + "bio_hashtags": _bd_list(node, "bio_hashtags"), + "post_hashtags": _bd_list(node, "post_hashtags"), + "pronouns": str(_first(node, "pronouns", default="") or ""), + } + # ⛔ THE PII RAIL, ENFORCED HERE RATHER THAN TRUSTED. Nothing above reads a `BD_NEVER_MAP` + # field, and this asserts it stays that way after the next edit — a map is exactly the kind + # of code somebody extends in a hurry. + return {k: v for k, v in out.items() if k not in BD_NEVER_MAP} + + +def _bd_post_identity(p): + """One entry of a Profiles row's `posts[]` → our post shape. None when it has no shortcode. + + ⚠ IDENTITY ONLY — AND THAT IS THE FINDING THIS WHOLE RUNG IS SHAPED AROUND. MEASURED + 2026-08-05 across 24 posts on two accounts: `caption`, `content_type`, `id`, `image_url`, + `url` are populated 24/24, and **`likes`, `comments` and `datetime` are `None` 24/24.** So the + profile scrape can fill `ut_ig_posts` and CANNOT fill `ut_ig_post_snapshots` — the engagement + time series §3's append law exists for needs the POSTS dataset, one record per post. That is + why `postMetrics` is a separate, opt-in, ~13×-cost rung rather than a free by-product. + ⚠ `is_pinned` was `True` on all 24 — a constant, not a measurement. Not mapped. + ⚠ Blank-valued keys are OMITTED rather than written, so a later engagement pull that DOES know + `posted_at` fills it in instead of being overwritten by this rung's silence. + """ + code = ig_shortcode(p.get("url")) if isinstance(p, dict) else "" + if not code: + return None + out = {"shortcode": code, "url": f"https://www.instagram.com/p/{code}/", + "type": _bd_type(p.get("content_type"), p.get("type"))} + caption = _first(p, "caption", "description", default="") + if isinstance(caption, dict): + caption = _first(caption, "text", default="") + if caption: + out["caption"] = str(caption) + return out + + +def _bd_post_metrics(row): + """One POSTS-dataset row → the engagement half. None when it has no shortcode. + + MEASURED 2026-08-05 on three real post rows: `likes` (308 / 3,736), `num_comments` (37 / 134) + and `date_posted` (`2026-05-04T17:51:58.000Z`) all populated. + ⚠ NO VIEW COUNT CAME BACK — `video_play_count` / `video_view_count` are declared in the schema + and the KEY WAS ABSENT from every row, including a Reel. So `views` reads None and the column + stays blank, which is the honest answer; it is not folded into likes and it is not zeroed. + ⚠ The candidates below are all COUNTS OF VIEWS in different vendor typings (`video_view_count` + is declared `text`, the others `number`). A candidate list may span types only when the + MEANING is identical — never the way `like_and_view_counts_disabled`, a boolean, would have + turned "likes are hidden" into "1 like". + ⛔ `latest_comments` IS PRESENT ON THESE ROWS AND IS NEVER READ (R11 / `BD_NEVER_MAP`). + """ + if not isinstance(row, dict): + return None + code = str(_first(row, "shortcode", "content_id", default="") or "") or \ + ig_shortcode(row.get("url")) + if not code: + return None + posted = str(_first(row, "date_posted", "taken_at", default="") or "") + partner = row.get("partnership_details") + out = { + "shortcode": code, + "url": str(_first(row, "url", default=f"https://www.instagram.com/p/{code}/")), + "influencer_key": str(_first(row, "user_posted", default="") or ""), + "posted_at": posted.replace("T", " ")[:16], + "type": _bd_type(row.get("content_type"), row.get("product_type")), + "caption": str(_first(row, "description", "caption", default="") or ""), + "likes": _ig_int(_first(row, "likes", "like_count")), + "comments": _ig_int(_first(row, "num_comments", "comment_count", "comments")), + "views": _ig_int(_first(row, "video_play_count", "video_view_count", + "engagement_score_view", "play_count")), + # Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) — and + # it is the one field here that answers a commercial question the counts cannot. + "paid_partnership": _bd_flag(row, "is_paid_partnership"), + "partner": (str(_first(partner, "username", "profile_id", default="") or "") + if isinstance(partner, dict) else ""), + "hashtags": _bd_list(row, "hashtags"), + "alt_text": str(_first(row, "alt_text", default="") or ""), + } + # ⛔ SAME RAIL AS THE PROFILE MAP — and it matters MORE here, because a Posts row genuinely + # arrives carrying `latest_comments`: fifteen third-party comment texts with their authors, + # sent whether we asked or not (R11). + return {k: v for k, v in out.items() if k not in BD_NEVER_MAP} + + +def _bd_rows(payload): + """A vendor answer → the list of row dicts inside it, whatever envelope it arrived in. + + MEASURED shapes: a bare list (`/v3/scrape`), NDJSON (already listified by `bd_call`), and + `{"data": [...]}` / `{"results": [...]}` on the snapshot reads. An unreadable envelope yields + NOTHING rather than a guess. + """ + if isinstance(payload, list): + return [x for x in payload if isinstance(x, dict)] + if isinstance(payload, dict): + for k in ("data", "results", "records", "items"): + v = payload.get(k) + if isinstance(v, list): + return [x for x in v if isinstance(x, dict)] + return [payload] if payload else [] + return [] + + +def _bd_deferral(payload): + """The snapshot id when the 'sync' endpoint DEFERRED, else ''. + + ⛔ MEASURED 2026-08-05, AND IT IS THE TRAP OF THIS WHOLE RUNG. `/v3/scrape` is documented and + named as the synchronous call, and for a small batch it is — two profile URLs came back + inline. Give it more work and it answers **200** with no rows and a note instead: + + {"snapshot_id": "sd_…", "message": "Your request is still in progress and cannot be + retrieved in this call. Use the provided Snapshot ID to track progress…"} + + Read naively that is a 200 with a body, so a parser looking for "did it error?" sails past + it, `_bd_rows` wraps the envelope as ONE unusable row, and the caller reports that the vendor + returned nothing — **while a snapshot we have already been billed for sits on the account, + finishing, and is never collected.** That is the D-26 failure shape (paid work abandoned) + arriving through the success path. + ⚠ It is a TIME threshold, not a row count: 2 URLs answered inline, 6 and 10 deferred. + """ + if isinstance(payload, dict) and payload.get("snapshot_id") and not payload.get("account"): + return str(payload["snapshot_id"]) + return "" + + +def bd_scrape(dataset_id, urls, wait=None): + """Scrape a batch. `(rows, note)`; a note means it did not answer. + + Sends the SYNC call and, when the vendor defers it (see `_bd_deferral`), **collects the + snapshot it handed back** rather than discarding it. One call carries many URLs, which is + what keeps the per-profile pacing floor from turning a 20-profile run into an hour. + """ + payload, note = bd_call(BD_PATH_SCRAPE, {"dataset_id": dataset_id}, + body=[{"url": u} for u in urls if u]) + if note: + return [], note + sid = _bd_deferral(payload) + if not sid: + return _bd_rows(payload), "" + + # --- THE DEFERRED PATH. Poll the scraper namespace (`sd_…` at `/datasets/v3/…` — crossing it + # with the corpus namespace is a flat 404 about a snapshot that is alive). + budget = BD_SCRAPE_WAIT if wait is None else float(wait) + waited = 0.0 + while True: + got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) + if not gnote: + rows = _bd_rows(got) + # The status document is itself JSON, so "did it deliver?" is decided by whether + # what came back looks like ROWS — never by the HTTP code. + if rows and not _bd_deferral(got) and not ( + len(rows) == 1 and str(rows[0].get("status") or "") in + ("running", "building", "collecting")): + return rows, "" + if waited >= budget: + break + time.sleep(BD_SCRAPE_POLL) + waited += BD_SCRAPE_POLL + # ⛔ NAME THE SNAPSHOT. It has been paid for and it is still finishing; a note that said only + # "no rows" would throw away both the data and the way to get it. + return [], (f"Bright Data deferred this batch to snapshot {sid} and it was not ready within " + f"{int(budget)}s — the records are collected, not lost; ask for a smaller batch " + f"or collect {sid} from the control panel") + + +def pull_profile_bd(url, max_posts=24, post_metrics=False, log=print): + """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides. + + Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner + treats every rung identically. `blocked` here means the VENDOR rung did not answer — the + caller decides whether to drop to the anonymous ladder. + + `post_metrics=True` adds the second, PAID-PER-POST call against the Posts dataset, which is + the ONLY source of likes/comments (see `_bd_post_identity`). It is off by default because it + multiplies the bill by roughly the post count. + """ + handle = ig_handle(url) + if not handle: + return {"state": "error", "note": f"{url!r} is not an Instagram profile URL", + "profile": {}, "posts": [], "via": ""} + rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"]) + if note: + return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", + "note": note} + node = rows[0] if rows else {} + profile = _bd_profile(node, handle) + if profile.get("followers") is None and profile.get("following") is None: + # It answered 200 with something we could not read. Say THAT — not "0 followers". + # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and + # is discarded, so a row where it is the only "readable" field is a row we cannot read. + return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", + "note": "Bright Data answered, but no follower/following counts were readable in " + "it (the field names may have moved — see instagram-capture.md)"} + posts = [] + for p in (node.get("posts") or [])[:max_posts]: + got = _bd_post_identity(p) + if got: + posts.append(got) + + if post_metrics and posts: + time.sleep(min(PACE_SECONDS, 1.0)) # the vendor is paid, but it is still someone's API + rows2, note2 = bd_scrape(BD_DS_POSTS, [p["url"] for p in posts]) + if note2: + # The engagement half refusing does NOT lose the identity half — the profile and the + # post rows still land, and the run says which part was not readable. + note = f"post metrics unavailable: {note2}" + else: + by_code = {} + for r in rows2: + got = _bd_post_metrics(r) + if got: + by_code[got["shortcode"]] = got + for p in posts: + extra = by_code.get(p["shortcode"]) + if extra: + # The metrics row is RICHER (it knows posted_at); merge it over the identity + # row, dropping keys it could not read so a blank never overwrites a value. + p.update({k: v for k, v in extra.items() if v not in (None, "")}) + if not by_code: + note = "post metrics were requested but the Posts dataset returned no rows" + + # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften + # because we are paying: a run that wrote a follower count and no posts must not paint green + # over a posts table that did not grow. + return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts, + "via": "brightdata", + "note": note or ("" if posts else "profile read; Bright Data returned no posts")} + + +# --------------------------------------------------------------------------------------------- +# DISCOVERY — sourcing handles we do NOT already know (owner ruling R7, DEBT D-23) +# --------------------------------------------------------------------------------------------- +# The engine until now only ENRICHED a profile set somebody typed in. This queries the vendor's +# **620,000,000-record pre-collected Profiles corpus** with a real query language and returns +# handles nobody here has ever seen. MEASURED end-to-end 2026-08-04: `followers 10k–200k AND +# biography includes "floral"` returned five real profiles, one of them a verified 20.8k-follower +# Georgia/Florida floral-design company — Fisch Floral's actual market. +# +# ⛔ FIVE PROPERTIES, EACH OF THEM A MEASURED FAILURE MODE RATHER THAN A PREFERENCE: +# +# 1. **A DIFFERENT ROUTE AND A DIFFERENT NAMESPACE.** `POST /datasets/filter` — ⚠ with NO `/v3/` +# segment; `/datasets/v3/filter` is a 404 that once got written down as "the corpus is +# unreachable". Its snapshots are `snap_…` and are read at `/datasets/snapshot/…`; the +# scraper's are `sd_…` at `/datasets/v3/snapshot/…`, and the two 404 each other. +# 2. **`records_limit` IS REQUIRED.** Unbounded queries die `NOT_ENOUGH_FUNDS` (code 104, +# `per_set` billing). ⚠ And bounding is NOT sufficient: **scan time tracks PREDICATE BREADTH, +# not row count** — a `records_limit:10` query on a bare `followers > 10000` was still +# `building` 50 minutes later. So the predicate is capped too, and the runner tolerates a +# snapshot that is not ready when it looks. +# 3. **IT IS SLOW, AND THE RUN HANDS OFF RATHER THAN HOLDING A THREAD.** MEASURED delivery +# latency on the narrow query: **19.6 minutes** (`created` 12:38:00 → `delivery_time` +# 12:57:42). Blocking a worker thread for twenty minutes on a free-tier container to poll is +# the wrong shape, so a run polls for a short budget and then PERSISTS the snapshot id; the +# next run collects it. A `partial` that says "still building, the next run collects it" is +# the honest report of exactly what happened. +# 4. **PROMOTION IS MANUAL, ALWAYS (R7).** The measured result set was ~40% on-target and +# included a hashtag-aggregator account that is not a person. Auto-promoting a candidate into +# the tracked profile set would silently multiply the enrichment bill AND pollute the +# snapshot series with rows nobody chose. The `tracked` column is NEVER written here. +# 5. **SERIAL.** `429 too_many_parallel_jobs` is real and the original probe's most important +# test died on it. +# +# ⚠ THE PRICE IS NOT MEASURED AND MAY NOT BE PRESENTED AS IF IT WERE. The funds gate fires BEFORE +# the API returns a number, `price: 0` means "not priced" rather than "free", and +# `/customer/balance` answers 403 for our token — so there is no balance to read and no spend to +# report. Every estimate below is derived from the PRICING PAGE and says so. + +#: The table a discovery run writes. Upsert by handle, so re-finding a profile is free. +DISCOVER_TABLE = "ut_ig_candidates" +#: A hard ceiling on one run's ask — and it is DERIVED, not chosen. A corpus row MEASURED at +#: ~35 KB (`file_size` 175,617 for 5 rows), so 1,000 records ≈ 33 MB would have crossed +#: `BD_MAX_KB` and come back as a truncated document **after being billed**. 500 leaves real +#: headroom. ⚠ If the row size grows, this number is the thing to re-derive. +BD_MAX_RECORDS = 500 +#: How long ONE run waits on a building snapshot before handing off to the next run (see 3 above). +BD_FILTER_WAIT = float(os.environ.get("AIOS_BD_FILTER_WAIT") or 120) +BD_FILTER_POLL = float(os.environ.get("AIOS_BD_FILTER_POLL") or 15) + +#: The operator set, enumerated BY REJECTION — send a bogus one and the API's own validation error +#: lists the legal set. The cheapest kind of measurement, and it makes this list a fact. +BD_FILTER_OPS = ("=", "!=", "<", "<=", ">", ">=", "in", "not_in", "includes", "not_includes", + "array_includes", "not_array_includes", "is_null", "is_not_null") +#: Operators that take NO value (everything else requires one). +BD_NULLARY_OPS = ("is_null", "is_not_null") + +#: Fields a predicate may name. MEASURED-accepted (24 of 25 probed; **`country_code` is REJECTED** +#: by the API and is therefore absent rather than offered-and-broken). +#: ⛔ `email_address`, `phone_number` and `business_email` ARE filterable and are DELIBERATELY +#: MISSING. Selecting people BY CONTACT DETAIL across 620M records is the materially heavier +#: privacy posture D-24 flags — a different licence question from per-URL enrichment, and one the +#: owner has not been asked. A vocabulary that cannot express it cannot be asked for it by +#: accident; adding them back is a decision, not a typo. +BD_FILTER_FIELDS = ("followers", "following", "posts_count", "avg_engagement", "biography", + "category_name", "business_category_name", "is_business_account", + "is_professional_account", "is_verified", "account", "full_name", + "external_url", "bio_hashtags", "post_hashtags", "profile_url", + "related_accounts", "profile_name", "id", "fbid", "highlights_count") +#: ⭐ The three the FIND SURFACE leads with, and the reason is measurement rather than taste: +#: these are the fields seen carrying values on real CORPUS rows. `category_name` and +#: `related_accounts` are filterable and were null/empty on every row we have looked at — a filter +#: on an unpopulated field returns nothing and looks exactly like "no such influencers exist". +BD_FILTER_LEAD = ("followers", "biography", "avg_engagement") + +#: ⛔ C4 (wave 22) — THE TOO-BIG GUARD, and its unit is the PREDICATE, not the row count. +#: MEASURED (2026-08-04): `records_limit: 10` over a bare `followers > 10000` was still +#: `building` 50 minutes later — scan time tracks PREDICATE BREADTH; bounding the rows does not +#: bound the scan. What actually narrows a 620M-row corpus is CONTENT: a text/array match on a +#: field that discriminates. Numeric ranges and booleans partition the corpus into slabs the +#: scanner still has to walk, so they refine a search and cannot BE one. +BD_NARROWING_FIELDS = frozenset({ + "biography", "account", "full_name", "profile_name", "category_name", + "business_category_name", "external_url", "bio_hashtags", "post_hashtags", + "related_accounts", "id", "fbid", "profile_url"}) +#: The ops that actually pin content — `!=`/`not_includes` on a text field matches nearly the +#: whole corpus, which is breadth wearing a condition's clothes. +BD_NARROWING_OPS = frozenset({"=", "in", "includes", "array_includes"}) +#: Fields MEASURED carrying values on real corpus rows — D-25's fill-rate table, 50 mixed +#: profiles (snapshot `snap_msfrlbmt7qvej8uby`, collected 2026-08-05). The bar is ≥60% +#: populated: below it a filter misses more corpus than it matches, and the surface should +#: say so. MEASURED AND EXCLUDED: category_name 48%, business_category_name 34%, +#: related_accounts 22%, post_hashtags 18%, bio_hashtags 10% — and the PII trio the map never +#: carries anyway read 2%/0%/0%, so even the thing R3 forbids would barely have worked. +#: ⚠ posts_count was 100% populated on CORPUS rows — the fabricated-zero finding +#: (`_bd_posts_count`) is a SCRAPE-path fact and both stay true. +BD_POPULATED_FIELDS = frozenset({ + "account", "followers", "following", "posts_count", "avg_engagement", "biography", + "external_url", "is_business_account", "is_professional_account", "is_verified", + "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) +BD_MIN_NARROWING = 1 + + +def predicate_narrows(p): + """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" + return (str((p or {}).get("name") or "") in BD_NARROWING_FIELDS + and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) + + +def narrowing_refusal(preds): + """C4's server law: the sentence when a predicate set has ZERO narrowing predicates, else + ''. One function so create/patch (via `clean_predicates`) and RUN (legacy stored configs + predate the law) refuse in the same words.""" + if any(predicate_narrows(p) for p in preds or []): + return "" + return ("add at least one CONTENT condition (biography includes …, account =, a hashtag) — " + "number and yes/no filters alone make the vendor scan the whole 620-million-row " + "corpus, and breadth is what costs: the measured `followers > 10000` search was " + "still building 50 minutes in") + + +def filter_meta(): + """C4's per-field flags for the Find surface: `[{name, populated, narrowing}]` over the + same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, R3).""" + return [{"name": n, "populated": n in BD_POPULATED_FIELDS, + "narrowing": n in BD_NARROWING_FIELDS} for n in BD_FILTER_FIELDS] + +CANDIDATE_FIELDS = [ + field_def("handle", "Handle"), field_def("profile_url", "Profile", "url"), + field_def("full_name", "Name"), field_def("followers", "Followers"), + field_def("following", "Following"), field_def("avg_engagement", "Avg engagement"), + field_def("bio", "Bio"), field_def("external_url", "Link in bio", "url"), + field_def("verified", "Verified"), field_def("category", "Category"), + field_def("found_count", "Times found"), field_def("first_found", "First found"), + field_def("last_found", "Last found"), + # C6 (wave 22): the candidate pool is PER-USER — the upsert key is the COMPOUND + # (handle, created_by), so two people discovering the same profile each get their own row, + # their own found_count and their own review card. Stamped by the RUNNER (the automation's + # creator), never by `_candidate_row`. + field_def("created_by", "Found by"), + # ⛔ WRITTEN BY A HUMAN, NEVER BY THIS ENGINE (R7). It is declared here so the column exists + # and can be ticked; `_candidate_row` never emits the key, and `upsert_rows` only writes the + # keys it is given — so a re-find can never un-tick somebody's decision either. + field_def("tracked", "Tracked", "checkbox"), +] + + +def clean_predicates(raw): + """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. + + Refuses rather than coerces: a predicate naming a field the API rejects comes back as a + 400 with the field named, because the alternative — dropping it — turns "find me verified + accounts in Georgia" into "find me any account" and bills for the difference. + """ + out = [] + for p in (raw or [])[:12]: + if not isinstance(p, dict): + continue + name = _s(p.get("name") or p.get("field"), 60).strip() + op = _s(p.get("operator") or p.get("op"), 20).strip() + if name not in BD_FILTER_FIELDS: + return None, (f"{name or 'that field'!r} cannot be used to search — the searchable " + f"fields are: {', '.join(BD_FILTER_FIELDS)}") + if op not in BD_FILTER_OPS: + return None, f"{op or 'that comparison'!r} is not one of: {', '.join(BD_FILTER_OPS)}" + entry = {"name": name, "operator": op} + if op not in BD_NULLARY_OPS: + v = p.get("value") + if v is None or (isinstance(v, str) and not v.strip()): + return None, f"give a value to compare {name!r} against" + entry["value"] = v.strip() if isinstance(v, str) else v + out.append(entry) + if not out: + # ⭐ WAVE 24 · AMENDMENT A2 — AN EMPTY FILTER IS INCOMPLETE, NOT WRONG, so it STORES. + # This used to refuse, which was right while a wizard collected the filters before the + # automation existed. C-TRIG law 1 inverts that: picking the `ig_profile_match` trigger + # CREATES the automation, and the filters are typed afterwards — so a save-time refusal + # here meant the trigger could never be picked at all. It is the A3 + # stored-inert-with-`configured:false` pattern this module already runs on. + # + # ⛔ NOTHING IS LOST AT THE MONEY DOOR, which is why this is safe rather than convenient: + # `run_discover_instagram` ALREADY calls `narrowing_refusal(preds)` before starting a + # search, deliberately, because "a stored config can predate the law, and the vendor + # bills for breadth whether the filter was saved yesterday or last month". The RUN is the + # wall. Every MALFORMED predicate above is still refused here, and the narrowing law + # below still refuses a non-empty filter that narrows nothing — only EMPTY changed. + return [], None + # C4 (wave 22): breadth is refused at WRITE time too — see `narrowing_refusal` for the + # measured hang this guards against. Same sentence at create/patch and at run. + guard_err = narrowing_refusal(out) + if guard_err: + return None, guard_err + return out, None + + +def discover_estimate(records_limit): + """The cost preview a Find node shows BEFORE it runs. **SPEC, never measured** — see the + section header. Returned as structured data so the UI cannot accidentally drop the caveat.""" + n = max(0, int(records_limit or 0)) + return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), + "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", + "note": "estimated from Bright Data's published dataset rate — the API never returns " + "a price before a run, and this account's token cannot read a balance"} + + +def bd_filter_start(predicates, operator="and", records_limit=5, dataset_id=BD_DS_PROFILES): + """Start ONE corpus query. Returns `(snapshot_id, note)`. + + ⚠ ALWAYS THE COMPOUND `{operator, filters:[…]}` SHAPE, even for a single condition. A bare + `{name, operator, value}` is a plausible-looking second shape that has never been sent from + here, and a one-element `filters` list is a trivial extension of the three-element body that + HAS been measured end-to-end. One shape, and it is the tested one. + """ + body = {"dataset_id": dataset_id, "records_limit": int(records_limit), + "filter": {"operator": operator, "filters": list(predicates)}} + payload, note = bd_call(BD_PATH_FILTER, body=body) + if note: + return "", note + sid = str((payload or {}).get("snapshot_id") or "") if isinstance(payload, dict) else "" + if not sid: + return "", "Bright Data accepted the search but named no snapshot to collect it from" + return sid, "" + + +def bd_filter_status(snapshot_id): + """`(status, size, note)` for a corpus snapshot. `status` ∈ building | ready | failed | ''.""" + payload, note = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snapshot_id}") + if note: + return "", 0, note + d = payload if isinstance(payload, dict) else {} + status = str(d.get("status") or "") + err = str(d.get("error") or "") + if status == "failed" and err: + # NOT_ENOUGH_FUNDS is the one everybody will hit; say what it means, not just its name. + extra = (" — the matched set is too large for this account's balance; narrow the " + "conditions or lower the limit" if "FUNDS" in err.upper() else "") + return status, 0, f"the search failed: {err}{extra}" + return status, int(d.get("dataset_size") or 0), "" + + +def bd_filter_rows(snapshot_id): + """The rows of a READY corpus snapshot. `(rows, note)`. + + ⚠ `/download` answered *"Snapshot is building. Try again in a few minutes"* while the status + endpoint ALREADY said `ready` — delivery lags readiness by minutes. So a download that does + not parse as rows is reported as not-yet-collectable rather than as an error. + """ + payload, note = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snapshot_id}/download", + {"format": "json"}) + if note: + return [], note + rows = _bd_rows(payload) + if not rows: + return [], "the search is ready but its rows are not downloadable yet — the vendor's " \ + "delivery lags its own `ready` status by minutes" + return rows, "" + + +def _candidate_row(row, stamp): + """One corpus row → a `ut_ig_candidates` row. None when it has no handle. + + ⛔ NEVER EMITS `tracked` OR `found_count`. `tracked` is the human's decision (R7) and + `found_count` is arithmetic the runner does against what is already stored — writing either + from here would overwrite a person's choice or reset a counter to 1 on every re-find. + """ + if not isinstance(row, dict): + return None + handle = str(_first(row, "account", "username", default="") or "").strip() + if not handle: + return None + return { + "handle": handle, + "profile_url": str(_first(row, "profile_url", "url", + default=f"https://www.instagram.com/{handle}/")), + "full_name": str(_first(row, "full_name", "profile_name", default="") or ""), + "followers": _s(_ig_int(_first(row, "followers"))), + "following": _s(_ig_int(_first(row, "following"))), + # ⭐ POPULATED ON CORPUS ROWS and null on scrape rows — the two paths differ, and this is + # the path where it carries values (measured on all five of the 2026-08-04 result set). + "avg_engagement": _s(_first(row, "avg_engagement", default="")), + "bio": _s(_first(row, "biography", "bio", default=""), 500), + "external_url": _s(_bd_first_url(_first(row, "external_url", "external_urls")), 300), + "verified": "1" if _first(row, "is_verified", default=False) else "", + "category": _s(_first(row, "category_name", "business_category_name", default="")), + "last_found": stamp, + } + + +def pull_profile(url, max_posts=24, log=print, tier="anonymous", fallback=True, + post_metrics=False): + """Everything readable about one public profile, cheapest-honest rung last. + + Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error. + `partial` means the identity was read but the media was not — which is the outcome the + anonymous surface most often gives, and saying so is the point. + + **RUNG 0 IS THE PAID ONE (R1's hybrid).** With `tier="brightdata"` the vendor is tried FIRST + and, when it answers, this returns exact counts with no `approx` tag. When it does not answer + the attempt is RECORDED and the anonymous ladder runs underneath it — that is what + "fail-closed on the key, and the anonymous rung still works" means in C4. + + `fallback=False` turns that safety net off on purpose: it means *"exact numbers or nothing"*. + An influencer table where some rows are exact and some are `og:description` roundings, with + no way to tell them apart at a glance, is a table you cannot do arithmetic on — so a user who + needs the guarantee can have it, and gets an honest `blocked` instead of a quiet downgrade. + (The `source`/`approx` columns tell them apart anyway; this makes it impossible to forget.) + """ + handle = ig_handle(url) + if not handle: + return {"state": "error", "note": f"{url!r} is not an Instagram profile URL", + "profile": {}, "posts": [], "via": ""} + attempts = [] + + if clean_tier(tier) == "brightdata": + paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics, log=log) + if paid["state"] in ("ok", "partial"): + return paid + attempts.append(f"brightdata:{_s(paid.get('note'), 90)}") + if not fallback: + return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", + "note": "the paid rung did not answer and the anonymous fallback is turned " + f"off for this automation ({paid.get('note') or 'blocked'})"} + time.sleep(PACE_SECONDS) + + # RUNG 1 — the web client's own profile endpoint. Public, no cookie; the one that used to + # carry both the profile AND the first page of media in a single answer. + api = f"https://www.instagram.com/api/v1/users/web_profile_info/?username={handle}" + status, body, note = _ig_get(api, headers={ + "X-IG-App-ID": IG_APP_ID, "Accept": "*/*", + "Referer": f"https://www.instagram.com/{handle}/"}) + attempts.append(f"web_profile_info:{status or note}") + if 200 <= status < 300 and body: + try: + node = ((json.loads(body.decode("utf-8", "replace")) or {}) + .get("data") or {}).get("user") or {} + except Exception: + node = {} + if node: + profile = _profile_from_web_api(node, handle) + media = node.get("edge_owner_to_timeline_media") or {} + posts = _posts_from_edges(media.get("edges"))[:max_posts] + page = media.get("page_info") or {} + if page.get("has_next_page") and page.get("end_cursor") and len(posts) < max_posts: + more, pnote = _paginate(node.get("id"), page.get("end_cursor"), + max_posts - len(posts), log) + posts.extend(more) + if pnote: + attempts.append(pnote) + state = "ok" if posts else "partial" + return {"state": state, "profile": profile, "posts": posts, + "via": "web_profile_info", + "note": "" if posts else "profile read; media edges empty (" + + ", ".join(attempts) + ")"} + + time.sleep(PACE_SECONDS) + + # RUNG 2 — the profile HTML. Its og:description carries the three counts even when every + # JSON surface refuses, so it is the honest floor: identity without media. + status, body, note = _ig_get(f"https://www.instagram.com/{handle}/") + attempts.append(f"html:{status or note}") + if 200 <= status < 300 and body: + text = body.decode("utf-8", "replace") + if "/accounts/login" in text[:4000] and "og:description" not in text: + return {"state": "blocked", "profile": {}, "posts": [], "via": "html", + "note": "Instagram served the login wall (" + ", ".join(attempts) + ")"} + desc = "" + m = re.search(r']+property="og:description"[^>]+content="([^"]*)"', text) + if m: + desc = m.group(1) + counts = _OG_COUNTS.search(desc or "") + title = "" + mt = re.search(r']+property="og:title"[^>]+content="([^"]*)"', text) + if mt: + title = mt.group(1) + if counts: + profile = { + "username": handle, "full_name": title.split("(")[0].strip(), + # ⚠ NO BIO. `og:description` on a profile page is + # "204K Followers, 5,245 Following, 750 Posts - See Instagram photos and videos + # from Inayma (@inayma)" — the tail is Instagram's own boilerplate, not the + # person's biography. Storing it would fill a Bio column with the same sentence + # for every influencer, which reads as data and is not. Measured 2026-08-04. + "bio": "", + "followers": _loose_count(counts.group(1)), + "following": _loose_count(counts.group(2)), + "posts_count": _loose_count(counts.group(3)), + "verified": "", + # The page abbreviates ("204K"), so the number is the page's rounding, not a + # count we read. Recorded on the row rather than left for someone to discover + # when 204,000 fails to reconcile with anything. + "approx": "1", + } + return {"state": "partial", "profile": profile, "posts": [], "via": "og:description", + "note": "counts are the page's own abbreviations; media is not anonymously " + "readable (" + ", ".join(attempts) + ")"} + if status in (401, 403, 429) or status == 0: + return {"state": "blocked", "profile": {}, "posts": [], "via": "", + "note": f"Instagram refused the anonymous read ({', '.join(attempts)})"} + return {"state": "error", "profile": {}, "posts": [], "via": "", + "note": f"nothing anonymously readable ({', '.join(attempts)})"} + + +#: The historical anonymous media query. Kept because trying it and RECORDING the refusal is +#: what makes "as deep as anonymously accessible" (R7) a measurement instead of an assumption. +IG_MEDIA_QUERY_HASH = "e769aa130647d2354c40ea6a439bfc08" + + +def _paginate(user_id, cursor, want, log=print): + """Walk further back through the media edges while the anonymous surface allows it.""" + out, note = [], "" + while user_id and cursor and len(out) < want: + time.sleep(PACE_SECONDS) + variables = json.dumps({"id": str(user_id), "first": min(50, want - len(out)), + "after": cursor}, separators=(",", ":")) + url = (f"https://www.instagram.com/graphql/query/?query_hash={IG_MEDIA_QUERY_HASH}" + f"&variables={requests.utils.quote(variables)}") + status, body, err = _ig_get(url, headers={"X-IG-App-ID": IG_APP_ID}) + if not (200 <= status < 300) or not body: + note = f"graphql:{status or err}" + break + try: + media = (((json.loads(body.decode("utf-8", "replace")) or {}).get("data") or {}) + .get("user") or {}).get("edge_owner_to_timeline_media") or {} + except Exception: + note = "graphql:unparseable" + break + edges = media.get("edges") or [] + if not edges: + note = "graphql:empty" + break + out.extend(_posts_from_edges(edges)) + page = media.get("page_info") or {} + if not page.get("has_next_page"): + break + cursor = page.get("end_cursor") + return out[:want], note + + +# --------------------------------------------------------------------------------------------- +# RUNNERS +# --------------------------------------------------------------------------------------------- + +def cap_note(capped, missing=()): + """The sentence(s) a cap breach adds to a run summary. + + `capped` = `[(table_key, rows_lost), …]` — the ROW ceiling was hit. + `missing` = `[table_key, …]` — the TABLE ceiling was hit: the database could not + be created at all. + + ⛔ IT NAMES THE TABLE. "142 rows skipped" sends somebody to look at the source page; "the + ut_ig_post_snapshots database is FULL" sends them to the actual problem. A loud failure that + does not say WHERE is only marginally better than a silent one. + + ⛔ AND THE TABLE CEILING IS NOW AS LOUD AS THE ROW ONE (closes DEBT D-11). `ut_ensure` refused + SILENTLY at `MAX_UT_TABLES`: it returned a key for a database it had not created, the + runners' `if tgt is not None` write guard then skipped that bucket, and the run reported + success over a database that does not exist. Exactly the silent-stop failure the row cap + already had, one level up — and harder to spot, because the table is not there to look at. + """ + out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} " + f"from this run were NOT written" for k, n in (capped or []) if n] + out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its " + f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" + for k in (missing or [])] + return "; ".join(out) + + +def ut_missing(rt, *keys): + """Which of `keys` do NOT exist after an `ut_ensure` — the table-ceiling detector (D-11).""" + have = ut_all(rt) + return [k for k in keys if k and k not in have] + + +def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """Automation #1 (R6): a public page → a blank database, re-runnable, UPSERT by key.""" + cfg = defn.get("config") or {} + fmap, key_field = cfg.get("fieldMap") or {}, cfg.get("keyField") + dry = bool(cfg.get("dryRun")) + # Per-NODE outcomes for the canvas. MEASURED as the run walks, never inferred afterwards from + # the rollup — inferring is exactly how a green dot ended up over an empty table (see the + # rollup comment in `run_field_instagram`). + steps = {"trigger": "ok", "fetch": "idle", "extract": "idle", "write": "idle"} + step(f"Fetching {urlparse(str(cfg.get('url') or '')).hostname or 'the page'}") + status, final, body = fetch(cfg["url"]) + if not (200 <= status < 300): + steps["fetch"] = "error" + return ("error", f"{cfg['url']} answered {status} — nothing was written", + {"status": status}, [], steps) + steps["fetch"] = "ok" + soup = _soup(body) + if cfg.get("extract") == "jsonld": + blocks = jsonld(soup) + flat = [] + for b in blocks: + if isinstance(b, list): + flat.extend([x for x in b if isinstance(x, dict)]) + elif isinstance(b, dict): + items = b.get("itemListElement") + flat.extend([x for x in items if isinstance(x, dict)] if isinstance(items, list) + else [b]) + raw_rows = [{k: _scalar(v) for k, v in d.items()} for d in flat] + else: + found = tables(soup) + idx = max(0, min(int(cfg.get("tableIndex") or 0), len(found) - 1)) if found else 0 + raw_rows = [r for r in (found[idx] if found else []) if isinstance(r, dict)] + if not raw_rows: + steps["extract"] = "error" + return ("error", "the page parsed but the selected table had no rows", {"rows": 0}, [], + steps) + steps["extract"] = "ok" + step(f"Extracting {len(raw_rows)} row{'' if len(raw_rows) == 1 else 's'}") + + incoming = [] + for r in raw_rows: + mapped = {tgt: _s(r.get(src, ""), 500) for src, tgt in fmap.items()} + if any(v for v in mapped.values()): + incoming.append(mapped) + + fields = [field_def(tgt, src if len(src) <= 60 else src[:60]) + for src, tgt in fmap.items()] + # the key column leads, so the table opens on the identity it is upserted by + fields.sort(key=lambda f: 0 if f["key"] == key_field else 1) + label = cfg.get("targetLabel") or defn.get("name") or "Scraped table" + if dry: + # THE WRITE NODE IS OFF — read, compute, report, touch nothing. Not even `ut_ensure`, + # which would create the table: a dry run that leaves a new empty database behind is not + # a dry run. + table_key = ut_key_for(label, cfg.get("targetTable") or None) + else: + table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) + t = ut_get(rt, table_key) or {} + rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, + cap=row_cap(table_key)) + touched = [rid for rid, row in rows.items() + if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] + # R5 — the engine moves each record down its winning lane, over the row's FINAL values. + # Blank ⇒ no lane matched: the record shows in the derived Write bucket, never a guess. + lanes = cfg.get("lanes") or [] + if lanes: + skey = stage_field_key(defn.get("id")) + stamp = _iso() + counts["routed"] = 0 + for rid in touched: + row = rows.get(rid) or {} + label = route_record(lanes, row) + if label: + counts["routed"] += 1 + if str(row.get(skey) or "") != label: + row[skey] = label + row[skey + "_at"] = stamp if label else "" + if not dry: + ut_write_rows(rt, table_key, rows) + if lanes: + ensure_stage_field(rt, table_key, defn, username) + counts["source_rows"] = len(incoming) + affected = touched[:200] + summary = (f"{counts['inserted']} new, {counts['updated']} updated, " + f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " + f"(kept)") + if lanes: + summary += f", {counts['routed']}/{len(touched)} routed to lanes" + capped = [(table_key, counts.get("capped", 0))] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + note = cap_note(capped, missing) + if note: + summary += f" — {note}" + steps["write"] = "partial" + if dry: + summary = f"DRY RUN (the Write step is turned off) — would have written: {summary}" + steps["write"] = "skipped" + elif not note: + steps["write"] = "ok" + state = "partial" if (counts.get("capped") or counts.get("skipped") or missing + or dry) else "ok" + log(f"[aios-auto] scrape_db {defn.get('id')} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def run_field_instagram(rt, defn, username="automation", log=print, step=_no_step, + rows=None): + """Automation #2 (R7 + R1's hybrid): for every row of a blank database that carries a profile + URL, pull the public profile — the PAID rung first when the tier says so, the anonymous + ladder underneath it — write a status string into the automation column, and append a + timestamped row to each of the three IG tables.""" + cfg = defn.get("config") or {} + table_key, fkey = cfg.get("targetTable"), cfg.get("fieldKey") + tier = clean_tier(cfg.get("tier")) or "anonymous" + paid = tier == "brightdata" + fallback = not cfg.get("noFallback") + post_metrics = bool(cfg.get("postMetrics")) + dry = bool(cfg.get("dryRun")) + steps = {"trigger": "ok", "source": "idle", "capture": "idle", + "capture_paid": "idle" if paid else "skipped", + "capture_anon": "idle" if (not paid or fallback) else "skipped", + "capture_metrics": "idle" if post_metrics else "skipped", + "write": "idle"} + t = ut_get(rt, table_key) + if not t: + steps["source"] = "error" + return ("error", f"{table_key} is not a database in this workspace", {}, [], steps) + url_field = cfg.get("urlField") or _auto_url_field(t, fkey) + if not url_field: + steps["source"] = "error" + return ("error", "the automation column has no URL field bound to it", {}, [], steps) + steps["source"] = "ok" + rows = dict(t.get("rows") or {}) + # the relational tables the pull lands in (R7): every row timestamped for time-range filters + if dry: + # The Write node is off: resolve the keys, create nothing. (`ut_ensure` writes.) + snap_key, post_key, ps_key = ("ut_ig_snapshots", "ut_ig_posts", "ut_ig_post_snapshots") + else: + snap_key = ut_ensure(rt, "IG snapshots", SNAPSHOT_FIELDS, username, + key="ut_ig_snapshots", flow_tag=str(defn.get("id") or "")) + post_key = ut_ensure(rt, "IG posts", POST_FIELDS, username, key="ut_ig_posts", + flow_tag=str(defn.get("id") or "")) + ps_key = ut_ensure(rt, "IG post snapshots", POST_SNAPSHOT_FIELDS, username, + key="ut_ig_post_snapshots", flow_tag=str(defn.get("id") or "")) + missing = [] if dry else ut_missing(rt, snap_key, post_key, ps_key) + snaps = dict((ut_get(rt, snap_key) or {}).get("rows") or {}) + posts = dict((ut_get(rt, post_key) or {}).get("rows") or {}) + psnaps = dict((ut_get(rt, ps_key) or {}).get("rows") or {}) + + counts = {"profiles": 0, "ok": 0, "partial": 0, "blocked": 0, "error": 0, + "posts": 0, "new_posts": 0, "paid": 0, "capped": 0, "metrics": 0, + "missing_tables": len(missing)} + cells, affected, notes = {}, [], [] + # ⛔ ACCUMULATE HERE, UPSERT ONCE PER TABLE AFTER THE LOOP. This used to call `upsert_rows` + # once per POST, which is O(existing) per call — survivable only while the cap was 5000 rows. + # `MAX_UT_IG_ROWS` makes the same loop hundreds of millions of dict copies, i.e. an automation + # that no longer finishes. **Raising a cap and batching its writer are ONE change.** (W19-C.) + in_snaps, in_posts, in_psnaps = [], [], [] + paid_tried = paid_answered = anon_tried = anon_answered = False + targets = [(rid, str(r.get(url_field, "") or "").strip()) + for rid, r in rows.items() if str(r.get(url_field, "") or "").strip()] + for i, (rid, url) in enumerate(targets): + if i: + time.sleep(PACE_SECONDS) # ≥2 s between profiles (R7) + # ITEM 6: the other genuinely long runner — ≥2 s of pacing per profile means a 60-profile + # column automation blocks for minutes by design. A counting step is the difference + # between "working through them" and "stuck". + step(f"Capturing profile {i + 1} of {len(targets)}") + counts["profiles"] += 1 + res = pull_profile(url, max_posts=cfg.get("maxPosts") or 24, log=log, + tier=tier, fallback=fallback, post_metrics=post_metrics) + pulled = _iso() + via = res.get("via") or "" + read_ok = res["state"] in ("ok", "partial") + # WHICH RUNG ANSWERED — tracked per profile so the canvas's rung dots are a measurement. + if paid: + paid_tried = True + if via == "brightdata" and read_ok: + paid_answered = True + counts["paid"] += 1 + elif fallback: + anon_tried = True + anon_answered = anon_answered or read_ok + else: + anon_tried = True + anon_answered = anon_answered or read_ok + if read_ok: + counts["ok" if res["state"] == "ok" else "partial"] += 1 + prof = res["profile"] + # ⚠ A SNAPSHOT ROW IS APPEND-KEYED (`@`), so every key is written + # every time and a blank is unambiguous: it means THIS PULL did not read it. That is + # the opposite of the identity rows below, which are upserted and therefore omit + # blanks so a cheap rung cannot erase an expensive one's value. + snap_row = { + "snapshot_key": f"{prof['username']}@{pulled}", + "influencer_key": prof["username"], "pulled_at": pulled, + "source": _s(res.get("via")), "approx": _s(prof.get("approx")), + "bio": _s(prof.get("bio"), 500), + "external_url": _s(prof.get("external_url"), 300), + } + for fd in SNAPSHOT_FIELDS: + k = fd["key"] + if k not in snap_row: + snap_row[k] = _s(prof.get(k), 400) + in_snaps.append(snap_row) + for p in res.get("posts") or []: + # ⚠ A BLANK IS OMITTED, NOT WRITTEN. `posted_at` / `caption` / `type` are + # readable on some rungs and not others — the Bright Data PROFILE row carries + # post identity with `datetime: None` (measured 24/24), while the engagement rung + # knows `date_posted`. `upsert_rows` writes exactly the keys it is handed, so + # sending "" would let a cheap run ERASE what an expensive one learned. + ident = {"shortcode": p["shortcode"], "influencer_key": prof["username"], + "url": _s(p.get("url"), 300)} + for k, n in (("posted_at", 200), ("type", 200), ("caption", 800), + ("paid_partnership", 8), ("partner", 120), ("hashtags", 400), + ("alt_text", 400)): + if p.get(k): + ident[k] = _s(p.get(k), n) + in_posts.append(ident) + # ⛔ AN ENGAGEMENT SNAPSHOT IS APPENDED ONLY WHEN SOMETHING WAS MEASURED. The + # post series is the append table that FILLS (maxPosts rows per profile per + # pull), and with the paid engagement rung OFF every one of those rows would + # carry three blanks — pure noise, eating a 200k ceiling and drawing a chart of + # nothing. A row in a time series should mean "this was true then"; a row that + # means "nobody looked" belongs nowhere. + metrics = {k: p.get(k) for k in ("likes", "comments", "views")} + if any(v is not None for v in metrics.values()): + in_psnaps.append({ + "post_snapshot_key": f"{p['shortcode']}@{pulled}", + "shortcode": p["shortcode"], "pulled_at": pulled, + "likes": _s(metrics["likes"]), "comments": _s(metrics["comments"]), + "views": _s(metrics["views"])}) + counts["metrics"] += 1 + counts["posts"] += len(res.get("posts") or []) + detail = f"{prof.get('followers') or '?'} followers" + if prof.get("approx"): + detail += " (approx)" + if res.get("posts"): + detail += f", {len(res['posts'])} posts" + elif res["state"] == "partial": + detail += ", posts not readable on the rung that answered" + cells[rid] = f"{res['state']} · {_stamp()} · {detail}" + elif res["state"] == "blocked": + counts["blocked"] += 1 + cells[rid] = f"blocked · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "blocked") + else: + counts["error"] += 1 + cells[rid] = f"error · {_stamp()} · {_s(res.get('note'), 90)}" + notes.append(res.get("note") or "error") + affected.append(rid) + + # --- THE THREE UPSERTS. Once each, over the whole run's accumulated rows. + snaps, c_snap = upsert_rows(snaps, in_snaps, "snapshot_key", cap=row_cap(snap_key)) + posts, c_post = upsert_rows(posts, in_posts, "shortcode", cap=row_cap(post_key)) + psnaps, c_ps = upsert_rows(psnaps, in_psnaps, "post_snapshot_key", cap=row_cap(ps_key)) + counts["new_posts"] = c_post["inserted"] + capped = [(snap_key, c_snap["capped"]), (post_key, c_post["capped"]), + (ps_key, c_ps["capped"])] + counts["capped"] = sum(n for _k, n in capped) + + # R5 — route each processed record down its winning lane, judged over the row's FINAL + # values (the fresh status cell included, so a lane on the cell itself is answerable). + lanes = cfg.get("lanes") or [] + stage_writes = {} + if lanes: + skey = stage_field_key(defn.get("id")) + stamp = _iso() + counts["routed"] = 0 + for rid, _url in targets: + merged = dict(rows.get(rid) or {}) + if rid in cells: + merged[fkey] = cells[rid] + label = route_record(lanes, merged) + if label: + counts["routed"] += 1 + if str((rows.get(rid) or {}).get(skey) or "") != label: + stage_writes[rid] = {skey: label, skey + "_at": stamp if label else ""} + + if not dry: + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, val in cells.items(): + tt.setdefault("rows", {}).setdefault(str(rid), {})[fkey] = val + for rid, vals in stage_writes.items(): + tt.setdefault("rows", {}).setdefault(str(rid), {}).update(vals) + for k, rws in ((snap_key, snaps), (post_key, posts), (ps_key, psnaps)): + tgt = cur.get(k) + if tgt is not None: + tgt["rows"] = rws + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") # ONE coalesced update for all four tables + if lanes: + ensure_stage_field(rt, table_key, defn, username) + + # --- C6 (R2): WRITE-THROUGH to the platform master. Three postures, never conflated: + # `ok` silent, `off` an honest aside (the tenant copy IS the story on this deployment), + # `error` a LOUD partial — a pooled history with silent holes is worse than none. + master_note = "" + if not dry and (in_snaps or in_posts or in_psnaps): + import ig_master + m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), + in_snaps, in_posts, in_psnaps) + if m_status == "error": + master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \ + f"is complete and the next run re-appends" + counts["master_failed"] = 1 + elif m_status == "ok": + counts["master"] = len(in_snaps) + len(in_psnaps) + + # ⚠ A RUN IS ONLY 'ok' IF EVERY PROFILE WAS. The first live run rolled up to 'ok' while + # every CELL said `partial`, because the rollup only asked about blocked/error — so the rail + # showed a green dot over a table with no posts in it. A summary that disagrees with the + # cells it summarises is the failure this whole module's honest-status rule exists to + # prevent, so `partial` now propagates. Measured 2026-08-04. + read = counts["ok"] + counts["partial"] + if not read: + state = "error" if counts["error"] else "partial" if counts["blocked"] else "ok" + elif counts["blocked"] or counts["error"] or counts["partial"]: + state = "partial" + else: + state = "ok" + # --- per-node outcomes for the canvas (see the same note in `run_scrape_db`) + if not targets: + cap_state = "idle" + elif not read: + cap_state = "error" if counts["error"] else "blocked" + else: + cap_state = "partial" if (counts["blocked"] or counts["error"] + or counts["partial"]) else "ok" + steps["capture"] = cap_state + steps["capture_paid"] = ("ok" if paid_answered + else "blocked" if paid_tried else "skipped") + steps["capture_anon"] = ("ok" if anon_answered + else "blocked" if anon_tried else "skipped") + # ⚠ MEASURED, LIKE EVERY OTHER DOT: it is `ok` only if an engagement row was actually + # appended. "It was switched on" is not the same fact as "it answered", and painting the + # second from the first is the fabrication `node_status` refuses to make. + # ⚠ AND `blocked` IS A CLAIM ABOUT THE VENDOR, so it needs something to have been ASKED. A + # run that found no posts to enrich did not have a rung refuse it — nothing was requested — + # so that reads `skipped`, the same word an off switch earns. + steps["capture_metrics"] = ("ok" if counts["metrics"] + else "blocked" if (post_metrics and counts["posts"]) + else "skipped") + + summary = (f"{read}/{counts['profiles']} profiles read, {counts['posts']} posts " + f"({counts['new_posts']} new)") + if lanes: + summary += f", {counts['routed']}/{len(targets)} routed to lanes" + if counts["paid"]: + summary += f", {counts['paid']} exact via Bright Data" + if counts["metrics"]: + summary += f", {counts['metrics']} post engagement snapshots" + elif post_metrics and counts["posts"]: + # ⚠ ASKED FOR AND NOT DELIVERED IS ITS OWN SENTENCE. Silence here would read as "there + # was no engagement", which is a claim about Instagram rather than about our run. + summary += ", no post engagement was readable" + if counts["partial"]: + summary += f", {counts['partial']} profile-only (posts not readable)" + if counts["blocked"]: + summary += f", {counts['blocked']} blocked" + note = cap_note(capped, missing) + if note: + summary += f" — {note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + elif not dry: + steps["write"] = "ok" + if master_note: + summary += f" — {master_note}" + state = "partial" if state != "error" else state + steps["write"] = "partial" + if dry: + summary = f"DRY RUN (the Write step is turned off) — would have written: {summary}" + steps["write"] = "skipped" + state = "partial" if state != "error" else state + if notes: + summary += f" — {notes[0][:120]}" + if not dry: + # C7: the fresh master rows are exactly what the table's metric cells summarise. + try: + compute_metric_cells(rt, table_key) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + return (state, summary, counts, affected, steps) + + +def _auto_url_field(table, fkey): + """The URL field an automation column is bound to: its own `automation.urlField`, else the + first url-typed column on the table. Never guessed silently — the caller reports which.""" + for f in table.get("fields") or []: + if f.get("key") == fkey: + bound = ((f.get("automation") or {}).get("urlField") or "").strip() + if bound: + return bound + for f in table.get("fields") or []: + if f.get("type") == "url": + return f.get("key") + return "" + + +def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_step, + rows=None): + """Automation #3 (owner ruling R7 / D-23): FIND handles nobody here has typed in. + + Two-phase by construction — see the DISCOVERY section header. A run either STARTS a corpus + query and hands its snapshot id to the next run, or COLLECTS one a previous run started. + Every outcome is a sentence about what actually happened; none of them is a green zero. + """ + cfg = defn.get("config") or {} + dry = bool(cfg.get("dryRun")) + limit = int(cfg.get("recordsLimit") or 5) + preds = list(cfg.get("predicates") or []) + auto_id = str(defn.get("id") or "") + steps = {"trigger": "ok", "find": "idle", "collect": "idle", "write": "idle"} + counts = {"asked": limit, "found": 0, "new": 0, "seen_again": 0, "capped": 0, + "missing_tables": 0} + + if not bd_ready(): + # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is + # the vendor's — so this is where the run stops, saying which env var is missing. + steps["find"] = "blocked" + return ("partial", "AIOS_BRIGHTDATA_KEY is not configured — profile search is closed " + "(nothing was searched and nothing was billed)", counts, [], steps) + + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + if not pending: + # C4 (wave 22): the guard holds at RUN too — a stored config can predate the law, and + # the vendor bills for breadth whether the filter was saved yesterday or last month. + guard_err = narrowing_refusal(preds) + if guard_err: + steps["find"] = "blocked" + return ("error", f"the search was not started — {guard_err}", counts, [], steps) + if dry: + est = discover_estimate(limit) + steps["find"] = steps["write"] = "skipped" + return ("partial", + f"DRY RUN (the Write step is turned off) — would search for up to {limit} " + f"profile{'' if limit == 1 else 's'} matching " + f"{_predicate_sentence(preds, cfg.get('operator'))}. Estimated cost " + f"${est['usd']} (ESTIMATE — {est['note']})", counts, [], steps) + step("Asking Bright Data to start the search") + sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit) + if note: + steps["find"] = "blocked" + return ("error", f"the search was not started — {note}", counts, [], steps) + pending = sid + # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes + # through `rt.update`, which can RAISE on an unavailable store — and that exception + # becomes an error run, so a `log()` placed after it never executes. The snapshot would + # then exist, be billed, and have its id in neither the store nor the logs. Logging first + # means the worst case is still recoverable by a human reading the Space output. + log(f"[aios-auto] discover {auto_id}: started {sid}") + # PERSIST BEFORE POLLING. A snapshot the vendor is already building does not stop + # existing because this process dies thirty seconds later, and a lost id is a set we + # paid attention to and can never collect. + set_state(rt, auto_id, {"pendingSnapshot": sid, "pendingSince": _iso()}) + steps["find"] = "ok" + + waited, status, size, note = 0.0, "", 0, "" + while True: + # ⭐ ITEM 6 — THE ONE PLACE THE LIVE STEP HAS TO MOVE. This loop blocks for up to + # BD_FILTER_WAIT (120 s) at the vendor, and D measured that this wait IS the whole of + # "Run once now is laggy / looks stuck". A counter here is what makes a legitimate + # two-minute wait distinguishable from a hung thread; without it the rendered step reads + # the same word for both, which is a progress indicator that cannot indicate progress. + step(f"Waiting for Bright Data to build the search — {int(waited)}s of " + f"{int(BD_FILTER_WAIT)}s") + status, size, note = bd_filter_status(pending) + if note: + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", note, counts, [], steps) + if status != "building" or waited >= BD_FILTER_WAIT: + break + time.sleep(BD_FILTER_POLL) + waited += BD_FILTER_POLL + step("Reading the profiles the search found") + + if status == "building": + # ⚠ THE HANDOFF, AND IT IS A SUCCESSFUL OUTCOME OF A SORT. Measured build latency is ~20 + # minutes; holding a worker thread that long on a free-tier container to poll would be + # the wrong shape. Say plainly that it is still running and who collects it. + steps["collect"] = "partial" + return ("partial", + f"the search is still running at Bright Data ({pending}) — searches take about " + f"20 minutes, and the NEXT run of this automation collects it. Nothing is lost " + f"and nothing is re-billed", counts, [], steps) + if status != "ready": + steps["collect"] = "error" + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + return ("error", f"the search ended as {status or 'unreadable'} and returned nothing", + counts, [], steps) + + rows, dnote = bd_filter_rows(pending) + if dnote: + # Delivery lags `ready` by minutes (measured). Keep the id; the next run collects it. + steps["collect"] = "partial" + return ("partial", f"the search finished with {size} match" + f"{'' if size == 1 else 'es'} but {dnote}", counts, [], steps) + set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) + steps["collect"] = "ok" + + stamp = _iso() + incoming = [c for c in (_candidate_row(r, stamp) for r in rows) if c] + counts["found"] = len(incoming) + # ⚠ COUNT THE ROWS WE COULD NOT USE. A corpus row with no handle cannot be a candidate, and + # dropping it silently means "the vendor delivered 50 and we kept 3" reads identically to + # "the vendor delivered 3". It also catches the shape where an error document comes back + # down the rows path and parses as one unusable row instead of an error. + counts["dropped"] = max(0, len(rows) - len(incoming)) + label = cfg.get("targetLabel") or "IG candidates" + table_key = (ut_key_for(label, cfg.get("targetTable") or DISCOVER_TABLE) if dry + else ut_ensure(rt, label, CANDIDATE_FIELDS, username, + key=cfg.get("targetTable") or DISCOVER_TABLE, + flow_tag=str(defn.get("id") or ""))) + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + # --- C6: the upsert key is the COMPOUND (handle, created_by) — per-user rows, per-user + # found_counts ("each user only works with their own data" without breaking the upsert + # law). Implemented as a synthesised key column that exists only inside this merge, so + # `upsert_rows` keeps its one-field contract and its id/cap arithmetic runs over the WHOLE + # table. A pre-wave row with no owner is ADOPTED by the first user who re-finds its handle + # (the `ut_ensure` owner-adoption move) rather than duplicated beside itself. + def _ck(handle, owner): + return f"{str(handle or '').strip()}\n{str(owner or '').strip()}" + + incoming_handles = {c["handle"] for c in incoming} + for r in existing.values(): + if not str(r.get("created_by") or "").strip() and \ + str(r.get("handle") or "").strip() in incoming_handles: + r["created_by"] = username + existing2 = {rid: {**r, "_ckey": _ck(r.get("handle"), r.get("created_by"))} + for rid, r in existing.items()} + # `found_count` is ARITHMETIC OVER WHAT IS STORED, not a value from the vendor — re-finding + # a profile is the signal that it keeps matching, so the counter grows instead of + # resetting. `first_found` is written only when the pair is new, so it never moves. + seen = {r["_ckey"]: r for r in existing2.values()} + skey = stage_field_key(auto_id) + for c in incoming: + c["created_by"] = username + c["_ckey"] = _ck(c["handle"], username) + prev_row = seen.get(c["_ckey"]) + if prev_row: + counts["seen_again"] += 1 + c["found_count"] = str((_ig_int(prev_row.get("found_count")) or 0) + 1) + else: + counts["new"] += 1 + c["found_count"] = "1" + c["first_found"] = stamp + # C2/R6 — a NEW candidate's card starts at the human gate. Stamped only at birth: + # a re-find must never pull a card somebody has already moved back into Review, + # for the same reason it never un-ticks `tracked`. + c[skey] = "Review" + c[skey + "_at"] = stamp + merged, mc = upsert_rows(existing2, incoming, "_ckey", cap=row_cap(table_key)) + merged = {rid: {k: v for k, v in r.items() if k != "_ckey"} + for rid, r in merged.items()} + counts["capped"] = mc["capped"] + missing = [] if dry else ut_missing(rt, table_key) + counts["missing_tables"] = len(missing) + if not dry and not missing: + ut_write_rows(rt, table_key, merged) + ensure_stage_field(rt, table_key, defn, username) + steps["write"] = "ok" + elif dry: + steps["write"] = "skipped" + + affected = [rid for rid, r in merged.items() + if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] + est = discover_estimate(counts["found"]) + summary = (f"{counts['found']} profile{'' if counts['found'] == 1 else 's'} found — " + f"{counts['new']} new, {counts['seen_again']} seen before. Est. cost " + f"${est['usd']} (ESTIMATE, not a billed figure). None is tracked until somebody " + f"ticks it") + if counts["dropped"]: + summary += (f". ⚠ {counts['dropped']} row{'' if counts['dropped'] == 1 else 's'} the " + f"search returned had no handle and could not be used") + cnote = cap_note([(table_key, counts["capped"])], missing) + if cnote: + summary += f" — {cnote}" + steps["write"] = "partial" + if dry: + summary = f"DRY RUN (the Write step is turned off) — would have written: {summary}" + state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" + if not dry and not missing: + try: + compute_metric_cells(rt, table_key) # C7 — see the field runner + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric compute {table_key} failed: {type(e).__name__}: {e}") + log(f"[aios-auto] discover {auto_id} -> {table_key}: {summary}") + return (state, summary, counts, affected, steps) + + +def _predicate_sentence(preds, operator="and"): + """A predicate list as something a person can read back. Used in summaries and the canvas.""" + joiner = " or " if str(operator or "").lower() == "or" else " and " + parts = [] + for p in preds or []: + v = p.get("value") + parts.append(f"{p.get('name')} {p.get('operator')}" + + ("" if v is None else f" {v}")) + return joiner.join(parts) or "no conditions" + + +#: How many records ONE `plain` run walks. Deliberately the ut table's OWN row ceiling rather +#: than a second, smaller number nobody could explain: a plain automation's records ARE its +#: table's rows. What this actually bounds is the APPEND tables (`MAX_UT_IG_ROWS`, 200k), which a +#: flow could be pointed at and which would not finish. +MAX_FLOW_RECORDS = MAX_UT_ROWS + + +def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): + """⭐ WAVE 24 / R6 — the runner for an automation with NO machine step: its flow IS the whole + automation. Returns the same 5-tuple every other runner does, so `run_now` needs no special + case and `apply_actions` still runs at the ONE call site that already exists. + + Before this existed, `RUNNERS.get("plain")` was None and pressing Run now committed + `error: unknown automation kind 'plain'` — on the kind every new automation now has. + + ⛔ AN UNBOUND FLOW ANSWERS `partial`, NEVER `ok`, and that is the load-bearing line here. + `apply_actions` returns immediately when there is no table, so a flow whose only action is + `create_record` into some other database does NOTHING — and a green dot over nothing is the + exact defect this module names in three other places. It reports the honest state and says + which fact is missing. + """ + table = _flow_table(defn) + if not table: + return ("partial", + "no database is bound yet — a plain automation walks the records of its target " + "database, and this one names none (pick one on the trigger, or in Properties)", + {}, [], {}) + ensure_stage_field(rt, table, defn, username) + all_rows = (ut_get(rt, table) or {}).get("rows") or {} + # ⛔ THE TRIGGER'S RECORDS WIN OVER THE WHOLE TABLE, and getting this wrong is a day-one bug + # on the kind every new automation now has. `plain` has no machine step, so it has nothing of + # its own to call "the records this run touched" — and walking the WHOLE table would mean + # "When a record is CREATED in ut_leads -> set status = New" writes `New` onto every lead the + # first time one is added. The trigger knows exactly which rows fired; `run_now` threads them + # here. `rows=None` (manual, schedule, Run now) still means the whole table, which is what + # those genuinely mean. + ids = ([r for r in (rows or []) if str(r) in all_rows] if rows + else sorted(all_rows, key=_rid_num)) + step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") + counts = {"records": len(ids)} + if len(ids) > MAX_FLOW_RECORDS: + # The cap is DISCLOSED, never silent ([[no-unverifiable-aggregates]]): the summary names + # both numbers so "it only processed some of them" is readable rather than deducible. + total = len(ids) + ids = ids[:MAX_FLOW_RECORDS] + return ("partial", + f"{total} records in {table} — this run walked the first " + f"{MAX_FLOW_RECORDS}", counts, ids, {}) + n = len(ids) + scope = "the trigger's" if rows else "all" + return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {table} ({scope})", + counts, ids, {}) + + +RUNNERS = {"plain": run_plain, "scrape_db": run_scrape_db, + "field_instagram": run_field_instagram, + "discover_instagram": run_discover_instagram} + + +# --------------------------------------------------------------------------------------------- +# THE SOURCE REGISTRY (DEBT D-9's seam, wave-20 item 6b) +# --------------------------------------------------------------------------------------------- +# WHY THIS EXISTS BEFORE THERE IS A SECOND SOURCE. The vendor swap that produced it (HikerAPI → +# Bright Data) touched a dozen places: a tier name, two node ids, a config flag, a readiness +# boolean on the wire, four client strings and two gate sections. That is what a hard-coded +# vendor costs, and TikTok (D-9) would have paid it again from scratch. +# +# So a SOURCE is declared once — what it is called, which vendor answers it, and which of the +# three verbs it can do — and the surfaces read the declaration instead of naming a vendor: +# +# probe "is this configured?" -> the honest readiness bit the UI shows +# capture enrich a handle we know -> `pull_profile`-shaped +# discover find handles we do not -> the corpus query +# +# ⚠ A SOURCE DECLARES ONLY WHAT IT CAN ACTUALLY DO. `discover: None` is not a gap to fill in +# later; it is the honest statement that this source has no discovery route, and a surface that +# offers one anyway is offering a button that must refuse. +SOURCES = { + "instagram": { + "key": "instagram", "label": "Instagram", "vendor": "Bright Data", + "probe": bd_ready, + "capture": pull_profile, + "discover": bd_filter_start, + "captureKind": "field_instagram", + "discoverKind": "discover_instagram", + "tiers": TIERS, + }, + # D-9 (TikTok) lands here as one more entry: a new field map and a new `capture`, with the + # canvas, the append tables, the caps and the honest-status contract already in place. +} + + +def source_status(): + """`[{key,label,vendor,ready,canDiscover}]` — what the surface says about each source. + + ⚠ A BOOLEAN, NEVER THE KEY (the wire rule the `hikerReady` bit already followed): a surface + needs exactly one bit to say "the paid rung is not configured" honestly, and shipping the + credential to a browser would put a billable secret in every user's devtools. + """ + return [{"key": s["key"], "label": s["label"], "vendor": s["vendor"], + "ready": bool(s["probe"]()), "canDiscover": bool(s.get("discover"))} + for s in SOURCES.values()] + + +# --------------------------------------------------------------------------------------------- +# THE CANVAS GRAPH (R5) — topology on the SERVER, pixels on the client +# --------------------------------------------------------------------------------------------- +# WHY THE TOPOLOGY LIVES HERE. The canvas draws what an automation DOES; the runners above are +# what it does. Two descriptions of one thing in two languages drift, and the drift is invisible — +# a canvas that still shows a step the engine stopped running looks completely fine. So the node +# list is derived from the DEFINITION by the same module that executes it, rides on the payload +# the way `CRON_PRESETS` and the kind list already do, and is asserted in `verify_automation.py`. +# The client owns layout (pixels, pan/zoom, hit targets) and nothing else. +# +# ⚠ BRANCHES RENDER, THEY DO NOT EXECUTE (R5, accepted at grill; DEBT D-7). The `capture` node is +# a real branch in BEHAVIOUR — the paid rung answers or the anonymous ladder does — but the engine +# walks it linearly. Nothing here should be read as a general branching runtime. + +#: A node's dot vocabulary. Deliberately WIDER than the run-level `STATES`: a node can be +#: `blocked` or `skipped` in ways a whole run cannot, and folding those into `partial` would throw +#: away the only information the dot is there to carry. +NODE_STATES = ("idle", "ok", "partial", "error", "blocked", "skipped") + +#: node id -> the switch it flips. A node ABSENT from this map has no switch, and that is a +#: deliberate answer rather than an omission: a "Fetch the page" step that can be turned off is +#: not an automation with a disabled step, it is a broken automation with a lie on it. The ones +#: here are all REAL — each changes what the next run does: +#: schedule the trigger fires on its cron, or only by hand +#: paid the paid rung is reached for, or it is not (this is the one that spends money) +#: fallback the anonymous ladder catches a refused paid rung, or the run says `blocked` +#: instead of quietly writing an approximate number +#: postMetrics likes/comments per post are bought, or the engagement series does not grow — +#: the expensive one, because it costs a vendor record PER POST rather than per +#: profile (measured: the profile row carries post identity and no engagement) +#: write DRY RUN — read everything, compute the counts, write nothing anywhere +NODE_TOGGLES = {"trigger": "schedule", "capture_paid": "paid", + "capture_anon": "fallback", "capture_metrics": "postMetrics", + "write": "write"} + + +def _cron_label(cron): + return next((p["label"] for p in CRON_PRESETS if p["cron"] == cron), cron or "") + + +def node_status(steps, nid): + """A node's dot — from the last run's MEASURED step outcome, or `idle`. + + ⛔ NEVER INFERRED FROM THE RUN'S OVERALL STATE, and this is a module-level function precisely + so a negative control can prove that. Every run stored before W19-C has no `steps` map at all, + and painting those nodes green because the run said `ok` would manufacture a measurement — the + same defect as the green dot over the empty posts table recorded in `run_field_instagram`. + No data ⇒ `idle`. An unrecognised word ⇒ `idle`, never passed through to the UI. + """ + v = str((steps or {}).get(nid) or "") + return v if v in NODE_STATES else "idle" + + +def graph(defn): + """One automation → `{nodes, edges}`, left to right. Pure over the definition.""" + defn = defn if isinstance(defn, dict) else {} + cfg = defn.get("config") or {} + sched = defn.get("schedule") or {} + runs = defn.get("runs") or [] + last = runs[0] if runs and isinstance(runs[0], dict) else {} + steps = last.get("steps") if isinstance(last.get("steps"), dict) else {} + + def node(nid, kind, title, subtitle, col, row=0, panel="", detail="", on=True): + return {"id": nid, "kind": kind, "title": title, "subtitle": subtitle, "detail": detail, + "col": col, "row": row, "panel": panel or nid, "enabled": bool(on), + "toggle": NODE_TOGGLES.get(nid, ""), "status": node_status(steps, nid)} + + on = bool(sched.get("enabled")) + trg = defn.get("trigger") or {} + if trg.get("key") in TRIGGER_KEYS and trg.get("key") not in ("manual", "schedule"): + # C3 (wave 22): the trigger node SAYS which trigger this flow has — the board and rail + # read the same node, so an event-triggered flow must not read "Schedule". The membership + # test is the KEY SET, not a hand-listed tuple: wave 23 added four triggers and the old + # tuple would have quietly rendered every one of them as "Schedule" (they are stored, so + # the else-branch was reachable) — green, compiling, and wrong on screen. + t_on = bool(trg.get("enabled", True)) and not trg.get("paused") + sub = TRIGGER_LABELS.get(trg["key"], trg["key"]) + watched = ", ".join(trg.get("fields") or []) or "any field" + # ⭐ WAVE 24 · laws 4/5 — `event_field` no longer names a FIELD (its condition is the + # whole of it) and `record_updated` no longer names a condition (its watched fields are). + # These two lines are the migration's visible half: a node still describing a `.field` + # the validator has stopped storing would be the surface and the store disagreeing. + det = {"event_field": (f"{trg.get('table', '')}" + + (f" — {_lane_sentence(trg.get('when'))}" + if trg.get("when") else "")), + "record_updated": f"{trg.get('table', '')} — {watched}", + "record_created": str(trg.get("table") or ""), + "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}", + "form_submitted": str(trg.get("table") or ""), + "webhook": "POST the hook URL to fire it", + "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), + cfg.get("operator"))[:80] + if cfg.get("predicates") + else "No filters yet — nothing to search for"), + "email": str(trg.get("query") or "")}.get(trg["key"], "") + if not trg.get("configured", True): + det = "Finish setting this trigger up before it can fire" + if trg.get("paused"): + det = str(defn.get("statusNote") or "paused") + nodes = [node("trigger", "trigger", sub, + "On" if t_on else "Off", 0, panel="trigger", on=t_on, detail=det)] + else: + nodes = [node("trigger", "trigger", "Schedule", + _cron_label(sched.get("cron")) if on else "Manual only", 0, + panel="schedule", on=on, + detail="" if on else "It still runs when you press Run now")] + edges = [] + + if defn.get("kind") == "field_instagram": + paid = clean_tier(cfg.get("tier")) == "brightdata" + fb = not cfg.get("noFallback") + anon_on = (not paid) or fb + metrics_on = bool(cfg.get("postMetrics")) + max_posts = cfg.get("maxPosts") or 24 + nodes += [ + node("source", "source", "Profile set", cfg.get("targetTable") or "No database", 1, + panel="source", + detail=f"URL column: {cfg.get('urlField')}" if cfg.get("urlField") + else "URL column: the one the field is bound to"), + node("capture", "branch", "Capture", + "Bright Data, then anonymous" if paid else "Anonymous only", 2, panel="capture", + detail=f"Up to {max_posts} posts per profile"), + node("capture_paid", "capture", "Bright Data", + "Exact counts, top posts" if paid else "Off — the paid rung is not used", + 3, row=-1, panel="capture", on=paid, + detail="" if bd_ready() else "No AIOS_BRIGHTDATA_KEY is configured"), + node("capture_anon", "capture", "Anonymous", + "Approximate counts, $0" if anon_on else "Off — exact numbers or nothing", + 3, row=1, panel="capture", on=anon_on, + detail="Local egress only — Instagram blocks datacenters"), + # ⚠ ITS OWN NODE BECAUSE IT IS ITS OWN BILL. The profile row carries post IDENTITY + # and no engagement (measured), so likes/comments are a SECOND vendor call per post. + # A switch that multiplies a run's cost by the post count deserves to be visible on + # the canvas rather than buried in a config panel. + node("capture_metrics", "capture", "Post engagement", + "Likes and comments per post" if metrics_on + else "Off — no likes/comments are bought", + 4, panel="capture", on=metrics_on, + detail=(f"~{max_posts}x the records of a profile-only run" + if metrics_on else + "ut_ig_post_snapshots only grows while this is on")), + node("write", "write", "Write", + cfg.get("targetTable") or "No database", 5, panel="write", + on=not cfg.get("dryRun"), + detail="+ ut_ig_snapshots · ut_ig_posts · ut_ig_post_snapshots"), + ] + edges = [{"from": "trigger", "to": "source", "label": ""}, + {"from": "source", "to": "capture", "label": ""}, + {"from": "capture", "to": "capture_paid", "label": "paid"}, + {"from": "capture", "to": "capture_anon", "label": "fallback"}, + {"from": "capture_paid", "to": "capture_metrics", "label": ""}, + {"from": "capture_anon", "to": "capture_metrics", "label": ""}, + {"from": "capture_metrics", "to": "write", "label": ""}] + # When there is no paid rung the anonymous one is the ONLY rung, so it must not carry a + # switch that would leave the automation with nothing to run. + if not paid: + for n in nodes: + if n["id"] == "capture_anon": + n["toggle"] = "" + elif defn.get("kind") == "discover_instagram": + limit = int(cfg.get("recordsLimit") or 0) + est = discover_estimate(limit) + pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") + nodes += [ + node("find", "source", "Find profiles", + _predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:60], 1, + panel="find", + detail=(f"Up to {limit} profiles · est. ${est['usd']} (ESTIMATE)" + if bd_ready() + else "No AIOS_BRIGHTDATA_KEY is configured — search is closed")), + node("collect", "capture", "Collect results", + f"Building: {pending}" if pending else "Searches take about 20 minutes", 2, + panel="find", + detail="A run that finds it still building hands off to the next run"), + node("write", "write", "Write candidates", + cfg.get("targetTable") or DISCOVER_TABLE, 3, panel="write", + on=not cfg.get("dryRun"), + detail="Nothing is tracked until somebody ticks it"), + ] + edges = [{"from": "trigger", "to": "find", "label": ""}, + {"from": "find", "to": "collect", "label": ""}, + {"from": "collect", "to": "write", "label": ""}] + elif defn.get("kind") == "scrape_db": + url = str(cfg.get("url") or "") + host = (urlparse(url).hostname or url or "No URL") if url else "No URL" + which = ("Structured data (JSON-LD)" if cfg.get("extract") == "jsonld" + else f"HTML table #{int(cfg.get('tableIndex') or 0)}") + mapped = len(cfg.get("fieldMap") or {}) + nodes += [ + node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), + node("extract", "capture", "Extract", which, 2, panel="columns", + detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " + f"{cfg.get('keyField') or '—'}"), + node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, + panel="write", on=not cfg.get("dryRun"), + detail=cfg.get("targetTable") or "a new database"), + ] + edges = [{"from": "trigger", "to": "fetch", "label": ""}, + {"from": "fetch", "to": "extract", "label": ""}, + {"from": "extract", "to": "write", "label": ""}] + # ⛔ WAVE 24 — THERE IS NO BARE `else` HERE ANY MORE, and its absence is the point. + # It used to be `scrape_db`'s branch, which meant a kind this function had never been taught + # about inherited a scrape's three machine nodes: `Fetch page / No URL`, `Extract / HTML + # table #0`, `Write / Scraped table`, over a definition with no URL and no field map. A + # surface that draws nothing is obviously incomplete; one that draws a fetch step that does + # not exist is confidently wrong, and it is wrong on the canvas, the builder and the board at + # once because all three read this. + # + # ⭐ SO: `plain` (R6) — and any future kind, until somebody writes its arm — ANSWERS THE + # TRIGGER NODE AND NOTHING ELSE: `{"nodes": [], "edges": []}`. Exactly one node, + # guaranteed. The Builder's node filter and `stages_for` (hence the Board) both depend on + # that promise, so it is stated rather than left to be inferred from the control flow. + return {"nodes": nodes, "edges": edges} + + +def toggle_node(rt, auto_id, node_id): + """Flip ONE node's switch. Returns `(defn, error)`. + + The node → config mapping lives on the SERVER for the reason the whole graph does: a client + that knew which field each node writes would be a second copy of that knowledge, free to + drift. The canvas just says "this node was clicked". + """ + prev = all_definitions(rt).get(str(auto_id)) + if prev is None: + return None, "no such automation" + target = next((n for n in graph(prev)["nodes"] if n["id"] == str(node_id)), None) + if target is None: + return None, f"there is no {node_id!r} step in this automation" + if not target.get("toggle"): + return None, (f"the {target['title']} step cannot be turned off — it is what this " + f"automation is") + cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) + which = target["toggle"] + trg_key = str((prev.get("trigger") or {}).get("key") or "") + if which == "schedule" and trg_key and trg_key not in TRIGGER_SCHEDULE_KEYS: + # The trigger node's switch flips THE TRIGGER when the flow has an event one — flipping + # the cron under a node labelled "When a field changes" would be a switch that lies. + # ⭐ WAVE 24 — DERIVED from `TRIGGER_SCHEDULE_KEYS` instead of the hand-listed tuple that + # was here, which omitted `record_updated`, `enters_view` and `form_submitted` and so + # told exactly that lie for all three. The set names the SMALL side (schedule-driven), so + # a trigger added to `TRIGGER_KEYS` defaults to flipping itself, which is the safe half. + trg = dict(prev.get("trigger") or {}) + trg["enabled"] = not (bool(trg.get("enabled", True)) and not trg.get("paused")) + trg["paused"] = False # a deliberate flip clears an auto-pause: human wins + return patch(rt, auto_id, {"trigger": trg}) + if which == "schedule": + sched["enabled"] = not sched.get("enabled") + elif which == "paid": + cfg["tier"] = ("anonymous" if clean_tier(cfg.get("tier")) == "brightdata" + else "brightdata") + elif which == "fallback": + cfg["noFallback"] = not cfg.get("noFallback") + elif which == "postMetrics": + cfg["postMetrics"] = not cfg.get("postMetrics") + elif which == "write": + cfg["dryRun"] = not cfg.get("dryRun") + return patch(rt, auto_id, {"config": cfg, "schedule": sched}) + + +# --------------------------------------------------------------------------------------------- +# THE BOARD (wave 22, contracts C1/C2 — owner ruling R1: RECORDS AS CARDS) +# --------------------------------------------------------------------------------------------- +# The automation detail IS a kanban board: columns = the flow's stages, cards = records of the +# target database, each record's position stored in a single-select STAGE FIELD auto-created on +# that database — so the same field drives a plain Kanban view with no board code at all. +# +# TWO KINDS OF STAGE, and conflating them is the mistake this section is built around: +# +# * RUN stages (trigger/fetch/find/collect/capture/write) are MACHINERY — a record passes +# through them but can never sit in one, the way a parcel is never "at" the conveyor belt. +# They come straight off `graph()` so the board and the runner cannot drift. +# * RECORD stages hold cards. They are the REVIEW gate (R6: discovery's review IS the +# `tracked` promotion), its terminal, and the R5 IF-routing lanes. Only these are choices +# of the stage field — a select offering "Fetch page" as a place to PUT a record would be +# offering a lie. +# +# ⛔ WHO WRITES THE STAGE FIELD (C2): the ENGINE, everywhere — except a review stage, where a +# HUMAN may advance a card to a member of the review stage's `next` and nothing else. That wall +# is enforced twice, once per door: `move_card` below (the board's own POST) and +# `grid_events.overlay_patch` (the plain grid/kanban door), which reads the law OUT OF THE FIELD +# DEFINITION (`automation.humanMoves` / `automation.arrive`) because platform code must not +# import this module. The engine STATES the law; the grid ENFORCES it mechanically. +# +# R7's approval-posture vocabulary rides every stage: `strict` = holds for a human, +# `auto` = the engine advances. Today review⇒strict and everything else⇒auto BY DERIVATION — +# the vocabulary is on the wire so a later wave can flip a stage without a client change. + +STAGE_FIELD_PREFIX = "stage_" +STAGE_KINDS = ("trigger", "action", "review", "branch") +POSTURES = ("auto", "strict") +#: One board answer carries at most this many cards PER STAGE; `counts` always carries the +#: true totals and `truncated` names the stages that overflowed — a cap nobody can see is the +#: silent-[:N] defect ([[no-unverifiable-aggregates]]), so this one is disclosed on the wire. +BOARD_CARD_CAP = 200 + +#: R5 lanes — the per-record IF-routing vocabulary. Enumerated HERE (and shipped on the wire) +#: so the client can never grow its own union of operators (the wave-9 silent-drop class). +MAX_LANES = 6 +LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", + "is_empty", "is_not_empty") +LANE_NULLARY_OPS = ("is_empty", "is_not_empty") + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — CONDITION TREES. `Cond = leaf | {all: [Cond…]} | {any: [Cond…]}` +# +# A leaf is exactly what wave 22 called a lane condition (`{field, op, value?}`), which is why +# this generalisation needed no migration: a stored leaf IS a valid tree, `cond_match` dispatches +# on shape, and nothing at rest is rewritten until the owner saves that automation. (The doc +# calls this "legacy single-condition lanes auto-wrap as {all:[leaf]} on read" — wrapping is the +# same function applied one level up, so the cheaper honest version is to evaluate the leaf where +# it lies and never touch the bytes.) +# +# ⛔ DEPTH IS BOUNDED AND THE BOUND IS ENFORCED AT WRITE TIME, not at evaluation time. An +# unbounded tree is an unbounded evaluation on a hook that runs inside somebody's keystroke, and +# "the server got slow" is the failure nobody traces back to a filter somebody nested 40 deep. +# ⛔ REFUSE-NEVER-COERCE all the way down (`clean_predicates`' discipline): an empty group, an +# unknown comparison, a valueless compare are all REFUSED with the reason named. A group that +# quietly dropped its unanswerable leaf would WIDEN — the tri-state scar the filter engine +# carries ([[cg-filter-engine-sql-port]]), reproduced here where nothing would report it. +MAX_COND_DEPTH = 3 +MAX_COND_CHILDREN = 12 +COND_GROUP_KEYS = ("all", "any") + + +def clean_cond(raw, depth=0, where=""): + """Validate one condition TREE. Returns `(cond|None, error)`; `(None, None)` means "no + condition", which is legal everywhere a condition is optional (the catch-all lane, an + unconditioned trigger, an action group that always runs).""" + if raw in (None, "", {}): + return None, None + at = f" on {where}" if where else "" + if not isinstance(raw, dict): + return None, f"the condition{at} must be an object" + conj = [k for k in COND_GROUP_KEYS if k in raw] + if len(conj) > 1: + return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the " + f"other") + if conj: + key = conj[0] + if depth + 1 > MAX_COND_DEPTH: + return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — " + f"the group{at} is deeper") + kids_raw = raw.get(key) + if not isinstance(kids_raw, list) or not kids_raw: + return None, f"the '{key}' group{at} needs at least one condition inside it" + if len(kids_raw) > MAX_COND_CHILDREN: + return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — " + f"the '{key}' group{at} has {len(kids_raw)}") + kids = [] + for child in kids_raw: + c, err = clean_cond(child, depth + 1, where) + if err: + return None, err + if c is None: + return None, (f"an empty condition sits inside the '{key}' group{at} — " + f"finish it or remove it") + kids.append(c) + return {key: kids}, None + field = _s(raw.get("field"), 80).strip() + op = _s(raw.get("op") or raw.get("operator"), 20).strip() + if not field: + return None, f"the condition{at} names no field" + if op not in LANE_OPS: + return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) + cond = {"field": field, "op": op} + if op not in LANE_NULLARY_OPS: + v = raw.get("value") + if v is None or (isinstance(v, str) and not v.strip()): + return None, f"give a value to compare {field!r} against{at}" + cond["value"] = v.strip() if isinstance(v, str) else v + return cond, None + + +def cond_fields(cond): + """Every field name a tree reads — the set a caller must have on the row image before the + answer means anything.""" + if not isinstance(cond, dict): + return set() + for key in COND_GROUP_KEYS: + if key in cond: + out = set() + for child in cond.get(key) or []: + out |= cond_fields(child) + return out + f = cond.get("field") + return {f} if f else set() +#: Lane labels a flow already uses for its fixed stages. A lane literally called "Review" would +#: collide with the stage the label is a choice FOR, and the board could no longer tell a +#: routed card from a gated one. +_RESERVED_STAGE_LABELS = frozenset({"review", "tracked", "declined", "write", "profile set", + "schedule", "stage"}) + + +def stage_field_key(auto_id): + """The stage field's key on the target database. Derived, never chosen — two automations on + one table must mint two distinct fields, and a rename must never move the key.""" + return STAGE_FIELD_PREFIX + re.sub(r"[^a-z0-9_]+", "_", str(auto_id or "").lower())[:40] + + +def clean_lanes(raw): + """Validate the R5 lane list. Returns `(lanes, error)` — refuses, never coerces + (the `clean_predicates` discipline: a condition we cannot answer is refused at WRITE time + with the reason named, because dropping or bending it would route records nobody asked to + route).""" + if raw in (None, ""): + return [], None + if not isinstance(raw, list): + return None, "lanes must be a list" + if len(raw) > MAX_LANES: + return None, f"an automation may have at most {MAX_LANES} lanes" + out, seen_ids, defaults = [], set(), 0 + for entry in raw: + if not isinstance(entry, dict): + return None, "each lane must be an object with a label and an optional condition" + label = " ".join(_s(entry.get("label"), 60).split()) + if not label: + return None, "every lane needs a label" + if label.strip().lower() in _RESERVED_STAGE_LABELS: + return None, (f"{label!r} is the name of a fixed stage — pick another label for " + f"the lane") + lid = "lane_" + (re.sub(r"[^a-z0-9]+", "_", label.lower()).strip("_") or "x")[:40] + base, n = lid, 2 + while lid in seen_ids: + lid = f"{base}_{n}" + n += 1 + seen_ids.add(lid) + cond, cerr = clean_cond(entry.get("when"), where=repr(label)) + if cerr: + return None, cerr + if cond is None: + defaults += 1 + if defaults > 1: + return None, ("only one lane may have no condition — it is the catch-all, " + "and two catch-alls would race for every record") + out.append({"id": lid, "label": label, "when": cond}) + return out, None + + +def _lane_num(v): + try: + return float(str(v).replace(",", "")) + except (TypeError, ValueError): + return None + + +def lane_match(cond, row): + """Does `row` satisfy this condition TREE? None matches everything (the catch-all). + + Named for the lane that first needed it; it is now C4's whole evaluator, and every wave-22 + caller (`route_record`, `_settle_eval`, `_seed_event_state`) gained tree support by keeping + this name rather than growing a second entry point that could disagree with it. + + ⛔ REFUSE-NEVER-COERCE, per record: an ordering comparison whose either side does not parse + as a number answers False — the record simply does not enter the lane — never "treat blank + as 0", which is the `toNum(null)=0` widening the 2026-08-03 owner wave was about. A blank + cell is an unknown, and an unknown cannot be less than 10,000. + """ + if cond is None: + return True + if isinstance(cond, dict): + if "all" in cond: + return all(lane_match(c, row) for c in cond.get("all") or []) + if "any" in cond: + return any(lane_match(c, row) for c in cond.get("any") or []) + raw = (row or {}).get(cond.get("field")) + op = cond.get("op") + if op == "is_empty": + return raw in (None, "") + if op == "is_not_empty": + return raw not in (None, "") + want = cond.get("value") + if op in (">", ">=", "<", "<="): + a, b = _lane_num(raw), _lane_num(want) + if a is None or b is None: + return False + return {"<": a < b, "<=": a <= b, ">": a > b, ">=": a >= b}[op] + have_s, want_s = str(raw if raw is not None else ""), str(want if want is not None else "") + if op == "includes": + return want_s.lower() in have_s.lower() + if op == "not_includes": + return want_s.lower() not in have_s.lower() + a, b = _lane_num(raw), _lane_num(want) + same = (a is not None and b is not None and a == b) or have_s == want_s + return same if op == "=" else (not same) if op == "!=" else False + + +def route_record(lanes, row): + """The winning lane's LABEL for one record, or '' when nothing matches. + + First declared match wins — declaration order is the priority order, the same rule every + rule engine the owner has used applies, and the catch-all (condition None) is simply the + first lane nothing above it beat. + """ + for lane in lanes or []: + if lane_match(lane.get("when"), row): + return lane["label"] + return "" + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4 — ACTIONS (owner ruling R3). The half of the Airtable builder that DOES things. +# +# `flow.actions` is an ordered list walked per RECORD, after the flow's machine steps have run. +# A `group` holds nested actions behind a condition — Airtable's "conditional action group", +# which the owner explicitly asked to be NESTABLE (R3 supersedes the wave-22 research brief's +# rec-8 "no nesting" verdict; that recommendation was about keeping a CANVAS legible, and this +# builder is a column, not a canvas). +# +# ⛔ THE CATALOG IS SERVER-OWNED AND INCLUDES WHAT WE HAVE NOT BUILT. `ACTION_CATALOG` carries a +# `ready` flag per kind, so the "+ Add advanced logic or action" menu paints Send email / Slack / +# Run script / Generate with AI faded with a reason instead of omitting them — the same honesty +# rule the trigger list follows, and the same enforcement: `clean_actions` REFUSES an unready +# kind with a sentence, so the faded state is a wall and not a styling choice. +MAX_ACTIONS = 20 # per automation, counting nested ones +MAX_GROUP_DEPTH = 2 # a group inside a group inside a group is a flowchart, not a flow +#: ⭐ WAVE 24 · C-FORK — branches per If / then. The SAME number as `MAX_LANES` and deliberately +#: so: a lane and a branch are the same act (route this record by condition), on two surfaces, +#: and two different ceilings for one act would be a difference a person has to learn for no +#: reason. Named separately so they can diverge on evidence rather than by accident. +MAX_BRANCHES = MAX_LANES +MAX_ACTION_VALUES = 20 # cells one update/create action may write +FIND_LIMIT_MAX = 100 +#: ⭐ WAVE 24 · C-ACT — THE MENU'S GROUP ORDER IS THE SERVER'S. The client sorts by `groupOrder` +#: and never by a literal list of group names: a client-side ordering is a second copy of this +#: table, and the way it fails is that a group added here renders last, or not at all, with +#: nothing red anywhere. +ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} +ACTION_CATALOG = [ + # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, BUILT IN WAVE 25 (DEBT D-51). ──── + # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row + # is a WALL, not a styling choice — and each `detail` says the true reason rather than a + # placeholder, because "coming soon" on five rows is how a menu stops being believed. + {"kind": "web_goto", "label": "Open a page", "group": "Web action", "ready": False, + "detail": "Runs a real browser inside a job — building next (wave 25)"}, + {"kind": "web_fill", "label": "Fill a field", "group": "Web action", "ready": False, + "detail": "Needs the recorder that captures a selector instead of a coordinate"}, + {"kind": "web_click", "label": "Click something", "group": "Web action", "ready": False, + "detail": "Needs the recorder that captures a selector instead of a coordinate"}, + {"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": False, + "detail": "Needs the browser job — the scrape action reads static pages today"}, + {"kind": "web_repair", "label": "Repair a broken step", "group": "Web action", + "ready": False, + "detail": "Self-healing selectors arrive with the browser engine (Scrapling)"}, + # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── + {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, + "detail": "Write values onto the record walking the flow"}, + {"kind": "create_record", "label": "Create record", "group": "Database", "ready": True, + "detail": "Add a row to another database"}, + {"kind": "find_records", "label": "Find records", "group": "Database", "ready": True, + "detail": "Look rows up by condition; the run log opens them"}, + # ── 3. CONNECTED ───────────────────────────────────────────────────────────────────────── + {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, + "detail": "Needs a send scope on the Gmail connection"}, + {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, + "detail": "Needs the Slack connector"}, + # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── + # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism, + # and R8 made it a FORK with lettered branches, which is a thing people already have a name + # for. The KIND is untouched — renaming it would orphan every stored action for a caption. + {"kind": "group", "label": "If / then", "group": "Advanced logic", "ready": True, + "detail": "Send the record down one of several branches, by condition"}, + {"kind": "review", "label": "Manual review", "group": "Advanced logic", "ready": True, + "detail": "Hold the record for a person — or an AI — to decide where it goes next"}, + {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", + "ready": False, "detail": "Run the same actions on every item in a list"}, + {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, + "detail": "Not built — a sandbox is its own decision"}, + {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", + "ready": False, "detail": "Review stages can already be decided by AI"}, +] +REVIEW_DECIDERS = ("user", "ai") +MAX_AI_PROMPT = 600 + + +def action_catalog(): + """The catalog as the wire carries it — a copy, because a caller that mutated the module + constant would change every later reader's answer. + + ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from + `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two + different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST + (not first) — a new group appearing above "Web action" because its order defaulted to 0 is + the failure that would look deliberate. + """ + last = max(ACTION_GROUP_ORDER.values()) + 1 + return [{**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)} + for a in ACTION_CATALOG] + + +def _action_label(act): + for row in ACTION_CATALOG: + if row["kind"] == act.get("kind"): + return row["label"] + return str(act.get("kind") or "Action") + + +def clean_actions(raw, depth=0, _seen=None, _count=None): + """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. + + Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder + survives a Save. A missing or colliding one is minted `act_`; minting on every clean would + move the selection under the person editing it. + """ + if raw in (None, ""): + return [], None + if not isinstance(raw, list): + return None, "actions must be a list" + seen = _seen if _seen is not None else set() + count = _count if _count is not None else [0] + out = [] + for entry in raw: + if not isinstance(entry, dict): + return None, "each action must be an object with a kind" + kind = _s(entry.get("kind"), 30).strip() + row = next((r for r in ACTION_CATALOG if r["kind"] == kind), None) + if row is None: + return None, (f"{kind or 'that action'!r} is not one of: " + + ", ".join(a["kind"] for a in ACTION_CATALOG)) + if not row["ready"]: + return None, (f"{row['label']!r} is on the menu but not built yet — " + f"{row['detail'][0].lower()}{row['detail'][1:]}") + count[0] += 1 + if count[0] > MAX_ACTIONS: + return None, f"an automation runs at most {MAX_ACTIONS} actions" + aid = _s(entry.get("id"), 40).strip() + if not re.fullmatch(r"act_[a-z0-9_]{1,32}", aid or "") or aid in seen: + n = 1 + while f"act_{n}" in seen: + n += 1 + aid = f"act_{n}" + seen.add(aid) + when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") + if cerr: + return None, cerr + cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} + cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count) + if cerr: + return None, cerr + out.append({"id": aid, "kind": kind, + "enabled": bool(entry.get("enabled", True)), + "when": when, "config": cfg}) + return out, None + + +def _clean_action_config(kind, cfg, depth, seen, count): + """One action's `config`, per kind. Returns `(config, error)`.""" + if kind == "group": + # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, + # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an + # if with no else, so expressing "otherwise" meant a second group carrying the negation + # by hand — two conditions a later edit could put out of step with each other. + if depth + 1 > MAX_GROUP_DEPTH: + return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — " + f"past that a flow is a flowchart and belongs on the board") + raw_branches = cfg.get("branches") + if not isinstance(raw_branches, list) or not raw_branches: + # ⛔ MIGRATION, NOT A REFUSAL. The shipped shape is exactly one implicit branch, and + # the live automations carry it — refusing it would 400 every Save of a flow that + # was legal yesterday. Read-side, so nothing has to be rewritten in the store. + raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions")}] + if len(raw_branches) > MAX_BRANCHES: + return None, (f"an If / then has at most {MAX_BRANCHES} branches — past that the " + f"record is being routed, which is what the board's lanes are for") + out_branches, bseen = [], set() + for i, br in enumerate(raw_branches): + if not isinstance(br, dict): + return None, "each branch of an If / then is an object" + cond, cerr = clean_cond(br.get("cond"), where="the branch") + if cerr: + return None, cerr + if cond is None and i != len(raw_branches) - 1: + # ⛔ ONE Otherwise, and it is LAST. `_walk` takes the first branch that matches + # and a null condition matches everything, so a catch-all above another branch + # would make every branch below it dead code — silently, and only for the + # records that reached it. Refused with the reason rather than reordered: this + # module does not quietly rewrite what somebody built. + return None, ("only the LAST branch of an If / then can be the Otherwise leg — " + "a catch-all above another branch makes the ones below it " + "unreachable") + kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count) + if kerr: + return None, kerr + if not kids: + return None, "a branch with no actions inside it does nothing" + bid = _s(br.get("id"), 40).strip() + if not re.fullmatch(r"b[a-z0-9_]{0,32}", bid or "") or bid in bseen: + bid = f"b{i + 1}" + bseen.add(bid) + label = " ".join(_s(br.get("label"), 40).split()) or ( + "Otherwise" if cond is None else _branch_letter(i)) + out_branches.append({"id": bid, "label": label, "cond": cond, "actions": kids}) + return {"branches": out_branches}, None + if kind in ("update_record", "create_record"): + values = cfg.get("values") + if not isinstance(values, dict) or not values: + return None, f"the {kind.replace('_', ' ')} action writes no values" + if len(values) > MAX_ACTION_VALUES: + return None, f"one action writes at most {MAX_ACTION_VALUES} cells" + clean_vals = {} + for k, v in values.items(): + key = _s(k, 80).strip() + if not key: + return None, "a value is written to a field with no name" + if v is not None and not isinstance(v, (str, int, float, bool)): + return None, (f"the value for {key!r} must be text or a number — " + f"an automation writes cells, not objects") + clean_vals[key] = _s(v, 500) if isinstance(v, str) else v + out = {"values": clean_vals} + # ⭐ WAVE 24 — an OPTIONAL self-given name, so a SEEDED action can say what it is + # ("Create an Instagram record", C-TRIG law 3) instead of wearing the catalog's generic + # "Create record". Follows the `review` arm below, which has carried `config.label` since + # wave 23 — one precedent, not a new mechanism, and it lives inside the untyped `config` + # bag so no shared client type has to change for it. + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + if kind == "create_record": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the create record action names no database" + if not table.startswith(UT_PREFIX): + return None, (f"actions write to blank databases (ut_*) this wave — " + f"{table!r} is not one") + out["table"] = table + return out, None + if kind == "find_records": + table = _s(cfg.get("table"), 60).strip() + if not table: + return None, "the find records action names no database" + cond, cerr = clean_cond(cfg.get("cond"), where="the find") + if cerr: + return None, cerr + try: + limit = int(cfg.get("limit") or 25) + except (TypeError, ValueError): + return None, "the find records limit must be a whole number" + if limit < 1 or limit > FIND_LIMIT_MAX: + return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" + return {"table": table, "cond": cond, "limit": limit}, None + if kind == "review": + by = _s(cfg.get("decidedBy"), 10).strip() or "user" + if by not in REVIEW_DECIDERS: + return None, f"a review is decided by one of: {', '.join(REVIEW_DECIDERS)}" + label = " ".join(_s(cfg.get("label"), 60).split()) or "Review" + if label.strip().lower() in _RESERVED_STAGE_LABELS and label != "Review": + return None, f"{label!r} is the name of a fixed stage — name the review something else" + nxt = cfg.get("next") + outs = [] + if isinstance(nxt, list): + for entry in nxt[:MAX_LANES]: + lbl = " ".join(_s(entry, 60).split()) + if lbl and lbl not in outs: + outs.append(lbl) + if not outs: + outs = ["Approved", "Declined"] + prompt = _s(cfg.get("prompt"), MAX_AI_PROMPT).strip() + if by == "ai" and not prompt: + return None, ("an AI review needs a prompt — tell it what to decide, or set the " + "review back to a person") + return {"decidedBy": by, "label": label, "next": outs, "prompt": prompt}, None + return {}, None + + +def _branch_letter(i): + """0 -> 'A', 1 -> 'B', … (R8's lettering). Past 'Z' it doubles rather than wrapping, so a + label is never reused — `MAX_BRANCHES` makes that unreachable, and a silent collision is + worse than an ugly name.""" + return chr(65 + i % 26) * (1 + i // 26) + + +def group_branches(act): + """The branches of a group action, MIGRATING the pre-wave-24 `{cond, actions}` shape. + + ⭐ ONE READER for the fork's shape, because the alternative is what this module keeps paying + for: `clean_actions`, `walk_actions` and the runner would each need to know that a stored + group might carry either shape, and the one that forgot would silently skip a live + automation's actions. A definition cleaned since wave 24 always carries `branches`; one read + straight out of the store (the runner reads definitions, not payloads) may not. + """ + cfg = (act or {}).get("config") or {} + brs = cfg.get("branches") + if isinstance(brs, list) and brs: + return [b for b in brs if isinstance(b, dict)] + return [{"id": "b1", "label": "A", "cond": cfg.get("cond"), + "actions": cfg.get("actions") or []}] + + +def walk_actions(actions): + """Every action in the tree, depth-first, groups included. One walker, so "how many actions + does this flow have" and "which review stages exist" cannot answer differently. + + ⚠ WAVE 24: a group's children live under its BRANCHES now. This walker feeds the action + COUNT, `review_actions` (hence the board's stages) and `compose_sentence` — so a version + that still read `config.actions` would report a fork's contents as zero actions, hide every + review inside a branch from the board, and let `MAX_ACTIONS` be exceeded without noticing. + """ + for act in actions or []: + yield act + if act.get("kind") == "group": + for br in group_branches(act): + for kid in walk_actions(br.get("actions")): + yield kid + + +def review_actions(defn): + """The review actions of one definition, in flow order — the stages a card can REST at.""" + flow = (defn or {}).get("flow") or {} + return [a for a in walk_actions(flow.get("actions")) if a.get("kind") == "review" + and a.get("enabled", True)] + + +def interpolate(text, row): + """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an + action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would + put template syntax in front of a customer, and a blank is the visible failure.""" + if not isinstance(text, str) or "{{" not in text: + return text + def _sub(m): + v = (row or {}).get(m.group(1).strip()) + return "" if v is None else str(v) + return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) + + +def stages_for(defn): + """One automation → `(stages, routes)` — C1's vocabulary, derived from `graph()` so the + board can never disagree with the runner about what the flow does. + + Stage extras beyond the graph node: `posture` (R7), `record` (may this stage HOLD cards), + `assignable` (may a card be PLACED here — i.e. is its label a choice of the stage field). + `record` without `assignable` marks the derived entry buckets (Profile set / Write): a card + shows there because of what is true about the row, not because anything stamped it. + """ + defn = defn if isinstance(defn, dict) else {} + cfg = defn.get("config") or {} + g = graph(defn) + kind_map = {"trigger": "trigger", "branch": "branch"} + stages, routes = [], [] + for n in g["nodes"]: + stages.append({ + "id": n["id"], "label": n["title"], + "kind": kind_map.get(n["kind"], "action"), + "col": n["col"], "row": n["row"], "enabled": n["enabled"], + "toggle": n["toggle"], "status": n["status"], "detail": n["detail"], + # C1 amendment (2026-08-05): the node's `panel` rides VERBATIM — B joins stage → + # config face on this key, never by guessing that ids match. + "panel": n.get("panel") or n["id"], + "posture": "auto", "record": False, "assignable": False, + }) + for e in g["edges"]: + routes.append({"from": e["from"], "to": e["to"], "label": e.get("label") or "", + "condition": None}) + maxcol = max((s["col"] for s in stages), default=0) + + def _mark(sid, record=True, assignable=False): + for s in stages: + if s["id"] == sid: + s["record"], s["assignable"] = record, assignable + + if defn.get("kind") == "discover_instagram": + # R6: the review gate lives at the END of discovery, and the human card-move IS the + # `tracked` promotion — R7-W20's tick, become a gesture. Nothing here auto-promotes. + # C3-A2(5): the gate has TWO exits — Approve (→ Tracked) and Decline (→ Declined, a + # TERMINAL state, never a delete: a candidate somebody rejected is a decision worth + # keeping, and deleting the row would un-decide it on the next re-find). + stages.append({"id": "review", "label": "Review", "kind": "review", + "col": maxcol + 1, "row": 0, "enabled": True, "toggle": "", + "status": "idle", "panel": "review", "posture": "strict", + "record": True, "assignable": True, + "detail": "Cards wait here for a person — moving one on IS the " + "tracked promotion"}) + stages.append({"id": "tracked", "label": "Tracked", "kind": "action", + "col": maxcol + 2, "row": 0, "enabled": True, "toggle": "", + "status": "idle", "panel": "tracked", "posture": "auto", + "record": True, "assignable": True, + "detail": "The set the Instagram column automation enriches"}) + stages.append({"id": "declined", "label": "Declined", "kind": "action", + "col": maxcol + 2, "row": 1, "enabled": True, "toggle": "", + "status": "idle", "panel": "declined", "posture": "auto", + "record": True, "assignable": True, + "detail": "Rejected candidates stay here — a kept decision, not a " + "delete"}) + routes.append({"from": "write", "to": "review", "label": "", "condition": None}) + routes.append({"from": "review", "to": "tracked", "label": "Approve", + "condition": None}) + routes.append({"from": "review", "to": "declined", "label": "Decline", + "condition": None}) + else: + # ⭐ WAVE 24 — a `plain` automation's derived entry bucket is the TRIGGER, because it has + # no machine node to be one. Without this arm `_card_stage`'s fallback hands every card + # the id `write`, which does not exist on a plain board: the cards would count against a + # stage that is in `counts` and absent from `stages`, i.e. records the board says exist + # and cannot show. The two functions have to agree, so they are edited together. + entry = {"field_instagram": "source", "plain": "trigger"}.get(defn.get("kind"), "write") + _mark(entry) + _mark("write") + for i, lane in enumerate(cfg.get("lanes") or []): + cond = lane.get("when") + stages.append({"id": lane["id"], "label": lane["label"], "kind": "branch", + "col": maxcol + 1, "row": i, "enabled": True, "toggle": "", + "status": "idle", "panel": "lanes", "posture": "auto", + "record": True, "assignable": True, + "detail": _lane_sentence(cond)}) + routes.append({"from": "write", "to": lane["id"], + "label": lane["label"] if cond is None else _lane_sentence(cond), + "condition": dict(cond) if cond else None}) + + # ── WAVE 23 · C4/C6 — REVIEW ACTIONS ARE STAGES. ───────────────────────────────────────── + # A review action is the one action kind that HOLDS a record rather than doing something to + # it, so it has to exist on the board: a flow whose review lived only in the builder would + # gate records at a place the kanban cannot show, which is the whole complaint the board was + # built to answer. Its declared exits become assignable stages too — a gate with nowhere to + # go is a hole records fall into. + tail = max((s["col"] for s in stages), default=0) + for i, act in enumerate(review_actions(defn)): + acfg = act.get("config") or {} + rid = act["id"] + stages.append({"id": rid, "label": acfg.get("label") or "Review", "kind": "review", + "col": tail + 1 + (i * 2), "row": 0, "enabled": True, "toggle": "", + "status": "idle", "panel": "review_action", "posture": "strict", + "record": True, "assignable": True, "actionId": rid, + "decidedBy": acfg.get("decidedBy") or "user", + "detail": ("An AI decides where each card goes — a person can override it" + if acfg.get("decidedBy") == "ai" + else "Cards wait here for a person to move them on")}) + for j, label in enumerate(acfg.get("next") or []): + oid = f"{rid}_out_{j}" + stages.append({"id": oid, "label": label, "kind": "action", + "col": tail + 2 + (i * 2), "row": j, "enabled": True, "toggle": "", + "status": "idle", "panel": "review_action", "posture": "auto", + "record": True, "assignable": True, "actionId": rid, + "detail": f"Where {acfg.get('label') or 'Review'} sends a card"}) + routes.append({"from": rid, "to": oid, "label": label, "condition": None}) + return stages, routes + + +def terminal_stage_ids(stages, routes): + """Stages a record can REST at with nowhere further to go — the ends C5's `ending` acts on. + Derived from the routes rather than declared, so a flow cannot gain an exit without this + noticing (and a lane that stops being terminal stops being reset).""" + assignable = {s["id"] for s in stages if s.get("assignable")} + onward = {r["from"] for r in routes if r["to"] in assignable and r["to"] != r["from"]} + return [sid for sid in assignable if sid not in onward] + + +def _stage_of(stages, sid): + for s in stages: + if s["id"] == sid: + return s + return None + + +# ───────────────────────────────────────────────────────────────────────────────────────────── +# WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. +# +# ⛔ ONE COALESCED WRITE PER TABLE PER RUN. The runner accumulates patches in memory and commits +# them once at the end (the 256-commits/hr budget every writer in this module respects). A +# per-action `rt.update` would be correct and would also spend the tenant's whole commit budget +# on one busy flow. +# ⛔ A REVIEW SUSPENDS THAT RECORD AND NOTHING ELSE. When a card reaches a review action the +# runner stamps its stage and stops walking THAT record — later actions belong to the branch a +# human (or the AI) has not chosen yet. Other records keep going; a gate is not a global pause. +# ⛔ EVERY WRITE HERE IS MACHINE-ORIGIN and goes through this module's own writers, never the +# human doors — which is exactly what keeps A2(2)'s loop prevention structural: an action's write +# cannot fire an event trigger, its own or a sibling's. + +def _act_row_patch(patches, table, rid, values): + patches.setdefault(table, {}).setdefault(str(rid), {}).update(values) + + +def _flow_table(defn): + """The database a flow's actions and ending operate on: the automation's own target, or the + table its event trigger watches when the flow has no target of its own.""" + cfg = (defn or {}).get("config") or {} + if defn.get("kind") == "discover_instagram": + return cfg.get("targetTable") or DISCOVER_TABLE + return cfg.get("targetTable") or ((defn.get("trigger") or {}).get("table") or "") + + +def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print): + """Walk `flow.actions` for each of `row_ids`, then apply the ENDING. Returns the run counts. + + ⚠ The ending runs even when there are NO actions, and that is not a detail: a discovery + automation's terminal stages (`tracked`/`declined`) come from `stages_for`, not from actions, + so an early return on an empty action list would have silently disabled the owner's loop + feature on the flagship flow — the one kind most likely to want `auto_reset`. Caught in + review; the tests all happened to pass a non-empty action list, which is why it was green. + """ + flow = (defn or {}).get("flow") or {} + actions = flow.get("actions") or [] + counts = {"actionsRun": 0, "updated": 0, "created": 0, "found": 0, "review": 0, + "aiDecided": 0, "cycles": 0} + if not table_key: + return counts + table = str(table_key) + rows = dict((ut_get(rt, table) or {}).get("rows") or {}) + if not rows: + return counts + stages, routes = stages_for(defn) + skey = stage_field_key(defn.get("id")) + stamp = _iso() + patches, creates, ai_budget = {}, {}, [AI_DECISIONS_PER_RUN] + + def _walk(acts, row, rid, depth=0): + """Returns True when the record was SUSPENDED (a review holds it).""" + for act in acts: + if not act.get("enabled", True): + continue + if not lane_match(act.get("when"), row): + continue + kind = act.get("kind") + cfg = act.get("config") or {} + counts["actionsRun"] += 1 + if kind == "group": + # ⭐ WAVE 24 · C-FORK — FIRST MATCHING BRANCH WINS, and only that one runs. + # Declaration order is priority order, the same rule `route_record` applies to + # the board's lanes, and the Otherwise leg (`cond: null`) is simply the branch + # nothing above it beat — `lane_match(None, row)` is True, and `clean_actions` + # has already guaranteed a null condition can only be LAST. + for br in group_branches(act): + if lane_match(br.get("cond"), row): + if _walk(br.get("actions") or [], row, rid, depth + 1): + return True + break + elif kind == "update_record": + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + row.update(vals) # later actions see the write, as they must + _act_row_patch(patches, table, rid, vals) + counts["updated"] += 1 + elif kind == "create_record": + target = str(cfg.get("table") or "") + vals = {k: interpolate(v, row) for k, v in (cfg.get("values") or {}).items()} + creates.setdefault(target, []).append(vals) + counts["created"] += 1 + elif kind == "find_records": + found = find_records(rt, cfg.get("table"), cfg.get("cond"), + int(cfg.get("limit") or 25)) + counts["found"] += len(found) + elif kind == "review": + label = cfg.get("label") or "Review" + at = str(row.get(skey) or "").strip() + if at == label: + return True # already waiting here — do not re-stamp + if at and at in (cfg.get("next") or []): + # ⛔ ALREADY DECIDED — the card went through this gate on an earlier run, so + # the flow continues past it. Without this the next run would stamp a + # finished card back to the review gate and hold it there again, forever: + # a card could never leave a flow that runs on a schedule. Found by this + # wave's own terminal-ending negative control, which is the argument for + # writing the NC in the same change as the feature. + continue + decided = "" + if cfg.get("decidedBy") == "ai" and ai_budget[0] > 0: + ai_budget[0] -= 1 + decided = ai_decide(rt, defn, act, row, rid, log=log) + if decided: + counts["aiDecided"] += 1 + _act_row_patch(patches, table, rid, + {skey: decided, skey + "_at": stamp}) + row[skey] = decided + continue # the AI moved it on — keep walking + _act_row_patch(patches, table, rid, {skey: label, skey + "_at": stamp}) + row[skey] = label + counts["review"] += 1 + return True + return False + + held = [] + if actions: + for rid in list(row_ids or [])[:FLOOD_LIMIT]: + row = dict(rows.get(str(rid)) or {}) + if not row: + continue + if _walk(actions, row, str(rid)): + held.append(str(rid)) + counts["cycles"] = _apply_ending(rt, defn, table, rows, patches, stages, routes, stamp) + _commit_action_writes(rt, table, patches, creates, username, log) + if held: + notify_review(rt, defn, held, log=log) + return counts + + +#: C6 cap: AI decisions ONE run may make. A flow that suddenly matches 500 records must not turn +#: into 500 paid calls before anybody notices — the rest simply wait for a human, which is the +#: behaviour the feature degrades to everywhere else too. +AI_DECISIONS_PER_RUN = 25 + + +def ai_decide(rt, defn, act, row, row_id="", log=print): + """R4/C6: let the model pick this card's next stage. Returns the chosen stage LABEL, or "" + to leave the card for a person. + + ⛔ FAIL-CLOSED IN EVERY DIRECTION: no provider configured, a network failure, a malformed + answer, or a label the review does not offer all return "" — and "" means the card sits at + the review gate exactly as it would with no AI at all. The feature can be broken, absent or + wrong and the worst outcome is a human doing the work. + """ + cfg = act.get("config") or {} + options = list(cfg.get("next") or []) + if not options: + return "" + try: + import ai_review + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] ai review unavailable: {type(e).__name__}: {e}") + return "" + fields = [f.get("key") for f in + (ut_get(rt, ((defn.get("config") or {}).get("targetTable") + or (defn.get("trigger") or {}).get("table") or "")) or {} + ).get("fields") or []] + choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, + row=row, fields=[f for f in fields if f], + label=cfg.get("label") or "Review") + if not choice: + if meta.get("problem"): + log(f"[aios-auto] ai review declined to answer: {meta['problem']}") + return "" + review_audit(rt, defn.get("id"), row_id, cfg.get("label") or "Review", + choice, meta.get("provider") or "ai", by="ai", + note=meta.get("reason") or "", model=meta.get("model") or "") + return choice + + +def notify_review(rt, defn, row_ids, log=print): + """C6: one notification per RUN naming the count — never one per record. + + A discovery pass that gates 300 candidates would otherwise put 300 rows in somebody's inbox + and the inbox would stop being read, which costs more than the notification gains. The board + is where the individual cards live; the bell says "there is work", once. + """ + if not row_ids: + return + owner = str((defn or {}).get("createdBy") or "").strip() + if not owner: + return + n = len(row_ids) + try: + import core.alerts as alerts + alerts.notify(owner, + f"{defn.get('name') or 'An automation'} — {n} " + f"record{'' if n == 1 else 's'} waiting for review", + topic="automation", key=str(defn.get("id") or ""), st=rt) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] review notification failed: {type(e).__name__}: {e}") + + +def find_records(rt, table_key, cond, limit=25): + """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts + the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" + rows = (ut_get(rt, str(table_key or "")) or {}).get("rows") or {} + out = [] + for rid, row in rows.items(): + if lane_match(cond, row or {}): + out.append(str(rid)) + if len(out) >= max(1, min(int(limit or 25), FIND_LIMIT_MAX)): + break + return out + + +def _apply_ending(rt, defn, table, rows, patches, stages, routes, stamp): + """C5: send terminal cards back round, per the ending mode. Returns cycles started. + + A RESET CLEARS THE STAMP rather than writing a start stage, so the card falls back to the + DERIVED bucket `_card_stage` would give a record that had never been stamped at all — which + is what "back to the first cycle" means, and it needs no opinion about which stage is first. + """ + ending = ((defn or {}).get("flow") or {}).get("ending") or {} + mode = ending.get("mode") or "terminal" + if mode in ("terminal", "manual_reset"): + return 0 + terminals = set(terminal_stage_ids(stages, routes)) + if not terminals: + return 0 + labels = {s["label"] for s in stages if s["id"] in terminals} + skey = stage_field_key(defn.get("id")) + cutoff = None + if mode == "timed_reset": + cutoff = _now() - _dt.timedelta(hours=int(ending.get("hours") or 24)) + started = 0 + for rid, row in rows.items(): + merged = {**(row or {}), **((patches.get(table) or {}).get(str(rid)) or {})} + if str(merged.get(skey) or "").strip() not in labels: + continue + if cutoff is not None: + at = _parse_iso(str(merged.get(skey + "_at") or "")) + if at is None or at > cutoff: + continue + n = (_ig_int(merged.get(cycles_field_key(defn.get("id")))) or 0) + 1 + _act_row_patch(patches, table, rid, + {skey: "", skey + "_at": stamp, + cycles_field_key(defn.get("id")): n}) + started += 1 + return started + + +def _commit_action_writes(rt, table, patches, creates, username, log): + """The ONE write. Row patches merge into the target's rows; creates append to their own + tables through the ordinary cap-respecting path.""" + if not patches and not creates: + return + for tkey, rowpatch in (patches or {}).items(): + cur = dict((ut_get(rt, tkey) or {}).get("rows") or {}) + for rid, vals in rowpatch.items(): + cur[str(rid)] = {**(cur.get(str(rid)) or {}), **vals} + ut_write_rows(rt, tkey, cur) + for tkey, new_rows in (creates or {}).items(): + t = ut_get(rt, tkey) + if t is None: + log(f"[aios-auto] create_record: {tkey} no longer exists — {len(new_rows)} skipped") + continue + cur = dict(t.get("rows") or {}) + nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 + cap = row_cap(tkey) + for vals in new_rows: + if len(cur) >= cap: + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap") + break + cur[str(nxt)] = dict(vals) + nxt += 1 + ut_write_rows(rt, tkey, cur) + + +def _lane_sentence(cond, top=True): + """One condition tree → the sentence the board and the field definition carry. + + C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a + lane label never claims a flat comparison the tree does not make. The server composes it for + the same reason it composes every other stage `detail` — a client paraphrase of a structure + the engine evaluates is a second implementation of the same sentence, free to drift from it. + """ + if cond is None: + return "Everything else" if top else "" + if isinstance(cond, dict): + for key, joiner in (("all", " and "), ("any", " or ")): + if key in cond: + parts = [_lane_sentence(c, False) for c in cond.get(key) or []] + parts = [p for p in parts if p] + if not parts: + return "" + inner = joiner.join(parts) + return inner if top or len(parts) == 1 else f"({inner})" + v = cond.get("value") + return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") + + +def _stage_choices(stages): + """The stage field's select vocabulary: the ASSIGNABLE stage labels, in board order.""" + return [s["label"] for s in sorted(stages, key=lambda s: (s["col"], s["row"])) + if s.get("assignable")] + + +def ensure_stage_field(rt, table_key, defn, username="automation"): + """C2: the stage field on the target database — created/refreshed idempotently, skipped + entirely when the flow has nothing a record can be PLACED in (a select with an empty + vocabulary is a column that can hold no value; it appears when the first lane or review + stage does). + + The field carries the WHOLE law the grid door needs (`automation.humanMoves` — from-label → + admissible to-labels — and `automation.arrive` — to-label → extra cell writes, which is how + the kanban drag Review→Tracked also sets `tracked`), because `grid_events` may not import + this module: the engine states the law where the enforcer can read it. + """ + key = str(table_key or "") + if not key or ut_get(rt, key) is None: + return "" + stages, _routes = stages_for(defn) + choices = _stage_choices(stages) + if not choices: + return "" + fkey = stage_field_key(defn.get("id")) + review_next = {} + for s in stages: + if s["kind"] != "review": + continue + nxt = [r["to"] for r in _routes if r["from"] == s["id"]] + labels = [t["label"] for t in stages if t["id"] in nxt and t.get("assignable")] + if labels: + review_next[s["label"]] = labels + auto = {"flowId": str(defn.get("id") or ""), "stageField": True, + "humanMoves": review_next} + if defn.get("kind") == "discover_instagram": + # Leaving Tracked is a human decision too (an admin demoting a mistake), so the flag + # follows the CARD in both directions — the engine itself still never writes it (R7). + auto["arrive"] = {"Tracked": {"tracked": "1"}, "Review": {"tracked": ""}, + "Declined": {"tracked": ""}} + field = {"key": fkey, "label": f"Stage — {defn.get('name') or defn.get('id')}"[:80], + "type": "select", "source": "overlay", "options": choices, "automation": auto} + # ⭐ WAVE 23 (C5, D-35's minimum) — THE CYCLE COLUMN, created beside the stage field when + # the flow can loop. The owner asked to SEE how many times a record has been round, and + # `_apply_ending` was writing the cell without anything ever declaring the column: the value + # would sit in the row, correct and invisible, because a ut grid renders the DEFINITION's + # fields. A machine-written cell with no field definition is the shape of a number nobody + # can find. Tagged `automation` so it greys out and refuses human writes like every other + # machine column (item 4 / R9). + cyc_key = cycles_field_key(defn.get("id")) + loops = (((defn.get("flow") or {}).get("ending") or {}).get("mode") or "terminal") != \ + "terminal" + cyc_field = {"key": cyc_key, "label": "Cycles"[:80], "type": "int", "source": "overlay", + "automation": {"flowId": str(defn.get("id") or ""), "cyclesField": True}} + + have, have_cyc = None, False + for f in (ut_get(rt, key) or {}).get("fields") or []: + if f.get("key") == fkey: + have = f + if f.get("key") == cyc_key: + have_cyc = True + if have is not None and have.get("options") == choices and \ + (have.get("automation") or {}) == auto and have_cyc == loops: + return fkey # nothing would change — save the commit + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + t = cur.get(key) + if t is None: + return cur + for f in t.setdefault("fields", []): + if f.get("key") == fkey: + f["options"] = choices + f["automation"] = auto + f["type"] = "select" + break + else: + t["fields"].append(field) + if loops and not any(f.get("key") == cyc_key for f in t["fields"]): + t["fields"].append(cyc_field) + # ⚠ A flow switched BACK to `terminal` keeps its cycle column rather than dropping it: + # the counts already in those cells are a record of work that really happened, and a + # schema change must not be a way to lose data nobody asked to delete. + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return fkey + + +def _rid_num(rid): + return int(rid) if str(rid).isdigit() else 10 ** 9 + + +def _card_stage(defn, stages, row, skey): + """Which stage holds this record. The STAMPED label wins; a blank or unrecognised value + falls to the DERIVED bucket — what is true about the row, not what anything wrote: + discovery splits on the human `tracked` decision, an Instagram column on whether its cell + has ever been written, and a scrape lands where its rows are born (Write).""" + val = str((row or {}).get(skey) or "").strip() + if val: + for s in stages: + if s.get("assignable") and s["label"] == val: + return s["id"] + kind = defn.get("kind") + if kind == "discover_instagram": + return "tracked" if str((row or {}).get("tracked") or "").strip() in ("1", "true", "True") \ + else "review" + if kind == "field_instagram": + fkey = (defn.get("config") or {}).get("fieldKey") or "" + return "write" if str((row or {}).get(fkey) or "").strip() else "source" + if kind == "plain": + # No machine step wrote anything, so there is nothing true ABOUT the row to derive a + # bucket from — an unstamped record is simply at the start of the flow. Must stay equal + # to the entry `stages_for` marks for `plain`, or a card names a stage nobody renders. + return "trigger" + return "write" + + +def _card_next(stages, routes, row, sid): + """Where THIS card may go from `sid` — every outgoing route whose condition the record + satisfies (declaration order kept, null = catch-all), targets limited to assignable stages. + Evaluated per record so R5's winning path is decided on the SERVER, once.""" + assignable = {s["id"] for s in stages if s.get("assignable")} + out = [] + for r in routes: + if r["from"] != sid or r["to"] not in assignable or r["to"] == sid: + continue + cond = r.get("condition") + if cond is not None and not lane_match(cond, row): + continue + if r["to"] not in out: + out.append(r["to"]) + return out + + +def _card_title(defn, row): + kind = defn.get("kind") + cfg = defn.get("config") or {} + if kind == "discover_instagram": + return str(row.get("handle") or row.get("full_name") or "?") + if kind == "scrape_db": + return str(row.get(cfg.get("keyField") or "") or "?") + url_f = cfg.get("urlField") or "" + return str(row.get("name") or row.get("handle") or row.get(url_f) or "?") + + +def _card_sub(defn, row): + kind = defn.get("kind") + if kind == "discover_instagram": + f = str(row.get("followers") or "").strip() + nm = str(row.get("full_name") or "").strip() + return f"{nm} · {f} followers" if (nm and f) else (nm or (f and f + " followers") or "") + if kind == "field_instagram": + return _s(row.get((defn.get("config") or {}).get("fieldKey") or ""), 90) + fmap = (defn.get("config") or {}).get("fieldMap") or {} + keys = [k for k in fmap.values() if k != (defn.get("config") or {}).get("keyField")] + return _s(row.get(keys[0]) if keys else "", 90) + + +def board(rt, defn, admin=False): + """C1's whole payload: `{stages, routes, cards, stageField, counts, truncated}`. + + `counts` carries the TRUE per-stage totals for every record stage (zeros included); + `cards` is capped per stage at `BOARD_CARD_CAP` and any stage that overflowed is NAMED in + `truncated` — the cap is disclosed, never silent. Cards ride in row-id order so the board + is stable across polls. + """ + stages, routes = stages_for(defn) + cfg = defn.get("config") or {} + table_key = str(cfg.get("targetTable") or "") + skey = stage_field_key(defn.get("id")) + t = ut_get(rt, table_key) + counts = {s["id"]: 0 for s in stages if s.get("record")} + # C1 amendment (2026-08-05): an automation with NO BOUND STAGE FIELD answers + # `stageField: null, cards: []` — the stages still ride (they come from the definition), + # and an empty board is a state, never an error. Bound = the field exists on the table. + bound = any(f.get("key") == skey for f in ((t or {}).get("fields") or [])) + if t is None or not bound: + return {"stages": stages, "routes": routes, "cards": [], "stageField": None, + "counts": counts, "truncated": []} + rows = t.get("rows") or {} + cards, truncated, per = [], [], {} + for rid in sorted(rows, key=_rid_num): + row = rows.get(rid) or {} + sid = _card_stage(defn, stages, row, skey) + counts[sid] = counts.get(sid, 0) + 1 + if per.get(sid, 0) >= BOARD_CARD_CAP: + if sid not in truncated: + truncated.append(sid) + continue + per[sid] = per.get(sid, 0) + 1 + stage = next((s for s in stages if s["id"] == sid), None) + nxt = _card_next(stages, routes, row, sid) + cards.append({ + "rowId": str(rid), "stageId": sid, + "title": _s(_card_title(defn, row), 80), + "sub": _s(_card_sub(defn, row), 90), + "movedAt": str(row.get(skey + "_at") or ""), + "canAdvance": bool(stage and stage["kind"] == "review" and nxt), + "next": nxt, + }) + return {"stages": stages, "routes": routes, "cards": cards, + "stageField": {"table": table_key, "key": skey}, + "counts": counts, "truncated": truncated} + + +def move_card(rt, auto_id, row_id, to, username="", admin=False): + """One card move, SERVER-decided (C1): a review-stage card advances only to a member of its + per-record `next`; every other move is an admin repair onto an assignable stage. Returns + `(fresh_board, None)` or `(None, (status, code, sentence))` — the route maps the tuple. + + The write lands in the DEFINITION rows (the shared truth the engine and every viewer read), + stamped with `movedAt`; on a discovery board arriving at Tracked sets `tracked` and leaving + it clears it — the human decision travelling with the card (R6/R7).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None, (404, "unknown_automation", "no automation with that id") + stages, routes = stages_for(defn) + cfg = defn.get("config") or {} + table_key = str(cfg.get("targetTable") or "") + t = ut_get(rt, table_key) + if t is None: + return None, (404, "no_target", f"{table_key or 'the target database'} does not exist " + f"yet — run the automation once") + skey = stage_field_key(defn.get("id")) + if not any(f.get("key") == skey for f in (t.get("fields") or [])): + return None, (400, "unbound", + "this automation has no stage field on its database yet — save or run " + "it once to bind the board") + row = (t.get("rows") or {}).get(str(row_id)) + if row is None: + return None, (404, "unknown_row", "that record is not in the target database") + target = next((s for s in stages if s["id"] == str(to or "")), None) + if target is None or not target.get("assignable"): + return None, (400, "bad_stage", + "cards can only be placed in a review, lane or terminal stage — the " + "run stages are machinery a record passes through") + cur_id = _card_stage(defn, stages, row, skey) + cur = next((s for s in stages if s["id"] == cur_id), None) + from_review = cur is not None and cur["kind"] == "review" + if from_review: + allowed = _card_next(stages, routes, row, cur_id) + if target["id"] not in allowed and not admin: + return None, (400, "not_next", + "a review card advances only to its listed next stage") + elif not admin: + return None, (403, "admin_only", + "only a review card can be advanced by hand — moving anything else is " + "an admin repair") + values = {skey: target["label"], skey + "_at": _iso()} + if defn.get("kind") == "discover_instagram": + values["tracked"] = "1" if target["id"] == "tracked" else "" + + def _up(cur_b): + cur_b = cur_b if isinstance(cur_b, dict) else {} + tt = cur_b.get(table_key) + if tt is not None: + tt.setdefault("rows", {}).setdefault(str(row_id), {}).update(values) + return cur_b + + rt.update(UT_STORE_KEY, _up, flush="sync") + if from_review: + review_audit(rt, defn.get("id"), row_id, cur["label"], target["label"], username) + return board(rt, defn, admin=admin), None + + +#: How many review decisions an automation remembers. Bounded like `runs` — an audit that can +#: grow a definition without limit is a serialisation cost wearing a compliance hat. +MAX_REVIEWS = 100 + + +def review_audit(rt, auto_id, row_id, from_label, to_label, username, + by="user", note="", model=""): + """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to + the definition (newest first, bounded) by BOTH doors: `move_card` here, and the grid door + in `core.grid_events`, which writes the same shape mechanically off the field's `flowId`. + + ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that + made it and the one-line reason it gave. One audit log, not two — a reader asking "who + decided this card" must not have to know there are two places to look, and the moment an + AI decision is invisible beside a human one the log stops being an audit. + """ + entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), + "from": _s(from_label, 60), "to": _s(to_label, 60), + "by": "ai" if by == "ai" else "user"} + if note: + entry["note"] = _s(note, 300) + if model: + entry["model"] = _s(model, 60) + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["reviews"] = ([entry] + list(d.get("reviews") or []))[:MAX_REVIEWS] + return cur + + rt.update(STORE_KEY, _up, flush="sync") + + +# --------------------------------------------------------------------------------------------- +# METRIC FIELDS (wave 22, contract C7) + THE SUBJECT PURGE (D-24) +# --------------------------------------------------------------------------------------------- +# A metric field is `{measure, window, agg}` over the MASTER series (C6's pooling benefit — +# the value reflects every pull the platform has, not just this tenant's). Computed by the +# ENGINE and stored as machine cells (C7 amendment: run+tick time — the ut wire lives in files +# no session owns this wave), human-write-refused at the grid door, and every number drills to +# the exact snapshot rows behind it. +# +# ⛔ NO DATA IS BLANK, NEVER ZERO. A handle the master has never seen, a window with no +# snapshots in it, a post series with no measured engagement — all read as an EMPTY cell. Zero +# is a measurement ("they have none"); blank is an admission ("we have not looked / it was not +# readable") — the `_bd_posts_count` law, one layer up. + +METRIC_MEASURES = ("followers", "avg_engagement", "likes", "comments") +METRIC_WINDOWS = ("latest", "last_3_posts", "last_7d", "last_30d") +#: Which measures read the PROFILE series vs the POST series — and which windows/aggs each +#: side can answer. A profile count over `last_3_posts` is a question the data cannot answer; +#: refused at clean time, never bent (mirrored in `core.user_tables`, gate-pinned). +PROFILE_MEASURES = ("followers", "avg_engagement") +METRIC_AGGS = ("avg", "sum", "latest") + + +def metric_value(series, measure, window, agg="", today=None): + """One metric over one handle's master series → `(value_string_or_None, drill_rows)`. + + `today` is a PARAMETER ([[date-window-vocabulary]]) — the caller decides the reference + day; post-count windows count back from the newest post. `drill_rows` are the exact rows + the number came from, so the route can honour [[no-unverifiable-aggregates]] without + recomputing differently.""" + series = series or {} + today = today or _now() + if measure in PROFILE_MEASURES: + rows = [r for r in series.get("snapshots") or [] + if str(r.get(measure) if r.get(measure) is not None else "").strip() != ""] + if window in ("last_7d", "last_30d"): + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + rows = [r for r in rows + if (_parse_iso(r.get("pulled_at")) or _dt.datetime.min) >= floor] + if not rows: + return None, [] + if window == "latest" or agg == "latest": + picked = [rows[-1]] + else: + picked = rows + vals = [_lane_num(r.get(measure)) for r in picked] + vals = [v for v in vals if v is not None] + if not vals: + return None, [] + out = vals[-1] if (window == "latest" or agg == "latest") else \ + (sum(vals) if agg == "sum" else sum(vals) / len(vals)) + if measure == "avg_engagement": + # The vendor's rate is 0-1; the pct cell renders POINTS (the semantic-pct vs + # transform-pct scar) — scaled exactly once, here. + return f"{out * 100:.2f}", picked + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), picked + # --- post measures: select POSTS by window, then each post's LATEST measured snapshot. + posts = [p for p in series.get("posts") or [] if str(p.get("posted_at") or "").strip()] + posts.sort(key=lambda p: str(p.get("posted_at"))) + if window == "latest": + picked_posts = posts[-1:] + elif window == "last_3_posts": + picked_posts = posts[-3:] + else: + days = 7 if window == "last_7d" else 30 + floor = today - _dt.timedelta(days=days) + picked_posts = [p for p in posts + if (_parse_iso(str(p.get("posted_at")).replace(" ", "T")) + or _dt.datetime.min) >= floor] + vals, drill = [], [] + for p in picked_posts: + snaps = (series.get("postSnapshots") or {}).get(str(p.get("shortcode") or "")) or [] + for snap in reversed(snaps): + raw = snap.get(measure) + v = _lane_num(raw) + if v is not None and str(raw).strip() != "": + vals.append(v) + drill.append(snap) + break + if not vals: + return None, [] + out = sum(vals) if (agg or "sum") == "sum" else \ + (vals[-1] if agg == "latest" else sum(vals) / len(vals)) + return (f"{out:.0f}" if float(out).is_integer() else f"{out:.2f}"), drill + + +def metric_fields_of_table(t): + return [f for f in ((t or {}).get("fields") or []) if isinstance(f.get("metric"), dict)] + + +def _table_handle(row, url_field): + return str(row.get("handle") or ig_handle(str(row.get(url_field or "") or "")) + or "").strip().lower() + + +def compute_metric_cells(rt, table_key, today=None): + """Recompute every metric cell on ONE table from the master series. One coalesced write, + only when something actually changed (the flush-ceiling law); zero reads when the table + has no metric fields or the master is off. Returns the number of rows touched.""" + t = ut_get(rt, table_key) + mfields = metric_fields_of_table(t) + if not mfields: + return 0 + import ig_master + if not ig_master.configured(): + return 0 + url_field = next((f.get("key") for f in (t.get("fields") or []) + if f.get("type") == "url"), "") + rows = t.get("rows") or {} + handles = {rid: _table_handle(row or {}, url_field) for rid, row in rows.items()} + series = ig_master.series_for({h for h in handles.values() if h}) + changes = {} + for rid, row in rows.items(): + s = series.get(handles.get(rid) or "") + for f in mfields: + bag = f["metric"] + val, _drill = (metric_value(s, bag.get("measure"), bag.get("window"), + bag.get("agg") or "", today=today) + if s else (None, [])) + want = "" if val is None else str(val) + if str((row or {}).get(f["key"], "")) != want: + changes.setdefault(str(rid), {})[f["key"]] = want + if not changes: + return 0 + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + tt = cur.get(table_key) + if tt is not None: + for rid, vals in changes.items(): + tt.setdefault("rows", {}).setdefault(rid, {}).update(vals) + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + return len(changes) + + +def refresh_metrics(rt, today=None, log=print): + """The tick half of the C7 amendment: `today` advances at tick cadence, so a date-window + metric can never go staler than one tick while a scheduler exists. Cheap by construction — + a table without metric fields costs a dict scan and nothing else.""" + touched = 0 + for tk, t in ut_all(rt).items(): + if metric_fields_of_table(t): + try: + touched += compute_metric_cells(rt, tk, today=today) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tk} failed: {type(e).__name__}: {e}") + return touched + + +def purge_subject(rt, handle): + """D-24: right-to-erasure for ONE Instagram subject — every row about them leaves the + tenant's four `ut_ig_*` tables AND the platform master (R2 made the master half + non-optional: a purge that missed the pooled copy would not be erasure). Returns + `{table: removed}` counts, master rows prefixed `master:` — every count drills to what is + now ABSENT, which is the one aggregate whose drill is emptiness.""" + subject = str(handle or "").strip().lstrip("@").lower() + if not subject: + return {} + counts = {} + tables = ut_all(rt) + post_rows = (tables.get("ut_ig_posts") or {}).get("rows") or {} + codes = {str(r.get("shortcode") or "") for r in post_rows.values() + if str((r or {}).get("influencer_key") or "").strip().lower() == subject} + + keeps = { + "ut_ig_snapshots": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_posts": lambda r: str((r or {}).get("influencer_key") + or "").strip().lower() != subject, + "ut_ig_post_snapshots": lambda r: str((r or {}).get("shortcode") or "") not in codes, + DISCOVER_TABLE: lambda r: str((r or {}).get("handle") + or "").strip().lower() != subject, + } + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + for tk, keep in keeps.items(): + t = cur.get(tk) + if t is None: + continue + rows = t.get("rows") or {} + nxt = {rid: r for rid, r in rows.items() if keep(r)} + counts[tk] = len(rows) - len(nxt) + t["rows"] = nxt + return cur + + rt.update(UT_STORE_KEY, _up, flush="sync") + import ig_master + for bucket, n in (ig_master.purge_handle(subject) or {}).items(): + counts[f"master:{bucket}"] = n + return counts + + +# --------------------------------------------------------------------------------------------- +# TRIGGERS (wave 22, contract C3 + amendment A2 — owner ruling R4; closes D-33) +# --------------------------------------------------------------------------------------------- +# Six ways an automation starts, exactly: manual | schedule | event_field | record_created | +# webhook | email. The first two are what always existed (Run now; cron via the tick). The four +# new ones are EVENTS, and A2 makes their discipline LAW rather than taste: +# +# * **EDGE, NEVER LEVEL (A2(1)).** A condition trigger fires on entering the matching state, +# not for being in it. Implemented as per-record ARMED state over successive evaluations +# (`state.eventDisarmed`): a record fires when it matches while armed, DISARMS, and re-arms +# only by evaluating False — Airtable's documented leave-and-re-enter rule, without needing +# a before-image of a row whose truth is spread over strata. Enabling a trigger SEEDS the +# disarmed set with everything currently matching, so already-matching records do not fire +# (`_seed_event_state`). A settle window coalesces write bursts (the per-keystroke scar). +# * **LOOP PREVENTION IS STRUCTURAL (A2(2)).** The hooks live on the HUMAN doors only +# (`grid_events.overlay_patch`, `user_tables.add_row`); the engine's own writers +# (`ut_write_rows`, the runners' coalesced updates, `patch_cells` from `move_card`) never +# emit — so an automation's write cannot fire event triggers, its own or a sibling's, by +# construction. The circuit breaker on top (>60 fires/5 min auto-pauses with the reason as +# a statusNote) catches whatever construction did not foresee. +# * **FLOOD HOLD (A2(3)).** One evaluation yielding more than 100 candidate records holds +# instead of running — a partial run entry names the count and the deliberate way through +# (Run now). C4's discovery guard is this rule's special case. +# * **REFIRE DEFAULTS (A2(4)), hard-coded this wave:** record_created fires once per record +# EVER (a high-water mark over row ids, so an undo-restored row cannot re-fire); +# event_field fires every transition. + +# ── WAVE 23 · C3 — the trigger vocabulary v2 (owner ruling R2). ─────────────────────────────── +# Airtable's phrasing, because the owner asked for Airtable's builder and a trigger list that +# renames the same events is a second vocabulary to learn for no gain. +# +# ⚠ `event_field` KEPT ITS KEY and changed its LABEL to "When a record matches conditions". +# Renaming the key would have orphaned every stored trigger in production for a caption; the key +# is the contract with the store, the label is the contract with the reader, and they are allowed +# to disagree. What genuinely widened is its SHAPE: the watched field is now OPTIONAL, so the +# trigger covers Airtable's condition-only form (any write to the table, evaluated against a C4 +# tree) as well as wave 22's watch-one-field form. Both are the same edge rule underneath. +# +# ⛔ PLANNED ≠ STORABLE. `button_clicked` / `comment_added` ride the wire so the picker can show +# them faded with a reason (R2: "never a dead control") — and `clean_trigger` REFUSES them with a +# sentence. A vocabulary that renders an option the validator rejects is the wave-9 silent-drop +# class wearing a friendlier face; here the two lists are separate on purpose and the refusal +# names the state rather than pretending the key is unknown. +# ── WAVE 24 · C-TRIG (owner ruling R6). ─────────────────────────────────────────────────────── +# ⭐ INSTAGRAM DISCOVERY BECOMES A TRIGGER. It was a KIND you picked in a create wizard; the +# wizard is deleted, and "when an Instagram profile fits a criteria" is the honest shape anyway — +# it is the event this flow starts from. Picking it sets the definition's kind to +# `discover_instagram` (law 1), which is the ONLY way that kind is reachable now. +# +# ⛔ IT IS NOT A TABLE TRIGGER AND NOT A ROW TRIGGER. It watches nothing: it MAKES rows, on the +# schedule (or on Run now), so it stays out of both lists below and its node switch flips the +# CRON — see `TRIGGER_SCHEDULE_KEYS`. +TRIGGER_KEYS = ("manual", "schedule", "event_field", "record_updated", "record_created", + "enters_view", "webhook", "email", "form_submitted", "ig_profile_match") +TRIGGER_PLANNED = ("button_clicked", "comment_added") +TRIGGER_LABELS = { + "manual": "Manual", + "schedule": "At a scheduled time", + "event_field": "When a record matches conditions", + "record_updated": "When a record is updated", + "record_created": "When a record is created", + "enters_view": "When a record enters a view", + "webhook": "When a webhook is received", + "email": "When an email arrives", + "form_submitted": "When a form is submitted", + "ig_profile_match": "When an Instagram profile fits a criteria", + "button_clicked": "When a button is clicked", + "comment_added": "When a comment is added", +} +#: Triggers that watch a database and therefore need one named before they can fire. +TRIGGER_TABLE_KEYS = ("event_field", "record_updated", "record_created", "enters_view", + "form_submitted") +#: ⭐ WAVE 24 — triggers whose NODE SWITCH means the CRON rather than the trigger itself. +#: `manual`/`schedule` are not stored at all; `ig_profile_match` is stored and IS schedule-driven, +#: so flipping its node must flip the schedule. +#: +#: ⚠ THIS REPLACES A HAND-LISTED TUPLE IN `toggle_node` THAT WAS ALREADY WRONG. It read +#: `("event_field", "record_created", "webhook", "email")` — omitting `record_updated`, +#: `enters_view` and `form_submitted`, all three of which have been storable since wave 23. For +#: those, clicking the trigger node's switch flipped the CRON under a node labelled "When a +#: record is updated": a switch that lies, which is exactly what the tuple at `graph()` warns +#: about eight lines into its own comment. Derived from one named set now, so a trigger added to +#: `TRIGGER_KEYS` cannot silently join the wrong side of it. +TRIGGER_SCHEDULE_KEYS = ("manual", "schedule", "ig_profile_match") +#: Triggers the ROW HOOKS drive (as opposed to the tick, or an inbound HTTP call). Named once so +#: `grid_hook` and the gates read the same list instead of two matching `in (...)` tuples. +TRIGGER_ROW_KEYS = ("event_field", "record_updated", "record_created", "enters_view") +MAX_WATCH_FIELDS = 12 +#: The settle window for field-change bursts (A2(1)). 0 evaluates INLINE — the gates run there, +#: and so would a deployment that prefers immediacy over coalescing. +EVENT_SETTLE_SECONDS = float(os.environ.get("AIOS_EVENT_SETTLE_SECONDS") or 15) +FIRE_LIMIT = 60 # A2(2): fires per window before the breaker pauses +FIRE_WINDOW_SECONDS = 300 +FLOOD_LIMIT = 100 # A2(3): candidate records one evaluation may act on +EMAIL_SEEN_CAP = 500 # message-id dedupe memory per automation +EMAIL_MAX_PER_POLL = 25 # bounded by construction — a poll is a tick guest +CONSECUTIVE_FAILURE_PAUSE = 5 # airtable-brief rec 6: a dead credential must not burn quota + +EMAIL_FIELDS = [ + field_def("email_id", "Email id"), field_def("email_from", "From"), + field_def("email_subject", "Subject"), field_def("email_date", "Date"), + field_def("email_snippet", "Snippet"), field_def("email_seen_at", "Seen at"), +] + + +def clean_trigger(raw, previous=None): + """Validate a definition's `trigger`. Returns `(trigger|None, error)` — None is legal and + means what it always meant: manual + whatever `schedule` says. + + ⚠ A3 (2026-08-05): the stored/wire name is `key` (`kind` accepted on input for symmetry + with the definition's own vocabulary). And an INCOMPLETE event trigger is STORED INERT + rather than refused — the picker writes `{key}` first and the table/field after, the + wave-18 unconfigured-automation-column precedent exactly; `configured: false` rides the + wire so the surface says "finish setting this up" instead of snapping back to Manual. It + cannot fire while incomplete (the hooks match on the table it does not name), which is the + fail-closed direction. MALFORMED parts (an unknown comparison, a valueless compare, a + condition on a field the trigger does not watch) are still refused with the sentence — + incomplete is a state, wrong is not. + """ + if raw in (None, "", {}): + return (dict(previous) if isinstance(previous, dict) and previous else None), None + if not isinstance(raw, dict): + return None, "the trigger must be an object" + prev = previous if isinstance(previous, dict) else {} + key = _s(raw.get("key") or raw.get("kind") or prev.get("key") or prev.get("kind"), + 30).strip() + if key in TRIGGER_PLANNED: + # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence + # says WHY rather than "unknown trigger", because the picker legitimately showed it. + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — " + f"it renders so you can see it is coming, and it cannot be saved") + if key not in TRIGGER_KEYS: + return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) + if key in ("manual", "schedule"): + # Not stored: `schedule` already owns the cron and Run now always works. Storing a + # no-op trigger would be a second copy of that fact, free to disagree with it. + return None, None + out = {"key": key, + "enabled": bool(raw["enabled"]) if "enabled" in raw else + bool(prev.get("enabled", True)), + "paused": bool(raw["paused"]) if "paused" in raw else bool(prev.get("paused"))} + if key in TRIGGER_TABLE_KEYS: + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("event triggers watch blank databases (ut_*) this wave — " + f"{table!r} is not one") + out["table"] = table + if key == "event_field": + # ⭐ WAVE 24 · C-TRIG LAW 4 (owner item 7) — THE WATCHED FIELD IS GONE. "When a record + # matches conditions" is a CONDITION trigger and nothing else: the field picker made it a + # second, quieter way to express the same narrowing, and the owner asked for one. + # ⚠ MIGRATION, NEVER A REFUSAL (law 6). A stored `field` is simply not read, so it is + # dropped on this definition's next clean — silently, and exactly once, because nothing + # writes the key back. A refusal here would have 400'd the live automations that carry it. + cond, cerr = clean_cond(raw.get("when") if "when" in raw else prev.get("when"), + where="the trigger") + if cerr: + return None, cerr + out["when"] = cond + if key == "record_updated": + # Airtable's shape: watch named fields, or leave the list empty for "any field". Empty + # is the WIDER reading and it is the default there too, so it stays the default here. + watch_raw = raw.get("fields") if "fields" in raw else prev.get("fields") + if watch_raw in (None, ""): + watch = [] + elif not isinstance(watch_raw, list): + return None, "the watched-field list must be a list of field keys" + else: + watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] + if len(watch) > MAX_WATCH_FIELDS: + return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " + f"fields — leave the list empty to watch every field") + out["fields"] = watch + # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A + # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: + # "a record was updated" is an event, and asking it to also be a filter was the overlap + # with `event_field` the owner asked to end. Stated loudly in the contract AND here so + # nobody restores it as a bug fix. + # ⚠ Same migration shape as law 4: a stored `when` stops being read, so `_row_gate` + # naturally returns "no gate" for it — the write itself becomes the event — rather than + # this needing a second removal anywhere. + if key == "enters_view": + out["viewId"] = _s(raw.get("viewId") if "viewId" in raw else prev.get("viewId"), + 80).strip() + if key == "form_submitted": + # Blank = any form on that database. Naming one narrows to it, which is what a table + # carrying an intake form AND a correction form needs. + out["formToken"] = _s(raw.get("formToken") if "formToken" in raw + else prev.get("formToken"), 64).strip() + if key == "webhook": + # The token is MINTED here, once, and survives every later patch — rotating it on + # every Save would silently break the external caller the URL was given to. + out["token"] = _s(prev.get("token"), 64) or _secrets_token() + # ⭐ WAVE 24 — DEBT D-41: the request BODY, mapped onto record fields by config. + # ⚠ BOTH HALVES ARE OPTIONAL, and that is what keeps this additive: a webhook with no + # map behaves exactly as it did — it fires the flow and reads nothing — so the live + # webhook automations are untouched. `webhook` deliberately stays OUT of + # `TRIGGER_TABLE_KEYS`: joining it would make a table REQUIRED for `configured`, and + # every existing webhook trigger would go unconfigured and stop firing. + table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() + if table and not table.startswith(UT_PREFIX): + return None, ("a webhook writes into a blank database (ut_*) — " + f"{table!r} is not one") + out["table"] = table + fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw + else prev.get("fieldMap")) + if ferr: + return None, ferr + out["fieldMap"] = fmap + if key == "email": + out["query"] = _s(raw.get("query") if "query" in raw else prev.get("query"), + 200).strip() or "in:inbox is:unread" + out["configured"] = _trigger_configured(out) + return out, None + + +#: D-41 ceilings. 40 mapped cells is `MAX_ACTION_VALUES` doubled — a webhook payload is somebody +#: else's schema and is legitimately wider than an action's hand-written value list. +MAX_BODY_FIELDS = 40 +MAX_BODY_DEPTH = 5 + + +def clean_body_map(raw): + """D-41: `{"": ""}` for a webhook. Returns `(map, error)`. + + Paths are DOTTED into nested objects (`customer.email`). ⛔ NO ARRAY INDEXING in v1, stated + rather than half-supported: `items.0.sku` would read as working for the first element and + silently write nothing the day a payload arrives with the list empty, which is the shape of + bug this module keeps paying for. A path that resolves to nothing writes nothing. + """ + if raw in (None, ""): + return {}, None + if not isinstance(raw, dict): + return None, "the webhook field map must be an object of {body path: field key}" + if len(raw) > MAX_BODY_FIELDS: + return None, f"a webhook maps at most {MAX_BODY_FIELDS} values onto a record" + out = {} + for path, field in raw.items(): + p = _s(path, 200).strip() + if not p: + return None, "a webhook mapping has an empty body path" + if len(p.split(".")) > MAX_BODY_DEPTH: + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload — " + f"map a shallower value") + fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") + if not fk: + return None, f"the value at {p!r} is not mapped to a field" + out[p] = fk[:60] + return out, None + + +def body_value(body, path): + """One dotted path into a decoded JSON body, or None. Scalars only — a mapped value that is + an object or a list answers None rather than a stringified `{...}` in a cell, because the + Row contract is scalar and a serialised dict in a grid cell is unreadable and unfilterable.""" + cur = body + for part in str(path or "").split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur if isinstance(cur, (str, int, float, bool)) else None + + +def webhook_row(rt, defn, body): + """D-41: write ONE record from a webhook payload. Returns `(row_id, mapped_count, note)`. + + ⚠ `note` EXISTS BECAUSE THE CAP WAS SILENT. A table at its row ceiling returned the same + `("", 0)` as "no map configured" and the door answered a cheerful 200 — indistinguishable, + from the only side the caller is on, from a payload whose paths did not resolve. That is the + D-11 class (a table that quietly stops growing), and the caller here is a machine that will + keep posting. The note rides the 200: the flow still fires, and the answer says why no row + was written. + + ⛔ THROUGH THE ENGINE'S OWN WRITER, so it emits no row events — the structural loop + prevention law (A2(2)). A sibling automation watching this table does NOT fire on a + webhook-written row, exactly as it does not fire on a scrape's rows. The webhook's OWN flow + fires, because `hook_fire` fires it explicitly, which is the difference between "this + trigger fired" and "a write happened". + """ + trg = (defn or {}).get("trigger") or {} + table, fmap = str(trg.get("table") or ""), dict(trg.get("fieldMap") or {}) + if not table or not fmap or not isinstance(body, dict): + return "", 0, "" # no map configured — nothing to report + t = ut_get(rt, table) + if t is None: + return "", 0, f"{table} no longer exists, so nothing was written" + values = {} + for path, fkey in fmap.items(): + v = body_value(body, path) + if v is not None: + values[fkey] = _s(v, 500) if isinstance(v, str) else v + if not values: + return "", 0, ("none of the mapped paths resolved to a value in this payload — " + "check the paths against what you are sending") + rows = dict((t.get("rows") or {})) + if len(rows) >= row_cap(table): + return "", 0, (f"{table} is at its {row_cap(table)}-row limit, so no record was " + f"written (the flow still ran)") + rid = str(max([int(r) for r in rows if str(r).isdigit()] or [0]) + 1) + rows[rid] = values + ut_write_rows(rt, table, rows) + return rid, len(values), "" + + +def _trigger_configured(trg, config=None): + """Is this trigger complete enough to fire? One reader, because "configured" is asserted in + three places (the wire, the graph node, the hooks) and three copies of a boolean is how a + surface says "ready" about a trigger the engine skips. + + ⭐ WAVE 24 (A2) — `config` is OPTIONAL and only `ig_profile_match` reads it, because that is + the one trigger whose configuration lives in the DEFINITION's config (the discovery filters) + rather than on the trigger. `clean_trigger` calls this without it and so answers + conservatively (False); `clean_definition` calls it again with the validated config and + refines. Conservative-then-refined is the fail-closed order — the reverse would flash + "ready" on a trigger with nothing to search for. + """ + key = str((trg or {}).get("key") or "") + if key == "ig_profile_match": + return bool((config or {}).get("predicates")) + if key in TRIGGER_TABLE_KEYS and not trg.get("table"): + return False + if key == "event_field": + # C-TRIG law 4: the CONDITION is now the whole of it. No condition = "fire on anything, + # ever" — which is not a trigger, it is a description of the table. + # ⚠ STATED CONSEQUENCE OF THE MIGRATION: a live automation that narrowed by FIELD alone + # and carried no condition becomes `configured: false` on its next clean. It stops + # firing, and it SAYS SO — the graph node reads "Finish setting this trigger up before it + # can fire" and `configured` rides the wire. Visibly unfinished, never silently inert. + return bool(trg.get("when")) + if key == "enters_view": + return bool(trg.get("viewId")) + return True + + +def _secrets_token(): + import secrets as _sec + return _sec.token_urlsafe(24) + + +# --- the circuit breaker (A2(2)) — process memory, like _RUNNING: a counter that outlives the +# process would keep punishing an automation for a storm that ended with the restart. +_FIRES = {} +_FIRES_LOCK = threading.Lock() + + +def _breaker_trips(tenant, auto_id, now=None): + now = now if now is not None else time.time() + key = (tenant, str(auto_id)) + with _FIRES_LOCK: + log = [t for t in _FIRES.get(key, []) if now - t < FIRE_WINDOW_SECONDS] + log.append(now) + _FIRES[key] = log + return len(log) > FIRE_LIMIT + + +def _pause_trigger(rt, auto_id, note): + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + trg = d.get("trigger") + if isinstance(trg, dict): + trg["paused"] = True + d["statusNote"] = _s(note, 200) + return cur + rt.update(STORE_KEY, _up, flush="sync") + + +def trigger_fire(rt, tenant, auto_id, log=print, rows=None): + """One trigger firing — breaker first, then the ordinary async run. False when it did not + start (breaker, or already running — both are answers, not errors).""" + if _breaker_trips(tenant, auto_id): + note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " + f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's " + f"subject in a loop") + _pause_trigger(rt, auto_id, note) + log(f"[aios-auto] breaker: {auto_id} {note}") + return False + return run_async(rt, tenant, auto_id, username="automation", log=log, rows=rows) + + +# --- the settle buffer (A2(1)) — per (tenant, automation), coalescing a burst into ONE +# evaluation. With EVENT_SETTLE_SECONDS == 0 the evaluation is INLINE (deterministic for gates). +_SETTLE = {} +_SETTLE_LOCK = threading.Lock() + + +def _settle_buffer(rt, tenant, auto_id, row_id, field="", after=None, log=print): + """Buffer ONE touched row for a coalesced evaluation, carrying the written cell. + + ⚠ WAVE 23 — THE WRITTEN VALUE IS STILL LOAD-BEARING, and an earlier draft of this wave + dropped it on the theory that `_settle_eval` could just read the row. It cannot, and the + reason is worth stating because it is invisible from this file: an ordinary ut cell typed at + the grid door lands in the editor's PER-USER OVERLAY stratum + (`grid_events.overlay_patch` → `table_store.patch_overlay`), not in the `user_tables` + definition rows. Only stage-field writes go through `patch_cells` to the shared rows. So for + the common case the value that just changed exists ONLY in this event, and a definition-row + read sees the pre-write value — the trigger would evaluate stale and never fire. + `_settle_eval` therefore MERGES: the definition row underneath (which is what lets a C4 tree + read the record's other columns) with the written cells on top. + """ + key = (tenant, str(auto_id)) + cell = {str(field): after} if field else {} + if EVENT_SETTLE_SECONDS <= 0: + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + _settle_eval(rt, tenant, auto_id, log=log) + return + with _SETTLE_LOCK: + buf = _SETTLE.setdefault(key, {"rows": {}}) + buf["rows"].setdefault(str(row_id), {}).update(cell) + timer = buf.get("timer") + if timer is not None: + timer.cancel() # the burst continues — push the window out + timer = threading.Timer(EVENT_SETTLE_SECONDS, _settle_eval, + args=(rt, tenant, auto_id), kwargs={"log": log}) + timer.daemon = True + buf["timer"] = timer + timer.start() + + +def view_filter(rt, table_key, view_id): + """`(tree, fields, problem)` for one saved view on a user table — the substrate the + `enters_view` trigger tests membership against (C3-v2 / owner R2). + + Personal strata first (`find_view`), then the shared bucket, because a view somebody shared + is exactly the kind an automation gets pointed at. A view that has been deleted answers a + PROBLEM rather than an empty tree: an empty tree matches everything, so degrading to one + would turn "when a record enters Overdue" into "on every write", which is the widening this + module refuses everywhere else. + """ + key = str(table_key or "") + vid = str(view_id or "").strip() + if not key or not vid: + return None, [], "the trigger names no view" + try: + import core.table_store as table_store + tops = table_store.make(f"{key}_table_workspace", st=rt) + found = tops.find_view(vid) + view = (found[1] if found else None) or tops.shared_view(vid) + except Exception as e: # noqa: BLE001 + return None, [], f"the view could not be read ({type(e).__name__})" + if not isinstance(view, dict): + return None, [], f"view {vid!r} no longer exists on {key}" + cfg = view.get("config") or {} + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + return tree, list((ut_get(rt, key) or {}).get("fields") or []), "" + + +def _row_gate(rt, defn, trg): + """The MATCH gate for a row-event trigger: `(row -> bool) | None`, plus a problem string. + + None means the trigger has NO match gate — the write itself is the event (wave 22's "the + field changed", and `record_updated` over any field). A problem means the gate cannot be + built, and the caller must then fire NOTHING: a gate we cannot evaluate is not a gate that + passes. + """ + key = trg.get("key") + if key == "enters_view": + tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId")) + if problem: + return None, problem + import harness.filter_eval as filter_eval + return (lambda row: filter_eval.matches(tree, row, fields)), "" + when = trg.get("when") + if when: + return (lambda row: lane_match(when, row)), "" + return None, "" + + +def _settle_eval(rt, tenant, auto_id, log=print): + """Evaluate one settled burst: edge over per-record armed state, flood hold, then fire.""" + with _SETTLE_LOCK: + buf = _SETTLE.pop((tenant, str(auto_id)), None) + written = dict((buf or {}).get("rows") or {}) + touched = list(written) + if not touched: + return + d = all_definitions(rt).get(str(auto_id)) + trg = (d or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True): + return + gate, problem = _row_gate(rt, d, trg) + if problem: + # LOUD, once, and it does not fire. A trigger pointed at a deleted view is a broken + # automation, not a quiet no-op — the note is what the surface shows instead of "On". + if (d.get("statusNote") or "") != problem: + _pause_trigger(rt, auto_id, problem) + log(f"[aios-auto] trigger gate: {auto_id} {problem}") + return + fired = [] + if gate is not None: + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + disarmed = set((d.get("state") or {}).get("eventDisarmed") or []) + nxt = set(disarmed) + for rid in touched: + # The merge (see `_settle_buffer`): the shared definition row underneath so a C4 + # tree can read the record's other columns, the just-written cells on top because + # for an ordinary ut column this event is the ONLY place that value exists yet. + if gate({**(rows.get(rid) or rows.get(str(rid)) or {}), + **(written.get(rid) or {})}): + if rid not in disarmed: + fired.append(rid) # false→true THIS evaluation: the edge + nxt.add(rid) + else: + nxt.discard(rid) # left the state — re-armed (Airtable's rule) + if nxt != disarmed: + set_state(rt, auto_id, {"eventDisarmed": sorted(nxt)[:5000]}) + if not fired: + return + else: + fired = list(touched) # "the field changed" — the burst is the edge + if len(fired) > FLOOD_LIMIT: + _commit_run(rt, auto_id, "partial", + f"the trigger matched {len(fired)} records in one evaluation — more than " + f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " + f"process them deliberately", {"held": len(fired)}, True) + return + trigger_fire(rt, tenant, auto_id, log=log, rows=fired) + + +def _seed_event_state(rt, defn): + """A2(1)'s enable rule: records ALREADY matching when the trigger is set start DISARMED, so + turning the trigger on fires nothing — the first fire needs a real false→true transition. + Evaluated over the definition rows (the machine-written truth this trigger class watches). + + Wave 23: one seeder for all three gated triggers, reading the SAME `_row_gate` the evaluation + reads. Two implementations of "does this row match" is how a seed disagrees with the edge it + is supposed to arm, and the symptom would be a flood of fires the moment somebody enables it. + """ + trg = (defn or {}).get("trigger") or {} + if trg.get("key") not in ("event_field", "record_updated", "enters_view") \ + or not trg.get("configured"): + return + gate, problem = _row_gate(rt, defn, trg) + if gate is None or problem: + return # no match gate ⇒ nothing to arm; a broken gate seeds nothing + rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {} + matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {})) + set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]}) + + +def grid_hook(evt): + """THE listener the human write doors emit into (registered onto + `core.user_tables.ROW_HOOKS` by `routes_automation` at import — the one place that may + import both sides). Never raises into a write path; a broken trigger listener must not + break typing into a cell.""" + try: + st = evt.get("st") + if st is None: + return + tenant = str(getattr(st, "key", "") or "royal-imports") + table = str(evt.get("table") or "") + kind = str(evt.get("type") or "") + defs = all_definitions(st) + field = str(evt.get("field") or "") + rid = str(evt.get("rowId") or "") + for aid, d in defs.items(): + trg = (d or {}).get("trigger") or {} + key = str(trg.get("key") or "") + if trg.get("paused") or not trg.get("enabled", True) \ + or not trg.get("configured", True) \ + or key not in TRIGGER_ROW_KEYS \ + or str(trg.get("table") or "") != table: + continue + if kind == "record_created" and key == "record_created": + hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0 + ridn = int(rid) if rid.isdigit() else None + if ridn is None or ridn <= hw: + continue # once per record EVER (A2(4)) — undo-proof + set_state(st, aid, {"rcHighwater": ridn}) + trigger_fire(st, tenant, aid, rows=[rid]) + elif kind == "event_field" and key == "event_field": + # ⭐ WAVE 24 · law 4 — the per-FIELD narrowing is gone with the stored key. Every + # write on the watched table settles, and the CONDITION decides whether it fires + # (`_row_gate`). The old `not trg.get("field") or str(...) == field` test would + # now always take its first arm anyway; leaving a read of a key the validator no + # longer writes is the drift seat this migration exists to close. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif kind == "event_field" and key == "record_updated" and ( + not (trg.get("fields") or []) or field in (trg.get("fields") or [])): + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + elif key == "enters_view": + # BOTH kinds feed it: a row can enter a view by being edited into its filter or + # by being CREATED already inside it. Listening only to edits would silently miss + # every new record — the half of the definition a reader assumes is covered. + _settle_buffer(st, tenant, aid, rid, field, evt.get("after")) + except Exception as e: # noqa: BLE001 + print(f"[aios-auto] trigger hook failed: {type(e).__name__}: {e}") + + +def form_fired(rt, table_key, row_id, values=None, form_token=""): + """⭐ THE FROZEN SIGNATURE session D calls from the public form door (contract C9/W23-W7). + + One submitted form row → every `form_submitted` automation watching that database fires. + Returns the list of automation ids that started, so the door can log what it set off (and so + the gate can assert it, rather than asserting a side effect nobody can see). + + ⛔ THIS IS A HUMAN DOOR, deliberately: an anonymous submission is a person filling in a form, + so it fires triggers exactly like typing into a cell does. The loop-prevention law is not + weakened by that �� the engine's own writers still never reach here (only `routes_forms` calls + it), so an automation cannot create a form row and re-fire itself. + + `values` is accepted and unused today: the row is already written when this is called, and + the flow reads it from the table. It stays in the signature because the caller HAS it and a + later refire policy ("only when field X was submitted") needs it — a parameter added later + would mean changing D's call site in a wave that does not own it. + """ + started, table = [], str(table_key or "") + if not table: + return started + tenant = str(getattr(rt, "key", "") or "royal-imports") + for aid, d in all_definitions(rt).items(): + trg = (d or {}).get("trigger") or {} + if trg.get("key") != "form_submitted" or trg.get("paused") \ + or not trg.get("enabled", True) or not trg.get("configured", True): + continue + if str(trg.get("table") or "") != table: + continue + want = str(trg.get("formToken") or "") + if want and not hmac.compare_digest(want, str(form_token or "")): + continue # this automation watches a DIFFERENT form on that table + if trigger_fire(rt, tenant, aid, rows=[str(row_id)] if row_id else None): + started.append(aid) + return started + + +def hook_fire(rt, tenant, auto_id, token, body=None): + """The webhook trigger's decision, separated from FastAPI so the gate can drive it. + Returns `(status, payload)` — 404 unknown, 403 wrong/missing token or wrong trigger kind, + 409 already running, 200 started. + + ⭐ WAVE 24 (D-41): `body` is the decoded JSON payload, or None when the caller sent none or + sent something that is not JSON. It is written onto a record BEFORE the flow fires — the + flow's actions walk the record the webhook just created, which is the whole point of mapping + it. `body=None` is the pre-wave behaviour exactly, so an existing caller is unaffected. + """ + import hmac as _hmac + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return 404, {"error": "unknown_automation"} + trg = defn.get("trigger") or {} + want = str(trg.get("token") or "") + if trg.get("key") != "webhook" or not want: + return 403, {"error": "no_webhook", "message": + "this automation has no webhook trigger"} + if trg.get("paused") or not trg.get("enabled", True): + return 403, {"error": "webhook_off", "message": "the webhook trigger is turned off"} + if not _hmac.compare_digest(want, str(token or "")): + return 403, {"error": "bad_token", "message": "that token is not valid"} + # D-41: map the payload onto a record FIRST, so the flow that fires next walks it. + row_id, mapped, note = webhook_row(rt, defn, body) + started = trigger_fire(rt, tenant, auto_id, rows=[row_id] if row_id else None) + out = {"started": bool(started), "at": _iso(), + # Answered even when zero, so a caller wiring a map up can see whether their paths + # resolved. Silence here would make "my JSON is not landing" undebuggable from the + # outside, which is the only side the caller is on. + "rowId": row_id or None, "mapped": mapped} + if note: + out["note"] = note # a 200 that wrote no row SAYS which reason it was + return 200, out + + +def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): + """The email trigger's tick half (C3): poll the CREATOR's Gmail through C5's seam, write a + row per NEW matching message, fire the flow. `_list`/`_read` are injection points so the + gate drives this without a network; production leaves them None. + + Fail-closed and QUIET when unconnected: the statusNote says so ONCE (not a run entry per + tick — 96 identical failures a day is a klaxon, not a status). Bounded everywhere: at most + `EMAIL_MAX_PER_POLL` new messages per tick, the flood hold above that, one coalesced write. + """ + trg = defn.get("trigger") or {} + if trg.get("key") != "email" or trg.get("paused") or not trg.get("enabled", True): + return None + import oauth_connect + creator = str(defn.get("createdBy") or "").strip() or "admin" + token, err = oauth_connect.google_creds(rt, creator) + if err: + if (defn.get("statusNote") or "") != err: + def _note(cur): + cur = cur if isinstance(cur, dict) else {} + dd = cur.get(str(auto_id)) + if dd is not None: + dd["statusNote"] = _s(err, 200) + return cur + rt.update(STORE_KEY, _note, flush="sync") + return None + lister = _list or (lambda q, n: oauth_connect.gmail_list(token, q, n)) + reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) + ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) + if lerr: + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}", + {}, True) + seen = set((defn.get("state") or {}).get("emailSeen") or []) + fresh = [m for m in ids if m not in seen] + if not fresh: + return None # nothing new is not a run — no history spam + if len(fresh) > FLOOD_LIMIT: + return _commit_run(rt, auto_id, "partial", + f"{len(fresh)} new emails matched in one poll — more than the " + f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " + f"the query, or press Run now after adjusting it", + {"held": len(fresh)}, True) + fresh = fresh[:EMAIL_MAX_PER_POLL] + rows_in, notes = [], [] + for mid in fresh: + row, rerr = reader(mid) + if row: + rows_in.append(row) + elif rerr: + notes.append(rerr) + table_key = (defn.get("config") or {}).get("targetTable") or "" + if not table_key: + return _commit_run(rt, auto_id, "error", + "the email trigger has nowhere to write — the automation names no " + "target database", {}, False) + ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", + EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) + skey = stage_field_key(auto_id) + stamp = _iso() + for r in rows_in: + r[skey + "_at"] = stamp + existing = dict((ut_get(rt, table_key) or {}).get("rows") or {}) + merged, counts = upsert_rows(existing, rows_in, "email_id", cap=row_cap(table_key)) + ut_write_rows(rt, table_key, merged) + new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] + set_state(rt, auto_id, {"emailSeen": new_seen}) + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — " + f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") + if notes: + summary += f" — {notes[0][:100]}" + entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, + True, affected=list(merged)[:200]) + trigger_fire(rt, tenant, auto_id, log=log) + return entry + + +def compose_sentence(defn): + """The one-sentence server-composed summary (airtable-brief rec 7): rendered from the + definition so it cannot lie about what runs.""" + cfg = defn.get("config") or {} + trg = defn.get("trigger") or {} + sched = defn.get("schedule") or {} + kind = defn.get("kind") + if trg.get("key") == "event_field": + # Law 4: no watched field any more, so the sentence stops naming one. It said + # "When None changes on ut_x" the moment the key stopped being stored. + head = f"When a record in {trg.get('table')} matches conditions" + elif trg.get("key") == "record_updated": + head = f"When a record in {trg.get('table')} is updated" + elif trg.get("key") == "ig_profile_match": + head = "When an Instagram profile fits the criteria" + elif trg.get("key") == "record_created": + head = f"When a record is created in {trg.get('table')}" + elif trg.get("key") == "webhook": + head = "When the webhook is called" + elif trg.get("key") == "email": + head = f"When an email matches {trg.get('query')}" + elif sched.get("enabled"): + head = f"{_cron_label(sched.get('cron'))}" + else: + head = "When you press Run now" + if kind == "scrape_db": + host = urlparse(str(cfg.get("url") or "")).hostname or "the page" + body = f"read {host} and upsert rows into {cfg.get('targetTable') or 'a new database'}" + elif kind == "field_instagram": + body = (f"capture Instagram profiles for {cfg.get('targetTable') or 'the database'}" + + (", exact counts first" if clean_tier(cfg.get("tier")) == "brightdata" + else ", anonymous only")) + elif kind == "discover_instagram": + body = (f"search Instagram for up to {cfg.get('recordsLimit') or 0} profiles into " + f"{cfg.get('targetTable') or DISCOVER_TABLE}") + else: + # ⭐ WAVE 24 — `plain` describes itself from its FLOW, because the flow is all it has. + # ⛔ THIS BRANCH USED TO BE `discover_instagram`'s, so before the arm above existed every + # plain automation would have introduced itself as "search Instagram for up to 0 profiles + # into ut_ig_candidates" — a sentence composed from a definition that says none of it, + # on the one surface whose whole promise is that it "cannot lie about what runs". + n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) + tbl = cfg.get("targetTable") or trg.get("table") or "" + body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) + if n else "do nothing yet — this automation has no actions") + lanes = cfg.get("lanes") or [] + tail = f", then route each record across {len(lanes)} lanes" if lanes else "" + return f"{head}, {body}{tail}." + + +def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Execute one automation SYNCHRONOUSLY. The route wraps this in a thread; the tick calls it + directly. Returns the run entry, or None when it was already running (the 409).""" + defn = all_definitions(rt).get(str(auto_id)) + if defn is None: + return None + if not _claim(tenant, auto_id): + return None + # ⛔ THE TABLES AN AUTOMATION MAKES BELONG TO THE AUTOMATION'S CREATOR, not to whoever + # happened to press Run — and above all not to the scheduler, which is not a person and + # cannot own anything (see `ut_ensure`). Without this the owner of a database was decided by + # whether a human or a cron got to the first run first. + owner = str(defn.get("createdBy") or "").strip() + if owner and username in MACHINE_OWNERS: + username = owner + try: + _step(tenant, auto_id, "running") + runner = RUNNERS.get(defn.get("kind")) + if runner is None: + return _commit_run(rt, auto_id, "error", + f"unknown automation kind {defn.get('kind')!r}", {}, False) + try: + # ⭐ WAVE 24 (item 6, on D's measurement) — THE LIVE STEP, closed over this run. + # `status.step` was already on the wire and D's half renders it; measured against the + # code, it was set exactly ONCE ("running") and never again, so the word would have + # been identical whether a run was mid-vendor-wait or genuinely hung. Rendering a + # constant as a progress indicator is worse than rendering nothing: it looks like an + # answer. The runners move it now, and the 120 s Bright Data wait counts out loud. + state, summary, counts, affected, steps = runner( + rt, defn, username=username, log=log, + step=lambda text: _step(tenant, auto_id, text), rows=rows) + except Refused as e: + return _commit_run(rt, auto_id, "error", f"refused: {e}", {}, False) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] run {auto_id} failed: {type(e).__name__}: {e}") + return _commit_run(rt, auto_id, "error", + f"{type(e).__name__}: {str(e)[:200]}", {}, False) + # ⭐ WAVE 23 (C4/C5) — THE FLOW RUNS HERE, after the machine steps and before the run is + # committed, over the records this run actually touched. ONE call site rather than three + # inside the runners: every kind gets actions and endings for free, and a fourth runner + # cannot forget to opt in. + # + # ⚠ Its absence was the wave's most expensive near-miss: actions were stored, validated, + # wired to the wire and covered by twelve gate checks that all called `apply_actions` + # DIRECTLY — so the whole feature was green and unreachable. A person would have built a + # flow, pressed Run now, and watched nothing happen. The gate now drives `run_now`. + # + # A failing action must not fail the RUN: the machine steps already wrote their rows and + # reporting that as an error would misdescribe what happened. It degrades to `partial` + # with the reason in the summary — the cap_note discipline. + try: + a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [], + username=username, log=log) + counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}} + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") + state = "partial" if state != "error" else state + summary = f"{summary} — the actions did not finish ({type(e).__name__})" + return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, + steps) + finally: + _release(tenant, auto_id) + + +def run_async(rt, tenant, auto_id, username="automation", log=print, rows=None): + """Start a run on a background thread. False when one is already in flight (→ 409).""" + if running(tenant, auto_id): + return False + th = threading.Thread(target=run_now, args=(rt, tenant, auto_id, username, log, rows), + daemon=True, name=f"automation-{tenant}-{auto_id}") + th.start() + return True + + +# --------------------------------------------------------------------------------------------- +# THE TICK + the in-process scheduler +# --------------------------------------------------------------------------------------------- + +def due_ids(rt, now=None): + """Which of this tenant's automations a tick at `now` should start. Pure over the store.""" + return sorted(aid for aid, d in all_definitions(rt).items() if is_due(d, now)) + + +def tick(rt, tenant, now=None, log=print): + """Fire every due automation for ONE tenant + poll every email trigger (C3). Returns the + ids started. The email polls are bounded and fail-quiet per automation — one broken + mailbox connection must not stop the tenant's schedules.""" + started = [] + for aid in due_ids(rt, now): + if run_async(rt, tenant, aid, username="scheduler", log=log): + started.append(aid) + for aid, d in all_definitions(rt).items(): + if (d.get("trigger") or {}).get("key") == "email": + try: + if email_poll(rt, tenant, aid, d, log=log) is not None: + started.append(aid) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] email poll {tenant}/{aid} failed: {type(e).__name__}: {e}") + # C7's amendment: date-window metrics advance with the tick, so `today` is never staler + # than one tick while a scheduler exists. A tenant without metric fields pays a dict scan. + try: + refresh_metrics(rt, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] metric refresh {tenant} failed: {type(e).__name__}: {e}") + if started: + log(f"[aios-auto] tick {tenant}: started {', '.join(started)}") + return started + + +def tick_all(now=None, log=print): + """Every registered tenant. Fail-quiet per tenant: one tenant's broken store must not stop + the others' schedules.""" + from harness import runtime as _rt + out = {} + for slug in _rt.known_tenants(): + try: + out[slug] = tick(_rt.get_runtime(slug), slug, now=now, log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] tick {slug} skipped: {type(e).__name__}: {e}") + return out + + +#: How often the in-process scheduler wakes. A minute is the cron resolution; anything finer +#: would be a busy loop against a vocabulary that cannot express it. +TICK_SECONDS = int(os.environ.get("AIOS_AUTOMATION_TICK_SECONDS") or 60) +_SCHEDULER = [None] + + +def scheduler_loop(log=print): + """The resync-daemon pattern (`api/main.py:313`): sleep FIRST, then work. + + Sleeping first is deliberate and load-bearing for the gate battery — `verify_api` and + `verify_seam` import this module through `main.py`, run against fake stores in seconds and + exit. A loop that ticked on entry would fire inside them. + """ + while True: + time.sleep(TICK_SECONDS) + try: + tick_all(log=log) + except Exception as e: # noqa: BLE001 + log(f"[aios-auto] scheduler tick failed: {type(e).__name__}: {e}") + + +def start_scheduler(log=print): + """Start the loop once per process. **OPT-IN: `AIOS_AUTOMATIONS=1`.** + + ⚠ AMENDED 2026-08-03 (D), and the reason is a measurement rather than a preference. The wave + brief specified this thread DEFAULT-ON for the API. Then `verify_api.py` was timed: **196 + seconds**, i.e. more than three 60 s tick intervals. A default-on scheduler therefore fires + two or three times inside a gate that imports `main.py` — and while a tick over a fake store + finds nothing due, `tick_all` reaches `runtime.get_runtime()`, which BUILDS tenants and + mutates the LRU cache that `verify_api`'s own isolation assertions read. A background thread + that can move a gate's subject is a flaky suite waiting to happen. + + So it takes the convention `main.py:353` already established for exactly this hazard — + `AIOS_PREWARM=1` — for exactly the reason stated there: env-gated rather than a startup + event, so importing this module in a gate can never fire anything. The DEPLOY sets it; the + external EventBridge tick (R5) does not depend on it either way, since that POSTs the + endpoint rather than riding this thread. + """ + if os.environ.get("AIOS_AUTOMATIONS") != "1": + return False + if _SCHEDULER[0] is not None: + return False + th = threading.Thread(target=scheduler_loop, kwargs={"log": log}, + daemon=True, name="automation-scheduler") + _SCHEDULER[0] = th + th.start() + return True