diff --git "a/api/automation_engine.py" "b/api/automation_engine.py" --- "a/api/automation_engine.py" +++ "b/api/automation_engine.py" @@ -37,7 +37,13 @@ from urllib.parse import urljoin, urlparse import requests -import providers +# ⭐ `providers` IS DELIBERATELY NOT IMPORTED HERE ANY MORE (wave 27 item 23). The capability +# router had exactly one consumer in this file — the views top-up inside the Bright Data rung — +# and that rung now lives in `connectors_ig`. So the vendor-routing seam is reached only by the +# connector that routes, which is the shape the split was for: this file no longer knows that +# there is more than one vendor, or that vendors cost money. Re-adding this import is therefore +# a signal, not a convenience — it means something in the automation runtime started making a +# vendor decision, and that belongs one layer down. try: # the parser the extraction half needs from bs4 import BeautifulSoup @@ -122,6 +128,24 @@ DEFAULT_KIND = "plain" #: the ruling retires the door, not the two automations behind it (see `create`). #: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. RETIRED_KINDS = ("scrape_db", "field_instagram") +#: ⛔ DEBT D-65 — WHAT TO DO INSTEAD, said at every door that refuses one of these. The kinds were +#: not deleted, they were REPLACED by actions any flow can take, and a refusal that does not name +#: the replacement sends somebody looking for a bug in a decision made on purpose. One sentence +#: per kind, in one place, because `create` and `clean_definition` both say it. +RETIRED_KIND_REPLACEMENT = { + "field_instagram": "Add the 'Enrich Instagram profile' action to any automation instead", + "scrape_db": "Add a scrape step to any automation instead", +} +#: ⛔ D-65 — THE RETIRED KINDS THAT ACTUALLY HAVE SOMEWHERE ELSE TO GO, and the distinction is the +#: whole reason this is a second, narrower tuple rather than a reuse of `RETIRED_KINDS`. +#: `field_instagram` was REPLACED: W25/R4 shipped `enrich_instagram` as a `ready:true` action any +#: flow can take, so refusing the kind costs a person nothing but a different click. +#: ⚠ `scrape_db` IS DELIBERATELY ABSENT. Its replacement — the five `web_*` actions — is declared +#: `ready:false` and is DEBT D-51, so refusing a PATCH to it would delete the only way to build a +#: scrape automation and call it tidying up. W24/R6 retired its CREATE door and kept the patch +#: door open on purpose, and there is a gate check that says so in those words. A kind may only be +#: walled off at a door once something else answers the same need. +#: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database", "field_instagram": "Instagram profile column", "discover_instagram": "Find Instagram profiles"} @@ -1059,6 +1083,12 @@ IG_FIELD_DESCRIPTIONS = { "country_code": "Country code reported for the profile.", "created_by": "User whose automation first added this lead.", "enriched_at": "Date the profile was last enriched.", + # ⭐ ITEM 16. The description says GUESS out loud, because the column is one and the number + # beside it is the only thing that says how much of one. + "location_guess": "Most common city tagged across this profile's captured posts - a guess, " + "not a stated location.", + "location_confidence": "Share of this profile's geotagged posts that agree on the guessed " + "city.", "external_url": "Website linked from the Instagram biography.", "external_url_title": "Title Instagram shows for the biography link.", "fbid": "Facebook ID associated with the Instagram account.", @@ -1540,6 +1570,74 @@ def _pin_unique(flow): "actions": [{**first, "config": {**cfg, "uniqueOn": IG_PINNED_UNIQUE}}] + acts[1:]} +#: The two steps `_ensure_ig_action` / `_ensure_enrich_step` plant, as `(kind, id)`. Named here so +#: the seeder and the un-seeder cannot disagree about what "seeded" means. +_IG_SEED_IDS = {"create_record": "act_1", "enrich_instagram": "act_enrich"} + + +def _is_untouched_ig_seed(action, table): + """Is this action still EXACTLY what the Instagram trigger planted? (wave 27 item 15) + + ⛔ THE COMPARISON IS AGAINST A FRESHLY BUILT SEED, not against a list of remembered keys, and + that is the only version that stays true: `_ensure_ig_action` and `_ensure_enrich_step` build + the same dicts one screen above, so a step gaining a config key next wave gains it here too. + A remembered key list would quietly start calling every seeded action "edited". + + ⛔ AND THE FRESH SEED GOES THROUGH `clean_actions` FIRST, which cost a red check to learn. The + stored action has been cleaned — `_clean_action_config` normalises it and adds the keys the + kind declares (`profileField: ""` on an enrich step, for one) — so comparing against the RAW + dict `_ensure_enrich_step` writes reports every seeded enrich step as "edited by a person", + and the un-seeding silently never fires for it. Comparing cleaned against cleaned is the only + version where the two sides are the same kind of object. + + ⚠ `enabled` IS DELIBERATELY NOT PART OF THE TEST for the enrich step. Its ruling says the + TOGGLE is how you turn it off — so a person who switched it off has expressed an opinion about + a step they still want, and an off seed is still a seed. + """ + if not isinstance(action, dict): + return False + kind = str(action.get("kind") or "") + if _IG_SEED_IDS.get(kind) != str(action.get("id") or ""): + return False + seeded, _seed_err = clean_actions( + (_ensure_ig_action({"actions": []}, table).get("actions") or [])) + fresh = {a.get("kind"): a for a in (seeded or [])} + seed = fresh.get(kind) + if not seed: + return False + cfg, seed_cfg = dict(action.get("config") or {}), dict(seed.get("config") or {}) + if kind == "enrich_instagram": + cfg.pop("enabled", None) + seed_cfg.pop("enabled", None) + return cfg == seed_cfg and (action.get("when") or None) is None + + +def _drop_ig_seeds(flow_raw, table): + """Strip the trigger's own seeded steps, keeping any the person has since made theirs. + + ⭐⭐ WAVE 27 ITEM 15 (owner) — *"stale pinned create_record on trigger change"*. THE COMMENT + THAT USED TO SIT AT THE KIND FLIP ARGUED THE OPPOSITE and is rewritten there; it read: *"it + does not delete the seeded actions … a create_record the person has since re-pointed at their + own table with their own values is THEIR action now"*. That reasoning is still correct and is + exactly what this function preserves — but it was applied to EVERY seeded action, including + the ones nobody had ever opened, and the result is what the owner reported: switch an + Instagram search to Manual and you are left holding a "Save the profile" step writing + `{{handle}}` into `ut_ig_candidates` on a flow that no longer produces handles. That is not + somebody's work being protected; it is the machine's own leftover, pointed at a table the + automation has nothing to do with any more. + + ⛔ SO THE TEST IS "DID ANYBODY TOUCH IT", NOT "WAS IT SEEDED" — the same guard shape as + `_vestigial_name_field` and `_retired_tracked_field`, and for the same reason: this is a + branch that destroys something, so the conservative half is the load-bearing half. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + kept = [a for a in actions if not _is_untouched_ig_seed(a, table)] + if len(kept) == len(actions): + return flow_raw + return {**flow, "actions": kept} + + def ig_action_pinned(defn, index): """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" @@ -1563,6 +1661,7 @@ def clean_definition(raw, previous=None, username=""): # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a # dead end the moment the wizard was deleted and the client's create body became {name}. kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND + drop_ig_seeds = False if (trigger or {}).get("key") == "ig_profile_match": # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and @@ -1600,13 +1699,37 @@ def clean_definition(raw, previous=None, username=""): # `discover_instagram`: `scrape_db` and `field_instagram` are surviving kinds whose # triggers are their own business, and clobbering them would retire them by accident. # - # ⛔ IT DOES NOT DELETE THE SEEDED ACTIONS. They stop being pinned, so one click removes - # them — but a `create_record` the person has since re-pointed at their own table with - # their own values is THEIR action now, and quietly destroying it because a trigger moved - # is the silent data loss this module refuses everywhere else. + # ⭐⭐ WAVE 27 ITEM 15 — IT NOW DROPS THE SEEDS NOBODY TOUCHED, and the paragraph this + # replaces argued the other way, so here is why it was half right. It read: *"it does not + # delete the seeded actions … a create_record the person has since re-pointed at their own + # table with their own values is THEIR action now, and quietly destroying it is the silent + # data loss this module refuses everywhere else."* Every word of that is still true and is + # exactly what `_is_untouched_ig_seed` protects. What it got wrong was applying the + # protection to actions NOBODY HAD EVER OPENED: the owner's report is a flow switched to + # Manual still carrying a "Save the profile" step writing `{{handle}}` into + # `ut_ig_candidates` — the machine's own leftover on a flow that no longer produces + # handles, unpinned but still there, still runnable, and still pointed at a table that has + # nothing to do with this automation. + # ⚠ APPLIED BELOW, at the flow, not here: `flow_raw` is not resolved yet at this line. kind = DEFAULT_KIND + drop_ig_seeds = True if kind not in KINDS: return None, f"unknown automation kind {kind!r}" + # ⛔ DEBT D-65 — THE REFUSAL THAT BELONGS HERE IS NOT SHIPPED, AND THAT IS A DECISION. + # D-65 offers two exits for `field_instagram`: convert a surviving definition to a `plain` + # flow carrying `enrich_instagram`, or REFUSE it here with a sentence naming the replacement. + # The refusal was built and then REVERTED, because W24/R6 deliberately made `patch()` the way + # a retired kind legally comes into existence — `create` refuses, `patch` accepts, and the two + # live automations save through this function every time their owner edits them. The gate's + # own fixture helper says so in the strongest terms available: *"If R6's refusal ever moved + # from `create` into `clean_definition` (the tempting simplification), this helper would go + # red across a dozen sections, which is the alarm that change deserves."* It did, and the + # alarm worked. + # ⚠ WHAT SHIPPED INSTEAD is the half that costs nothing and was the actual complaint: every + # door that DOES refuse now names the replacement (`RETIRED_KIND_REPLACEMENT`), so nobody + # meets "unknown automation kind" for a decision made on purpose. The rest of D-65 is an + # owner-visible behaviour change — either new `field_instagram` mints stop working, or stored + # ones are rewritten under their owner — and that is a ruling, not a refactor. name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) if not name: return None, "name the automation" @@ -1643,6 +1766,12 @@ def clean_definition(raw, previous=None, username=""): # permanent, and it is deliberate rather than a widened condition nobody noticed. if (trigger or {}).get("key") == "ig_profile_match": flow_raw = _ensure_ig_action(flow_raw, config.get("targetTable") or DISCOVER_TABLE) + elif drop_ig_seeds: + # ITEM 15: the trigger just STOPPED being the Instagram one. Un-seed what the trigger + # planted and nobody has since made theirs — measured against the table the seeds were + # planted for, which is the PREVIOUS config's, not the one being saved. + flow_raw = _drop_ig_seeds( + flow_raw, (prev.get("config") or {}).get("targetTable") or DISCOVER_TABLE) flow, ferr = clean_flow(flow_raw, prev.get("flow")) if ferr: return None, ferr @@ -1662,10 +1791,39 @@ def clean_definition(raw, previous=None, username=""): # a stored definition carrying two different values is reconciled on its next clean, # silently and exactly once, because nothing writes the losing value back. A refusal here # would 400 every Save of a definition that was legal yesterday. + # ⭐⭐ WAVE 27 ITEM 11 — WHICHEVER SIDE MOVED WINS, and R2's "the action wins" is now the + # TIE-BREAK rather than the whole rule. The owner's report: change the database on the + # TRIGGER, save, and the picker springs back to the old one with no message. The cause is + # the branch below read unconditionally — the pinned action still named yesterday's table, + # so it overwrote a change the person had just made in front of it, silently, every time. + # + # ⛔ R2 IS NOT REVERSED, and the distinction is which QUESTION the code asks. R2 settled + # "when the two disagree, whose value is real?" — the action's, because R2b puts the + # preset list on the action's own panel, so that is the one they were looking at. That + # answers a definition at REST. A save is different: it carries an INTENT, and the honest + # question is which side this particular save changed. So: + # · the trigger-side picker moved -> it wins, and the pinned action is re-pointed to + # follow it (leaving the action behind would just re-revert on the next save); + # · only the action moved, or neither did and they were already inconsistent -> R2's + # migration, unchanged, reconciling a stored definition exactly once. + # · BOTH moved in one payload -> the action still wins. That is R2 literally, and it is + # also the only reading that cannot lose a value: the action's table is the one with + # the field mapping attached to it. first = (flow.get("actions") or [{}])[0] pinned = str((first.get("config") or {}).get("table") or "") \ if first.get("kind") == "create_record" else "" - if pinned and pinned != config.get("targetTable"): + prev_target = str((prev.get("config") or {}).get("targetTable") or "") + prev_pinned = str(((((prev.get("flow") or {}).get("actions") or [{}])[0] + ).get("config") or {}).get("table") or "") + target_moved = bool(prev_target) and config.get("targetTable") != prev_target + action_moved = bool(prev_pinned) and pinned != prev_pinned + if pinned and target_moved and not action_moved: + # The person changed the trigger's database. Carry it INTO the pinned step so the two + # agree, instead of throwing their edit away to keep a stale copy consistent. + cfg1 = dict(first.get("config") or {}) + cfg1["table"] = str(config.get("targetTable") or "") + flow["actions"] = [{**first, "config": cfg1}, *(flow.get("actions") or [])[1:]] + elif pinned and pinned != config.get("targetTable"): config["targetTable"] = pinned # The stored label described a DIFFERENT database, so keeping it would caption the new # one with the old one's name. Blank falls back to the key everywhere, and `ut_ensure` @@ -1925,8 +2083,8 @@ def create(rt, raw, username=""): if defn["kind"] in RETIRED_KINDS: _unmint(rt, minted) return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " - f"longer created — the ones already using it keep working. Start a new " - f"automation and pick its trigger instead") + f"longer created — the ones already using it keep working. " + f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") defn["id"] = _new_id(existing) def _up(cur): @@ -2135,9 +2293,49 @@ def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=No return cur rt.update(STORE_KEY, _up, flush="sync") + _notify_outcome(rt, auto_id, state, entry) return entry +def _notify_outcome(rt, auto_id, state, entry): + """⭐⭐ WAVE 27 ITEM 31 / CONTRACT C5 — the bell rings on EVERY outcome, good and bad (I7). + + Owner: notify on both success and error. `_commit_run` is the ONE place a run's outcome is + written, so it is the only place this can go without a second definition of "what happened". + + ⛔ AMENDMENT A1 MOVED TWO REQUIREMENTS HERE AND THE FIRST IS A CROSS-TENANT DEFECT IF MISSED. + 1. **`st=` IS PASSED EXPLICITLY.** `core.alerts.notify`'s `st=None` default resolves to the + MODULE-level store, which is tenant #0's (Royal Imports). The automation tick runs on a + background thread with no session, so an omitted `st` files nurilab's automation failures + in Royal Imports' inbox — D-16's class, silently, forever. + 2. **THE OWNER IS THE AUTOMATION'S `createdBy`, NEVER THE RUNNER'S IDENTITY.** The scheduler + has no session at all, so the alternative is not "the wrong person" but "nobody", and + `notify` returns None on a blank owner — the notification would simply never exist, on + exactly the runs nobody was watching. + ⚠ AND NO SERVER ADMISSION WAS NEEDED. A1 measured that `routes_alerts._TOPICS` gates only + view-alert CREATION; `notify()` writes straight into the notifications bucket and + `GET /notifications` filters by nothing, so an `automation` topic already reaches the bell. + ⚠ FAILURE HERE IS SWALLOWED. A notification that cannot be filed must not turn a run that + SUCCEEDED into one that raised — the run entry is already committed one line above, and the + bell is a courtesy on top of it. + """ + try: + defn = (rt.get(STORE_KEY) or {}).get(str(auto_id)) or {} + owner = str(defn.get("createdBy") or "").strip() + if not owner: + return + from core import alerts as _alerts + _alerts.notify( + owner=owner, + label=_s(defn.get("name") or "Automation", 80), + topic="automation", + key=str(auto_id), + detail=f"{state}: {entry.get('summary') or ''}"[:300], + st=rt) + except Exception: # noqa: BLE001 + pass + + # --------------------------------------------------------------------------------------------- # INSTAGRAM (R7, extended by R1) — PUBLIC DATA ONLY, NEVER AN INSTAGRAM LOGIN # --------------------------------------------------------------------------------------------- @@ -2162,8 +2360,6 @@ def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=No # so a run history answers "what still works anonymously?" from data rather than from memory. PACE_SECONDS = float(os.environ.get("AIOS_IG_PACE_SECONDS") or 2.5) -IG_APP_ID = "936619743392459" # the public web client id the browser itself sends -IG_MAX_RETRIES = 2 #: ⭐ MAP THE SCHEMA, NOT JUST WHAT WAS POPULATED ON THE PROBE — and the reason is this page's #: own headline. **HISTORY IS UNBUYABLE.** No vendor sells "followers on 1 January"; every number @@ -2352,1182 +2548,37 @@ POST_SNAPSHOT_FIELDS = [ ] -def ig_handle(url): - """The handle out of a profile URL (or a bare handle). '' when it is not one.""" - raw = str(url or "").strip() - if not raw: - return "" - if raw.startswith("@"): - return re.sub(r"[^A-Za-z0-9._]", "", raw[1:])[:40] - if "instagram.com" not in raw: - return re.sub(r"[^A-Za-z0-9._]", "", raw)[:40] if "/" not in raw else "" - path = urlparse(raw if "//" in raw else "https://" + raw).path.strip("/") - first = (path.split("/") or [""])[0] - if first in ("p", "reel", "reels", "explore", "accounts", "stories", ""): - return "" - return re.sub(r"[^A-Za-z0-9._]", "", first)[:40] - - -def _ig_get(url, headers=None, timeout=20.0): - """One anonymous GET with 429 backoff. Returns `(status, body_bytes, note)`.""" - delay = PACE_SECONDS - for attempt in range(IG_MAX_RETRIES + 1): - try: - status, _final, body = fetch(url, timeout=timeout, headers=headers, max_kb=4096) - except Refused: - raise - except Exception as e: # noqa: BLE001 - return 0, b"", f"{type(e).__name__}: {str(e)[:120]}" - if status == 429 and attempt < IG_MAX_RETRIES: - time.sleep(delay) - delay *= 2 # exponential, never a tight retry - continue - return status, body, "" - return 429, b"", "rate limited after retries" - - -def _ig_int(v): - try: - return int(v) - except Exception: - return None - - -def _ig_zero_is_blank(v): - """Like `_ig_int`, but a vendor `0` reads as NOT MEASURED rather than as a measurement. - - ⛔ FOR THE PAID VIEW/PLAY RUNG ONLY. `POST_FIELDS` states the rule — *"BLANK, NEVER ZERO … a 0 - here would claim a post nobody watched"* — but `_ig_int(0)` returns `0`, so the rule was - prose with nothing enforcing it. MEASURED 2026-08-08: `sriyynntt` came back with `views: 0` on - all twelve reels and those zeros were published as measurements; they were removed by the - REPEAT rule, not the zero rule, i.e. by luck. A single vendor `0` on one reel still landed. - ⚠ DO NOT reach for this on `likes` or `comments`. Zero likes is a real, common and useful - measurement; blanking it would destroy data to satisfy a rule that was never about it. - """ - n = _ig_int(v) - return None if n == 0 else n - - -def _profile_from_web_api(node, handle): - return { - "username": node.get("username") or handle, - "full_name": node.get("full_name") or "", - "bio": node.get("biography") or "", - "followers": _ig_int((node.get("edge_followed_by") or {}).get("count")), - "following": _ig_int((node.get("edge_follow") or {}).get("count")), - "posts_count": _ig_int((node.get("edge_owner_to_timeline_media") or {}).get("count")), - "verified": "1" if node.get("is_verified") else "", - } - - -def _posts_from_edges(edges): - out = [] - for e in edges or []: - n = (e or {}).get("node") or {} - code = n.get("shortcode") - if not code: - continue - caption = "" - for ce in ((n.get("edge_media_to_caption") or {}).get("edges") or []): - caption = ((ce or {}).get("node") or {}).get("text") or "" - break - ts = n.get("taken_at_timestamp") - out.append({ - "shortcode": code, - "posted_at": (_dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M") - if isinstance(ts, (int, float)) else ""), - "type": ("video" if n.get("is_video") else - "carousel" if n.get("__typename") == "GraphSidecar" else "image"), - "caption": caption, - "url": f"https://www.instagram.com/p/{code}/", - "likes": _ig_int((n.get("edge_liked_by") or n.get("edge_media_preview_like") - or {}).get("count")), - "comments": _ig_int((n.get("edge_media_to_comment") or {}).get("count")), - }) - return out - - -_OG_COUNTS = re.compile( - r"([\d.,KMkm]+)\s+Followers?,\s*([\d.,KMkm]+)\s+Following,\s*([\d.,KMkm]+)\s+Posts?") - - -def _loose_count(txt): - """'1.2M' / '12,345' -> int. IG's og:description abbreviates, so the snapshot is honest - about being approximate rather than pretending to a precision it does not have.""" - t = str(txt or "").strip().replace(",", "") - mult = 1 - if t[-1:].lower() == "k": - mult, t = 1_000, t[:-1] - elif t[-1:].lower() == "m": - mult, t = 1_000_000, t[:-1] - try: - return int(float(t) * mult) - except Exception: - return None - - # --------------------------------------------------------------------------------------------- -# BRIGHT DATA — THE PAID RUNG (owner ruling R1 / wave-20 D-21, contract C4) +# THE INSTAGRAM CONNECTOR — moved out (wave 27 item 23). See `connectors_ig.py`. # --------------------------------------------------------------------------------------------- -# WHY A VENDOR AT ALL, stated once so nobody re-opens it: the anonymous ladder above is honest but -# it is ALSO geographically fenced — Instagram blocks datacenter egress, so from the HF Space or a -# Lambda the ladder degrades to `blocked` and the whole automation becomes a thing that only works -# on the owner's desk. The vendor takes the egress risk, returns EXACT counts (not -# og:description's "204K") and reaches post-level data the anonymous surface has closed. +# Bright Data, Apify, the corpus routes and the anonymous ladder used to be ~1,900 lines HERE. +# They are a CONNECTOR: they know what a vendor's rows look like. Nothing in the automation +# runtime needs that, and the runtime is what this file is for. # -# ⚠ WHY THIS VENDOR AND NOT THE PREVIOUS ONE. HikerAPI was crypto-only with no legal entity — -# no invoice, no counterparty, nothing that can go on a subprocessor list at the EU/enterprise -# gates. Bright Data is nameable, litigated Meta and won on logged-off public scraping, and runs -# KYC. That is a POSTURE purchase, not a feature one; the price is ~2.5× and it is the right -# trade. Full comparison: `.claude/wiki/research/instagram-capture.md` §2a. -# -# ⛔ THE FOUR RAILS TRANSFER UNCHANGED (contract C4 — they were written vendor-agnostic): -# 1. **FAIL-CLOSED on the key.** No `AIOS_BRIGHTDATA_KEY` ⇒ this rung refuses with an honest -# blocked ATTEMPT and the anonymous ladder still runs. Never a crash, never a silent skip. -# 2. **The key is never logged, never in a URL, never in an exception.** It travels in the -# `Authorization: Bearer` header only, and every error below quotes a STATUS, never the -# request. A secret that reaches a log line has leaked to everyone who can read logs. -# 3. **The base URL is guarded like any other.** `AIOS_BRIGHTDATA_BASE` is env-provided, so it -# goes through the same SSRF rail — an env var pointed at the metadata endpoint would -# otherwise be a credential-bearing request to it. -# 4. **A rung failure is a RUNG failure.** `bd_call` swallows its own `Refused`/transport -# errors, because `run_now` turns a `Refused` into a whole-run error — and a mistyped base -# URL must not kill a run whose anonymous rung would have answered. -# 5. **Exact ⇒ NO `approx` TAG.** That tag is the anonymous rung's rounding disclosure. -# -# ⚠ THE KEY IS READ LAZILY, INSIDE THE CALL. A module-level snapshot would be taken at import, -# which makes the "key absent ⇒ blocked, not crash" gate untestable (nothing could flip it between -# checks) and would also mean a deploy that sets the secret after import silently has no key. -# -# ⛔ TWO SNAPSHOT NAMESPACES THAT 404 EACH OTHER (measured; the trap that costs an hour): -# the SCRAPER `/datasets/v3/trigger` + `/v3/scrape` -> `sd_…` read at `/datasets/v3/…` -# the CORPUS `/datasets/filter` (⚠ NO `/v3/`) -> `snap_…` read at `/datasets/…` -# Cross them and you get a flat "404 Snapshot does not exist" about a snapshot that is alive. - -#: Pinned from live probes against `api.brightdata.com` (2026-08-04 and 2026-08-05) — see -#: `.claude/wiki/research/instagram-capture.md` §2c/§2d for the recipe and its provenance tags. -#: They live in ONE named block so a vendor rename is a one-line fix rather than a hunt. -BD_BASE_DEFAULT = "https://api.brightdata.com" -BD_DS_PROFILES = "gd_l1vikfch901nx3by4" # Instagram – Profiles. 36 fields, 620M records -BD_DS_POSTS = "gd_lk5ns7kz21pck8jpis" # Instagram – Posts. 43 fields -BD_DS_REELS = "gd_lyclm20il4r5helnj" # Instagram Reels: views and play counts -BD_DS_COMMENTS = "gd_ltppn085pokosxh13" # Instagram Comments: opt-in full comment engagement -BD_PATH_SCRAPE = "/datasets/v3/scrape" # SYNC: rows come back inline. param: dataset_id -BD_PATH_TRIGGER = "/datasets/v3/trigger" # ASYNC: -> {"snapshot_id": "sd_…"} -BD_PATH_SNAPSHOT = "/datasets/v3/snapshot" # /?format=json -> the rows -BD_PATH_FILTER = "/datasets/filter" # ⚠ NO /v3/ — the CORPUS query (discovery) -BD_PATH_FILTER_SNAPSHOT = "/datasets/snapshot" # / -> status -#: A sync scrape MEASURED at 15–23 s for one URL and ~40 s for two, so the timeout is generous; -#: it is a ceiling against a hung socket, not a latency budget. -BD_TIMEOUT = 300.0 -#: The READ ceiling for one vendor answer, in KB. `fetch` truncates SILENTLY at this size, so it -#: is load-bearing arithmetic rather than a round number: MEASURED `file_size` was 175,617 bytes -#: for 5 corpus rows ⇒ **~35 KB per profile row**, and `BD_MAX_RECORDS` is set from it with -#: headroom (500 × 35 KB ≈ 17 MB against 32 MB). A truncated body is REPORTED as truncated in -#: `bd_call` — never allowed to surface as "that is not JSON". -BD_MAX_KB = 32768 -#: How long a run waits on a scrape batch the vendor DEFERRED (see `_bd_deferral`). Much shorter -#: than the corpus filter's budget because a scrape snapshot is minutes, not tens of minutes — -#: and unlike the corpus path there is no handoff to a later run, so this is the whole wait. -BD_SCRAPE_WAIT = float(os.environ.get("AIOS_BD_SCRAPE_WAIT") or 180) -BD_SCRAPE_POLL = float(os.environ.get("AIOS_BD_SCRAPE_POLL") or 10) -# A Profile response is the enrichment's identity half, so it retains the normal bounded -# synchronous wait. Post/Reel engagement can create several snapshots per profile; action -# runners hand those off immediately and the scheduler collects the already-paid work later. -BD_METRIC_SCRAPE_WAIT = float(os.environ.get("AIOS_BD_METRIC_SCRAPE_WAIT") or 0) -#: ⚠ SPEC, NOT MEASURED — off the pricing page, never returned by the API (the funds gate fires -#: BEFORE a price is quoted, and `price: 0` means "not priced", not "free"). Every surface that -#: shows a number derived from this MUST say it is an estimate. `/customer/balance` answers 403 -#: for our token, so there is no way to check a balance or a spend from here either. -BD_RECORD_PRICE_SPEC = 0.0025 - - -def bd_key(): - """The key, or ''. Read fresh every call — see the module note above on why.""" - return (os.environ.get("AIOS_BRIGHTDATA_KEY") or "").strip() - - -def bd_base(): - return (os.environ.get("AIOS_BRIGHTDATA_BASE") or BD_BASE_DEFAULT).strip().rstrip("/") - - -def bd_ready(): - """Is the paid rung configured at all? The UI reads this to say so honestly.""" - return bool(bd_key()) - - -def _bd_why(raw, key=""): - """The vendor's OWN words for a refusal, as `" — …"`, or `""` when it gave none. - - ⛔ THIS WAS THROWN AWAY, AND THE COST WAS MEASURED RATHER THAN IMAGINED. nurilab's live - discovery automation failed four runs across two days with `the profile source answered 400` - while the body it discarded said, in full: - - {"validation_errors":["Filter logical groups can have a maximum of 4 rules."]} - - — the exact instruction needed to fix it, in our hands, deleted one line before the customer. - A bare status code sends somebody to read a vendor's schema for a fault we were already - holding the answer to; it is the same defect as a green dot over nothing, pointed the other - way. Every non-2xx says WHY now, and the named cases above (402/401/403/429) keep their own - plainer sentences because those are ours to explain, not the vendor's. - """ - try: - text = bytes(raw or b"").decode("utf-8", "replace").strip()[:600] - except Exception: # noqa: BLE001 - return "" - if not text: - return "" - why = "" - try: - d = json.loads(text) - except Exception: # noqa: BLE001 - d = None - if isinstance(d, dict): - errs = d.get("validation_errors") or d.get("errors") - if isinstance(errs, list) and errs: - why = "; ".join(str(e) for e in errs) - else: - why = str(d.get("error") or d.get("message") or d.get("detail") or "") - why = why or text - # ⚠ A BODY WE DID NOT WRITE GETS THE SAME TREATMENT AS AN EXCEPTION'S str(). `bd_call` already - # refuses to put the latter on the wire in case it carries the key; promising that a vendor - # never echoes an Authorization header back is not a guarantee this module can make. - if key and key in why: - why = why.replace(key, "***") - return f" — {_s(why, 300)}" - - -def bd_call(path, params=None, body=None): - """One authenticated vendor call. GET when `body` is None, POST otherwise. - - Returns `(payload, note)`; a non-empty `note` means it did NOT answer, and the note is safe to - show a user and safe to write to a log. - - Never raises. Every refusal shape — no key, a guarded base, a 402, a 429, a transport error, - unparseable JSON — comes back as a note so the caller can record the attempt and drop to the - next rung. - """ - key = bd_key() - if not key: - # ⛔ THE FAIL-CLOSED PATH (C4). Named precisely, because "blocked" with no reason sends - # somebody to read Instagram's status page instead of setting an env var. - return None, "AIOS_BRIGHTDATA_KEY is not configured — the paid rung is closed" - qs = "&".join(f"{k}={requests.utils.quote(str(v))}" - for k, v in (params or {}).items() if v not in (None, "")) - url = f"{bd_base()}{path}" + (f"?{qs}" if qs else "") - hdrs = {"Authorization": f"Bearer {key}", "Accept": "application/json"} - try: - if body is None: - status, _final, raw = fetch(url, timeout=BD_TIMEOUT, max_kb=BD_MAX_KB, headers=hdrs) - else: - status, raw = fetch_json(url, body, timeout=BD_TIMEOUT, max_kb=BD_MAX_KB, - headers=hdrs) - except Refused as e: - # The BASE was refused by the SSRF rail — a configuration fault, not a vendor outage. - return None, f"the profile source's address was refused by the URL rail: {e}" - except Exception as e: # noqa: BLE001 - # ⚠ `type(e).__name__` only. A requests exception's str() can carry the full URL, and the - # Authorization header is one refactor away from being a query param; this line must not - # be what leaks it. - return None, f"the profile source did not answer ({type(e).__name__})" - if status == 402: - return None, "the profile search is out of credit" - if status in (401, 403): - return None, f"the profile source refused our key ({status})" - if status == 429: - # ⚠ MEASURED during the discovery probe: `too_many_parallel_jobs`. The filter API has a - # concurrency cap and the probe's most important test died on it — hence every caller - # below is SERIAL by construction. - return None, "the profile source is busy with another search; they run one at a time" - if not (200 <= status < 300): - return None, f"the profile source answered {status}{_bd_why(raw, key)}" - # ⛔ A BODY THAT REACHED THE READ CEILING IS A TRUNCATION, AND IT MUST SAY SO. `fetch` reads - # at most `max_kb` and returns what it got — silently. A truncated JSON document then fails - # `json.loads`, falls through to the NDJSON branch, and comes back as "answered with - # something that is not JSON": a sentence that sends somebody to read the vendor's schema - # looking for a fault that is OUR byte ceiling — after a result set that has already been - # paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow. - if len(raw) >= BD_MAX_KB * 1024: - return None, (f"the profile source's answer exceeded this client's {BD_MAX_KB // 1024} MB read " - f"ceiling and was TRUNCATED — ask for fewer records; nothing was parsed " - f"from a partial document") - text = raw.decode("utf-8", "replace").strip() - if not text: - return None, f"the profile source answered {status} with an empty body" - try: - return json.loads(text), "" - except Exception: # noqa: BLE001 - pass - # NDJSON is the other shape this API uses for row sets. Tried SECOND, so a genuine JSON error - # is not silently reinterpreted as a one-line NDJSON document. - rows = [] - for line in text.splitlines(): - line = line.strip() - if not line: - continue - try: - rows.append(json.loads(line)) - except Exception: # noqa: BLE001 - return None, "the profile source answered with something we could not read" - return (rows, "") if rows else (None, "the profile source answered with something we could not read") - - -def _first(node, *names, default=None): - """The first of `names` present on `node` with a real value. - - ⚠ THE TOLERANCE IS THE POINT, not laziness. The vendor's `components/schemas` was not readable - on 2026-08-04 (recorded as RECALLED in the digest), so every media field name below is a - hypothesis. Reading through a candidate list means a rename costs one more candidate instead - of a rung that returns nothing and cannot say why. **A field that matches nothing returns - `default` (None), never 0** — an invented zero is a measurement, and this module's whole - honest-status contract exists to stop those. - """ - if not isinstance(node, dict): - return default - for n in names: - v = node.get(n) - if v not in (None, ""): - return v - return default - - -#: Source-data retention is intentionally unfiltered: if Bright Data returned it in an already -#: paid Profile, Post, Reel, or Comment response, it belongs in that record's source document. -#: The one cost boundary is the *request*: the full Comments dataset is only called when an owner -#: enables `commentMetrics`. - -#: `bd_call` refuses a partial response at this same transport ceiling. A single source document -#: therefore fits a JSON cell; Source data is never shortened by `_s`, which would corrupt JSON -#: and silently lose fields. -BD_SOURCE_PAYLOAD_MAX = BD_MAX_KB * 1024 - -#: The vendor's content vocabulary → ours. MEASURED values: profiles' `posts[].content_type` is -#: `Image`/`Reel`/`Carousel`; the Posts dataset adds `product_type` `clips`/`carousel_container`. -_BD_TYPES = {"image": "image", "images": "image", "photo": "image", "photos": "image", - "video": "video", "videos": "video", "reel": "video", "reels": "video", - "clip": "video", "clips": "video", "igtv": "video", "carousel": "carousel", - "carousel_container": "carousel", "sidecar": "carousel", "posts": "image"} - - -def _bd_type(*raw): - """The first recognisable content type among `raw`, defaulting to `image`.""" - for v in raw: - hit = _BD_TYPES.get(str(v or "").strip().lower()) - if hit: - return hit - return "image" - - -def ig_shortcode(url): - """The shortcode out of a post/reel permalink. '' when it is not one. - - The `ut_ig_posts` identity key. The vendor gives a post URL (sometimes without its trailing - slash) and a numeric `pk`; the shortcode is what both datasets agree on and what a human can - paste into a browser, so it stays the key. - """ - raw = str(url or "").strip() - if not raw: - return "" - path = urlparse(raw if "//" in raw else "https://" + raw).path.strip("/").split("/") - if len(path) >= 2 and path[0] in ("p", "reel", "reels", "tv"): - return re.sub(r"[^A-Za-z0-9_-]", "", path[1])[:40] - return "" - - -def _bd_first_url(v): - """A link-in-bio value → one string. - - ⚠ MEASURED 2026-08-05: `external_url` is an **ARRAY** on the real row - (`["http://linktr.ee/utopianevents/"]`), not the string the previous vendor returned. A dict - (`external_url_title`) is also in the family. Read through the shape rather than assuming one. - """ - if isinstance(v, list): - v = next((x for x in v if x), "") - if isinstance(v, dict): - v = _first(v, "url", "link", default="") - return str(v or "") - - -def _bd_posts_count(node): - """`posts_count`, or None — **and a vendor ZERO IS DISCARDED.** - - ⛔ MEASURED 2026-08-05: the Profiles dataset returned `posts_count: 0` for BOTH probe accounts - — `inayma` (203,800 followers, 12 posts returned in the same row) and `utopian_events` (20,924 - followers, 12 posts). The POSTS dataset carries the true value for the same two accounts - (1,215 and 751), so the zero is not the account being empty; it is a measurement failure - wearing a number. Writing it would be the invented measurement this module's honest-status - contract exists to forbid — and a `_first`-style candidate list cannot help, because the - vendor DID answer, with the wrong answer. - - ⚠ THE ASYMMETRY IS DELIBERATE AND IT COSTS SOMETHING: an account that genuinely has zero posts - also reads blank. "Not read" is honest about an unknown; "0" is a claim, and on the only - evidence we have that claim is wrong 2 times out of 2. - ⛔ `len(posts)` IS NOT A SUBSTITUTE. The profile row carries the TOP 12 — a cap, not a count. - """ - v = _ig_int(_first(node, "posts_count", "media_count", "post_count")) - return None if v == 0 else v - - -def _bd_flag(node, *names): - """A vendor boolean → `'1'` / `''` / None. **None when the key is ABSENT or null**, which is - the distinction that matters: `is_business_account: false` is a measurement and `''` records - it; a missing key is not, and must not arrive as "false".""" - for n in names: - v = node.get(n) if isinstance(node, dict) else None - if v is None: - continue - return "1" if v is True or str(v).strip().lower() in ("true", "1", "yes") else "" - return None - - -def _bd_list(node, *names): - """An array field → a comma-joined string, or None when it is absent/empty.""" - for n in names: - v = node.get(n) if isinstance(node, dict) else None - if isinstance(v, list) and v: - return ", ".join(str(x) for x in v if x)[:400] - return None - - -def _bd_tagged_location(node): - """Bright Data's optional Post `location` -> one filterable content-context string. - - The documented Posts response is an ordered array such as - `["Capanema", "Pará", "Brasil"]`; retain the order rather than guessing which component is - a city or country. Reels do not promise this field, so a blank means it was not returned. - """ - raw = _first(node, "location", "location_details") - if isinstance(raw, dict): - raw = [raw.get(k) for k in ("name", "city", "region", "state", "country", "slug")] - if not isinstance(raw, list): - raw = [raw] - parts = [] - for value in raw: - if isinstance(value, dict): - value = _first(value, "name", "city", "region", "state", "country", "slug") - text = str(value or "").strip() - if text and text not in parts: - parts.append(text) - return ", ".join(parts)[:400] or None - - -def _bd_source_payload(row): - """One complete Bright Data response row as valid JSON, without field filtering. - - Source data is the lossless provider record, not a second relationship cache. A Profile's - embedded ``posts`` array stays here *and* each parseable post becomes a linked Post record; - keeping both preserves data while Links remain the interactive relational view. `bd_call` - rejects partial bodies at ``BD_SOURCE_PAYLOAD_MAX``, and the table-layer JSON ceiling matches - it, so this function never truncates a paid source response. - """ - if not isinstance(row, dict): - return None - try: - packed = json.dumps({str(k): v for k, v in row.items()}, ensure_ascii=False, - sort_keys=True, separators=(",", ":")) - except (TypeError, ValueError): - return None - # A source row is bounded by the complete raw HTTP response. Leave this defensive assertion - # explicit: should the transport ceiling and JSON-cell ceiling ever diverge, fail visibly - # rather than shortening a document and pretending every paid field was retained. - if len(packed.encode("utf-8")) > BD_SOURCE_PAYLOAD_MAX: - raise ValueError("a Bright Data source row exceeded the guarded response ceiling") - return packed - - -def _bd_comment(row, influencer_key="", shortcode="", fallback_key=""): - """One Bright Data Comment response -> one linked Comment record with engagement. - - The dedicated Comments dataset is invoked only by the default-off `commentMetrics` switch. - This mapper is also used for comment arrays embedded in an already-paid Post/Reel response, - which adds no request. The full provider object is preserved in Source data; Likes and Replies - are promoted for the grid and future rollups. - """ - if not isinstance(row, dict): - return None - url = str(_first(row, "url", "post_url", default="") or "") - code = str(shortcode or "") or ig_shortcode(url) - comment_id = str(_first(row, "comment_id", "id", "pk", default="") or fallback_key or "") - if not code or not comment_id: - return None - posted = str(_first(row, "comment_date", "date_of_comment", "created_at", "created_time", - default="") or "") - return { - "comment_key": comment_id, - "shortcode": code, - "influencer_key": str(influencer_key or _first(row, "post_user", default="") or ""), - "commented_at": posted.replace("T", " ")[:16], - "likes": _ig_int(_first(row, "likes_number", "likes", "like_count", "likes_count")), - "replies": _ig_int(_first(row, "replies_number", "num_replies", "replies", - "reply_count", "replies_count")), - "source_payload": _bd_source_payload(row), - } - - -def _bd_embedded_comments(post, influencer_key=""): - """Comment rows carried inside a paid Profile/Post/Reel record, with no extra request.""" - if not isinstance(post, dict): - return [] - shortcode = str(post.get("shortcode") or "") or ig_shortcode(post.get("url")) - if not shortcode: - return [] - out, seen = [], set() - # Known Bright Data arrays plus any future list-valued comment field. Source data still keeps - # unknown shapes even if it cannot be normalised into a Comment row. - arrays = [(str(k), v) for k, v in post.items() - if isinstance(v, list) and (str(k) in ("latest_comments", "top_comments") - or "comment" in str(k).lower())] - for source_key, values in arrays: - for index, value in enumerate(values): - if not isinstance(value, dict): - continue - try: - raw = json.dumps(value, ensure_ascii=False, sort_keys=True, - separators=(",", ":")) - except (TypeError, ValueError): - continue - fallback = "embedded:" + hashlib.sha256( - f"{shortcode}:{source_key}:{index}:{raw}".encode("utf-8")).hexdigest()[:24] - got = _bd_comment(value, influencer_key=influencer_key, shortcode=shortcode, - fallback_key=fallback) - if got and got["comment_key"] not in seen: - seen.add(got["comment_key"]) - out.append(got) - return out - - -def _bd_profile(node, handle): - """A Profiles row → our snapshot shape. Every name below is MEASURED off a real row - (2026-08-05, 35 keys); the candidates guard renames, not guesses. - - ⚠ Several of these were **null on both probe rows** (`avg_engagement`, `category_name`, - `business_category_name`, `is_professional_account`, `bio_hashtags`, `post_hashtags`). They - are mapped ANYWAY — see the note on `SNAPSHOT_FIELDS`: n=2 is not evidence enough to drop a - column permanently, and history is unbuyable, so a field omitted today is lost for today. - ⚠ They ARE populated on the CORPUS (`/datasets/filter`) rows, which is a different path with - a different population; that is a reason to capture them, not a licence to promise a - scrape-side feature before D-25 measures the fill rate. - """ - out = { - "username": str(_first(node, "account", "username", default=handle) or handle), - "full_name": str(_first(node, "full_name", "profile_name", default="") or ""), - "bio": str(_first(node, "biography", "bio", default="") or ""), - "followers": _ig_int(_first(node, "followers", "follower_count")), - "following": _ig_int(_first(node, "following", "following_count")), - "posts_count": _bd_posts_count(node), - "verified": "1" if _first(node, "is_verified", default=False) else "", - "external_url": _bd_first_url(_first(node, "external_url", "external_urls", - "external_url_title")), - # ⛔ NO `approx` KEY. These counts are exact, and carrying the anonymous rung's rounding - # tag onto them would be a lie in the other direction (R1: "drop the approx tag when - # exact"). Its ABSENCE is what tells the snapshot row it is trustworthy. - "ig_id": str(_first(node, "id", "fbid", default="") or ""), - "profile_url": str(_first(node, "profile_url", "url", default="") or ""), - "avg_engagement": _first(node, "avg_engagement"), - "category": str(_first(node, "category_name", default="") or ""), - "business_category": str(_first(node, "business_category_name", default="") or ""), - "is_business": _bd_flag(node, "is_business_account"), - "is_professional": _bd_flag(node, "is_professional_account"), - "is_private": _bd_flag(node, "is_private"), - "highlights_count": _ig_int(_first(node, "highlights_count")), - "bio_hashtags": _bd_list(node, "bio_hashtags"), - "pronouns": str(_first(node, "pronouns", default="") or ""), - # ⭐ 2026-08-07 (owner instruction: *"have ALL Fields available to us from Bright Data to - # be pre-set Fields for us and populated"*) — THE REST OF THE PROFILES SCHEMA. - # - # Every key below was MEASURED on a real row (2026-08-05, 35 row keys vs 36 declared; - # `wave20-split.md`), and the first four were explicitly logged as *"populated on scrape - # and NOT in any previous list"* — so these are not speculative names, they are fields the - # vendor was already sending and this map was already throwing away. - # Every unpromoted provider key — including contact and image fields — stays complete in - # Source data. This keeps the locked grid readable without throwing away paid data. - "profile_name": str(_first(node, "profile_name", default="") or ""), - "is_joined_recently": _bd_flag(node, "is_joined_recently"), - "has_channel": _bd_flag(node, "has_channel"), - "partner_id": str(_first(node, "partner_id", default="") or ""), - # ⚠ An OBJECT on the real row (`external_url_title`), which is why it goes through the - # same reader as the url itself rather than `str()` — a `str(dict)` would write the - # literal `{'title': …}` into a column somebody reads. - "external_url_title": _bd_first_url(_first(node, "external_url_title")), - # ⚠ `fbid` is the Facebook-side id, a SECOND opaque identifier beside `ig_id`. Text for - # the same reason `ig_id` is: it is a name, not a quantity, and some exceed 2^53. - "fbid": str(_first(node, "fbid", default="") or ""), - # ⚠ MEASURED `None` (not `[]`) on the real row — the graph-walk discovery seed is dead - # (§2c). Mapped anyway: history is unbuyable, so a field not captured today is lost for - # today, and n=1 is not evidence enough to drop a column permanently. - "related_accounts": _bd_list(node, "related_accounts"), - # ⚠ `country_code` is the ONE field the corpus filter API REJECTS as a predicate, and it - # was `None` on all five corpus rows. It is a row field regardless, and reading it costs - # nothing; a blank here means not read, like every other blank in this map. - "country_code": str(_first(node, "country_code", default="") or ""), - "source_payload": _bd_source_payload(node), - } - return out - - -def _bd_post_identity(p): - """One entry of a Profiles row's `posts[]` → our post shape. None when it has no shortcode. - - ⚠ IDENTITY ONLY — AND THAT IS THE FINDING THIS WHOLE RUNG IS SHAPED AROUND. MEASURED - 2026-08-05 across 24 posts on two accounts: `caption`, `content_type`, `id`, `image_url`, - `url` are populated 24/24, and **`likes`, `comments` and `datetime` are `None` 24/24.** So the - profile scrape can fill `ut_ig_posts` and CANNOT fill `ut_ig_post_snapshots` — the engagement - time series §3's append law exists for needs the POSTS dataset, one record per post. That is - why `postMetrics` is a separate, opt-in, ~13×-cost rung rather than a free by-product. - ⚠ `is_pinned` was `True` on all 24 — a constant, not a measurement. Not mapped. - ⚠ Blank-valued keys are OMITTED rather than written, so a later engagement pull that DOES know - `posted_at` fills it in instead of being overwritten by this rung's silence. - """ - code = ig_shortcode(p.get("url")) if isinstance(p, dict) else "" - if not code: - return None - kind = _bd_type(p.get("content_type"), p.get("type")) - # The URL route is part of Bright Data's endpoint contract. A Reel sent through the Posts - # scraper can return likes/comments while omitting its view fields; `/reel/{shortcode}` is - # the input documented for the Reels scraper. - route = "reel" if kind == "video" else "p" - out = {"shortcode": code, "url": f"https://www.instagram.com/{route}/{code}/", - "type": kind} - caption = _first(p, "caption", "description", default="") - if isinstance(caption, dict): - caption = _first(caption, "text", default="") - if caption: - out["caption"] = str(caption) - out["source_payload"] = _bd_source_payload(p) - embedded = _bd_embedded_comments(p) - if embedded: - out["embedded_comments"] = embedded - return out - - -def _bd_post_metrics(row): - """One POSTS-dataset row → the engagement half. None when it has no shortcode. - - MEASURED 2026-08-05 on three real post rows: `likes` (308 / 3,736), `num_comments` (37 / 134) - and `date_posted` (`2026-05-04T17:51:58.000Z`) all populated. - ⚠ NO VIEW COUNT CAME BACK — `video_play_count` / `video_view_count` are declared in the schema - and the KEY WAS ABSENT from every row, including a Reel. So `views` reads None and the column - stays blank, which is the honest answer; it is not folded into likes and it is not zeroed. - ⚠ The candidates below are all COUNTS OF VIEWS in different vendor typings (`video_view_count` - is declared `text`, the others `number`). A candidate list may span types only when the - MEANING is identical — never the way `like_and_view_counts_disabled`, a boolean, would have - turned "likes are hidden" into "1 like". - Embedded comment arrays are retained in Source data and converted to linked Comment rows - without triggering the separate full Comments-dataset request. - """ - if not isinstance(row, dict): - return None - code = str(_first(row, "shortcode", "content_id", default="") or "") or \ - ig_shortcode(row.get("url")) - if not code: - return None - posted = str(_first(row, "date_posted", "taken_at", default="") or "") - partner = row.get("partnership_details") - out = { - "shortcode": code, - "url": str(_first(row, "url", default=f"https://www.instagram.com/p/{code}/")), - "influencer_key": str(_first(row, "user_posted", default="") or ""), - "posted_at": posted.replace("T", " ")[:16], - "type": _bd_type(row.get("content_type"), row.get("product_type")), - "caption": str(_first(row, "description", "caption", default="") or ""), - "likes": _ig_int(_first(row, "likes", "like_count")), - "comments": _ig_int(_first(row, "num_comments", "comment_count", "comments")), - # ⛔⛔ `views` IS NOT READ HERE, AND THE FIELD NAME IS THE WHOLE TRAP (2026-08-08, §4e of - # instagram-capture.md). Bright Data's `views` is an ACCOUNT-level number: it rides in the - # vendor's own profile-context block — - # "input": {"url": …, "posts_count": 386, "followers": 6133, "following": 2850, - # "video_play_count": null, "views": null, "country_code": null} - # — next to followers/following/posts_count, and on the delivered row it is CONSTANT - # across every reel exactly as those are. MEASURED: one value across 12 distinct reels - # whose likes ran 3→271, on 18 of 19 creators, and 9753→9754 between two captures two - # hours apart (so it is a live account counter, not a stale cache). - # ⚠ THIS IS NOT A COLLECTION-MODE PROBLEM AND MUST NOT BE "FIXED" BY REWRITING THE CALL. - # The documented discover-by-profile contract was bought and tested: same constant, same - # null play count. Reading `views` here promoted a profile metric into a post column, and - # that — not a vendor fault — is what put one number on many unrelated posts. - # `video_play_count` is the genuine per-reel field. It is NULL on 91 of 91 records across - # 20 accounts in this cohort and BOTH collection modes, so Plays stays blank; blank is the - # honest answer for a paid rung that did not answer, and always was. - "plays": _ig_zero_is_blank(_first(row, "video_play_count", "play_count")), - # Sponsored-post detection. MEASURED populated (`True`, with the brand alongside) — and - # it is the one field here that answers a commercial question the counts cannot. - "paid_partnership": _bd_flag(row, "is_paid_partnership"), - "partner": (str(_first(partner, "username", "profile_id", default="") or "") - if isinstance(partner, dict) else ""), - "hashtags": _bd_list(row, "hashtags"), - "alt_text": str(_first(row, "alt_text", default="") or ""), - "tagged_location": _bd_tagged_location(row), - "source_payload": _bd_source_payload(row), - } - embedded = _bd_embedded_comments(row, influencer_key=out["influencer_key"]) - if embedded: - out["embedded_comments"] = embedded - return out - - -def _bd_rows(payload): - """A vendor answer → the list of row dicts inside it, whatever envelope it arrived in. - - MEASURED shapes: a bare list (`/v3/scrape`), NDJSON (already listified by `bd_call`), and - `{"data": [...]}` / `{"results": [...]}` on the snapshot reads. An unreadable envelope yields - NOTHING rather than a guess. - """ - if isinstance(payload, list): - return [x for x in payload if isinstance(x, dict)] - if isinstance(payload, dict): - for k in ("data", "results", "records", "items"): - v = payload.get(k) - if isinstance(v, list): - return [x for x in v if isinstance(x, dict)] - return [payload] if payload else [] - return [] - - -def _bd_deferral(payload): - """The snapshot id when the 'sync' endpoint DEFERRED, else ''. - - ⛔ MEASURED 2026-08-05, AND IT IS THE TRAP OF THIS WHOLE RUNG. `/v3/scrape` is documented and - named as the synchronous call, and for a small batch it is — two profile URLs came back - inline. Give it more work and it answers **200** with no rows and a note instead: - - {"snapshot_id": "sd_…", "message": "Your request is still in progress and cannot be - retrieved in this call. Use the provided Snapshot ID to track progress…"} - - Read naively that is a 200 with a body, so a parser looking for "did it error?" sails past - it, `_bd_rows` wraps the envelope as ONE unusable row, and the caller reports that the vendor - returned nothing — **while a snapshot we have already been billed for sits on the account, - finishing, and is never collected.** That is the D-26 failure shape (paid work abandoned) - arriving through the success path. - ⚠ It is a TIME threshold, not a row count: 2 URLs answered inline, 6 and 10 deferred. - """ - if isinstance(payload, dict) and payload.get("snapshot_id") and not payload.get("account"): - return str(payload["snapshot_id"]) - return "" - - -def bd_scrape(dataset_id, urls, wait=None, deferred=None): - """Scrape a batch. `(rows, note)`; a note means it did not answer. - - Sends the SYNC call and, when the vendor defers it (see `_bd_deferral`), **collects the - snapshot it handed back** rather than discarding it. One call carries many URLs, which is - what keeps the per-profile pacing floor from turning a 20-profile run into an hour. - """ - requested_urls = [str(u) for u in (urls or []) if u] - payload, note = bd_call(BD_PATH_SCRAPE, {"dataset_id": dataset_id}, - body=[{"url": u} for u in requested_urls]) - if note: - return [], note - sid = _bd_deferral(payload) - if not sid: - return _bd_rows(payload), "" - - # --- THE DEFERRED PATH. Poll the scraper namespace (`sd_…` at `/datasets/v3/…` — crossing it - # with the corpus namespace is a flat 404 about a snapshot that is alive). - budget = BD_SCRAPE_WAIT if wait is None else float(wait) - waited = 0.0 - while True: - got, gnote = bd_call(f"{BD_PATH_SNAPSHOT}/{sid}", {"format": "json"}) - if not gnote: - rows = _bd_rows(got) - # The status document is itself JSON, so "did it deliver?" is decided by whether - # what came back looks like ROWS — never by the HTTP code. - if rows and not _bd_deferral(got) and not ( - len(rows) == 1 and str(rows[0].get("status") or "") in - ("running", "building", "collecting")): - return rows, "" - if waited >= budget: - break - time.sleep(BD_SCRAPE_POLL) - waited += BD_SCRAPE_POLL - # ⛔ NAME THE SNAPSHOT. It has been paid for and it is still finishing; a note that said only - # "no rows" would throw away both the data and the way to get it. - if isinstance(deferred, list): - deferred.append({"snapshotId": sid, "datasetId": str(dataset_id), - "urls": requested_urls}) - return [], (f"the profile source deferred this batch to {sid} and it was not ready within " - f"{int(budget)}s — the records are collected, not lost; ask for a smaller batch " - f"or collect {sid} from the control panel") - - -# ============================================================================================= -# APIFY — THE SECOND PROVIDER (owner ruling 2026-08-08). Registered in `providers.py`; this is -# only the wire. +# ⭐ THE LIST BELOW IS A DEPENDENCY STATEMENT, NOT A CONVENIENCE. It is every name the automation +# runtime still reaches for after the split — twenty, down from the seventy-one that used to be +# defined here — so `git diff` on this block is how anyone sees the engine growing a new vendor +# dependency. Re-exporting them is also REQUIRED rather than tidy: `routes_automation` and +# `routes_connectors` call `engine.bd_ready()` on the module object, and this is what keeps that +# true without those files having to learn where the wire went. +# ⚠ SO THE RE-EXPORT ALSO MEANS `engine.bd_call` STILL RESOLVES, and a gate that only checked +# that would be green whether or not anything moved. `verify_automation`'s `section_split` +# therefore asserts `__module__` on the whole moved set — it derives the split from the RUNNING +# system instead of trusting this import line ([[gate-answers-the-wrong-question]]). # -# ⭐ IT EXISTS FOR EXACTLY ONE CAPABILITY TODAY: `ig_post_views`. Bright Data answers profile, -# likes and comments correctly and cannot answer a per-reel view count on ANY route; Apify's -# `videoPlayCount` matched a browser-read ground truth to the digit (137,684 vs ~137K and 299,493 -# vs 299K). So this runs as a TOP-UP over Bright Data's rows, never as a replacement for them — -# re-buying the profile and the likes from a second vendor is the expensive mistake the capability -# split exists to prevent. -# ⛔ `videoPlayCount`, NEVER `videoViewCount`. Apify ships BOTH, and the wrong one (10,678) agrees -# with Bright Data's useless `views` (10,638) for the same reel whose true count is ~137K. Two -# vendors' junk fields agreeing is what makes picking by name so dangerous here. -# ============================================================================================= - -APIFY_BASE = "https://api.apify.com/v2" -#: The general Instagram scraper. `directUrls` accepts /p/ and /reel/ permalinks alike. -APIFY_ACTOR_POSTS = os.environ.get("AIOS_APIFY_ACTOR") or "apify~instagram-scraper" -#: Seconds to let a synchronous actor run before giving up. Apify bills the run either way, so a -#: timeout here is money already spent — generous on purpose. -APIFY_WAIT = float(os.environ.get("AIOS_APIFY_WAIT") or 240) - - -def apify_key(): - """The key, or ''. Read fresh every call, exactly as `bd_key` is.""" - return (os.environ.get("AIOS_APIFY_KEY") or "").strip() - - -def apify_ready(): - return bool(apify_key()) - - -def apify_posts(urls): - """Post/reel permalinks -> (canonical rows, note). A note means it did NOT answer. - - Returns rows already normalised through `providers.normalize_post_apify`, so nothing - downstream ever sees an Apify-shaped key. That is the owner's schema ruling enforced at the - only place it can be: the boundary. - """ - want = [str(u) for u in (urls or []) if u] - if not want: - return [], "" - key = apify_key() - if not key: - return [], "AIOS_APIFY_KEY is not configured — the Apify rung is closed" - url = (f"{APIFY_BASE}/acts/{APIFY_ACTOR_POSTS}/run-sync-get-dataset-items" - f"?token={requests.utils.quote(key)}&timeout={int(APIFY_WAIT)}") - body = {"directUrls": want, "resultsType": "posts", "resultsLimit": max(1, len(want)), - "addParentData": False} - try: - status, raw = fetch_json(url, body, timeout=APIFY_WAIT + 60, max_kb=BD_MAX_KB) - except Refused as e: - return [], f"Apify's address was refused by the URL rail: {e}" - except Exception as e: # noqa: BLE001 - # ⚠ TYPE NAME ONLY — the token is in the query string of `url`, so a str(e) that echoed - # the request would put a live credential in the run log. - return [], f"Apify did not answer ({type(e).__name__})" - if status == 402: - return [], "the Apify account is out of credit" - if status in (401, 403): - return [], f"Apify refused our key ({status})" - if status >= 400: - return [], f"Apify answered {status}" - try: - items = json.loads(raw.decode("utf-8", "replace")) - except Exception: # noqa: BLE001 - return [], "Apify's answer was not readable JSON" - if not isinstance(items, list): - return [], "Apify answered with something other than a list of items" - rows = [providers.normalize_post_apify(x) for x in items] - return [r for r in rows if r], "" - - -#: ⭐⭐ 2026-08-07 — HOW LONG THE CORPUS RUNG MAY WAIT. A corpus query is a scan over 620M rows -#: and takes MINUTES, not seconds (§2d MEASURED ~4 min for a narrow one). It is a FALLBACK behind -#: a scrape that already failed, so the budget is generous — but bounded, because an enrich step -#: that never returns is an automation that never finishes. -BD_CORPUS_WAIT = float(os.environ.get("AIOS_BD_CORPUS_WAIT") or 420) -BD_CORPUS_POLL = float(os.environ.get("AIOS_BD_CORPUS_POLL") or 15) - - -def bd_corpus_profile(handle, log=print): - """One profile out of the PRE-COLLECTED corpus. `(node|None, note)`. - - ⭐⭐ WHY THIS RUNG EXISTS, and it is measured rather than defensive. On 2026-08-07 the SCRAPE - path answered `Parse error: Invalid URL` for `inayma` on **four consecutive attempts and - through both input shapes** (a URL with and without its trailing slash, and the `user_name` - discovery input, which the vendor converts to the same URL internally) — while a control - profile in the SAME batch came back with 36 keys. So the URL form was not the problem and - neither was the account: the anonymous ladder read it fine. The vendor simply could not - scrape that profile. - ⇒ **The same dataset answered instantly by the OTHER route.** `POST /datasets/filter` with - `account = inayma` returned the full 33-key row — exact followers (203,818 against the - anonymous rung's rounded 204,000), biography, `is_verified`, `avg_engagement`, `fbid`, - `highlights_count`. One dataset, two routes, and only one of them was broken. - - ⚠ THE ROWS ARE PRE-COLLECTED, so this is not a live read — it is the vendor's own last - capture, and its population differs from the scrape path's (§2d: `avg_engagement` 88% - populated on the corpus vs 0% on scrape). That is why it is a FALLBACK and not the default: - fresher-but-thinner beats staler-but-fuller when both work. - ⛔ THE SNAPSHOT NAMESPACES ARE NOT INTERCHANGEABLE. A corpus query mints `snap_…` and is read - at `/datasets/snapshot/{id}`; the scraper mints `sd_…` and is read at - `/datasets/v3/snapshot/{id}`. Crossing them returns a flat 404 about a snapshot that is alive - and building — the trap §2c records, and the reason these paths are spelled out here. - """ - h = str(handle or "").strip().lstrip("@").lower() - if not h: - return None, "no handle to look up" - payload, note = bd_call(BD_PATH_FILTER, None, - {"dataset_id": BD_DS_PROFILES, "records_limit": 1, - "filter": {"name": "account", "operator": "=", "value": h}}) - if note: - return None, note - snap = str((payload or {}).get("snapshot_id") or "").strip() - if not snap: - return None, "the profile corpus accepted the query but named no snapshot" - deadline = time.time() + BD_CORPUS_WAIT - while time.time() < deadline: - time.sleep(BD_CORPUS_POLL) - got, perr = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snap}", None, None) - status = str((got or {}).get("status") or "").strip() - if perr or status in ("failed", "error"): - return None, (perr or f"the corpus query failed ({(got or {}).get('error') or status})") - if status == "ready": - # ⚠ READY IS NOT DELIVERABLE. §2d MEASURED `/download` answering "Snapshot is - # building. Try again in a few minutes" while status already said `ready` — delivery - # lags readiness. So the download is polled too, inside the same budget. - while time.time() < deadline: - rows, derr = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snap}/download", - {"format": "json"}, None) - if derr: - time.sleep(BD_CORPUS_POLL) - continue - if isinstance(rows, dict): - rows = [rows] - if isinstance(rows, list) and rows: - return rows[0], "" - time.sleep(BD_CORPUS_POLL) - break - return None, (f"the profile corpus did not deliver {h!r} within " - f"{int(BD_CORPUS_WAIT)}s (snapshot {snap} is still building — it is not lost)") - - -def _tag_metric_deferrals(items, start, kind, influencer_key): - """Attach only the context a later snapshot collector needs to preserve the IG graph.""" - for item in items[start:]: - if isinstance(item, dict): - item["kind"] = kind - item["influencer"] = str(influencer_key or "").strip().lstrip("@").lower() - item["requestedAt"] = _iso() - - -def pull_profile_bd(url, max_posts=DEFAULT_POSTS_PER_PULL, post_metrics=False, - comment_metrics=False, log=print, pending_metrics=None): - """The paid rung: one profile, EXACT counts, plus the top posts the anonymous surface hides. - - Same return contract as `pull_profile` (`{state, profile, posts, via, note}`) so the runner - treats every rung identically. `blocked` here means the VENDOR rung did not answer — the - caller decides whether to drop to the anonymous ladder. - - `post_metrics=True` adds the paid Post/Reel calls. `comment_metrics=True` independently adds - the separate paid Comments-dataset call. Both default off; the latter is never inferred from - an enabled post-metrics switch. Embedded comment previews returned by either paid record are - retained without that extra Comments request. - """ - handle = ig_handle(url) - if not handle: - return {"state": "error", "note": f"{url!r} is not an Instagram profile URL", - "profile": {}, "posts": [], "comments": [], "via": ""} - rows, note = bd_scrape(BD_DS_PROFILES, [f"https://www.instagram.com/{handle}/"]) - node = rows[0] if rows else {} - profile = _bd_profile(node, handle) if node else {} - unreadable = profile.get("followers") is None and profile.get("following") is None - via = "brightdata" - if note or unreadable: - # ⭐⭐ 2026-08-07 — THE CORPUS IS TRIED BEFORE THIS RUNG GIVES UP, and it is the difference - # between a blank row and a full one. MEASURED: the scraper answered `Parse error: Invalid - # URL` for `inayma` four times running and through both input shapes, while a control - # profile in the same batch returned 36 keys — and `POST /datasets/filter` on the SAME - # dataset returned that profile's complete 33-key row. A rung that reports `blocked` - # while the vendor is holding the answer on another route is a rung that gave up early. - # ⚠ The FIRST failure is kept in the note either way: if the corpus also misses, the - # person needs to know the scrape was tried and what it said, not only the fallback. - # ⚠ THE UNREADABLE-ROW DIAGNOSTIC IS CARRIED, NOT DROPPED. "It answered 200 with something - # we could not parse" and "it refused" are different problems with different fixes, and - # the field-names hint is the one that tells the next reader to go and diff a real row - # against the schema. Losing it behind the corpus fallback would make a mapping - # regression look like a vendor outage. - scrape_note = note or ("the scrape answered, but no follower/following counts were " - "readable in it (the field names may have moved — see " - "instagram-capture.md)") - cnode, cnote = bd_corpus_profile(handle, log=log) - if cnode: - node = cnode - profile = _bd_profile(node, handle) - via = "brightdata:corpus" - # ⚠ The corpus is PRE-COLLECTED, so the row is the vendor's last capture rather than - # a read taken now. Said out loud on the attempt, because "when was this true" is the - # one question a stored measurement must always be able to answer. - note = (f"the live scrape did not answer ({_s(scrape_note, 120)}), so this came from " - f"the vendor's pre-collected profile corpus — the values are its last " - f"capture, not a read taken just now") - else: - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "note": f"{scrape_note}; the profile corpus did not answer either ({cnote})"} - if profile.get("followers") is None and profile.get("following") is None: - # It answered 200 with something we could not read. Say THAT — not "0 followers". - # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and - # is discarded, so a row where it is the only "readable" field is a row we cannot read. - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "note": "the profile source answered, but no follower/following counts were readable in " - "it (the field names may have moved — see instagram-capture.md)"} - posts = [] - for p in (node.get("posts") or [])[:max_posts]: - got = _bd_post_identity(p) - if got: - posts.append(got) - - if post_metrics and posts: - time.sleep(min(PACE_SECONDS, 1.0)) # the vendor is paid, but it is still someone's API - # Posts and Reels are different Bright Data endpoints. Sending every permalink to the - # Posts endpoint was why Inayma's five video rows got likes/comments but no views: the - # documented Reels response is where `views` and `video_play_count` live. Split the batch - # without duplicating any record, so enabling post metrics still buys one row per post. - ordinary = [p for p in posts if p.get("type") != "video"] - videos = [p for p in posts if p.get("type") == "video"] - rows2, metric_notes, ordinary_rows = [], [], [] - # `None` keeps direct callers backwards-compatible: they wait for their answer. An - # automation passes one run-level list, which makes a deferred metric batch durable and - # lets this profile finish rather than parking the whole enrichment on it. - metric_wait = BD_METRIC_SCRAPE_WAIT if isinstance(pending_metrics, list) else None - if ordinary: - at = len(pending_metrics) if isinstance(pending_metrics, list) else 0 - got_rows, got_note = bd_scrape(BD_DS_POSTS, [p["url"] for p in ordinary], - wait=metric_wait, deferred=pending_metrics) - if isinstance(pending_metrics, list): - _tag_metric_deferrals(pending_metrics, at, "posts", profile.get("username") or handle) - rows2.extend(got_rows) - ordinary_rows.extend(got_rows) - if got_note: - metric_notes.append(f"posts: {got_note}") - if videos: - at = len(pending_metrics) if isinstance(pending_metrics, list) else 0 - got_rows, got_note = bd_scrape(BD_DS_REELS, [p["url"] for p in videos], - wait=metric_wait, deferred=pending_metrics) - if isinstance(pending_metrics, list): - _tag_metric_deferrals(pending_metrics, at, "posts", profile.get("username") or handle) - rows2.extend(got_rows) - if got_note: - metric_notes.append(f"reels: {got_note}") - # Bright Data's Profiles feed has used more than one media spelling. If one of those - # unrecognised labels sent a Reel to Posts, the Posts response itself identifies the - # mistake (`content_type: Reel`) but cannot supply Views. Correct it immediately with a - # Reels read, then let the richer response win below. This costs a replacement record only - # for a proven misroute; correctly classified images/carousels are never re-read. - repair_reels = [] - for row in ordinary_rows: - mapped = _bd_post_metrics(row) - if mapped and mapped.get("type") == "video": - repair_reels.append(f"https://www.instagram.com/reel/{mapped['shortcode']}/") - if repair_reels: - at = len(pending_metrics) if isinstance(pending_metrics, list) else 0 - got_rows, got_note = bd_scrape(BD_DS_REELS, list(dict.fromkeys(repair_reels)), - wait=metric_wait, deferred=pending_metrics) - if isinstance(pending_metrics, list): - _tag_metric_deferrals(pending_metrics, at, "posts", profile.get("username") or handle) - rows2.extend(got_rows) - if got_note: - metric_notes.append(f"reel recovery: {got_note}") - if metric_notes and not rows2: - # The engagement half refusing does NOT lose the identity half — the profile and the - # post rows still land, and the run says which part was not readable. - note = f"post metrics unavailable: {'; '.join(metric_notes)}" - else: - by_code = {} - for r in rows2: - got = _bd_post_metrics(r) - if got: - by_code[got["shortcode"]] = got - for p in posts: - extra = by_code.get(p["shortcode"]) - if extra: - # The metrics row is RICHER (it knows posted_at); merge it over the identity - # row, dropping keys it could not read so a blank never overwrites a value. - p.update({k: v for k, v in extra.items() if v not in (None, "")}) - if not by_code: - note = "post metrics were requested but the Posts dataset returned no rows" - elif metric_notes: - note = f"some post metrics were unavailable: {'; '.join(metric_notes)}" - - # ⭐⭐ THE VIEWS TOP-UP — capability routing, not provider failover (owner ruling - # 2026-08-08, `providers.py`). Bright Data has already answered profile + likes + comments - # above and is declared INCAPABLE of `ig_post_views`, so the chain for that ONE capability - # resolves to Apify and only the video permalinks are re-bought. Re-running the whole - # profile on the second vendor would pay twice for the 90% that already worked, which at - # thousands of tenants x twelve posts is the entire bill. - # ⚠ VIDEO ONLY. A carousel or image has no view count, so sending it would buy a record - # that can only come back blank. - video_urls = [p["url"] for p in posts - if p.get("type") == "video" and p.get("url")] - if video_urls: - def _views_work(prov, _urls=tuple(video_urls)): - if prov.key == "apify": - return apify_posts(list(_urls)) - # A provider in the chain with no runner here is a configuration error, not a - # vendor outage — say so rather than returning an empty list that reads as "the - # vendor had nothing". - return None, f"no {prov.key} runner is wired for ig_post_views" - - got_views, attempts = providers.run( - "ig_post_views", _views_work, - # ⛔ THE SATISFIED PREDICATE IS THE FALLBACK TRIGGER. Rows that come back with - # every `views` blank are a FAILURE for this capability even at HTTP 200 — which - # is exactly how Bright Data behaves, and why a chain that only caught exceptions - # would never have reached a second provider. - satisfied=lambda rows: bool(rows) and any(r.get("views") for r in rows), - log=log) - for row in (got_views or []): - target = next((p for p in posts - if p.get("shortcode") == row.get("shortcode")), None) - # ⛔ TAKE ONLY THE CAPABILITY THAT WAS ASKED FOR. This provider also returns - # likes/comments/caption, and letting them land would silently switch the source - # of columns Bright Data already answered — the schema would be stable but the - # PROVENANCE would flip halfway through a row. - if target and row.get("views") not in (None, ""): - target["views"] = row["views"] - # ⭐ LOCATION RIDES ALONG FOR FREE, AND ONLY INTO A GAP (2026-08-08). - # MEASURED: Bright Data's `location_details` is RICHER when present — it carries - # real coordinates (`lat -6.9246, lng 106.9292, name "Sukabumi"`) which Apify does - # not return at all — but it is populated on only 44 of 227 posts, and on - # `DblAkEbv0ry` it returned nothing while Apify returned "Jakarta, Indonesia". - # So Bright Data stays the source and this fills only what it left BLANK. - # ⛔ THE COST ARGUMENT IS WHY IT IS HERE AND NOT ITS OWN CAPABILITY: this Apify - # record has ALREADY been bought for the view count and carries `locationName` in - # the same payload. Routing location as a separate capability would buy a second - # record for a field that is already sitting in this response. - if target and not str(target.get("tagged_location") or "").strip() and str(row.get("tagged_location") or "").strip(): - target["tagged_location"] = row["tagged_location"] - used = next((a.provider for a in attempts if a.ok), "") - if used: - note = "; ".join(x for x in (note, f"view counts via {used}") if x) - elif attempts: - note = "; ".join(x for x in ( - note, "no provider could supply view counts: " - + "; ".join(f"{a.provider} {a.note}" for a in attempts)) if x) - - comments = [] - if comment_metrics and posts: - # This is the cost boundary: do not make the full comment request unless the stored - # enrichment action explicitly opted in. A batched call still bills provider records, but - # it prevents unnecessary HTTP fan-out and preserves every returned field per comment. - time.sleep(min(PACE_SECONDS, 1.0)) - comment_wait = BD_METRIC_SCRAPE_WAIT if isinstance(pending_metrics, list) else None - at = len(pending_metrics) if isinstance(pending_metrics, list) else 0 - comment_source, comment_note = bd_scrape(BD_DS_COMMENTS, [p["url"] for p in posts], - wait=comment_wait, deferred=pending_metrics) - if isinstance(pending_metrics, list): - _tag_metric_deferrals(pending_metrics, at, "comments", profile.get("username") or handle) - for row in comment_source: - got = _bd_comment(row, influencer_key=profile.get("username") or handle) - if got: - comments.append(got) - if comment_note: - note = "; ".join(x for x in (note, f"comment engagement unavailable: {comment_note}") if x) - elif not comments: - note = "; ".join(x for x in (note, - "comment engagement was requested but the Comments dataset returned no rows") if x) - - # IDENTITY WITHOUT MEDIA IS STILL `partial`, on the paid rung too. The rule does not soften - # because we are paying: a run that wrote a follower count and no posts must not paint green - # over a posts table that did not grow. - # ⚠ `via` CARRIES WHICH ROUTE ANSWERED — `brightdata` or `brightdata:corpus`. It lands in the - # snapshot row's `source` column, so a stored measurement can always say whether it was read - # live or taken from the vendor's pre-collected corpus. Two routes writing one indisting- - # uishable `source` would make the series unauditable exactly where it matters most. - return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts, - "comments": comments, - "via": via, - "note": note or ("" if posts else "profile read; no posts were returned")} +# ⚠ AND IT IS DELIBERATELY HERE, mid-file, rather than at the top. `connectors_ig` reaches back +# for the SSRF rail (`fetch`/`fetch_json`/`Refused`) and for `PACE_SECONDS`; it does so lazily, +# inside its functions, precisely so the two modules cannot cycle. This import sits BELOW the rail +# and below `PACE_SECONDS`, so even if somebody later makes one of those reach-backs a +# module-level import, the names it wants already exist and the cycle still resolves. +from connectors_ig import ( # noqa: E402 + BD_DS_COMMENTS, BD_DS_POSTS, BD_DS_REELS, BD_EXCLUDE_MAX, BD_PATH_SNAPSHOT, + BD_RECORD_PRICE_SPEC, + _bd_comment, _bd_deferral, _bd_first_url, _bd_post_metrics, _bd_rows, _bd_tagged_location, + _first, _ig_int, + bd_call, bd_filter_rows, bd_filter_start, bd_filter_status, bd_ready, depth_refusal, + ig_handle, pull_profile, +) # --------------------------------------------------------------------------------------------- @@ -4049,6 +3100,28 @@ PRESET_PROFILE_FIELDS = [ field_def("comments_captured", "Comments captured", "rollup", rollup={"link": "comments_link", "fn": "countall", "distinctBy": "comment_key"}), + # ⭐⭐ WAVE 27 ITEM 16 — WHERE THIS CREATOR ACTUALLY IS, DERIVED, FOR NOTHING. + # + # ⛔ THE VENDOR DOES NOT SELL THIS. `country_code` was MEASURED `None` on every corpus row we + # have ever looked at, and the filter API REFUSES it as a predicate — so residency is the one + # thing a buyer most wants and the one thing the profile row cannot answer. The competitor + # teardown found the same gap solved by GUESSING from bio words, two buckets deep + # ([[janney-ai-teardown]]). + # + # ⭐ WE ARE ALREADY HOLDING THE ANSWER AND WERE THROWING IT AWAY. Each post carries the place + # it was tagged in, and the enrich has ALREADY BOUGHT twelve of them: `tagged_location` on the + # post row, plus the vendor's fuller `location`/`location_details` inside the paid + # `source_payload` that `_bd_tagged_location` normalises away. The modal city across a + # creator's own posts is a far better residency signal than a word in a bio, and it costs a + # dict comprehension. **Zero new vendor spend** — this is the whole reason it is a v1. + # + # ⚠ IT IS A GUESS AND THE COLUMN SAYS SO, IN ITS NAME AND IN ITS NEIGHBOUR. A travel creator + # posting from twelve cities gets a low confidence rather than a confident wrong answer, and + # `location_confidence` is the number a person filters on before trusting the guess. A single + # geotagged post produces NO guess at all — n=1 dressed as a pattern is what + # `SEED_MIN_ROWS_SHARING` refuses fifty lines up, and it would read as 100% certain. + field_def("location_guess", "Location (guess)"), + field_def("location_confidence", "Location confidence", "pct"), # ⭐ R3's stamp. WITHOUT IT THE WHOLE SET IS UNREADABLE: a blank `followers` means "never # enriched" and a stale one means "enriched in March", and no cell on the row can tell them # apart. It is the single field that turns the other fifteen from numbers into measurements. @@ -4092,10 +3165,19 @@ CANDIDATE_FIELDS = [ field_def("found_count", "Times found", "int"), field_def("first_found", "First found", "date"), field_def("last_found", "Last found", "date"), - # C6 (wave 22): the candidate pool is PER-USER — the upsert key is the COMPOUND - # (handle, created_by), so two people discovering the same profile each get their own row, - # their own found_count and their own review card. Stamped by the RUNNER (the automation's - # creator), never by `_candidate_row`. + # ⛔ DEBT D-73 — THIS NOTE STATED A RETIRED LAW AS CURRENT FACT, directly above the field it + # describes, which is the first place anybody looks up what this column means. It read: *"the + # candidate pool is PER-USER — the upsert key is the COMPOUND (handle, created_by), so two + # people discovering the same profile each get their own row, their own found_count and their + # own review card."* Wave 26 · R4/R5 retired every clause of that. + # + # ⭐ THE CURRENT LAW: the identity is `(platform, handle)` and the TENANT is the unit. Two + # people in one workspace who discover the same profile share ONE row, one `found_count` and + # one history — because they are looking at one company's leads, not two private lists. + # `created_by` survives as an INFORMATIONAL stamp only: "Found by", answering who saw it + # first. It is no part of the dedup key, and a run must never branch on it. + # ⚠ Stamped by the RUNNER (the automation's creator), never by `_candidate_row` — unchanged, + # and the one clause of the old note that was still true. field_def("created_by", "Found by"), # ⛔ WAVE 26 · R6 — `tracked` WAS HERE AND IS DELETED. Owner, 2026-08-06: *"We already have # a Field called stage to track the progress of the Automation per Record. We don't need @@ -4222,6 +3304,7 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): wanted = {str(f.get("key")): dict(f) for f in wanted_fields} drops = set(drop_by_table.get(table_key) or ()) fields, seen = [], set() + drops = _guarded_drops(table, drops) for stored in table.get("fields") or []: key = str(stored.get("key") or "") if key in drops: @@ -4279,7 +3362,11 @@ def _reconcile_ig_graph_fields(rt, wanted_by_table, drop_by_table=None): continue table = snapshot.get(table_key) or {} by_key = {str(f.get("key") or ""): f for f in table.get("fields") or []} - drops = set(drop_by_table.get(table_key) or ()) + # ⚠ THE SAME GUARD, OR THE MIGRATION STOPS BEING IDEMPOTENT. This precheck exists to skip + # the commit when nothing would change; a spared `tracked` column left in the drop set + # answers "yes, work to do" on every single call, forever, and the schema pass would + # rewrite the table on every authenticated read without ever changing a byte. + drops = _guarded_drops(table, drop_by_table.get(table_key) or ()) if drops & set(by_key): needs = True break @@ -4484,6 +3571,78 @@ def _merge_candidates(rows): return out, merged +#: ⛔ DEBT D-74 + D-81 (wave 27 item 12) — `tracked` WAS DELETED FROM THE VOCABULARY AND NEVER +#: FROM THE TABLES. W26/R6 removed the column from `CANDIDATE_FIELDS` and from ~14 engine sites +#: *"we already have a Field called stage… we don't need another checkbox for this"* — and +#: MEASURED 2026-08-07 on `royal-imports/aios-nurilab-data`, `ut_beauty_influencer_leads` still +#: declared it, filled on 1 of 41 rows. A checkbox on the owner's flagship database that nothing +#: writes, nothing reads, and anybody can still click. +#: +#: ⚠ THE DECISION WAS WHICH, NOT WHETHER (D-74 stated both futures): sweep the ticks and lose the +#: record of who approved what before the wave, or leave them and let the next hand-made `tracked` +#: column silently inherit them. The wave takes the sweep — R6 deleted the concept, so a surviving +#: tick is not a decision anybody can act on, and an inert cell that reappears in the next EXPORT +#: as a column nobody declared is the worse half. +TRACKED_KEY = "tracked" + + +def _retired_tracked_field(fields): + """The retired `tracked` column when it is still the MACHINE's own, else None. + + ⛔ GUARDED THE WAY `_vestigial_name_field` IS, and for the same reason: this is a branch that + DESTROYS something. `tracked` was minted `{type: "checkbox"}` by the discovery preset, so a + column still declaring a checkbox is the one R6 retired. A column somebody has since RETYPED, + or bound a formula/link/rollup/measure to, is their work wearing an old key — it survives as + an ordinary column and merely stops being fed by anything, which was already true. + + ⚠ THE ASYMMETRY WITH THE TWO KEYS BESIDE IT IS DELIBERATE. `posts` and `post_hashtags` are + dropped unguarded: both are machine-derived cells with no history of anyone editing them. + `tracked` is the one a HUMAN ticked, so it is the one that gets the conservative test. + """ + f = next((f for f in fields if str(f.get("key") or "") == TRACKED_KEY), None) + if f is None or f.get("type") not in ("checkbox", "", None): + return None + if any(f.get(k) for k in ("formula", "link", "rollup", "measure", "metric")): + return None + return f + + +def _guarded_drops(table, drops): + """`drops` minus any key this table is allowed to keep — today, exactly `tracked` (D-81). + + ⛔ THE GUARD HAS TO LIVE WHERE THE DROP HAPPENS, and finding that out cost a red check. + `tracked` is deleted from TWO places: `migrate_ig_tables`' profile loop, which asks + `_retired_tracked_field` whether the column is still the machine's own, and the schema + reconciliation, which drops by KEY with no question asked. Guarding only the first was + decorative — the schema pass runs immediately afterwards on the same table and deleted the + retyped column the guard had just spared. Two paths to one deletion with one of them checked + is the shape where a control looks present and enforces nothing + ([[defects-that-mask-each-other]]). + """ + drops = set(drops or ()) + if TRACKED_KEY in drops: + stored = [f for f in (table or {}).get("fields") or [] + if str(f.get("key") or "") == TRACKED_KEY] + if stored and _retired_tracked_field(stored) is None: + drops.discard(TRACKED_KEY) + return drops + + +def _orphan_tracked_rows(fields, rows): + """Row ids carrying a `tracked` CELL that no field declares — D-74's actual subject. + + ⛔ THE COLUMN AND THE CELLS WERE RETIRED SEPARATELY, WHICH IS WHY THIS IS NOT COVERED BY THE + BRANCH ABOVE. R6 deleted the field DEFINITION from `CANDIDATE_FIELDS`; a table whose + declaration was already dropped keeps every `tracked` key in its row dicts, invisible on every + surface that renders from `fields` — and a migration that only looks at `fields` finds nothing + to do and `continue`s past the table. Storage, export and any future re-declaration of the key + all still see them. + """ + if any(str(f.get("key") or "") == TRACKED_KEY for f in fields): + return [] # the column above owns its own cells + return [rid for rid, r in rows.items() if TRACKED_KEY in (r or {})] + + def migrate_ig_tables(rt, log=print, only=""): """Bring every Instagram Profile and canonical child table onto one current contract. @@ -4502,7 +3661,7 @@ def migrate_ig_tables(rt, log=print, only=""): # platform", these are "how many TABLES changed shape". "pinned": 0, "flagged": 0, "droppedName": 0, "droppedRecentPosts": 0, "droppedPostHashtags": 0, - "droppedTracked": 0, "schema": 0} + "droppedTracked": 0, "droppedTrackedCells": 0, "schema": 0} want = {f["key"]: f["type"] for f in CANDIDATE_FIELDS} # ⛔ `only` NARROWS THE SET, IT DOES NOT BYPASS THE TEST — and skipping that cost six red # checks the first time. `ut_ensure` calls this with the key it is ABOUT TO CREATE, so on a @@ -4547,9 +3706,13 @@ def migrate_ig_tables(rt, log=print, only=""): want_flag = (flag_f is not None and other_profile is None and not isinstance(flag_f.get("profile"), dict)) vestigial = _vestigial_name_field(fields, rows) - retired = {f.get("key") for f in fields} & {"posts", "post_hashtags", "tracked"} + retired = {f.get("key") for f in fields} & {"posts", "post_hashtags"} + if _retired_tracked_field(fields) is not None: + retired.add(TRACKED_KEY) + orphan_tracked = _orphan_tracked_rows(fields, rows) if (not stale_keys and has_platform and not needs_stamp - and not want_pin and not want_flag and vestigial is None and not retired): + and not want_pin and not want_flag and vestigial is None and not retired + and not orphan_tracked): continue stats["tables"] += 1 if want_pin: @@ -4573,7 +3736,13 @@ def migrate_ig_tables(rt, log=print, only=""): r.pop(key, None) stats["droppedRecentPosts"] += int("posts" in retired) stats["droppedPostHashtags"] += int("post_hashtags" in retired) - stats["droppedTracked"] += int("tracked" in retired) + stats["droppedTracked"] += int(TRACKED_KEY in retired) + for rid in orphan_tracked: + # D-74: a tick whose column was already gone. Counted apart from the column drop + # because they answer different questions — "how many tables still declared it" and + # "how many rows were still carrying it after the declaration went". + rows[rid].pop(TRACKED_KEY, None) + stats["droppedTrackedCells"] += 1 for key in stale_keys: conv = _MIGRATE_CONVERT.get(key) @@ -4768,6 +3937,55 @@ def seed_rows(rt, table_key, view_id=""): return rows[:SEED_MAX_ROWS], "" +def automation_tables(defn): + """Every `ut_*` table THIS automation writes into — its config target plus every + `create_record` action's table, deduped, order preserved. + + ⚠ THE SAME TWO AUTHORITIES `observed_category_tables` READS, narrowed to one definition. + Split out rather than parameterised because the two questions are different: that one asks + "where could a Category value be, anywhere in this tenant" and is deliberately generous; this + one asks "which rows has THIS automation already found" and must not reach into a table it + does not write, or a nightly search would start excluding another automation's finds. + """ + keys, seen = [], set() + cfg = (defn or {}).get("config") or {} + for raw in [cfg.get("targetTable"), + *(((a or {}).get("config") or {}).get("table") + for a in walk_actions(((defn or {}).get("flow") or {}).get("actions")))]: + k = str(raw or "").strip() + if k.startswith(UT_PREFIX) and k not in seen: + seen.add(k) + keys.append(k) + return keys + + +def already_found_handles(rt, defn, cap=None): + """The handles this automation has already written, for the vendor-side exclusion (item 7). + + ⭐⭐ THE POINT IS THE BILL, not the row count. Discovery upserts by handle, so re-finding a + profile has always been harmless — and never free: the vendor bills for every record it + returns, so a nightly search over stable keywords pays again, in full, for the same accounts + every night and reports them as `seen_again`. MEASURED shape of the waste: nurilab's beauty + scout re-found its whole result set on every run. This is the list that stops it, sent as one + `not_in` the vendor evaluates BEFORE billing. + + ⚠ SCOPED TO THIS AUTOMATION'S OWN TABLES. Excluding handles another automation found would + hide profiles this one has never seen — cheaper, and wrong: two scouts with different keywords + are two questions, and one must not answer with "somebody already looked at that". + """ + cap = BD_EXCLUDE_MAX if cap is None else int(cap) + out, seen = [], set() + for table_key in automation_tables(defn): + for row in ((ut_get(rt, table_key) or {}).get("rows") or {}).values(): + h = str((row or {}).get("handle") or "").strip().lstrip("@").lower() + if h and h not in seen: + seen.add(h) + out.append(h) + if len(out) >= cap: + return out + return out + + def observed_category_tables(rt): """Every table a Category value could have been written into, DERIVED from the automations this tenant actually has — the default discovery table and the snapshot table are the FLOOR, @@ -4994,195 +4212,6 @@ def discover_estimate(records_limit): "a price before a run, and this account's token cannot read a balance"} -#: ⛔ THE VENDOR'S HARD CAP ON ONE LOGICAL GROUP — MEASURED 2026-08-06 against the live filter API -#: with nurilab's own seven-condition search, which had failed every run for two days: -#: -#: 400 {"validation_errors":["Filter logical groups can have a maximum of 4 rules."]} -#: -#: ⚠ EVERY ONE OF THOSE SEVEN CONDITIONS IS ACCEPTED ALONE (all seven probed individually, all -#: 200). The refusal is about the SHAPE of the group and nothing else — so no field, operator or -#: value in the builder is at fault, and no amount of editing the conditions would have found it. -#: What it means without `bd_group` below: the FIFTH condition anybody adds turns a working -#: discovery automation into one that fails forever, and the surface blames the search. -BD_GROUP_MAX = 4 - - -def bd_group(nodes, operator): - """`nodes` → an equivalent list in which NO logical group exceeds `BD_GROUP_MAX` rules. - - ⭐ SAFE BECAUSE AND AND OR ARE ASSOCIATIVE. Folding a run of same-operator rules into a nested - group of the SAME operator cannot change what the filter matches — `a∧b∧c∧d∧e` and - `(a∧b∧c∧d)∧e` are one predicate written two ways. That is why this is a re-SHAPING of the - owner's search rather than a reinterpretation of it, and why it needs no ruling: a condition - list means the same thing before and after. Nesting itself is already proven on this API - (`expand_predicates`' or-groups, `snap_msh14ix81n8hlpo3nr`). - - ⛔ A ONE-RULE CHUNK IS NEVER WRAPPED. `expand_predicates` already carries that law for the - one-value case — a one-element group is a shape nothing has ever been billed against — and a - balanced split reaches it whenever a level divides with a remainder of one. - """ - nodes = list(nodes or []) - if len(nodes) <= BD_GROUP_MAX: - return nodes - # ⚠ BALANCED, NOT "take three and nest the rest". The tail-recursive shape costs one level of - # DEPTH per extra rule (13 conditions ⇒ 5 deep) against a vendor whose nesting limit we have - # not measured and would discover the same way we discovered this one: in production, on - # somebody's automation. Chunking makes the depth log₄(n) — 16 conditions fit in two levels. - out = [] - for i in range(0, len(nodes), BD_GROUP_MAX): - chunk = nodes[i:i + BD_GROUP_MAX] - out.append(chunk[0] if len(chunk) == 1 else {"operator": operator, "filters": chunk}) - return bd_group(out, operator) - - -#: ⛔ DEBT D-68 — THE VENDOR'S NESTING CEILING, MEASURED IN PRODUCTION 2026-08-06: -#: 400 — "filter" failed custom validation because logical operators cannot be more than -#: 3 levels deep -#: ⭐ AND IT IS NOT AN INDEPENDENT BUDGET FROM `BD_GROUP_MAX`. A >4-rule group is fixed by -#: NESTING, so the 4-rule cap is what MANUFACTURES depth: five top-level conditions carrying -#: multi-value lists goes `AND`(split) → `OR`(values) → `OR`(split) and is refused. The real -#: envelope is about 4 top-level conditions x <=4 values each, which is why -#: `MAX_PREDICATE_VALUES = 12` was never reachable at a realistic condition count. -BD_MAX_DEPTH = 3 - - -def filter_depth(node): - """How many logical levels a vendor `filters` structure nests to. A bare rule is depth 1.""" - if isinstance(node, list): - return max((filter_depth(n) for n in node), default=0) - if isinstance(node, dict) and isinstance(node.get("filters"), list): - return 1 + filter_depth(node["filters"]) - return 1 - - -def depth_refusal(predicates, operator="and"): - """⭐ D-68 — refuse a search whose SHAPE the vendor will reject, before it is sent. - - Returns a sentence or `""`. - - ⛔ WHY THIS CANNOT BE A RULE-COUNT CHECK. The failure is in the emitted STRUCTURE, not in the - conditions: the same five conditions pass or fail depending on how many VALUES each carries, - because every multi-value list becomes its own OR level and every group over `BD_GROUP_MAX` - adds a split level. So the only honest test is to BUILD the shape and measure it — which is - cheap, because we build it anyway. - ⚠ Free to get wrong at the vendor and expensive to diagnose there: a rejected filter is never - billed, but the 400 says nothing about which condition to drop, so a person edits keywords for - an afternoon while the shape stays identical. That is what this sentence exists to prevent. - """ - try: - depth = filter_depth(bd_group(expand_predicates(predicates), operator or "and")) - except Exception: # noqa: BLE001 - return "" # never block a save on this check's own bug - if depth <= BD_MAX_DEPTH: - return "" - multi = [p for p in (predicates or []) - if isinstance((p or {}).get("value"), (list, tuple)) and len(p["value"]) > 1] - hint = (f"drop one of the {len(multi)} multi-value conditions, or shorten its list" - if multi else "drop one condition") - return (f"this search nests {depth} levels deep and the provider allows " - f"{BD_MAX_DEPTH} — {hint}. Every extra condition beyond {BD_GROUP_MAX}, and every " - f"list of values, adds a level") - - -def expand_predicates(predicates): - """Stored predicates → the vendor's `filters` list, expanding a value LIST into a nested OR. - - ⭐ MEASURED 2026-08-06: the filter API ACCEPTS a nested `{operator, filters:[…]}` inside - `filters` (`snap_msh14ix81n8hlpo3nr`), which is what makes "Bio contains any of floral, - flower, beauty" expressible WITHOUT setting the whole search to "match any". The alternative - — the global Match dropdown — would drag every other condition into the same union, so - "…and at least 10,000 followers" would silently become "…or at least 10,000 followers". - - ⛔ A ONE-VALUE CONDITION IS STILL SENT FLAT. Wrapping it in a one-element group would be a - shape nothing has ever been billed against, adopted for tidiness, on the path that spends - money. Only the several-values case takes the new shape. - """ - out = [] - for p in predicates or []: - v = (p or {}).get("value") - if isinstance(v, list) and len(v) > 1: - # ⚠ THE OR-GROUP IS SUBJECT TO THE SAME 4-RULE CAP as the outer one, and this is the - # leg that hits it first in practice: "Bio contains any of floral, flower, beauty, - # bouquet, wedding" is five keywords in ONE condition — a single row in the builder, - # and a group the vendor refuses. `bd_group` is applied to the union too. - out.append({"operator": "or", - "filters": bd_group( - [{"name": p["name"], "operator": p["operator"], "value": one} - for one in v], "or")}) - elif isinstance(v, list): - out.append({**p, "value": v[0]} if v else - {k: val for k, val in p.items() if k != "value"}) - else: - out.append(dict(p)) - return out - - -def bd_filter_start(predicates, operator="and", records_limit=5, dataset_id=BD_DS_PROFILES): - """Start ONE corpus query. Returns `(snapshot_id, note)`. - - ⚠ ALWAYS THE COMPOUND `{operator, filters:[…]}` SHAPE, even for a single condition. A bare - `{name, operator, value}` is a plausible-looking second shape that has never been sent from - here, and a one-element `filters` list is a trivial extension of the three-element body that - HAS been measured end-to-end. One shape, and it is the tested one. - """ - body = {"dataset_id": dataset_id, "records_limit": int(records_limit), - # ⭐ `bd_group` — the top-level group obeys the vendor's 4-rule cap (see its note). - # Applied HERE rather than inside `expand_predicates` so the expansion stays a pure - # stored-shape → wire-shape translation and the rebalance is one readable step on the - # one path that actually posts a filter. - "filter": {"operator": operator, - "filters": bd_group(expand_predicates(predicates), operator)}} - payload, note = bd_call(BD_PATH_FILTER, body=body) - if note: - return "", note - sid = str((payload or {}).get("snapshot_id") or "") if isinstance(payload, dict) else "" - if not sid: - return "", "the search was accepted but came back with nothing to collect it from" - return sid, "" - - -def bd_filter_status(snapshot_id): - """`(status, size, note)` for a corpus snapshot. `status` ∈ building | ready | failed | ''.""" - payload, note = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snapshot_id}") - if note: - return "", 0, note - d = payload if isinstance(payload, dict) else {} - status = str(d.get("status") or "") - err = str(d.get("error") or "") - if status == "failed" and "did not match any records" in err.lower(): - # ⭐ MEASURED 2026-08-06 (`snap_msh14ix81n8hlpo3nr`): a filter that matches NOTHING comes - # back `failed` — "Provided filter did not match any records" — not `ready` with zero - # rows. Passed through, that reached the customer as "the search failed", which is the - # single most likely outcome of typing a keyword that is too specific and the one most - # certain to be read as "the product is broken". It is an ANSWER, so it gets its own - # status and no error note. - return "empty", 0, "" - if status == "failed" and err: - # NOT_ENOUGH_FUNDS is the one everybody will hit; say what it means, not just its name. - extra = (" — the matched set is too large for this account's balance; narrow the " - "conditions or lower the limit" if "FUNDS" in err.upper() else "") - return status, 0, f"the search failed: {err}{extra}" - return status, int(d.get("dataset_size") or 0), "" - - -def bd_filter_rows(snapshot_id): - """The rows of a READY corpus snapshot. `(rows, note)`. - - ⚠ `/download` answered *"Snapshot is building. Try again in a few minutes"* while the status - endpoint ALREADY said `ready` — delivery lags readiness by minutes. So a download that does - not parse as rows is reported as not-yet-collectable rather than as an error. - """ - payload, note = bd_call(f"{BD_PATH_FILTER_SNAPSHOT}/{snapshot_id}/download", - {"format": "json"}) - if note: - return [], note - rows = _bd_rows(payload) - if not rows: - return [], "the search is ready but its rows are not downloadable yet — the vendor's " \ - "delivery lags its own `ready` status by minutes" - return rows, "" - - def _candidate_row(row, stamp): """One corpus row → a `ut_ig_candidates` row. None when it has no handle. @@ -5220,159 +4249,6 @@ def _candidate_row(row, stamp): } -def pull_profile(url, max_posts=DEFAULT_POSTS_PER_PULL, log=print, tier="anonymous", - fallback=True, - post_metrics=False, comment_metrics=False, pending_metrics=None): - """Everything readable about one public profile, cheapest-honest rung last. - - Returns `{state, profile, posts, via, note}` with state ∈ ok | partial | blocked | error. - `partial` means the identity was read but the media was not — which is the outcome the - anonymous surface most often gives, and saying so is the point. - - **RUNG 0 IS THE PAID ONE (R1's hybrid).** With `tier="brightdata"` the vendor is tried FIRST - and, when it answers, this returns exact counts with no `approx` tag. When it does not answer - the attempt is RECORDED and the anonymous ladder runs underneath it — that is what - "fail-closed on the key, and the anonymous rung still works" means in C4. - - `fallback=False` turns that safety net off on purpose: it means *"exact numbers or nothing"*. - An influencer table where some rows are exact and some are `og:description` roundings, with - no way to tell them apart at a glance, is a table you cannot do arithmetic on — so a user who - needs the guarantee can have it, and gets an honest `blocked` instead of a quiet downgrade. - (The `source`/`approx` columns tell them apart anyway; this makes it impossible to forget.) - """ - handle = ig_handle(url) - if not handle: - return {"state": "error", "note": f"{url!r} is not an Instagram profile URL", - "profile": {}, "posts": [], "comments": [], "via": ""} - attempts = [] - - if clean_tier(tier) == "brightdata": - paid = pull_profile_bd(url, max_posts=max_posts, post_metrics=post_metrics, - comment_metrics=comment_metrics, log=log, - pending_metrics=pending_metrics) - if paid["state"] in ("ok", "partial"): - return paid - attempts.append(f"brightdata:{_s(paid.get('note'), 90)}") - if not fallback: - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "note": "the paid rung did not answer and the anonymous fallback is turned " - f"off for this automation ({paid.get('note') or 'blocked'})"} - time.sleep(PACE_SECONDS) - - # RUNG 1 — the web client's own profile endpoint. Public, no cookie; the one that used to - # carry both the profile AND the first page of media in a single answer. - api = f"https://www.instagram.com/api/v1/users/web_profile_info/?username={handle}" - status, body, note = _ig_get(api, headers={ - "X-IG-App-ID": IG_APP_ID, "Accept": "*/*", - "Referer": f"https://www.instagram.com/{handle}/"}) - attempts.append(f"web_profile_info:{status or note}") - if 200 <= status < 300 and body: - try: - node = ((json.loads(body.decode("utf-8", "replace")) or {}) - .get("data") or {}).get("user") or {} - except Exception: - node = {} - if node: - profile = _profile_from_web_api(node, handle) - media = node.get("edge_owner_to_timeline_media") or {} - posts = _posts_from_edges(media.get("edges"))[:max_posts] - page = media.get("page_info") or {} - if page.get("has_next_page") and page.get("end_cursor") and len(posts) < max_posts: - more, pnote = _paginate(node.get("id"), page.get("end_cursor"), - max_posts - len(posts), log) - posts.extend(more) - if pnote: - attempts.append(pnote) - state = "ok" if posts else "partial" - return {"state": state, "profile": profile, "posts": posts, - "via": "web_profile_info", - "note": "" if posts else "profile read; media edges empty (" - + ", ".join(attempts) + ")"} - - time.sleep(PACE_SECONDS) - - # RUNG 2 — the profile HTML. Its og:description carries the three counts even when every - # JSON surface refuses, so it is the honest floor: identity without media. - status, body, note = _ig_get(f"https://www.instagram.com/{handle}/") - attempts.append(f"html:{status or note}") - if 200 <= status < 300 and body: - text = body.decode("utf-8", "replace") - if "/accounts/login" in text[:4000] and "og:description" not in text: - return {"state": "blocked", "profile": {}, "posts": [], "via": "html", - "note": "Instagram served the login wall (" + ", ".join(attempts) + ")"} - desc = "" - m = re.search(r']+property="og:description"[^>]+content="([^"]*)"', text) - if m: - desc = m.group(1) - counts = _OG_COUNTS.search(desc or "") - title = "" - mt = re.search(r']+property="og:title"[^>]+content="([^"]*)"', text) - if mt: - title = mt.group(1) - if counts: - profile = { - "username": handle, "full_name": title.split("(")[0].strip(), - # ⚠ NO BIO. `og:description` on a profile page is - # "204K Followers, 5,245 Following, 750 Posts - See Instagram photos and videos - # from Inayma (@inayma)" — the tail is Instagram's own boilerplate, not the - # person's biography. Storing it would fill a Bio column with the same sentence - # for every influencer, which reads as data and is not. Measured 2026-08-04. - "bio": "", - "followers": _loose_count(counts.group(1)), - "following": _loose_count(counts.group(2)), - "posts_count": _loose_count(counts.group(3)), - "verified": "", - # The page abbreviates ("204K"), so the number is the page's rounding, not a - # count we read. Recorded on the row rather than left for someone to discover - # when 204,000 fails to reconcile with anything. - "approx": "1", - } - return {"state": "partial", "profile": profile, "posts": [], "via": "og:description", - "note": "counts are the page's own abbreviations; media is not anonymously " - "readable (" + ", ".join(attempts) + ")"} - if status in (401, 403, 429) or status == 0: - return {"state": "blocked", "profile": {}, "posts": [], "via": "", - "note": f"Instagram refused the anonymous read ({', '.join(attempts)})"} - return {"state": "error", "profile": {}, "posts": [], "via": "", - "note": f"nothing anonymously readable ({', '.join(attempts)})"} - - -#: The historical anonymous media query. Kept because trying it and RECORDING the refusal is -#: what makes "as deep as anonymously accessible" (R7) a measurement instead of an assumption. -IG_MEDIA_QUERY_HASH = "e769aa130647d2354c40ea6a439bfc08" - - -def _paginate(user_id, cursor, want, log=print): - """Walk further back through the media edges while the anonymous surface allows it.""" - out, note = [], "" - while user_id and cursor and len(out) < want: - time.sleep(PACE_SECONDS) - variables = json.dumps({"id": str(user_id), "first": min(50, want - len(out)), - "after": cursor}, separators=(",", ":")) - url = (f"https://www.instagram.com/graphql/query/?query_hash={IG_MEDIA_QUERY_HASH}" - f"&variables={requests.utils.quote(variables)}") - status, body, err = _ig_get(url, headers={"X-IG-App-ID": IG_APP_ID}) - if not (200 <= status < 300) or not body: - note = f"graphql:{status or err}" - break - try: - media = (((json.loads(body.decode("utf-8", "replace")) or {}).get("data") or {}) - .get("user") or {}).get("edge_owner_to_timeline_media") or {} - except Exception: - note = "graphql:unparseable" - break - edges = media.get("edges") or [] - if not edges: - note = "graphql:empty" - break - out.extend(_posts_from_edges(edges)) - page = media.get("page_info") or {} - if not page.get("has_next_page"): - break - cursor = page.get("end_cursor") - return out[:want], note - - # --------------------------------------------------------------------------------------------- # RUNNERS # --------------------------------------------------------------------------------------------- @@ -5639,6 +4515,57 @@ PRESET_WRITTEN_ELSEWHERE = ( ) +#: A location guess needs at least this many GEOTAGGED posts to agree with. One post is not a +#: pattern and would render as 100% confident, which is the fabrication `SEED_MIN_ROWS_SHARING` +#: refuses on the discovery side for exactly the same reason. +LOCATION_MIN_POSTS = 2 + + +def _post_place(post): + """The city a post was tagged in, or ''. Reads the normalised field FIRST, the paid payload + second. + + ⛔ THE SECOND READ IS THE POINT AND IT COSTS NOTHING. `_bd_tagged_location` flattens the + vendor's `location` array into "Capanema, Pará, Brasil" and DISCARDS the rest — including the + `lat`/`lng`/`name` object Bright Data returns on some rows and the whole `location_details` + shape. The complete vendor row is retained in `source_payload` on the same record, already + paid for, so a post whose normalised field came back blank can still be read here. + ⚠ THE FIRST COMPONENT IS THE CITY. The vendor's array is ordered outward — city, region, + country — so the head is the narrowest thing it told us, and "Jakarta" is the answer to + "where is this person"; "Indonesia" mostly is not. + """ + if not isinstance(post, dict): + return "" + raw = str(post.get("tagged_location") or "").strip() + if not raw: + try: + payload = json.loads(post.get("source_payload") or "{}") + except Exception: # noqa: BLE001 + payload = {} + raw = str(_bd_tagged_location(payload) or "") if isinstance(payload, dict) else "" + return raw.split(",")[0].strip() + + +def location_guess(posts): + """`(city, confidence_pct)` from the places a creator's own posts were tagged in. + + `confidence` is the share of GEOTAGGED posts that agree on the modal city — deliberately not + the share of ALL posts, because a creator who geotags three of twelve is telling us about + three, and dividing by twelve would report a real signal as weak. How thin the evidence is + stays visible a different way: `LOCATION_MIN_POSTS` refuses to answer at all below two. + """ + places = [p for p in (_post_place(x) for x in posts or []) if p] + if len(places) < LOCATION_MIN_POSTS: + return "", None + counts = {} + for p in places: + counts[p] = counts.get(p, 0) + 1 + # ⚠ TIE-BROKEN BY FIRST APPEARANCE, never by sort order: two cities at 3/3 would otherwise be + # resolved alphabetically, which is an arbitrary answer wearing a confident number. + best = max(places, key=lambda p: (counts[p], -places.index(p))) + return best, round(100.0 * counts[best] / len(places), 1) + + def preset_cells(res, pulled): """C4/R3: one pull → the LATEST-value cells written onto the enriched record. @@ -5662,6 +4589,13 @@ def preset_cells(res, pulled): # arrived with no platform at all — and a blank half of the dedup key is how one account # becomes two rows. cells["platform"] = PLATFORM_INSTAGRAM + # ⭐ ITEM 16 — the residency guess, off posts this pull already paid for. Written only when + # there is one: a blank is "we could not tell", and overwriting a good guess from a run whose + # twelve posts happened to carry no geotag would make the column worse the more it ran. + place, confidence = location_guess((res or {}).get("posts")) + if place: + cells["location_guess"] = _s(place, 120) + cells["location_confidence"] = str(confidence) # R3: `enriched_at` is a `date` column now. It answers "how stale is this number", which is a # question in days; the full stamp keeps its precision on the snapshot series. cells["enriched_at"] = _day(pulled) @@ -5975,7 +4909,19 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " f"${est['usd']}", counts, [], steps) step("Starting the search") - sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit) + # ⭐ ITEM 7 — DO NOT PAY FOR WHAT WE ALREADY HAVE. The exclusion is engine-added: it never + # appears as a condition row, and `bd_filter_start` drops it rather than risk a shape the + # vendor refuses. Reported either way, because "why did this run find fewer" and "why did + # this run cost the same as last night" are both questions this line answers. + excl = {} + sid, note = bd_filter_start(preds, cfg.get("operator") or "and", limit, + exclude_handles=already_found_handles(rt, defn), + applied=excl) + if excl.get("excluded"): + step(f"excluded {excl['excluded']} already-found " + f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") + elif excl.get("dropped"): + step(f"searching without the already-found list — {excl['dropped']}") if note: steps["find"] = "blocked" return ("error", f"the search was not started — {note}", counts, [], steps) @@ -6031,11 +4977,21 @@ def run_discover_instagram(rt, defn, username="automation", log=print, step=_no_ except Exception: # noqa: BLE001 mins = 0 been = f" — {mins} min so far" if mins else "" + # ⭐ WAVE 27 ITEM 14 (owner) — THE BRAG IS GONE. The sentence used to explain the wait by + # naming the corpus size: *"the wait is the SEARCH over 620 million profiles, not the 10 + # rows"*. It was TRUE, and it was answering a question the person had not asked. Somebody + # watching a run that has taken eighteen minutes wants to know whether it is broken and + # when it ends; how big the index is, is our fact about our supplier. Keeping it also + # meant this line was the one place a vendor's catalogue size was quoted to a customer, so + # it would have gone stale the day the vendor grew — a number nobody would think to + # re-measure because nothing depends on it. + # ⚠ THE SHAPE OF THE ANSWER SURVIVES and is the part that mattered: the wait does not + # shrink when you ask for fewer records, and a person who does not know that reads + # "20 minutes for 10 rows" as a fault. return ("partial", f"Still searching at the provider{been}. A corpus search takes about 20 minutes " - f"however few records you asked for: the wait is the SEARCH over 620 million " - f"profiles, not the 10 rows. Nothing is lost, you are not charged twice, and the " - f"results are collected automatically as soon as they are ready", + f"however few records you asked for. Nothing is lost, you are not charged twice, " + f"and the results are collected automatically as soon as they are ready", counts, [], steps) if status == "empty": # A search that matched nothing RAN CORRECTLY. It is the ordinary result of a keyword @@ -6605,32 +5561,31 @@ def toggle_node(rt, auto_id, node_id): # --------------------------------------------------------------------------------------------- -# THE BOARD (wave 22, contracts C1/C2 — owner ruling R1: RECORDS AS CARDS) +# THE BOARD IS DELETED (wave 27 item 12, owner ruling R3) — what is left, and why it stayed # --------------------------------------------------------------------------------------------- -# The automation detail IS a kanban board: columns = the flow's stages, cards = records of the -# target database, each record's position stored in a single-select STAGE FIELD auto-created on -# that database — so the same field drives a plain Kanban view with no board code at all. -# -# TWO KINDS OF STAGE, and conflating them is the mistake this section is built around: -# -# * RUN stages (trigger/fetch/find/collect/capture/write) are MACHINERY — a record passes -# through them but can never sit in one, the way a parcel is never "at" the conveyor belt. -# They come straight off `graph()` so the board and the runner cannot drift. -# * RECORD stages hold cards. They are the REVIEW gate (R6: discovery's review IS the -# review gate) and the R5 IF-routing lanes. Only these are choices -# of the stage field — a select offering "Fetch page" as a place to PUT a record would be -# offering a lie. +# Wave 22 made the automation detail a kanban board: columns were the flow's stages, cards were +# records of the target database, and each record's position lived in a single-select STAGE FIELD +# the engine created on the customer's own database. Wave 26 deleted the board's built-in +# terminals; wave 27 deletes the rest — `board()`, `move_card()`, `stages_for()`, +# `ensure_stage_field()`, the four card builders, the lane vocabulary on the wire, the review +# branch of the action walk, and the endings that sent a terminal card back round. # -# ⛔ WHO WRITES THE STAGE FIELD (C2): the ENGINE, everywhere — except a review stage, where a -# HUMAN may advance a card to a member of the review stage's `next` and nothing else. That wall -# is enforced twice, once per door: `move_card` below (the board's own POST) and -# `grid_events.overlay_patch` (the plain grid/kanban door), which reads the law OUT OF THE FIELD -# DEFINITION (`automation.humanMoves` / `automation.arrive`) because platform code must not -# import this module. The engine STATES the law; the grid ENFORCES it mechanically. +# ⛔ THE REASON IS A DATA REASON, NOT A UI ONE, and it is the sentence to keep: the board wrote +# MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` stamp and a `_cycles` +# counter per automation — to render a view of state the RUN LOG already holds. The customer paid +# for that in columns they did not ask for, in a store commit on every scheduled run, and in a +# permission wall (`humanMoves`/`arrive`) that existed only to stop them editing the columns we +# had added. `migrate_ig_tables` now drops those columns wherever an automation created them. # -# R7's approval-posture vocabulary rides every stage: `strict` = holds for a human, -# `auto` = the engine advances. Today review⇒strict and everything else⇒auto BY DERIVATION — -# the vocabulary is on the wire so a later wave can flip a stage without a client change. +# ⚠ WHAT SURVIVES, AND IT IS DELIBERATE IN EACH CASE: +# * `lane_match` + `LANE_OPS` + the condition trees below — never board-only. They are the +# evaluator for ACTION conditions, trigger conditions and `find_records`, and the board was +# one of four callers. The name is the last thing the board left behind here. +# * `retire_automation_stage_fields` / `retire_automation_board_state` / `_without_retired_board` +# — the MIGRATION. It must outlive the thing it retires, or a definition written before the +# retirement walks into a runtime that no longer understands it. +# * `ai_decide` + `review_audit` — R3 keeps review "as an AI decision without lanes"; they are +# parked with no caller and say so at their own definitions. LANE_OPS = ("=", "!=", ">", ">=", "<", "<=", "includes", "not_includes", "is_empty", "is_not_empty") @@ -6838,8 +5793,10 @@ ACTION_CATALOG = [ # `field_instagram` was an automation you created to fill one column; this is a step any flow # can take, which is the shape it should always have had — enriching a profile is something # you do TO a record, not a species of automation. + # ⭐ WAVE 27 · C4 (item 33) — `connector` NESTS THIS ROW UNDER "Scraper" in the action menu, + # exactly as `TRIGGER_CONNECTOR` already nests the trigger picker. See `ACTION_CONNECTOR`. {"kind": "enrich_instagram", "label": "Enrich Instagram profile", "group": "Connected", - "ready": True, + "ready": True, "connector": "scraper", "detail": "Fill this record's Instagram columns from its profile, and add a point to its " "history"}, {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, @@ -6861,6 +5818,29 @@ ACTION_CATALOG = [ ] +#: ⭐ WAVE 27 · C4 (owner item 33) — WHICH CONNECTOR AN ACTION BELONGS TO, so the action menu can +#: nest exactly as the trigger picker already does (`reference/Airtable Automation 10.png`: an +#: "Integrations" header, one row per connector, a chevron into that connector's own submenu). +#: +#: ⚠ SAME SHAPE AS `TRIGGER_CONNECTOR`, DELIBERATELY, and the same warning applies: these are +#: GROUPING HANDLES for the picker, not `/connectors/directory` slugs. A client must group by the +#: key and render the label, never join it against the directory. +#: ⛔ AND THE LABEL IS NOT DECLARED HERE AT ALL — it is LOOKED UP in `TRIGGER_CONNECTOR` by key. +#: A "Scraper" nest in the trigger menu and a "Scrapers" nest in the action menu would be one +#: connector wearing two names on two screens somebody sees within a second of each other, and a +#: second literal is how that happens. This tuple says only WHICH connectors an action may name; +#: what they are CALLED has exactly one source. +ACTION_CONNECTORS = ("scraper",) + + +def _connector_meta(key): + """`{key, label}` for a connector key, from the one place either picker declares it.""" + key = str(key or "") + if key not in ACTION_CONNECTORS: + return None + return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) + + def action_catalog(): """The catalog as the wire carries it — a copy, because a caller that mutated the module constant would change every later reader's answer. @@ -6870,10 +5850,23 @@ def action_catalog(): different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST (not first) — a new group appearing above "Web action" because its order defaulted to 0 is the failure that would look deliberate. + + ⭐ WAVE 27 (C4): a row naming a `connector` is stamped with the full `{key, label}` the client + nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two + rows in one nest cannot disagree about what that nest is called. A row naming an unknown + connector loses the stamp instead of inventing a nest with a raw slug for a title. """ last = max(ACTION_GROUP_ORDER.values()) + 1 - return [{**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)} - for a in ACTION_CATALOG] + out = [] + for a in ACTION_CATALOG: + row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)} + conn = _connector_meta(a.get("connector")) + if conn: + row["connector"] = dict(conn) + else: + row.pop("connector", None) + out.append(row) + return out def _action_label(act): @@ -7178,56 +6171,6 @@ def walk_actions(actions): yield kid -def board_actions(defn): - """⭐ OWNER ITEM 1 (2026-08-06) — the actions a card WALKS, in flow order. - - Owner, verbatim: *"the stages should always be based on the Automation Actions path, so we - track every card as they go along the automation."* Before this, only `review` actions became - stages (wave 23 C4/C6) and everything else was invisible on the board — so a discovery flow's - kanban showed three columns (`Review`/`Tracked`/`Declined`) that corresponded to no step - anybody had added, which is exactly what prompted *"What exactly are those stages?"*. - - ⛔ THE PINNED DISCOVERY STEP IS EXCLUDED, and it must be: `apply_actions` DROPS `actions[0]` - for `discover_instagram` because the runner writes the candidates itself. A column for a step - the engine skips could never hold a card — a lane that is permanently empty for a structural - reason, which is the "green dot over nothing" defect wearing a different hat. - - ⚠ DISABLED ACTIONS ARE EXCLUDED for the same reason and no other: `_walk` skips them, so a - card cannot rest there. Switching one back on brings its column back. - """ - flow = (defn or {}).get("flow") or {} - acts = list(flow.get("actions") or []) - if ig_action_pinned(defn, 0) and acts: - acts = acts[1:] - out = [] - for a in walk_actions(acts): - if not isinstance(a, dict) or not a.get("enabled", True): - continue - # A GROUP is a fork, not a place — the card is inside one of its branches, never "at" it. - if a.get("kind") == "group": - continue - out.append(a) - return out - - -def action_stage_label(act, taken): - """One action's stage label: what the BUILDER calls that step, made unique. - - ⚠ UNIQUE MATTERS MORE THAN PRETTY. `_card_stage` resolves a stored cell back to a stage by - LABEL, and `_stage_choices` feeds those labels to a `select` column — so two steps sharing a - name would produce a duplicated option and a card that could be at either of them. Two - `Create record` steps become `Create record` and `Create record (step 2)`. - """ - cfg = act.get("config") or {} - base = _s(cfg.get("label") or _ACTION_LABELS.get(act.get("kind")) or act.get("kind"), 40) - if base not in taken: - return base - n = 2 - while f"{base} (step {n})" in taken: - n += 1 - return f"{base} (step {n})" - - def interpolate(text, row): """`{{field_key}}` → the record's value. Unknown keys resolve to EMPTY, deliberately: an action that wrote the literal `{{status}}` into a cell because somebody typo'd the key would @@ -7240,183 +6183,6 @@ def interpolate(text, row): return re.sub(r"\{\{([^}]{1,80})\}\}", _sub, text) -def stages_for(defn): - """One automation → `(stages, routes)` — C1's vocabulary, derived from `graph()` so the - board can never disagree with the runner about what the flow does. - - Stage extras beyond the graph node: `posture` (R7), `record` (may this stage HOLD cards), - `assignable` (may a card be PLACED here — i.e. is its label a choice of the stage field). - `record` without `assignable` marks the derived entry buckets (Profile set / Write): a card - shows there because of what is true about the row, not because anything stamped it. - """ - defn = defn if isinstance(defn, dict) else {} - cfg = defn.get("config") or {} - g = graph(defn) - kind_map = {"trigger": "trigger", "branch": "branch"} - stages, routes = [], [] - for n in g["nodes"]: - stages.append({ - "id": n["id"], "label": n["title"], - "kind": kind_map.get(n["kind"], "action"), - "col": n["col"], "row": n["row"], "enabled": n["enabled"], - "toggle": n["toggle"], "status": n["status"], "detail": n["detail"], - # C1 amendment (2026-08-05): the node's `panel` rides VERBATIM — B joins stage → - # config face on this key, never by guessing that ids match. - "panel": n.get("panel") or n["id"], - "posture": "auto", "record": False, "assignable": False, - }) - for e in g["edges"]: - routes.append({"from": e["from"], "to": e["to"], "label": e.get("label") or "", - "condition": None}) - maxcol = max((s["col"] for s in stages), default=0) - - def _mark(sid, record=True, assignable=False): - for s in stages: - if s["id"] == sid: - s["record"], s["assignable"] = record, assignable - - if defn.get("kind") == "discover_instagram": - # The review gate lives at the END of discovery: a found candidate waits for a person. - # ⚠ Nothing here auto-promotes, which was true before wave 26 and is now true by - # construction rather than by discipline — there is no terminal to promote INTO. - stages.append({"id": "review", "label": "Review", "kind": "review", - "col": maxcol + 1, "row": 0, "enabled": True, "toggle": "", - "status": "idle", "panel": "review", "posture": "strict", - "record": True, "assignable": True, - "detail": "Cards wait here for a person to decide where they go next"}) - # ⛔ WAVE 26 · R6 — THE BUILT-IN `Tracked` / `Declined` TERMINALS ARE DELETED, and this - # is a deliberate reduction in what the product asserts. Owner: *"drop the review board's - # terminals entirely"* — the automation stops having an opinion about what happens to a - # candidate after it is found. Discovery still lands every new card in Review; where it - # goes NEXT is the tenant's own vocabulary, added on the board itself (R8's option editor - # ships this same wave, which is what makes this a change of ownership rather than a loss - # of capability). - routes.append({"from": "write", "to": "review", "label": "", "condition": None}) - else: - # ⭐ WAVE 24 — a `plain` automation's derived entry bucket is the TRIGGER, because it has - # no machine node to be one. Without this arm `_card_stage`'s fallback hands every card - # the id `write`, which does not exist on a plain board: the cards would count against a - # stage that is in `counts` and absent from `stages`, i.e. records the board says exist - # and cannot show. The two functions have to agree, so they are edited together. - entry = {"field_instagram": "source", "plain": "trigger"}.get(defn.get("kind"), "write") - _mark(entry) - _mark("write") - for i, lane in enumerate(cfg.get("lanes") or []): - cond = lane.get("when") - stages.append({"id": lane["id"], "label": lane["label"], "kind": "branch", - "col": maxcol + 1, "row": i, "enabled": True, "toggle": "", - "status": "idle", "panel": "lanes", "posture": "auto", - "record": True, "assignable": True, - "detail": _lane_sentence(cond)}) - routes.append({"from": "write", "to": lane["id"], - "label": lane["label"] if cond is None else _lane_sentence(cond), - "condition": dict(cond) if cond else None}) - - # ── WAVE 23 · C4/C6 — REVIEW ACTIONS ARE STAGES. ───────────────────────────────────────── - # A review action is the one action kind that HOLDS a record rather than doing something to - # it, so it has to exist on the board: a flow whose review lived only in the builder would - # gate records at a place the kanban cannot show, which is the whole complaint the board was - # built to answer. Its declared exits become assignable stages too — a gate with nowhere to - # go is a hole records fall into. - # ⭐ OWNER ITEM 1 (2026-08-06) — EVERY ACTION IS A STAGE, not only the review ones. - # - # This block used to iterate `review_actions(defn)`. That was wave 23's C4/C6 answer to "a - # review has to exist on the board", and it was right as far as it went — but it left every - # other step invisible, so the board could not answer *where did this card get to*. The - # owner's ask is the general form of the same argument: the columns ARE the actions path. - # - # ⚠ WHAT MAKES THIS HONEST RATHER THAN DECORATIVE is the other half, in `apply_actions`: - # a record is STAMPED with the furthest action it reached. Columns nothing ever writes into - # would be a picture of a flow rather than a report on it, and the two are told apart only by - # whether cards ever appear in them. - tail = max((s["col"] for s in stages), default=0) - col, taken = tail + 1, set() - for act in board_actions(defn): - acfg = act.get("config") or {} - rid = act["id"] - label = action_stage_label(act, taken) - taken.add(label) - is_review = act.get("kind") == "review" - stages.append({"id": rid, "label": label, - "kind": "review" if is_review else "action", - "col": col, "row": 0, "enabled": True, "toggle": "", - "status": "idle", - "panel": "review_action" if is_review else "flow_action", - # POSTURE IS THE AUDIT RULE, so only a review claims `strict` — an - # ordinary step is somewhere a card passes through, not a gate somebody - # signs off, and marking every column strict would ask for an audited - # click on movements nobody decided. - "posture": "strict" if is_review else "auto", - "record": True, "assignable": True, "actionId": rid, - **({"decidedBy": acfg.get("decidedBy") or "user"} if is_review else {}), - "detail": ( - ("An AI decides where each card goes — a person can override it" - if acfg.get("decidedBy") == "ai" - else "Cards wait here for a person to move them on") - if is_review else - f"Cards that reached this step and went no further this run")}) - col += 1 - if not is_review: - continue - for j, nxt in enumerate(acfg.get("next") or []): - oid = f"{rid}_out_{j}" - olabel = nxt if nxt not in taken else f"{nxt} (step {j + 2})" - taken.add(olabel) - stages.append({"id": oid, "label": olabel, "kind": "action", - "col": col, "row": j, "enabled": True, "toggle": "", - "status": "idle", "panel": "review_action", "posture": "auto", - "record": True, "assignable": True, "actionId": rid, - "detail": f"Where {label} sends a card"}) - routes.append({"from": rid, "to": oid, "label": nxt, "condition": None}) - if acfg.get("next"): - col += 1 - # ⛔ NO AUTO-CHAINING ROUTES BETWEEN CONSECUTIVE STEPS, and the first version of this block had - # them. Two things went wrong and both were invisible on nurilab's own automation (its flow has - # only the pinned step, so no action stage exists to chain): - # - # 1. A ROUTE IS AN OFFER THE WRITE DOOR REFUSES. `_card_next` turns outgoing routes into the - # board's "move this card on" buttons, but `ensure_stage_field` builds `humanMoves` from - # REVIEW stages only — so every chained button would have been refused by `grid_events`. - # A control that is drawn and cannot work is worse than one that is absent. - # 2. IT MADE THE ASYMMETRY WORSE, not better: zipping a flat stage list pairs ADJACENT - # entries, so of a review's two exits only the SECOND got an onward route and the first - # silently became terminal. Two exits of one gate with different endings is not a thing - # anybody chose. - # - # The path still reads left-to-right: `col` is assigned in flow order above, and the board - # lays columns out by `col`. Order is what makes it a path; a drawn arrow was never the point. - return stages, routes - - -def terminal_stage_ids(stages, routes): - """Stages a record can REST at with nowhere further to go — the ends C5's `ending` acts on. - Derived from the routes rather than declared, so a flow cannot gain an exit without this - noticing (and a lane that stops being terminal stops being reset). - - ⛔ AN ORDINARY STEP IS PASSED THROUGH, NEVER RESTED AT (2026-08-06, owner item 1). Making every - action a board stage put a card at the LAST step of a completed run — and every one of those - stages has no outgoing route, so `auto_reset` read "reached the end" and cleared the stamp - while incrementing the cycle counter. MEASURED before this line existed: three runs of a - two-action `auto_reset` flow left `stage=''` and `cycles` at 1, 2, 3 — a card that never rests - anywhere and a counter climbing forever, silently, on every scheduled run. - "Terminal" means *the flow is done with this record until somebody moves it*; a step the run - walked straight through is the opposite of that. Only stages a card genuinely WAITS at — review - gates, their declared exits, and lanes — can end a flow. Nurilab's own automation could not - have shown this: its flow has only the pinned step, so it has no action stage at all. - """ - assignable = {s["id"] for s in stages - if s.get("assignable") and s.get("panel") != "flow_action"} - onward = {r["from"] for r in routes if r["to"] in assignable and r["to"] != r["from"]} - return [sid for sid in assignable if sid not in onward] - - -def _stage_of(stages, sid): - for s in stages: - if s["id"] == sid: - return s - return None - - # ───────────────────────────────────────────────────────────────────────────────────────────── # WAVE 23 · C4/C5/C6 — THE ACTION RUNNER. One record at a time, actions in declared order. # @@ -7796,25 +6562,22 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print rows = dict((ut_get(rt, table) or {}).get("rows") or {}) if not rows: return counts - # Board-stage tracking was deliberately removed. The flow still runs its actions in order; - # the run log is now the single operational record of what happened. + # ⛔ NOTHING HERE TRACKS WHERE A RECORD "GOT TO" ANY MORE (wave 27 item 12, ruling R3). The + # walk used to keep a `reached` map and stamp each record with the furthest action it made — + # a board column's worth of bookkeeping written into the tenant's own table on every run. + # The board is deleted, so the stamp had no reader, and a machine column nobody reads is a + # cell the customer has to look at and a store commit we pay for on every scheduled run. + # The RUN LOG is the operational record of what happened now, and it is the only one. patches, creates = {}, {} - # ⭐ OWNER ITEM 1 (2026-08-06) — WHERE EACH CARD GOT TO. `stages_for` now makes every action a - # board column; this is what puts cards in them. Keyed by row id, holding the LABEL of the - # furthest action that record reached this run, flushed once per record after the walk. - # - # ⛔ ONE STAMP PER RECORD, NOT ONE PER ACTION. Writing at each step would put N patches on the - # same cell in a single run and land the same final value, at N times the cost — and on a - # flood-limited walk over thousands of rows that is the difference between a run and a - # timeout. The last value is the only one that was ever going to survive. - # - # ⚠ A REVIEW STAMPS ITSELF AND RETURNS (see its branch), so it must WIN over this: it is a - # HOLD, and overwriting it with the step before would release a card nobody decided on. - reached = {} - def _walk(acts, row, rid, depth=0): - """Returns True when the record was SUSPENDED (a review holds it).""" + """Walk the actions in order for ONE record. + + ⚠ THE RETURN VALUE IS VESTIGIAL and is deliberately kept as `False`. It used to mean + "this record was SUSPENDED by a review gate" — the one branch that could stop a walk + early. With review retired nothing suspends anything, so every record walks its whole + flow; the signature stays so a future gate-style action has an obvious place to say so. + """ for act in acts: if not act.get("enabled", True): continue @@ -7913,68 +6676,27 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print found = find_records(rt, cfg.get("table"), cfg.get("cond"), int(cfg.get("limit") or 25)) counts["found"] += len(found) - elif kind == "review": - label = cfg.get("label") or "Review" - at = str(row.get(skey) or "").strip() - if at == label: - return True # already waiting here — do not re-stamp - if at and at in (cfg.get("next") or []): - # ⛔ ALREADY DECIDED — the card went through this gate on an earlier run, so - # the flow continues past it. Without this the next run would stamp a - # finished card back to the review gate and hold it there again, forever: - # a card could never leave a flow that runs on a schedule. Found by this - # wave's own terminal-ending negative control, which is the argument for - # writing the NC in the same change as the feature. - # - # ⛔ AND THE CARD IS AT ITS OWN EXIT, NOT AT THIS GATE (owner item 1, - # 2026-08-06). The end-of-walk stamp records the furthest ACTION a record - # reached, and the action here is the REVIEW — so without this line a card - # resting in `Done` is stamped back to `Gate` on the next scheduled run, and - # walks backwards through the flow forever. The very same negative control - # caught it, which is the second time this branch has been paid for by it. - reached[rid] = at - continue - decided = "" - if cfg.get("decidedBy") == "ai" and ai_budget[0] > 0: - ai_budget[0] -= 1 - decided = ai_decide(rt, defn, act, row, rid, log=log) - if decided: - counts["aiDecided"] += 1 - _act_row_patch(patches, table, rid, - {skey: decided, skey + "_at": stamp}) - row[skey] = decided - # ⛔ THE DECISION IS THE LANDING. Without this the end-of-walk stamp would - # overwrite the stage the AI just chose with the REVIEW's own label — - # silently un-deciding it, on the one path where a machine had already - # committed to an answer. - reached[rid] = decided - continue # the AI moved it on — keep walking - _act_row_patch(patches, table, rid, {skey: label, skey + "_at": stamp}) - row[skey] = label - counts["review"] += 1 - return True + # ⛔ THE `review` BRANCH IS DELETED (wave 27 item 12, owner ruling R3), and it had + # already stopped being reachable one wave earlier — which is the part worth reading. + # `_without_retired_board` strips every `review` action out of a definition on the + # READ path, so `all_definitions` and this function have not seen one since the board + # was retired. What was left behind was a branch referencing `skey`, `stamp` and + # `ai_budget` — three names with NO DEFINITION anywhere in this module. It was not + # dead code that merely wasted space: it was a `NameError` held back by a migration + # rather than by a guard, and any change that let one stored `review` action through + # would have crashed the whole action walk for every record in that flow. + # ⚠ `ai_decide` and `review_audit` SURVIVE as library code with no caller — R3 keeps + # review "as an AI decision without lanes", and the cheap-first provider ladder behind + # it is real, working, measured work. They are PARKED, deliberately and in writing, + # not orphaned; see their own notes. return False - held = [] if actions: for rid in list(row_ids or [])[:FLOOD_LIMIT]: row = dict(rows.get(str(rid)) or {}) if not row: continue - if _walk(actions, row, str(rid)): - held.append(str(rid)) - continue - # ⭐ OWNER ITEM 1 — the card lands on the furthest step it reached. Skipped for a HELD - # record (the `continue` above): its review already stamped its own label and holding - # it is the point, so writing the step before would quietly release it. - # - # ⚠ ONLY WHEN IT MOVED. Re-stamping the label a row already carries would mark the - # cell dirty on every scheduled run — the store commits it, the grid repaints it, and - # `stage_moved` tells every open browser to re-read — for a value that did not change. - landed = reached.get(str(rid)) - if landed and str(row.get(skey) or "").strip() != landed: - _act_row_patch(patches, table, str(rid), - {skey: landed, skey + "_at": stamp}) + _walk(actions, row, str(rid)) counts.update(_enrich_flush(rt, defn, username, enrich, log)) # ⭐ C5: the REALIZED numbers overwrite the walk's attempt count. `counts["created"]` was # incremented once per create the flow decided to make; what landed is what the store says, @@ -8050,6 +6772,16 @@ def _enrich_flush(rt, defn, username, acc, log): the write-through to the platform master. Forking any of those would give the enrich action a history that the metric fields (which read `ut_ig_snapshots`) could not see. """ + if "unbound" in (acc.get("notes") or []): + # ⛔ DEBT D-79, SECOND HALF — THE SILENT FAILURE, AND IT WAS THE WORSE HALF. An enrich step + # on a database with no profile column `continue`s BEFORE `enrich["profiles"] += 1`, so + # this function used to return `{}`, `run_now` merged only non-zero keys, and the run + # committed **`ok` — "N records walked"** having written nothing at all. The sentence + # naming the fix went to `log()`, i.e. the server console, which no customer reads. That + # is "green over nothing" on the one path where somebody is waiting for data. + # ⚠ IT IS A COUNT, not a flag, so `run_now`'s existing non-zero merge carries it without a + # special case — and so the run entry itself records that this happened. + return {"enrichUnbound": 1} if not acc.get("profiles"): return {} out = {"enriched": acc["ok"], "enrichBlocked": acc["blocked"]} @@ -8301,6 +7033,15 @@ def ai_decide(rt, defn, act, row, row_id="", log=print): answer, or a label the review does not offer all return "" — and "" means the card sits at the review gate exactly as it would with no AI at all. The feature can be broken, absent or wrong and the worst outcome is a human doing the work. + + ⚠⚠ PARKED SINCE WAVE 27 — THIS FUNCTION HAS NO CALLER, AND THAT IS RECORDED RATHER THAN + ACCIDENTAL. Its one caller was the `review` branch of the action walk, deleted with the board + under R3, which keeps review "as an AI decision without lanes". What is parked is genuinely + worth parking: the cheap-first provider ladder in `ai_review` (groq → cerebras → openrouter → + anthropic), the fail-closed posture above, and the audit shape `review_audit` writes. What is + MISSING is only the door — an action kind that asks a question and takes an answer, without a + stage column to write it into. **Do not delete this in a dead-code sweep without reading that + sentence first**; equally, do not treat it as shipped — nothing reaches it today. """ cfg = act.get("config") or {} options = list(cfg.get("next") or []) @@ -8328,29 +7069,6 @@ def ai_decide(rt, defn, act, row, row_id="", log=print): return choice -def notify_review(rt, defn, row_ids, log=print): - """C6: one notification per RUN naming the count — never one per record. - - A discovery pass that gates 300 candidates would otherwise put 300 rows in somebody's inbox - and the inbox would stop being read, which costs more than the notification gains. The board - is where the individual cards live; the bell says "there is work", once. - """ - if not row_ids: - return - owner = str((defn or {}).get("createdBy") or "").strip() - if not owner: - return - n = len(row_ids) - try: - import core.alerts as alerts - alerts.notify(owner, - f"{defn.get('name') or 'An automation'} — {n} " - f"record{'' if n == 1 else 's'} waiting for review", - topic="automation", key=str(defn.get("id") or ""), st=rt) - except Exception as e: # noqa: BLE001 - log(f"[aios-auto] review notification failed: {type(e).__name__}: {e}") - - def find_records(rt, table_key, cond, limit=25): """The `find_records` action's read: matching row ids, bounded and DISCLOSED (the caller puts the count in the run log, where it drills — [[no-unverifiable-aggregates]]).""" @@ -8364,42 +7082,6 @@ def find_records(rt, table_key, cond, limit=25): return out -def _apply_ending(rt, defn, table, rows, patches, stages, routes, stamp): - """C5: send terminal cards back round, per the ending mode. Returns cycles started. - - A RESET CLEARS THE STAMP rather than writing a start stage, so the card falls back to the - DERIVED bucket `_card_stage` would give a record that had never been stamped at all — which - is what "back to the first cycle" means, and it needs no opinion about which stage is first. - """ - ending = ((defn or {}).get("flow") or {}).get("ending") or {} - mode = ending.get("mode") or "terminal" - if mode in ("terminal", "manual_reset"): - return 0 - terminals = set(terminal_stage_ids(stages, routes)) - if not terminals: - return 0 - labels = {s["label"] for s in stages if s["id"] in terminals} - skey = stage_field_key(defn.get("id")) - cutoff = None - if mode == "timed_reset": - cutoff = _now() - _dt.timedelta(hours=int(ending.get("hours") or 24)) - started = 0 - for rid, row in rows.items(): - merged = {**(row or {}), **((patches.get(table) or {}).get(str(rid)) or {})} - if str(merged.get(skey) or "").strip() not in labels: - continue - if cutoff is not None: - at = _parse_iso(str(merged.get(skey + "_at") or "")) - if at is None or at > cutoff: - continue - n = (_ig_int(merged.get(cycles_field_key(defn.get("id")))) or 0) + 1 - _act_row_patch(patches, table, rid, - {skey: "", skey + "_at": stamp, - cycles_field_key(defn.get("id")): n}) - started += 1 - return started - - def _commit_action_writes(rt, table, patches, creates, username, log): """The ONE write. Row patches merge into the target's rows; creates land in their own tables — APPENDED, or UPSERTED on the action's `uniqueOn` key (C5 / owner ruling R1a). @@ -8490,12 +7172,16 @@ def _commit_action_writes(rt, table, patches, creates, username, log): def _lane_sentence(cond, top=True): - """One condition tree → the sentence the board and the field definition carry. + """One condition tree → the sentence a step's `detail` carries on the canvas. C4: a GROUP renders as its children joined by "and"/"or" and parenthesised when nested, so a - lane label never claims a flat comparison the tree does not make. The server composes it for - the same reason it composes every other stage `detail` — a client paraphrase of a structure - the engine evaluates is a second implementation of the same sentence, free to drift from it. + label never claims a flat comparison the tree does not make. The SERVER composes it, for the + same reason it composes every other `detail` — a client paraphrase of a structure the engine + evaluates is a second implementation of the same sentence, free to drift from it. + + ⚠ The name is board-era ("lane") and the board is gone; the caller is `graph()`, which is + live. Renamed nothing on purpose: this string is compared in a gate and read in a log, and a + rename would be churn on a working function to fix a word. """ if cond is None: return "Everything else" if top else "" @@ -8512,340 +7198,10 @@ def _lane_sentence(cond, top=True): return f"{cond.get('field')} {cond.get('op')}" + ("" if v is None else f" {v}") -def _stage_choices(stages): - """The stage field's select vocabulary: the ASSIGNABLE stage labels, in board order.""" - return [s["label"] for s in sorted(stages, key=lambda s: (s["col"], s["row"])) - if s.get("assignable")] - - -def ensure_stage_field(rt, table_key, defn, username="automation"): - """C2: the stage field on the target database — created/refreshed idempotently, skipped - entirely when the flow has nothing a record can be PLACED in (a select with an empty - vocabulary is a column that can hold no value; it appears when the first lane or review - stage does). - - The field carries the WHOLE law the grid door needs (`automation.humanMoves` — from-label → - admissible to-labels — and `automation.arrive` — to-label → extra cell writes, which is how - a drag can carry extra cell writes), because `grid_events` may not import - this module: the engine states the law where the enforcer can read it. - """ - key = str(table_key or "") - if not key or ut_get(rt, key) is None: - return "" - stages, _routes = stages_for(defn) - choices = _stage_choices(stages) - if not choices: - return "" - fkey = stage_field_key(defn.get("id")) - # ⭐ WAVE 26 · R6 + R8 — THE USER'S OWN OPTIONS ARE PART OF THE VOCABULARY, AND OF THE LAW. - # - # ⛔ WITHOUT THIS, R6 AND R8 DO NOT COMPOSE, and the result would be a board nobody can use. - # R6 deleted the built-in Tracked/Declined terminals, so a discovery review stage now has NO - # routed exits — which made `humanMoves` empty, which makes `grid_events` refuse EVERY drag. - # Meanwhile R8 (this same wave) gives people an "add an option" control on that very column. - # So a user would add "Contacted", drag a card to it, and the door would refuse a lane the - # product had just invited them to create. - # ⚠ The options are also PRESERVED rather than recomputed: this function rewrites the field - # definition on every save and run, so deriving `options` from the stages alone would silently - # delete every lane a person added — and take the cards sitting in them with it. - existing = next((f for f in ((ut_get(rt, key) or {}).get("fields") or []) - if f.get("key") == fkey), None) - user_added = [o for o in ((existing or {}).get("options") or []) - if o not in choices and str(o or "").strip()] - choices = choices + user_added - - review_next = {} - for s in stages: - if s["kind"] != "review": - continue - nxt = [r["to"] for r in _routes if r["from"] == s["id"]] - labels = [t["label"] for t in stages if t["id"] in nxt and t.get("assignable")] - # Anything on the column that is not this stage itself is a legal destination. A routed - # exit still ranks first so a flow that DOES declare its own terminals keeps its order. - labels += [o for o in choices if o not in labels and o != s["label"]] - if labels: - review_next[s["label"]] = labels - auto = {"flowId": str(defn.get("id") or ""), "stageField": True, - "humanMoves": review_next} - # ⛔ WAVE 26 · R6 — THE `arrive` MAP IS GONE WITH THE COLUMN IT MAINTAINED. It existed to keep - # `tracked` in step with a card's lane in both directions; with one progress field there is - # nothing to keep in step. ⚠ `grid_events` still reads `arrive` generically, so a future flow - # can reintroduce arrival side-effects — what is deleted is the built-in Instagram one, not - # the mechanism. - field = {"key": fkey, "label": f"Stage — {defn.get('name') or defn.get('id')}"[:80], - "type": "select", "source": "overlay", "options": choices, "automation": auto} - # ⭐ WAVE 23 (C5, D-35's minimum) — THE CYCLE COLUMN, created beside the stage field when - # the flow can loop. The owner asked to SEE how many times a record has been round, and - # `_apply_ending` was writing the cell without anything ever declaring the column: the value - # would sit in the row, correct and invisible, because a ut grid renders the DEFINITION's - # fields. A machine-written cell with no field definition is the shape of a number nobody - # can find. Tagged `automation` so it greys out and refuses human writes like every other - # machine column (item 4 / R9). - cyc_key = cycles_field_key(defn.get("id")) - loops = (((defn.get("flow") or {}).get("ending") or {}).get("mode") or "terminal") != \ - "terminal" - cyc_field = {"key": cyc_key, "label": "Cycles"[:80], "type": "int", "source": "overlay", - "automation": {"flowId": str(defn.get("id") or ""), "cyclesField": True}} - - have, have_cyc = None, False - for f in (ut_get(rt, key) or {}).get("fields") or []: - if f.get("key") == fkey: - have = f - if f.get("key") == cyc_key: - have_cyc = True - if have is not None and have.get("options") == choices and \ - (have.get("automation") or {}) == auto and have_cyc == loops: - return fkey # nothing would change — save the commit - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - t = cur.get(key) - if t is None: - return cur - for f in t.setdefault("fields", []): - if f.get("key") == fkey: - f["options"] = choices - f["automation"] = auto - f["type"] = "select" - break - else: - t["fields"].append(field) - if loops and not any(f.get("key") == cyc_key for f in t["fields"]): - t["fields"].append(cyc_field) - # ⚠ A flow switched BACK to `terminal` keeps its cycle column rather than dropping it: - # the counts already in those cells are a record of work that really happened, and a - # schema change must not be a way to lose data nobody asked to delete. - return cur - - rt.update(UT_STORE_KEY, _up, flush="sync") - return fkey - - def _rid_num(rid): return int(rid) if str(rid).isdigit() else 10 ** 9 -def _card_stage(defn, stages, row, skey): - """Which stage holds this record. The STAMPED label wins; a blank or unrecognised value - falls to the DERIVED bucket — what is true about the row, not what anything wrote: - a discovery candidate nobody has moved is in Review, an Instagram column derives from whether - its cell has ever been written, and a scrape lands where its rows are born (Write).""" - val = str((row or {}).get(skey) or "").strip() - if val: - for s in stages: - if s.get("assignable") and s["label"] == val: - return s["id"] - kind = defn.get("kind") - if kind == "discover_instagram": - # R6: there is no `tracked` tick left to derive a terminal from, and no terminal to derive - # it INTO. An unstamped candidate is simply one nobody has moved yet, which is Review. - return "review" - if kind == "field_instagram": - fkey = (defn.get("config") or {}).get("fieldKey") or "" - return "write" if str((row or {}).get(fkey) or "").strip() else "source" - if kind == "plain": - # No machine step wrote anything, so there is nothing true ABOUT the row to derive a - # bucket from — an unstamped record is simply at the start of the flow. Must stay equal - # to the entry `stages_for` marks for `plain`, or a card names a stage nobody renders. - return "trigger" - return "write" - - -def _card_next(stages, routes, row, sid): - """Where THIS card may go from `sid` — every outgoing route whose condition the record - satisfies (declaration order kept, null = catch-all), targets limited to assignable stages. - Evaluated per record so R5's winning path is decided on the SERVER, once.""" - assignable = {s["id"] for s in stages if s.get("assignable")} - out = [] - for r in routes: - if r["from"] != sid or r["to"] not in assignable or r["to"] == sid: - continue - cond = r.get("condition") - if cond is not None and not lane_match(cond, row): - continue - if r["to"] not in out: - out.append(r["to"]) - # ⭐ WAVE 26 · R6 + R8 — AN UNROUTED REVIEW GATE ADVANCES TO THE LANES THAT EXIST. - # - # ⛔ Without this the review card is a dead end. R6 deleted the routes `Approve`/`Decline` rode - # on, so `out` is empty for a discovery board — `canAdvance` goes false, the move door refuses - # everything, and the card can only be looked at. The lanes a person added under R8 are exactly - # the places it should go, and they carry no routes because nobody drew any. - # ⚠ Only for a stage with NO declared exits. A flow that DOES route its own gate keeps its - # declared destinations and their order — an explicit path is never widened by this. - if not out and any(s["id"] == sid and s["kind"] == "review" for s in stages): - out = [s["id"] for s in stages if s.get("assignable") and s["id"] != sid] - return out - - -def _card_title(defn, row): - kind = defn.get("kind") - cfg = defn.get("config") or {} - if kind == "discover_instagram": - return str(row.get("handle") or row.get("full_name") or "?") - if kind == "scrape_db": - return str(row.get(cfg.get("keyField") or "") or "?") - url_f = cfg.get("urlField") or "" - return str(row.get("name") or row.get("handle") or row.get(url_f) or "?") - - -def _card_sub(defn, row): - kind = defn.get("kind") - if kind == "discover_instagram": - f = str(row.get("followers") or "").strip() - nm = str(row.get("full_name") or "").strip() - return f"{nm} · {f} followers" if (nm and f) else (nm or (f and f + " followers") or "") - if kind == "field_instagram": - return _s(row.get((defn.get("config") or {}).get("fieldKey") or ""), 90) - fmap = (defn.get("config") or {}).get("fieldMap") or {} - keys = [k for k in fmap.values() if k != (defn.get("config") or {}).get("keyField")] - return _s(row.get(keys[0]) if keys else "", 90) - - -def _with_user_lanes(stages, table, skey): - """⭐ WAVE 26 · R6 + R8 — THE TENANT'S OWN OPTIONS, AS REAL STAGES. - - R6 removed the built-in Tracked/Declined terminals and handed the vocabulary to the tenant; - R8 gives them a control to add one on the column itself. Those two only compose if an added - option becomes a PLACE — a lane the board draws and `move_card` will accept. - - ⛔ WHY THIS IS ONE FUNCTION AND NOT TWO CALL SITES. `board` and `move_card` each derived their - stage list independently from `stages_for`, so building the lanes in the board alone would - have drawn a lane the move door then refused: the card sits there, the drag is rejected, and - the two surfaces disagree about what exists. Sibling readers written from one template - diverging exactly this way is the lesson D-10 closed with, and it is cheaper to not write the - second copy than to catch it later. - ⚠ `kind: "lane"`, not `"action"` — these are places a card can BE, never steps the engine - runs, so nothing tries to give them a config panel or a run status. - """ - known = {s["label"] for s in stages} - fld = next((f for f in ((table or {}).get("fields") or []) if f.get("key") == skey), None) - col = max((s["col"] for s in stages), default=0) - out = list(stages) - for i, opt in enumerate(o for o in ((fld or {}).get("options") or []) - if str(o or "").strip() and o not in known): - out.append({"id": f"opt_{i}", "label": opt, "kind": "lane", - "col": col + 1 + i, "row": 0, "enabled": True, "toggle": "", - "status": "idle", "panel": "", "posture": "auto", - "record": True, "assignable": True, - "detail": "A stage you added on this database"}) - return out - - -def board(rt, defn, admin=False): - """C1's whole payload: `{stages, routes, cards, stageField, counts, truncated}`. - - `counts` carries the TRUE per-stage totals for every record stage (zeros included); - `cards` is capped per stage at `BOARD_CARD_CAP` and any stage that overflowed is NAMED in - `truncated` — the cap is disclosed, never silent. Cards ride in row-id order so the board - is stable across polls. - """ - stages, routes = stages_for(defn) - cfg = defn.get("config") or {} - table_key = str(cfg.get("targetTable") or "") - skey = stage_field_key(defn.get("id")) - t = ut_get(rt, table_key) - counts = {s["id"]: 0 for s in stages if s.get("record")} - # C1 amendment (2026-08-05): an automation with NO BOUND STAGE FIELD answers - # `stageField: null, cards: []` — the stages still ride (they come from the definition), - # and an empty board is a state, never an error. Bound = the field exists on the table. - bound = any(f.get("key") == skey for f in ((t or {}).get("fields") or [])) - if t is None or not bound: - return {"stages": stages, "routes": routes, "cards": [], "stageField": None, - "counts": counts, "truncated": []} - stages = _with_user_lanes(stages, t, skey) - counts = {s["id"]: 0 for s in stages if s.get("record")} - rows = t.get("rows") or {} - cards, truncated, per = [], [], {} - for rid in sorted(rows, key=_rid_num): - row = rows.get(rid) or {} - sid = _card_stage(defn, stages, row, skey) - counts[sid] = counts.get(sid, 0) + 1 - if per.get(sid, 0) >= BOARD_CARD_CAP: - if sid not in truncated: - truncated.append(sid) - continue - per[sid] = per.get(sid, 0) + 1 - stage = next((s for s in stages if s["id"] == sid), None) - nxt = _card_next(stages, routes, row, sid) - cards.append({ - "rowId": str(rid), "stageId": sid, - "title": _s(_card_title(defn, row), 80), - "sub": _s(_card_sub(defn, row), 90), - "movedAt": str(row.get(skey + "_at") or ""), - "canAdvance": bool(stage and stage["kind"] == "review" and nxt), - "next": nxt, - }) - return {"stages": stages, "routes": routes, "cards": cards, - "stageField": {"table": table_key, "key": skey}, - "counts": counts, "truncated": truncated} - - -def move_card(rt, auto_id, row_id, to, username="", admin=False): - """One card move, SERVER-decided (C1): a review-stage card advances only to a member of its - per-record `next`; every other move is an admin repair onto an assignable stage. Returns - `(fresh_board, None)` or `(None, (status, code, sentence))` — the route maps the tuple. - - The write lands in the DEFINITION rows (the shared truth the engine and every viewer read), - stamped with `movedAt`; R6 removed the discovery board's built-in terminals, and leaving - it clears it — the human decision travelling with the card (R6/R7).""" - defn = all_definitions(rt).get(str(auto_id)) - if defn is None: - return None, (404, "unknown_automation", "no automation with that id") - stages, routes = stages_for(defn) - cfg = defn.get("config") or {} - table_key = str(cfg.get("targetTable") or "") - t = ut_get(rt, table_key) - if t is None: - return None, (404, "no_target", f"{table_key or 'the target database'} does not exist " - f"yet — run the automation once") - skey = stage_field_key(defn.get("id")) - if not any(f.get("key") == skey for f in (t.get("fields") or [])): - return None, (400, "unbound", - "this automation has no stage field on its database yet — save or run " - "it once to bind the board") - # R6 + R8: the same lane set the board draws, so this door cannot refuse a stage the board - # shows. One derivation, two readers. - stages = _with_user_lanes(stages, t, skey) - row = (t.get("rows") or {}).get(str(row_id)) - if row is None: - return None, (404, "unknown_row", "that record is not in the target database") - target = next((s for s in stages if s["id"] == str(to or "")), None) - if target is None or not target.get("assignable"): - return None, (400, "bad_stage", - "cards can only be placed in a review, lane or terminal stage — the " - "run stages are machinery a record passes through") - cur_id = _card_stage(defn, stages, row, skey) - cur = next((s for s in stages if s["id"] == cur_id), None) - from_review = cur is not None and cur["kind"] == "review" - if from_review: - allowed = _card_next(stages, routes, row, cur_id) - if target["id"] not in allowed and not admin: - return None, (400, "not_next", - "a review card advances only to its listed next stage") - elif not admin: - return None, (403, "admin_only", - "only a review card can be advanced by hand — moving anything else is " - "an admin repair") - # ⭐ WAVE 26 · R6 — ONE PROGRESS FIELD. A move used to write the card's lane AND mirror it - # into the `tracked` checkbox; that second write is gone with the column. `move_card` keeps a - # coherent job — it moves cards between the stages the flow actually has — and it is now the - # only writer of a record's progress, which is the whole point of the ruling. - values = {skey: target["label"], skey + "_at": _iso()} - - def _up(cur_b): - cur_b = cur_b if isinstance(cur_b, dict) else {} - tt = cur_b.get(table_key) - if tt is not None: - tt.setdefault("rows", {}).setdefault(str(row_id), {}).update(values) - return cur_b - - rt.update(UT_STORE_KEY, _up, flush="sync") - if from_review: - review_audit(rt, defn.get("id"), row_id, cur["label"], target["label"], username) - return board(rt, defn, admin=admin), None - - #: How many review decisions an automation remembers. Bounded like `runs` — an audit that can #: grow a definition without limit is a serialisation cost wearing a compliance hat. MAX_REVIEWS = 100 @@ -8854,13 +7210,18 @@ MAX_REVIEWS = 100 def review_audit(rt, auto_id, row_id, from_label, to_label, username, by="user", note="", model=""): """C3-A2(5): a review decision is AUDITED — who moved which card where, when. Appended to - the definition (newest first, bounded) by BOTH doors: `move_card` here, and the grid door - in `core.grid_events`, which writes the same shape mechanically off the field's `flowId`. + the definition (newest first, bounded). ⭐ WAVE 23 (R4/C6): an AI decision writes THE SAME ROW with `by: "ai"` plus the model that made it and the one-line reason it gave. One audit log, not two — a reader asking "who decided this card" must not have to know there are two places to look, and the moment an AI decision is invisible beside a human one the log stops being an audit. + + ⚠ PARKED SINCE WAVE 27, with `ai_decide` and for the same reason. Both of its doors are gone: + `move_card` was deleted with the board (R3), and the grid door in `core.grid_events` wrote + this shape off a stage field's `flowId` — a field the migration now drops. Kept because the + SHAPE is the contract a future decision action would write, and re-deriving an audit format + is how two of them end up existing. """ entry = {"ts": _iso(), "user": _s(username, 80), "rowId": str(row_id), "from": _s(from_label, 60), "to": _s(to_label, 60), @@ -10414,6 +8775,15 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): summary += (f" — {batches} post-engagement batch" f"{'' if batches == 1 else 'es'} still building; Views and other " "metrics will be collected automatically without another paid read") + if a_counts.get("enrichUnbound"): + # ⛔ D-79(2): AND THE FIX GOES IN THE SUMMARY, not only in the log. The run is + # `partial` because it genuinely did part of its job — it walked the records — and + # the sentence names the ONE thing that has to change, in the two places a person + # can change it. The old behaviour was `ok` with an empty table. + state = "partial" if state != "error" else state + summary += (" — the Instagram step did not run: this database has no profile " + "column. Name one on the step, or mark a text column as the " + "Instagram profile") if a_counts.get("enrichBlocked"): state = "partial" if state != "error" else state summary += f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked"