diff --git a/RELEASES.json b/RELEASES.json index 5d46e2104153b5ab209ee0ea846571f7e122ca56..ad239fc58580b5f535689dd7ec1644f6bf768e55 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,18 @@ { - "current": "v11 (3126617)", + "current": "v13 (314c0e3)", "releases": [ + { + "version": "v13", + "sha": "314c0e3", + "date": "2026-08-06", + "subject": "release v13" + }, + { + "version": "v12", + "sha": "4908f39", + "date": "2026-08-06", + "subject": "release v12" + }, { "version": "v11", "sha": "3126617", diff --git a/VERSION b/VERSION index ba8c682859242ccc42d979cac4c5ba4844b28105..6c1db79abe749cdfb8487a7a493a64957942e723 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v11 (3126617) +v13 (314c0e3) diff --git a/api/ai_review.py b/api/ai_review.py new file mode 100644 index 0000000000000000000000000000000000000000..312b7969e449e0ad71d727de4bb3e6b70f5c58db --- /dev/null +++ b/api/ai_review.py @@ -0,0 +1,207 @@ +"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). + +⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made +that somebody optionally a MODEL: the engine hands over the record's own values, the review's +prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of +those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit +log a human click writes to, tagged `by: "ai"` with the provider and model that made it. + +⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a +network failure, a slow answer, a malformed answer, or an answer naming a stage the review does +not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would +have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing +the work. Nothing here can move a card somewhere the review does not already permit. + +⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is +`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is +last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` +pins one; `AIOS_AI_MODEL` overrides the model. + +⚠ WHY RAW HTTP RATHER THAN THE `anthropic` SDK, stated because it is a deliberate deviation from +the /claude-api skill's default and not an oversight. Three of the four legs are OpenAI-chat-shaped +endpoints with no shared SDK, so a ladder built on the SDK would be one SDK leg beside three +hand-rolled ones — two implementations of the same call, the seam this repo keeps closing. And +`aios-web/api/requirements.txt` is PINNED to what the verify battery proves ([[pin-deps-space- +rebuilds]]): adding a dependency there rebuilds the container, which is a real deploy risk to take +for one leg of an optional feature. `requests` is already a dependency and `harness/analyst.py` +established per-provider raw HTTP as the house pattern. Booked as a DEBT line so the integrator +can overturn it deliberately rather than by drift. + +The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and +`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an +empty content list, so code that indexes `content[0]` unconditionally breaks on it. +""" +from __future__ import annotations + +import json +import os +import re + +import requests + +#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. +PROVIDERS = [ + {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", + "url": "https://api.groq.com/openai/v1/chat/completions", + "model": "llama-3.3-70b-versatile"}, + {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", + "url": "https://api.cerebras.ai/v1/chat/completions", + "model": "gpt-oss-120b"}, + {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", + "url": "https://openrouter.ai/api/v1/chat/completions", + "model": "openai/gpt-4o-mini"}, + # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as + # the backstop for a one-line classification, and this is the cheapest current Claude that + # does it well. A bigger model here would be spending the owner's money to pick between two + # labels it already has in front of it. + {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", + "url": "https://api.anthropic.com/v1/messages", + "model": "claude-haiku-4-5"}, +] +ANTHROPIC_VERSION = "2023-06-01" +TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) +MAX_FIELD_CHARS = 200 # per value handed to the model +MAX_FIELDS = 30 # columns handed to the model +MAX_REASON = 200 + + +def ladder(): + """The providers that are actually usable here, in order. Empty = the feature is off.""" + pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() + live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] + if pin: + live = [p for p in live if p["name"] == pin] + return live + + +def configured(): + return bool(ladder()) + + +def _record_text(row, fields): + """The record, as the model sees it. Values are truncated and the column set is bounded — + an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not + need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is + sitting at would be the model reading its own question back.""" + keys = [k for k in (fields or list((row or {}).keys())) + if not str(k).startswith("stage_")][:MAX_FIELDS] + lines = [] + for k in keys: + v = (row or {}).get(k) + if v is None or str(v).strip() == "": + continue + lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") + return "\n".join(lines) or "(this record has no filled-in values)" + + +def _instruction(prompt, options, label): + return ( + f"You are deciding what happens to one record waiting at a review step called " + f"{label!r} in a workflow.\n\n" + f"The person who built this workflow told you: {prompt}\n\n" + f"Choose EXACTLY ONE of these next steps, by its exact name:\n" + + "\n".join(f"- {o}" for o in options) + + "\n\nAnswer with one line of JSON and nothing else:\n" + '{"choice": "", "reason": ""}\n' + "If the record does not give you enough to decide, answer " + '{"choice": "", "reason": "why not"} and a person will decide instead.' + ) + + +def _parse(text, options): + """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is + DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss + resolved by string distance is how a card ends up somewhere nobody authorised.""" + raw = str(text or "").strip() + obj = None + m = re.search(r"\{.*\}", raw, re.S) + if m: + try: + obj = json.loads(m.group(0)) + except (ValueError, TypeError): + obj = None + if not isinstance(obj, dict): + return "", "" + choice = str(obj.get("choice") or "").strip() + reason = str(obj.get("reason") or "").strip()[:MAX_REASON] + for opt in options: + if choice.lower() == str(opt).lower(): + return str(opt), reason # the OFFERED spelling wins, never the model's + return "", reason + + +def _call_openai(p, model, system, user, timeout): + r = requests.post(p["url"], timeout=timeout, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": model, "max_tokens": 300, "temperature": 0, + "messages": [{"role": "system", "content": system}, + {"role": "user", "content": user}]}) + if r.status_code >= 400: + return "", f"{p['name']} answered {r.status_code}" + body = r.json() + choices = body.get("choices") or [] + if not choices: + return "", f"{p['name']} returned no choices" + return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "" + + +def _call_anthropic(p, model, system, user, timeout): + r = requests.post(p["url"], timeout=timeout, + headers={"x-api-key": os.environ[p["env"]].strip(), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + json={"model": model, "max_tokens": 300, "system": system, + "messages": [{"role": "user", "content": user}]}) + if r.status_code >= 400: + return "", f"anthropic answered {r.status_code}" + body = r.json() + # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so + # reading content[0] before this check turns a refusal into an IndexError inside a run. + if body.get("stop_reason") == "refusal": + return "", "anthropic declined to answer this record" + parts = [b.get("text") or "" for b in (body.get("content") or []) + if isinstance(b, dict) and b.get("type") == "text"] + if not parts: + return "", "anthropic returned no text" + return "".join(parts), "" + + +def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): + """Pick this record's next stage. Returns `(choice, meta)`. + + `choice` is "" whenever a person should decide — which is every failure mode there is. + `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. + """ + opts = [str(o) for o in (options or []) if str(o).strip()] + if not opts: + return "", {"problem": "the review offers no next stages"} + if not str(prompt or "").strip(): + return "", {"problem": "the review has no prompt for the model to follow"} + live = ladder() + if not live: + return "", {"problem": "no AI provider is configured on this deployment"} + system = _instruction(prompt, opts, label) + user = "Here is the record:\n\n" + _record_text(row, fields) + tmo = float(timeout or TIMEOUT_SECONDS) + override = (os.environ.get("AIOS_AI_MODEL") or "").strip() + problems = [] + for p in live: + model = override or p["model"] + try: + text, err = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( + p, model, system, user, tmo) + except Exception as e: # noqa: BLE001 + text, err = "", f"{p['name']} failed: {type(e).__name__}" + if err: + problems.append(err) + continue # ladder: a dead provider degrades to the next one + choice, reason = _parse(text, opts) + if not choice: + # The provider ANSWERED and declined (or answered unusably). That is a decision about + # this record, not a fault in the provider, so it does NOT fall through to a more + # expensive one — the card goes to a human, which is what the model just asked for. + return "", {"provider": p["name"], "model": model, + "problem": reason or "the model did not choose one of the stages"} + return choice, {"provider": p["name"], "model": model, "reason": reason} + return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} diff --git a/api/automation_engine.py b/api/automation_engine.py index d7b5fc43821353b8b855281dd72e13a02ea648fa..70d2264f9c821b32d7faa696e25a3c524671b192 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -24,6 +24,7 @@ the skill version takes a URL from a developer on a CLI; this takes one from a r from __future__ import annotations import datetime as _dt +import hmac import ipaddress import json import os @@ -93,7 +94,29 @@ IG_TABLE_PREFIX = "ut_ig_" #: that stays true when the next `ut_ig_*` table is not one. APPEND_TABLES = frozenset({"ut_ig_snapshots", "ut_ig_post_snapshots"}) -KINDS = ("scrape_db", "field_instagram", "discover_instagram") +#: ⭐ WAVE 24 (owner ruling R6) — `plain` IS WHAT AN AUTOMATION IS NOW, and it is the DEFAULT. +#: The create wizard is deleted, so nobody picks a kind any more: a new automation is a trigger +#: plus actions and NO machine step. The other three are MACHINE kinds — a scrape, an Instagram +#: column, an Instagram search — and they survive on the automations that already use them +#: (`discover_instagram` stays reachable through the `ig_profile_match` trigger; see C-TRIG). +#: +#: ⚠ ADDING A KIND HERE IS THE SMALL HALF, and the reason is worth reading before adding a fifth: +#: TWO dispatches in this file used to end in a bare `else` that belonged to a SPECIFIC kind +#: rather than to a default — `graph()`'s was `scrape_db`'s and `compose_sentence`'s was +#: `discover_instagram`'s. A kind added here alone would have inherited another kind's whole +#: description: three machine nodes it does not have, and a one-line summary about searching +#: Instagram for up to 0 profiles. Both are explicit arms now, and neither has an `else`. +KINDS = ("plain", "scrape_db", "field_instagram", "discover_instagram") +#: What a definition with no kind becomes (C-TRIG law 2) — the shape `POST /automations` stores +#: when the body names none, which after R6 is every create the client makes. +DEFAULT_KIND = "plain" +#: R6: no NEW automation may be either of these. They still RUN, still PATCH and still validate — +#: the ruling retires the door, not the two automations behind it (see `create`). +#: ⚠ `discover_instagram` is NOT here: it stays creatable, through the `ig_profile_match` trigger. +RETIRED_KINDS = ("scrape_db", "field_instagram") +KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database", + "field_instagram": "Instagram profile column", + "discover_instagram": "Find Instagram profiles"} EXTRACTS = ("table", "jsonld") STATES = ("idle", "running", "ok", "error", "partial") #: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). @@ -849,6 +872,25 @@ def clean_config(kind, raw, previous=None): def flag(name): return bool(raw[name]) if name in raw else bool(prev.get(name)) + if kind == "plain": + # ⭐ WAVE 24 / R6 — a plain automation has no machine step, so it has almost no config: + # the database its records walk, and the board's lane router. + # + # ⚠ THE TARGET IS OPTIONAL ON PURPOSE. An automation triggered by `record_updated` on + # `ut_foo` already knows its table from the trigger — `_flow_table` resolves target-else- + # trigger — so requiring a second copy of that fact here would be a mandatory field with + # exactly one legal answer, and a Save that refuses until you retype what you just picked. + # There is no `dryRun`: dry-run means "run the machine step but do not write", and there + # is no machine step to run. + target = _s(raw.get("targetTable"), 60).strip() + if target and not target.startswith(UT_PREFIX): + target = UT_PREFIX + target + lanes, lerr = clean_lanes(raw.get("lanes") if "lanes" in raw else prev.get("lanes")) + if lerr: + return None, lerr + return {"targetTable": target, + "targetLabel": _s(raw.get("targetLabel"), 60).strip(), + "lanes": lanes}, None if kind == "scrape_db": url = _s(raw.get("url"), 2000).strip() if not url: @@ -916,7 +958,12 @@ def clean_config(kind, raw, previous=None): "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") if limit > BD_MAX_RECORDS: return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" - preds, perr = clean_predicates(raw.get("predicates")) + # ⚠ THE JOIN IS RESOLVED BEFORE THE PREDICATES ARE JUDGED, because it is part of what + # makes them broad or narrow (see `narrowing_refusal`). Validating them first and reading + # the operator afterwards is how the OR hole survived: the guard was handed the branches + # and never told they were a union. + join = "or" if str(raw.get("operator") or "").lower() == "or" else "and" + preds, perr = clean_predicates(raw.get("predicates"), join) if perr: return None, perr if raw.get("lanes") if "lanes" in raw else prev.get("lanes"): @@ -928,7 +975,7 @@ def clean_config(kind, raw, previous=None): if not target.startswith(UT_PREFIX): target = UT_PREFIX + target return {"recordsLimit": limit, "predicates": preds, - "operator": "or" if str(raw.get("operator") or "").lower() == "or" else "and", + "operator": join, "targetTable": target, "targetLabel": _s(raw.get("targetLabel"), 60).strip() or "IG candidates", "dryRun": flag("dryRun")}, None @@ -949,34 +996,196 @@ def clean_schedule(raw, previous=None): return out +# ── WAVE 23 · C5 — the ENDING, and the cycle counter that makes a loop countable. ───────────── +# The owner's words: "we can make this automation a loop, so the ending should always be defined, +# either it ends somewhere deterministic like Closed/Failed, or it goes to reset automatically, +# or the user have to click a button to reset, or after a certain amount of time it can +# automatically reset to first cycle." +# +# ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition +# that predates this field must not start moving records on its own the day the code ships. A +# loop is a thing somebody turns on. +ENDING_MODES = ("terminal", "auto_reset", "manual_reset", "timed_reset") +MAX_RESET_HOURS = 720 # 30 days; past that "it resets eventually" is a lie +#: The per-record cycle counter's cell suffix, beside `stage_` and `stage__at`. +CYCLES_SUFFIX = "_cycles" + + +def cycles_field_key(auto_id): + return stage_field_key(auto_id) + CYCLES_SUFFIX + + +def clean_ending(raw, previous=None): + """Validate `{mode, hours?}`. Returns `(ending, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + mode = _s(raw.get("mode") or prev.get("mode"), 20).strip() or "terminal" + if mode not in ENDING_MODES: + return None, f"{mode!r} is not one of: " + ", ".join(ENDING_MODES) + out = {"mode": mode} + if mode == "timed_reset": + try: + hours = int(raw.get("hours") if raw.get("hours") is not None else prev.get("hours") + or 24) + except (TypeError, ValueError): + return None, "the reset delay must be a whole number of hours" + if hours < 1 or hours > MAX_RESET_HOURS: + return None, f"the reset delay is between 1 and {MAX_RESET_HOURS} hours" + out["hours"] = hours + return out, None + + +def clean_flow(raw, previous=None): + """Validate `{actions, ending}` — the builder's half of a definition. `(flow, error)`.""" + raw = raw if isinstance(raw, dict) else {} + prev = previous if isinstance(previous, dict) else {} + actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions")) + if err: + return None, err + ending, err = clean_ending(raw.get("ending") if "ending" in raw else prev.get("ending"), + prev.get("ending")) + if err: + return None, err + return {"actions": actions, "ending": ending}, None + + +#: AMENDMENT A1 — what the `ig_profile_match` kind-flip seeds as `recordsLimit` when the +#: definition carries none. +#: +#: ⚠ 25 BECAUSE THE INPUT SAYS 25 (2026-08-06). It was 10 — the size wave 21 proved live at +#: $0.15 — while `AutomationDetail` initialises its own box to 25, and the two never met: the +#: flow node read "Up to 10 profiles · about $0.025" beside a field reading 25, on a freshly +#: created automation. Two numbers for one fact, in the panel, before anybody had typed +#: anything. **Read off the screenshot; every assertion in the battery was green.** +#: +#: This is the same species as the default-versus-guard split fixed the same day: a value +#: decided in one file and a value decided in another, describing the same thing. Aligning the +#: constants closes the only window in which they can disagree — the seed — because every later +#: state comes from the stored config. +DISCOVER_SEED_RECORDS = 25 + + +def _ensure_ig_action(flow_raw, table): + """Create record is PERMANENT step 1 on `ig_profile_match` — it cannot be deleted or moved. + + ⭐ OWNER RULING 2026-08-06, AND IT REVERSES WAVE 24's LAW 3. That law seeded this action once, + on the edge into the trigger, and ended "deleting it is their call, not a refusal" — with a + comment warning that re-seeding on every clean would be "a control that will not take no for + an answer". The owner's answer: *"should ALWAYS have a 'Create record' Step 1, that can't be + deleted, because the nature of that automation is that it needs to first create a record in a + database somewhere from the list of profiles to fetch."* + + Which is right, and the earlier reasoning had the category wrong: a flow whose TRIGGER + produces rows has nowhere to put them until something writes them, so an Instagram search + with no Create record is not a customised automation — it is a search whose results are + discarded. That is not a preference to respect. + + ⚠ IT KEEPS THE USER'S EDITS. Presence and POSITION are guaranteed; the table it writes to and + the values it maps are theirs. An existing `create_record` further down is MOVED to the front + rather than duplicated — re-seeding a second one would quietly double every run's writes. + """ + flow = dict(flow_raw or {}) if isinstance(flow_raw, dict) else {} + actions = [a for a in (flow.get("actions") or []) if isinstance(a, dict)] + if actions and actions[0].get("kind") == "create_record": + return flow_raw if isinstance(flow_raw, dict) else flow + at = next((i for i, a in enumerate(actions) if a.get("kind") == "create_record"), -1) + if at > 0: + actions.insert(0, actions.pop(at)) + else: + # The seed must be a config `clean_actions` ACCEPTS, or the builder shows a red banner + # and no card: the wave-23 header records four kinds that shipped with illegal seeds and + # did exactly that. `create_record` needs a ut_-prefixed table and at least one value. + actions.insert(0, { + "id": "act_1", "kind": "create_record", "enabled": True, "when": None, + # `config.label` follows the `review` action's precedent in `_clean_action_config` — + # an action naming itself, inside the untyped config bag, so no shared TS type changes. + "config": {"table": str(table or DISCOVER_TABLE), + "label": "Save the profile", + "values": {"handle": "{{handle}}"}}, + }) + flow["actions"] = actions + return flow + + +def ig_action_pinned(defn, index): + """Is this action the one that cannot be removed? Derived, never stored — a stored `pinned` + flag is a second copy of the rule, and the copy is what a hand-written PATCH omits.""" + return (defn or {}).get("kind") == "discover_instagram" and index == 0 + + def clean_definition(raw, previous=None, username=""): """Whole-definition validation. Returns `(defn, error)`.""" raw = raw if isinstance(raw, dict) else {} prev = previous or {} - kind = _s(raw.get("kind") or prev.get("kind"), 40) + # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing + # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second + # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys + # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload + # invalid in BOTH its trigger and its config now answers with the trigger's sentence. + trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw + else prev.get("trigger"), prev.get("trigger")) + if terr: + return None, terr + # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this + # refused with "unknown automation kind ''" — correct while a wizard always sent one, and a + # dead end the moment the wizard was deleted and the client's create body became {name}. + kind = _s(raw.get("kind") or prev.get("kind"), 40) or DEFAULT_KIND + if (trigger or {}).get("key") == "ig_profile_match": + # LAW 1: the trigger IS how `discover_instagram` gets chosen now. Unconditional rather + # than "when the kind is unset" — a definition whose trigger says Instagram-discovery and + # whose kind says something else is not a preference to respect, it is two halves of one + # answer disagreeing, and the trigger is the half the person actually picked. + kind = "discover_instagram" if kind not in KINDS: return None, f"unknown automation kind {kind!r}" name = " ".join(_s(raw.get("name") or prev.get("name") or "", MAX_NAME).split()) if not name: return None, "name the automation" cfg_raw = raw.get("config") if isinstance(raw.get("config"), dict) else prev.get("config") + if kind == "discover_instagram" and prev.get("kind") != "discover_instagram": + # AMENDMENT A1: the kind FLIP seeds the one field `clean_config` insists on, or the very + # first Save after picking this trigger 400s — against the stored-inert-with- + # `configured:false` pattern the whole picker is built on (see `clean_trigger`'s A3 note). + # ⛔ THE REFUSAL ITSELF IS UNTOUCHED: an explicit 0 still gets its sentence. That guard + # exists because an unbounded query is the one shape the vendor refuses outright, and + # coercing a blank into a number would be exactly the silent widening it protects against. + # Seeding cannot spend — `clean_schedule` defaults `enabled` False, so nothing runs until + # a person presses Run now or arms the schedule. + cfg_raw = dict(cfg_raw or {}) + if not _ig_int(cfg_raw.get("recordsLimit")): + cfg_raw["recordsLimit"] = DISCOVER_SEED_RECORDS config, err = clean_config(kind, cfg_raw, prev.get("config")) if err: return None, err + if trigger: + # A2: re-ask completeness now that the config is validated — `ig_profile_match`'s own + # configuration IS the config, and `clean_trigger` could not see it. + trigger["configured"] = _trigger_configured(trigger, config) try: schedule = clean_schedule( raw.get("schedule") if isinstance(raw.get("schedule"), dict) else prev.get("schedule"), prev.get("schedule")) except ValueError as e: return None, str(e) - trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw - else prev.get("trigger"), prev.get("trigger")) - if terr: - return None, terr + # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) + flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") + # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only + # call was what made the action deletable; running it on every save is what makes step 1 + # permanent, and it is deliberate rather than a widened condition nobody noticed. + if (trigger or {}).get("key") == "ig_profile_match": + flow_raw = _ensure_ig_action(flow_raw, config.get("targetTable") or DISCOVER_TABLE) + flow, ferr = clean_flow(flow_raw, prev.get("flow")) + if ferr: + return None, ferr return { "id": _s(prev.get("id") or raw.get("id"), 40), "name": name, "kind": kind, "config": config, "schedule": schedule, "trigger": trigger, + # WAVE 23 · C4/C5 — the BUILDER's half: the ordered actions a record walks and what + # happens when it reaches the end. Absent on every definition written before this wave, + # which is exactly why `clean_flow` defaults it to "no actions, terminal ending": the + # code shipping must not put existing automations into a loop. + "flow": flow, "status": prev.get("status") or {"state": "idle", "lastRunAt": "", "lastSummary": ""}, "runs": list(prev.get("runs") or [])[:MAX_RUNS], # C3-A2(5): the review-decision audit survives a Save the way `state` does — an @@ -1027,13 +1236,108 @@ def _new_id(existing): return f"auto_{n}" +#: C2's three answers to "which database does this automation work on?" — the FIRST question the +#: create wizard asks (owner item 3: the kind is not the first thing a person picks, the data is). +TARGET_MODES = ("existing", "new", "automated") + + +def resolve_target(rt, raw, username=""): + """C2: turn the wizard's `target` into a bound table key. Returns `(config_patch, error)`. + + - `existing` — bind a database that is already there. + - `new` — mint a blank one now, so the automation has somewhere to write before its first + run instead of conjuring a table the person never agreed to. + - `automated` — leave it to the runner: a scraping automation MINTS its target on first run + (`ut_ensure`), stamped `source: "Automation"`, which is what makes it a machine database. + Nothing is created here, deliberately — an empty table created up front for a scrape that + never runs is litter nobody can explain. + """ + if not isinstance(raw, dict) or not raw: + return {}, None # no target block = the pre-wizard shape + mode = _s(raw.get("mode"), 20).strip() or "existing" + if mode not in TARGET_MODES: + return None, f"{mode!r} is not one of: " + ", ".join(TARGET_MODES) + if mode == "automated": + label = " ".join(_s(raw.get("label"), 60).split()) + return ({"targetLabel": label} if label else {}), None + if mode == "existing": + key = _s(raw.get("table"), 60).strip() + if not key: + return None, "choose the database this automation works on" + t = ut_get(rt, key) + if t is None: + return None, f"{key!r} is not a database in this workspace" + return {"targetTable": key, "targetLabel": t.get("label") or key}, None + label = " ".join(_s(raw.get("label"), 60).split()) + if not label: + return None, "name the new database" + import core.user_tables as _ut_new + key = _ut_new.create(label, username or "automation", st=rt) + if not key: + return None, ("the database could not be created — this workspace may be at its table " + "limit") + return {"targetTable": key, "targetLabel": label}, None + + +def _unmint(rt, key): + """Delete a database this call minted moments ago, because the call is refusing (D-50). + + ⚠ FAIL-QUIET BY DESIGN. The caller is already returning a refusal with a sentence the user + needs to read; a rollback that raised would replace that sentence with a 500 and lose the + reason the create failed in the first place. The worst case of a swallowed failure here is + the orphan we had before — strictly no worse, and the refusal still reaches the person. + """ + if not key: + return + try: + import core.user_tables as _ut_del + _ut_del.delete(key, st=rt) + except Exception: # noqa: BLE001 + pass + + def create(rt, raw, username=""): existing = all_definitions(rt) if len(existing) >= MAX_AUTOMATIONS: return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" + raw = dict(raw or {}) + # ⭐ WAVE 24 — DEBT D-50. `resolve_target` MINTS a database for `mode:"new"`, and validation + # happens after it, so a correct refusal ("a source URL is required") used to leave an + # orphaned `ut_*` table behind with no automation pointing at it. MEASURED at the W23 close: + # three refused creates, two orphaned databases, deleted by hand. A person mis-filling a form + # twice silently accumulated empty databases in their nav. + # + # ⛔ THE ROLLBACK IS DERIVED, NOT DECLARED, and that is the point of doing it this way. It + # does not test for `mode == "new"`; it asks "did the table this call's target resolved to + # exist BEFORE this call?" So it closes the hole for any future target mode that mints, and + # a fifth mode cannot reopen it by forgetting a branch. Scoped to THIS call's own target + # rather than "any table that appeared", so a concurrent create in another request is never + # collateral. + before = set(ut_all(rt) or {}) + patch_cfg, terr = resolve_target(rt, raw.pop("target", None), username) + if terr: + return None, terr + minted = str((patch_cfg or {}).get("targetTable") or "") + minted = minted if minted and minted not in before else "" + if patch_cfg: + raw["config"] = {**(raw.get("config") or {}), **patch_cfg} defn, err = clean_definition(raw, None, username) if err: + _unmint(rt, minted) return None, err + # ⭐ WAVE 24 · OWNER RULING R6 — "scrape_db and field_instagram survive on the automations + # that already use them and NO NEW ONE CAN BE CREATED." + # + # ⛔ HERE, IN `create`, AND NEVER IN `clean_definition`. A PATCH of one of the two live + # automations runs through `clean_definition` with `prev["kind"]` already set, so refusing + # the kind there would make both of them unsaveable — the ruling retires the door, not the + # automations behind it. Enforced on the server rather than left to the deleted wizard: a + # wall that holds only because the client stopped asking is a convention, not a wall. + if defn["kind"] in RETIRED_KINDS: + _unmint(rt, minted) + return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " + f"longer created — the ones already using it keep working. Start a new " + f"automation and pick its trigger instead") defn["id"] = _new_id(existing) def _up(cur): @@ -1054,7 +1358,12 @@ def patch(rt, auto_id, raw, username=""): if prev is None: return None, "no such automation" merged = dict(prev) - for k in ("name", "config", "schedule", "kind", "trigger"): + # ⚠ A HAND-MAINTAINED PATCHABLE-KEY LIST, and it is the silent-drop seat of this module: a + # key missing here is accepted by the route, validated by `clean_definition`, and then + # DISCARDED — the client shows a saved flow that the store never received, and nothing goes + # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only + # ordering that never has a window where the bug exists. + for k in ("name", "config", "schedule", "kind", "trigger", "flow"): if k in (raw or {}): merged[k] = raw[k] defn, err = clean_definition(merged, prev, username) @@ -1080,8 +1389,15 @@ def _stage_field_after_write(rt, defn, username): """C2's 'at automation create (or first bind)': when the target database already exists, the stage field appears/refreshes with the definition write — a patch that adds a lane grows the select's vocabulary in the same breath. A table that does not exist yet gets its - field from the first RUN instead (the runner ensures after `ut_ensure`).""" - tgt = str((defn.get("config") or {}).get("targetTable") or "") + field from the first RUN instead (the runner ensures after `ut_ensure`). + + ⭐ WAVE 24 — resolved through `_flow_table` rather than reading `config.targetTable` directly, + because a `plain` automation is allowed to take its table from its TRIGGER (see `clean_config` + on why the target is optional). Reading the config alone left exactly those automations + without a stage field, so their board stayed permanently empty — `board()` answers + `cards: []` until the field is bound, which looks like "no records" rather than "no field". + """ + tgt = str(_flow_table(defn) or "") if tgt and ut_get(rt, tgt) is not None: ensure_stage_field(rt, tgt, defn, username) @@ -1142,6 +1458,12 @@ def _step(tenant, auto_id, text): cur["step"] = text +def _no_step(_text): + """The default `step` sink for a runner nobody gave a live slot to (a gate driving a runner + directly). A runner must never REQUIRE the slot — the slot is process state, and the runner + is the part a test is allowed to call on its own.""" + + def _release(tenant, auto_id): with _RUN_LOCK: _RUNNING.pop((tenant, str(auto_id)), None) @@ -1484,23 +1806,23 @@ def bd_call(path, params=None, body=None): headers=hdrs) except Refused as e: # The BASE was refused by the SSRF rail — a configuration fault, not a vendor outage. - return None, f"the Bright Data base URL was refused by the URL rail: {e}" + return None, f"the profile source's address was refused by the URL rail: {e}" except Exception as e: # noqa: BLE001 # ⚠ `type(e).__name__` only. A requests exception's str() can carry the full URL, and the # Authorization header is one refactor away from being a query param; this line must not # be what leaks it. - return None, f"Bright Data did not answer ({type(e).__name__})" + return None, f"the profile source did not answer ({type(e).__name__})" if status == 402: - return None, "Bright Data answered 402 — the account is out of credit" + return None, "the profile search is out of credit" if status in (401, 403): - return None, f"Bright Data refused the key ({status})" + return None, f"the profile source refused our key ({status})" if status == 429: # ⚠ MEASURED during the discovery probe: `too_many_parallel_jobs`. The filter API has a # concurrency cap and the probe's most important test died on it — hence every caller # below is SERIAL by construction. - return None, "Bright Data answered 429 — too many parallel jobs; the calls are serialised" + return None, "the profile source is busy with another search; they run one at a time" if not (200 <= status < 300): - return None, f"Bright Data answered {status}" + return None, f"the profile source answered {status}" # ⛔ A BODY THAT REACHED THE READ CEILING IS A TRUNCATION, AND IT MUST SAY SO. `fetch` reads # at most `max_kb` and returns what it got — silently. A truncated JSON document then fails # `json.loads`, falls through to the NDJSON branch, and comes back as "answered with @@ -1508,12 +1830,12 @@ def bd_call(path, params=None, body=None): # looking for a fault that is OUR byte ceiling — after a result set that has already been # paid for. Naming it here is the same no-silent-caps rule the row and table ceilings follow. if len(raw) >= BD_MAX_KB * 1024: - return None, (f"Bright Data's answer exceeded this client's {BD_MAX_KB // 1024} MB read " + return None, (f"the profile source's answer exceeded this client's {BD_MAX_KB // 1024} MB read " f"ceiling and was TRUNCATED — ask for fewer records; nothing was parsed " f"from a partial document") text = raw.decode("utf-8", "replace").strip() if not text: - return None, f"Bright Data answered {status} with an empty body" + return None, f"the profile source answered {status} with an empty body" try: return json.loads(text), "" except Exception: # noqa: BLE001 @@ -1528,8 +1850,8 @@ def bd_call(path, params=None, body=None): try: rows.append(json.loads(line)) except Exception: # noqa: BLE001 - return None, "Bright Data answered with something that is not JSON" - return (rows, "") if rows else (None, "Bright Data answered with something that is not JSON") + return None, "the profile source answered with something we could not read" + return (rows, "") if rows else (None, "the profile source answered with something we could not read") def _first(node, *names, default=None): @@ -1844,7 +2166,7 @@ def bd_scrape(dataset_id, urls, wait=None): waited += BD_SCRAPE_POLL # ⛔ NAME THE SNAPSHOT. It has been paid for and it is still finishing; a note that said only # "no rows" would throw away both the data and the way to get it. - return [], (f"Bright Data deferred this batch to snapshot {sid} and it was not ready within " + return [], (f"the profile source deferred this batch to {sid} and it was not ready within " f"{int(budget)}s — the records are collected, not lost; ask for a smaller batch " f"or collect {sid} from the control panel") @@ -1875,7 +2197,7 @@ def pull_profile_bd(url, max_posts=24, post_metrics=False, log=print): # ⚠ `posts_count` is deliberately NOT part of this test: it arrives as a fabricated 0 and # is discarded, so a row where it is the only "readable" field is a row we cannot read. return {"state": "blocked", "profile": {}, "posts": [], "via": "brightdata", - "note": "Bright Data answered, but no follower/following counts were readable in " + "note": "the profile source answered, but no follower/following counts were readable in " "it (the field names may have moved — see instagram-capture.md)"} posts = [] for p in (node.get("posts") or [])[:max_posts]: @@ -1910,7 +2232,7 @@ def pull_profile_bd(url, max_posts=24, post_metrics=False, log=print): # over a posts table that did not grow. return {"state": "ok" if posts else "partial", "profile": profile, "posts": posts, "via": "brightdata", - "note": note or ("" if posts else "profile read; Bright Data returned no posts")} + "note": note or ("" if posts else "profile read; no posts were returned")} # --------------------------------------------------------------------------------------------- @@ -2014,6 +2336,147 @@ BD_POPULATED_FIELDS = frozenset({ "full_name", "profile_name", "highlights_count", "id", "fbid", "profile_url"}) BD_MIN_NARROWING = 1 +# ── THE FIELD SCHEMA — what a person sees, and what they are allowed to ask. ────────────────── +# ⛔ THE SURFACE USED TO SHOW THE VENDOR'S OWN COLUMN NAMES AND ALL FOURTEEN OPERATORS ON EVERY +# ROW. So `is_business_account` offered `>=`, `fbid` sat in the list with no explanation of what +# it is, and `not_array_includes` was a thing a customer was expected to reason about. Owner: +# *"look at each damn schema and only provide toggles operator that make sense"*. +# +# ⚠ THE LABEL IS NOW LOAD-BEARING IN BOTH DIRECTIONS. The old comment in `AutomationFind.tsx` +# defended raw names on the grounds that a refusal names the field exactly as the row does — and +# it was right, which is why `field_label()` is used by the REFUSALS too. Prettifying only the UI +# is how you get an error message about `bio_hashtags` on a row labelled "Hashtags in bio". +BD_FIELD_LABELS = { + "followers": "Followers", "following": "Following", "posts_count": "Posts", + "avg_engagement": "Engagement rate", "biography": "Bio", "category_name": "Category", + "business_category_name": "Business category", "is_business_account": "Business account", + "is_professional_account": "Professional account", "is_verified": "Verified", + "account": "Handle", "full_name": "Name", "profile_name": "Profile name", + "external_url": "Link in bio", "profile_url": "Profile link", + "bio_hashtags": "Hashtags in bio", "post_hashtags": "Hashtags in posts", + "related_accounts": "Related accounts", "id": "Instagram ID", "fbid": "Facebook ID", + "highlights_count": "Story highlights", +} +#: A one-line "what IS this" for the rows nobody can be expected to guess. Absent = self-evident; +#: a hint on all 21 rows is a hint on none of them (the wave-24 chip lesson). +BD_FIELD_HINTS = { + "account": "The @username", + "avg_engagement": "Likes and comments as a share of followers", + "id": "Instagram's internal number for the account — for matching a list you already have", + "fbid": "The linked Facebook id — for matching a list you already have", + "profile_name": "The name shown above the bio, when it differs from the account name", + "related_accounts": "Accounts Instagram suggests alongside this one", +} +#: The KIND decides the comparisons offered and the control drawn for the value. +BD_FIELD_KINDS = { + **{n: "number" for n in ("followers", "following", "posts_count", "avg_engagement", + "highlights_count")}, + **{n: "boolean" for n in ("is_business_account", "is_professional_account", "is_verified")}, + **{n: "tags" for n in ("bio_hashtags", "post_hashtags", "related_accounts")}, + **{n: "choice" for n in ("category_name", "business_category_name")}, + **{n: "text" for n in ("biography", "account", "full_name", "profile_name", "external_url", + "profile_url", "id", "fbid")}, +} +#: ⛔ ONLY WHAT MAKES SENSE, and the ORDER is the order they are offered in — the first entry of +#: each list is what `default_operator` must agree with. +BD_OPS_BY_KIND = { + # ⚠ `in`/`not_in` are DELIBERATELY ABSENT. "is any of" is what `=` becomes the moment a second + # value is typed (see `BD_MULTI_VALUE_OPS`), so offering both would be two controls for one + # idea — and the second one is the one written in vendor grammar. + "text": ("includes", "not_includes", "=", "is_not_null", "is_null"), + "choice": ("=", "!=", "is_not_null", "is_null"), + "number": (">=", "<=", "=", ">", "<"), + # A yes/no field has exactly one sensible comparison and a two-option value. `!=` on a boolean + # is `=` with the other value, written the confusing way. + "boolean": ("=",), + "tags": ("array_includes", "not_array_includes", "is_not_null", "is_null"), +} +#: Plain English for every comparison the surface can offer. The vendor's token stays on the wire +#: (it is what the API accepts); nobody has to read it. +BD_OP_LABELS = { + "includes": "contains", "not_includes": "does not contain", + "=": "is", "!=": "is not", "in": "is any of", "not_in": "is none of", + ">=": "at least", "<=": "at most", ">": "more than", "<": "less than", + "array_includes": "has any of", "not_array_includes": "does not have", + "is_null": "is empty", "is_not_null": "is not empty", +} +#: Yes/No, so a boolean is a two-item dropdown instead of a box you type `true` into. +BD_BOOLEAN_OPTIONS = [{"value": "true", "label": "Yes"}, {"value": "false", "label": "No"}] + +#: ⭐ THE OPERATORS THAT MAY CARRY SEVERAL VALUES — owner item 3, "if i want to include many +#: keywords like floral, flower, beauty". One condition row holds the list; `bd_filter_start` +#: expands it into a NESTED OR group, so it composes with the other conditions instead of forcing +#: the whole search to "match any" (which would also drag Followers into the union — the hole the +#: OR guard closed the same day). +#: +#: ⛔ POSITIVE OPERATORS ONLY, and this is a correctness line rather than a scope line. "does not +#: contain floral OR does not contain flower" matches nearly every account alive — a negative over +#: a list is an AND (De Morgan), and quietly OR-ing it would build a filter that reads like a +#: narrowing and behaves like the whole corpus. Negatives stay single-valued until someone needs +#: them enough to write the AND branch. +BD_MULTI_VALUE_OPS = frozenset({"includes", "=", "array_includes"}) +MAX_PREDICATE_VALUES = 12 + + +def field_label(name): + """The human name for a searchable field — used by the SURFACE and by every REFUSAL.""" + return BD_FIELD_LABELS.get(name) or str(name or "that field") + + +def field_kind(name): + return BD_FIELD_KINDS.get(name, "text") + + +def ops_for(name): + """The comparisons this field may be asked. A tuple, in offer order.""" + return BD_OPS_BY_KIND.get(field_kind(name), BD_OPS_BY_KIND["text"]) + +#: ⭐ WHAT A **FRESH** CONDITION ON A FIELD STARTS AS — and it lives HERE, beside the narrowing +#: law, because the two drifted apart and the drift cost a user their first automation. +#: +#: ⛔ THE SCAR, IN THREE WAVES. Wave 21 fixed a seeded condition the server refused for having no +#: value by making a new condition NULLARY (`is_not_null` — "this field has any value"), and its +#: comment called that "narrowing-in-the-right-direction". Wave 22 then wrote `BD_NARROWING_OPS` +#: and `is_not_null` is NOT IN IT, which silently made that default a condition the save door +#: always refuses. Wave 24 deleted the create wizard, so the Find panel became the ONLY way in — +#: and the refusal moved from a corner case to the first thing a new user meets. Three waves, one +#: sentence of drift, and nothing red anywhere at any point. +#: +#: So the rule is now DERIVED and ASSERTED (`verify_automation.py`): a fresh condition on a field +#: that CAN narrow must narrow. The client asks for `defaultOperator` per field and never picks +#: one itself — a client-side default is a second copy of this table, free to disagree with the +#: guard that judges it, which is exactly what happened. +BD_ARRAY_FIELDS = frozenset({"bio_hashtags", "post_hashtags", "related_accounts"}) +BD_BOOLEAN_FIELDS = frozenset({"is_business_account", "is_professional_account", "is_verified"}) +#: Narrowing fields whose values are IDENTIFIERS — a substring match on a handle or an id is a +#: worse question than an equality, and both narrow. +BD_EXACT_FIELDS = frozenset({"account", "id", "fbid", "profile_url"}) + + +def default_operator(name): + """The operator a NEW condition on `name` starts with. + + Narrowing wherever the field can narrow, so a fresh condition is ONE step (type a value) from + saveable rather than two (change the comparison, then type a value) — with the second step + being one the surface never told anyone to take. + """ + kind = field_kind(name) + if kind == "tags": + return "array_includes" + if kind in ("boolean", "choice"): + # ⚠ `choice` LANDS HERE AND NOT ON `includes`, which is the arm it used to fall through + # to (Category is a narrowing field). `includes` is not in `BD_OPS_BY_KIND["choice"]`, so + # the default would have been an operator the same module refuses to offer — the exact + # default-versus-guard split this function was written to close, reintroduced one commit + # later by adding a kind. Asserted per field in `verify_automation.py`. + return "=" + if kind == "number": + # `>=` cannot narrow the scan and is not pretending to — it is the comparison a person + # reaching for Followers means, and the guard's sentence is the honest answer when it is + # the ONLY condition present. + return ">=" + return "=" if name in BD_EXACT_FIELDS else "includes" + def predicate_narrows(p): """Does ONE predicate narrow the corpus scan? Content field + content op, nothing else.""" @@ -2021,23 +2484,65 @@ def predicate_narrows(p): and str((p or {}).get("operator") or "") in BD_NARROWING_OPS) -def narrowing_refusal(preds): - """C4's server law: the sentence when a predicate set has ZERO narrowing predicates, else - ''. One function so create/patch (via `clean_predicates`) and RUN (legacy stored configs - predate the law) refuse in the same words.""" - if any(predicate_narrows(p) for p in preds or []): +def narrowing_refusal(preds, operator="and"): + """C4's server law: the sentence when a predicate set does not narrow, else ''. One function + so create/patch (via `clean_predicates`) and RUN (legacy stored configs predate the law) + refuse in the same words. + + ⭐ THE JOIN IS PART OF THE LAW, and it was missed for two waves. The guard asked "does ANY + predicate narrow?" — a question that is only correct under AND. Under **OR** the result is a + UNION, so the query is exactly as broad as its WIDEST branch: `biography includes "florist" OR + followers >= 10000` saved clean and asked the vendor for every account over ten thousand + followers, which is the measured 50-minute / NOT_ENOUGH_FUNDS shape the guard exists to stop. + A money wall that a dropdown three rows above it can walk through is not a wall. + + So: under OR every branch must narrow; under AND one is enough. An EMPTY set never narrows + either way — `all([])` is True, which would have made "no conditions at all" the widest legal + query of the lot. + """ + preds = list(preds or []) + if str(operator or "").lower() == "or": + if preds and all(predicate_narrows(p) for p in preds): + return "" + return ("with Match set to any, every condition has to describe the account itself " + "(Bio, Handle, Name, a hashtag). Matching any means the results are added " + "together, so one follower or yes/no condition widens the whole search. Narrow " + "every condition, or set Match to all") + if any(predicate_narrows(p) for p in preds): return "" - return ("add at least one CONTENT condition (biography includes …, account =, a hashtag) — " - "number and yes/no filters alone make the vendor scan the whole 620-million-row " - "corpus, and breadth is what costs: the measured `followers > 10000` search was " - "still building 50 minutes in") + return ("add a condition that describes the account itself — Bio contains, Handle is, a " + "hashtag. Follower counts and yes/no conditions alone match too much of Instagram " + "to search") def filter_meta(): - """C4's per-field flags for the Find surface: `[{name, populated, narrowing}]` over the - same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, R3).""" - return [{"name": n, "populated": n in BD_POPULATED_FIELDS, - "narrowing": n in BD_NARROWING_FIELDS} for n in BD_FILTER_FIELDS] + """C4's per-field flags for the Find surface: `[{name, populated, narrowing, defaultOperator}]` + over the same 21 names `BD_FILTER_FIELDS` offers (the 3 PII fields stay structurally absent, + R3). + + `defaultOperator` rides here rather than being a client constant for the reason written over + `default_operator`: the client's own default was `is_not_null` for every field, which the + narrowing guard refuses on every field. + """ + return [{"name": n, + "label": field_label(n), + "hint": BD_FIELD_HINTS.get(n, ""), + "kind": field_kind(n), + "populated": n in BD_POPULATED_FIELDS, + "narrowing": n in BD_NARROWING_FIELDS, + "defaultOperator": default_operator(n), + # The comparisons THIS field may be asked, already in plain English and already in + # offer order. A client that filtered a global list by field kind would be a second + # copy of `BD_OPS_BY_KIND`, and the copy is what goes stale. + "operators": [{"value": o, "label": BD_OP_LABELS.get(o, o), + "nullary": o in BD_NULLARY_OPS, + # `in`/`is any of` and the tag operators take a LIST — the control + # has to know that, and deriving it from the token in the client is + # the same second-copy mistake one level down. + "multi": o in BD_MULTI_VALUE_OPS} + for o in ops_for(n)], + "options": BD_BOOLEAN_OPTIONS if field_kind(n) == "boolean" else []} + for n in BD_FILTER_FIELDS] CANDIDATE_FIELDS = [ field_def("handle", "Handle"), field_def("profile_url", "Profile", "url"), @@ -2059,7 +2564,7 @@ CANDIDATE_FIELDS = [ ] -def clean_predicates(raw): +def clean_predicates(raw, operator="and"): """Validate a discovery filter into the vendor's shape. Returns `(predicates, error)`. Refuses rather than coerces: a predicate naming a field the API rejects comes back as a @@ -2073,23 +2578,76 @@ def clean_predicates(raw): name = _s(p.get("name") or p.get("field"), 60).strip() op = _s(p.get("operator") or p.get("op"), 20).strip() if name not in BD_FILTER_FIELDS: - return None, (f"{name or 'that field'!r} cannot be used to search — the searchable " - f"fields are: {', '.join(BD_FILTER_FIELDS)}") - if op not in BD_FILTER_OPS: - return None, f"{op or 'that comparison'!r} is not one of: {', '.join(BD_FILTER_OPS)}" + return None, ("that field cannot be searched — the searchable ones are: " + + ", ".join(field_label(f) for f in BD_FILTER_FIELDS)) + # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly + # because every operator was legal on every field; a yes/no column offering "at least" + # is a question with no meaning that the vendor is nevertheless asked. + allowed = ops_for(name) + if op not in allowed: + return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r} — " + "it takes: " + + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) entry = {"name": name, "operator": op} if op not in BD_NULLARY_OPS: - v = p.get("value") - if v is None or (isinstance(v, str) and not v.strip()): - return None, f"give a value to compare {name!r} against" - entry["value"] = v.strip() if isinstance(v, str) else v + raw_v = p.get("value") + # ONE value or SEVERAL — several is the "contains any of" shape (owner item 3), and + # it is stored as a list rather than as N sibling predicates so the row a person sees + # and the row that is stored are the same thing. + vals = raw_v if isinstance(raw_v, list) else [raw_v] + vals = [v.strip() if isinstance(v, str) else v for v in vals] + vals = [v for v in vals if not (v is None or (isinstance(v, str) and not v))] + if not vals: + return None, f"give {field_label(name)} something to compare against" + if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: + return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value — " + f"{field_label(name)} can only be given a list with " + + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) + if o in BD_MULTI_VALUE_OPS)) + if len(vals) > MAX_PREDICATE_VALUES: + return None, (f"{field_label(name)} takes at most {MAX_PREDICATE_VALUES} values " + "in one condition") + if field_kind(name) == "boolean": + ok = {o["value"] for o in BD_BOOLEAN_OPTIONS} + bad = [v for v in vals if str(v).lower() not in ok] + if bad: + return None, f"{field_label(name)} is answered Yes or No" + # ⛔ A REAL BOOLEAN, NOT THE STRING. MEASURED 2026-08-06: the filter API answers + # **400** to `{"name": "is_verified", "operator": "=", "value": "true"}` and + # accepts `True` (`snap_msh2rp4kh3v833q0u`). Nobody had ever sent one — the field + # was a free-text box until this wave, so the Yes/No dropdown that makes it easy + # to ask is also the thing that would have made every boolean condition fail. + # The dropdown's option VALUES stay "true"/"false" (a