diff --git a/RELEASES.json b/RELEASES.json index 9cb41452c6982b95f99cb4396c10707db484fa74..39fec7d4e4bd0ca19e1c6a012a2bade2146f3a09 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,18 @@ { - "current": "4859c03", + "current": "c526ced", "releases": [ + { + "version": "v58", + "sha": "c526ced", + "date": "2026-08-29", + "subject": "release v58" + }, + { + "version": "v57", + "sha": "4859c03", + "date": "2026-08-29", + "subject": "release v57" + }, { "version": "v56", "sha": "2970b72", diff --git a/VERSION b/VERSION index 23930acb868cfe03c9dae7c0eec5bf5a6802de8d..af96199c1e9f5a7572855f1ca617127184651d01 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4859c03 +c526ced diff --git a/api/ai_review.py b/api/ai_review.py index 03f68bd48c54f4543883f221926d84a4cd135c1f..093c323357f15f25b5301e2db540196f9d7855ee 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -1,591 +1,591 @@ -"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). - -⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made -that somebody optionally a MODEL: the engine hands over the record's own values, the review's -prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of -those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit -log a human click writes to, tagged `by: "ai"` with the provider and model that made it. - -⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a -network failure, a slow answer, a malformed answer, or an answer naming a stage the review does -not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would -have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing -the work. Nothing here can move a card somewhere the review does not already permit. - -⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is -`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is -last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` -pins one; `AIOS_AI_MODEL` overrides the model. - -⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH -USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting -the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. -The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the -integrator can overturn it deliberately rather than by drift"*. This is that overturn. - -Its two reasons were real and are both answered rather than waved away: - 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is - not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers - the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no - Anthropic knowledge — it lost some. - 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the - import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the - same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH - manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. -⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that -half of the original reasoning never expired. - -The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and -`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an -empty content list, so code that indexes `content[0]` unconditionally breaks on it. -""" -from __future__ import annotations - -import json -import os -import re - -import requests - -#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. -PROVIDERS = [ - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED - # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent - # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo - # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. - {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", - "url": "https://api.groq.com/openai/v1/chat/completions", - "model": "openai/gpt-oss-120b"}, - {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", - "url": "https://api.cerebras.ai/v1/chat/completions", - "model": "gpt-oss-120b"}, - {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", - "url": "https://openrouter.ai/api/v1/chat/completions", - "model": "openai/gpt-4o-mini"}, - # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as - # the backstop for a one-line classification, and this is the cheapest current Claude that - # does it well. A bigger model here would be spending the owner's money to pick between two - # labels it already has in front of it. - {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", - "url": "https://api.anthropic.com/v1/messages", - # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, - # and a deployment whose side rungs are out of credit can raise the tier without a release. - "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, -] -ANTHROPIC_VERSION = "2023-06-01" -TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) -MAX_FIELD_CHARS = 200 # per value handed to the model -MAX_FIELDS = 30 # columns handed to the model -MAX_REASON = 200 - - -def ladder(): - """The providers that are actually usable here, in order. Empty = the feature is off.""" - pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() - live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] - if pin: - live = [p for p in live if p["name"] == pin] - return live - - -def configured(): - return bool(ladder()) - - -def _record_text(row, fields): - """The record, as the model sees it. Values are truncated and the column set is bounded — - an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not - need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is - sitting at would be the model reading its own question back.""" - keys = [k for k in (fields or list((row or {}).keys())) - if not str(k).startswith("stage_")][:MAX_FIELDS] - lines = [] - for k in keys: - v = (row or {}).get(k) - if v is None or str(v).strip() == "": - continue - lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") - return "\n".join(lines) or "(this record has no filled-in values)" - - -def _instruction(prompt, options, label): - return ( - f"You are deciding what happens to one record waiting at a review step called " - f"{label!r} in a workflow.\n\n" - f"The person who built this workflow told you: {prompt}\n\n" - f"Choose EXACTLY ONE of these next steps, by its exact name:\n" - + "\n".join(f"- {o}" for o in options) - + "\n\nAnswer with one line of JSON and nothing else:\n" - '{"choice": "", "reason": ""}\n' - "If the record does not give you enough to decide, answer " - '{"choice": "", "reason": "why not"} and a person will decide instead.' - ) - - -def _parse(text, options): - """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is - DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss - resolved by string distance is how a card ends up somewhere nobody authorised.""" - raw = str(text or "").strip() - obj = None - m = re.search(r"\{.*\}", raw, re.S) - if m: - try: - obj = json.loads(m.group(0)) - except (ValueError, TypeError): - obj = None - if not isinstance(obj, dict): - return "", "" - choice = str(obj.get("choice") or "").strip() - reason = str(obj.get("reason") or "").strip()[:MAX_REASON] - for opt in options: - if choice.lower() == str(opt).lower(): - return str(opt), reason # the OFFERED spelling wins, never the model's - return "", reason - - -# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. -# -# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s -# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token -# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach -# this transport rather than be bolted onto its caller — the caller never sees the body. -# -# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The -# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), -# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers -# shape this repo keeps paying for. -def _call_openai(p, model, system, user, timeout): - r = requests.post(p["url"], timeout=timeout, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": model, "max_tokens": 300, "temperature": 0, - "messages": [{"role": "system", "content": system}, - {"role": "user", "content": user}]}) - if r.status_code >= 400: - # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused - # before running the model has nothing to bill. The call is still COUNTED by `decide`. - return "", f"{p['name']} answered {r.status_code}", None - body = r.json() - choices = body.get("choices") or [] - if not choices: - return "", f"{p['name']} returned no choices", body - return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body - - -#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. -#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say -#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. -LAST_ANTHROPIC_TRANSPORT = "" - - -def _call_anthropic(p, model, system, user, timeout): - """One classification turn on the Messages API, through `providers.anthropic_send`. - - ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which - prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is - absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises - both transports to `(status, body)`, so every line below this call is untouched. - """ - global LAST_ANTHROPIC_TRANSPORT - import providers as _prov # noqa: PLC0415 - # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is - # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. - req = _prov.anthropic_request( - model=model, key=os.environ[p["env"]].strip(), system=system, - messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) - status, body, transport = _prov.anthropic_send(req, timeout=timeout) - LAST_ANTHROPIC_TRANSPORT = transport - if status >= 400 or status == 0: - # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the - # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" - # (which is what this line used to say, and what the owner quoted back at us) cannot recur. - return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None - body = body or {} - # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so - # reading content[0] before this check turns a refusal into an IndexError inside a run. - # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. - if body.get("stop_reason") == "refusal": - return "", "anthropic declined to answer this record", body - parts = [b.get("text") or "" for b in (body.get("content") or []) - if isinstance(b, dict) and b.get("type") == "text"] - if not parts: - return "", "anthropic returned no text", body - return "".join(parts), "", body - - -def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): - """Pick this record's next stage. Returns `(choice, meta)`. - - `choice` is "" whenever a person should decide — which is every failure mode there is. - `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. - - ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because - this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it - HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as - unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for - why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI - dependency). - """ - import usage_ledger # noqa: PLC0415 - opts = [str(o) for o in (options or []) if str(o).strip()] - if not opts: - return "", {"problem": "the review offers no next stages"} - if not str(prompt or "").strip(): - return "", {"problem": "the review has no prompt for the model to follow"} - live = ladder() - if not live: - return "", {"problem": "no AI provider is configured on this deployment"} - system = _instruction(prompt, opts, label) - user = "Here is the record:\n\n" + _record_text(row, fields) - tmo = float(timeout or TIMEOUT_SECONDS) - override = (os.environ.get("AIOS_AI_MODEL") or "").strip() - problems = [] - for p in live: - model = override or p["model"] - body = None - try: - text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( - p, model, system, user, tmo) - except Exception as e: # noqa: BLE001 - text, err = "", f"{p['name']} failed: {type(e).__name__}" - # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. - # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an - # unusable answer. A meter that books only successes reports a cheap week for a run that - # spent its budget being declined — and a declined call is billed. The only path that does - # NOT record is a transport that never reached the vendor (`body is None`), which spent - # nothing. - if body is not None: - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record("ai_review", p["name"], model, ins, outs, - total=usage_ledger.total_from(body), st=st, user=user) - if err: - problems.append(err) - continue # ladder: a dead provider degrades to the next one - choice, reason = _parse(text, opts) - if not choice: - # The provider ANSWERED and declined (or answered unusably). That is a decision about - # this record, not a fault in the provider, so it does NOT fall through to a more - # expensive one — the card goes to a human, which is what the model just asked for. - return "", {"provider": p["name"], "model": model, - "problem": reason or "the model did not choose one of the stages"} - return choice, {"provider": p["name"], "model": model, "reason": reason} - return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} - - -# ══════════════════════════════════════════════════════════════════ the FLOW WRITER -# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. -# -# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the -# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: -# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the -# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object -# in that vocabulary. -# -# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This -# is the one module in the API package that already holds an LLM ladder and its credentials, so a -# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs -# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 -# shipped three finished routers 404-dead behind green gates. The DOOR is a route on -# `routes_automation.py`, which is already mounted. -# -# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds -# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are -# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema -# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters -# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise -# and answers 200, so a draft accepted without the diff would show a person a flow that is not the -# flow the model described, with nothing anywhere saying so. -# -# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door -# can be proven end to end with no API key and no spend. A gate that can only run where a -# credential exists is a gate that never runs. - -#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: -#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you -#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, -#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. -FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") -MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 -MAX_PROMPT_CHARS = 2000 - - -def flow_providers(pin=None): - """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. - - ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and - the draft door used to read `prompt` off the body and nothing else — so the key the client sent - was accepted and dropped, and the picker was a control over nothing. - ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the - ladder stopped offering must not turn every later draft into an error; the caller is told which - rung actually answered, which is the honest half. - """ - by_name = {p["name"]: p for p in PROVIDERS} - live = [n for n in FLOW_PROVIDER_ORDER - if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] - wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() - if wanted and wanted in live: - return [by_name[wanted]] - return [by_name[n] for n in live] - - -def flow_schema(kinds, trigger_keys, table_keys): - """The tool schema the model fills — the catalog's OWN key lists as enums. - - ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the - transport rather than by a check afterwards. A model asked for "any action name" invents - `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a - 400 from `clean_actions` carrying a sentence about a word the person never typed. - ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a - schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a - transport failure (`routes_query._spec_schema` carries the same note, for the same measured - reason). - """ - return { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": ["flow", "refused"], - "description": "refused = this cannot be built from the steps available"}, - "refusal": {"type": "string", - "description": "when kind=refused: ONE plain sentence naming what is " - "missing, in the words a non-technical person would use"}, - "name": {"type": "string", "description": "a short title for the automation"}, - "trigger": {"type": "string", "enum": sorted(trigger_keys), - "description": "what starts this automation"}, - "table": {"type": "string", "enum": sorted(table_keys), - "description": "the database whose records this flow walks, if any"}, - "actions": { - "type": "array", - "maxItems": MAX_DRAFT_ACTIONS, - "items": { - "type": "object", - "properties": { - "kind": {"type": "string", "enum": sorted(kinds)}, - "why": {"type": "string", - "description": "one short sentence: why this step is here"}, - "config": {"type": "object", "additionalProperties": True, - "description": "the step's settings, using ONLY the keys named " - "for that kind in the system message"}, - }, - "required": ["kind"], - }, - }, - }, - "required": ["kind"], - } - - -def flow_system_prompt(catalog, required, triggers, tables): - """What the model is told it may build with — DERIVED, never written down twice. - - Every list here is the server's own: the catalog rows the menu paints, the required-key table - the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases - with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to - the catalog is offered here on the same deploy and a kind removed stops being offered. - """ - lines = ["You build small automations for a business tool. You are given the EXACT set of " - "steps this tool can perform. You may use nothing else.", - "", "THE STEPS YOU MAY USE:"] - # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the - # PHRASE is the only human wording of that requirement anywhere — the one the runner's own - # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a - # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing - # what to call the setting. Handing it one and hoping is how a draft comes back configured - # against a key the validator drops. - req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} - for row in catalog or []: - if not row.get("ready"): - continue - kind = str(row.get("kind") or "") - need = req.get(kind) or [] - lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" - + (f" REQUIRED settings: {', '.join(need)}" if need else "")) - lines += ["", "WHAT CAN START AN AUTOMATION:"] - for t in triggers or []: - if t.get("planned") or not t.get("ready", True): - continue - lines.append(f"- {t.get('key')}: {t.get('label')}") - lines += ["", "THE DATABASES THIS PERSON HAS:"] - for t in (tables or [])[:40]: - cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) - lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") - if not tables: - lines.append("- (none - do not name a database)") - lines += [ - "", - "RULES:", - "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " - "answer kind=refused and say plainly which capability is missing.", - "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " - "rather than inventing a web address, a CSS selector or a column name.", - "3. A web address must start with http:// or https://.", - "4. To use a value from the record the flow is walking, write it as {{Column name}}.", - "5. Name a database only from the list above, by its key.", - "6. Keep it short. Fewer steps that work beat more steps that guess.", - ] - return "\n".join(lines) - - -def _flow_from_tool_call(obj): - """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" - if not isinstance(obj, dict): - return None, "the assistant's answer could not be read" - if str(obj.get("kind") or "") == "refused": - return None, (str(obj.get("refusal") or "").strip() - or "the assistant could not build this from the steps available") - acts = obj.get("actions") - if not isinstance(acts, list) or not acts: - return None, ("the assistant did not produce any steps - try describing what should " - "happen, one action at a time") - out = [] - for a in acts[:MAX_DRAFT_ACTIONS]: - if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): - continue - cfg = a.get("config") - out.append({"kind": str(a["kind"]).strip(), - "config": cfg if isinstance(cfg, dict) else {}, - "why": str(a.get("why") or "").strip()[:200]}) - if not out: - return None, "the assistant's steps could not be read" - return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", - "trigger": str(obj.get("trigger") or "").strip(), - "table": str(obj.get("table") or "").strip(), - # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, - # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need - # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the - # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past - # the twelfth and said nothing, so a model that answered with a twenty-step journey had - # eight steps deleted between the answer and the screen, with no key anywhere in the - # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this - # ceiling is the draft door's own, which makes reporting it the whole obligation. - # ⚠ Found by the verifier that checked this ticket, not by a gate. - "asked": len([a for a in acts if isinstance(a, dict)]), - "actions": out}, "" - - -def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, - st=None, user="", model=None): - """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. - - Exactly one of `draft` and `refusal_sentence` is truthy — the same contract - `web_agent.run_step` keeps, so a caller has no third case to get wrong. - - ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for - the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and - `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and - D asserts it; the two keywords are what D has to pass for the line to be attributable. - """ - # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. - # It parses either way; a reader scanning for the declaration does not look inside a `for`. - global LAST_ANTHROPIC_TRANSPORT - import usage_ledger # noqa: PLC0415 - text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] - if not text: - return None, "type what you want the automation to do", None - kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) - if not kinds: - return None, "this deployment offers no automation steps to build with", None - trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) - if t.get("key") and not t.get("planned")}) or ["manual"] - table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] - tools = [{"type": "function", "function": { - "name": "build_automation", - "description": "Emit the automation, or refuse.", - "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] - messages = [{"role": "system", - "content": flow_system_prompt(catalog, required, triggers, tables)}, - {"role": "user", "content": text}] - - if chat is not None: - # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal - # branches as a live call; only the transport is replaced. - draft, refusal = _flow_from_tool_call(chat(messages, tools)) - return draft, refusal, "injected" - - provs = flow_providers(model) - if not provs: - # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was - # never built [[flag-shipped-without-its-writer]]. - return None, ("the assistant is not configured on this deployment, so an automation cannot " - "be drafted from a description yet"), None - tmo = float(timeout or TIMEOUT_SECONDS) - problems = [] - import providers as _prov - for p in provs: - # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read - # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, - # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the - # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* - # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: - # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter - # 402), so without this branch the drafter cannot answer at all. - # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this - # ladder runs. The parameter is the caller's to send, which is why the wire takes it. - # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch - # below reads one thing. The anthropic side gets there through `anthropic_send` (official - # SDK when installed, the identical raw POST when not); the openai side still posts here - # because there is no shared SDK across three different vendors on that wire. - global LAST_ANTHROPIC_TRANSPORT - try: - if p["shape"] == "anthropic": - req = _prov.anthropic_request( - model=p["model"], key=os.environ[p["env"]].strip(), system=None, - messages=messages, tools=tools, max_tokens=1500, tool_choice="required") - status, body, transport = _prov.anthropic_send(req, timeout=tmo) - LAST_ANTHROPIC_TRANSPORT = transport - else: - r = requests.post(p["url"], timeout=tmo, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1500}) - status = r.status_code - body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: {type(e).__name__}") - continue - if status != 200: - # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off - # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and - # the memo makes the next turn SKIP the empty account instead of paying for it again. - # ⚠ The body is re-serialised for the substring test because the SDK path never had a - # `.text` — that is the one thing this normalisation costs, and it costs nothing else. - detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] - if _prov.is_credit_failure(status, detail): - _prov.mark_no_credit(p["name"]) - problems.append(_prov.refusal_sentence(p["name"], status, detail)) - continue - try: - body = body or {} - if p["shape"] == "anthropic": - _text, args, _refused = _prov.anthropic_read(body) - if _refused: - problems.append(f"{p['name']}: {_refused}") - continue - else: - calls = (((body.get("choices") or [{}])[0].get("message") or {}) - .get("tool_calls") or []) - args = json.loads(calls[0]["function"]["arguments"]) if calls else None - except Exception as e: # noqa: BLE001 - problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") - continue - # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE - # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer - # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot - # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that - # branch by construction. The 200-with-junk case is rare and named here rather than - # silently rounded to zero. - ins, outs = usage_ledger.tokens_from(body) - usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, - total=usage_ledger.total_from(body), st=st, user=user) - draft, refusal = _flow_from_tool_call(args) - if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": - # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more - # expensive rung — the model has just said this cannot be built. `decide()` takes the - # same posture for the same reason. - return None, refusal, p["name"] - if draft is not None: - return draft, "", p["name"] - problems.append(f"{p['name']}: {refusal}") - return None, ("; ".join(problems)[:300] or "no provider answered"), None +"""AI REVIEW — let a model decide which stage a card moves to (wave 23, owner ruling R4/R14). + +⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made +that somebody optionally a MODEL: the engine hands over the record's own values, the review's +prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of +those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit +log a human click writes to, tagged `by: "ai"` with the provider and model that made it. + +⛔ FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a +network failure, a slow answer, a malformed answer, or an answer naming a stage the review does +not offer — all return `("", {...})`, and the caller leaves the card exactly where a human would +have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing +the work. Nothing here can move a card somewhere the review does not already permit. + +⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is +`groq → cerebras → openrouter → anthropic`; the first CONFIGURED provider wins, and Anthropic is +last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` +pins one; `AIOS_AI_MODEL` overrides the model. + +⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH +USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting +the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. +The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the +integrator can overturn it deliberately rather than by drift"*. This is that overturn. + +Its two reasons were real and are both answered rather than waved away: + 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is + not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers + the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no + Anthropic knowledge — it lost some. + 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the + import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the + same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH + manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. +⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that +half of the original reasoning never expired. + +The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and +`stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an +empty content list, so code that indexes `content[0]` unconditionally breaks on it. +""" +from __future__ import annotations + +import json +import os +import re + +import requests + +#: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. +PROVIDERS = [ + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED + # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent + # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo + # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. + {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", + "url": "https://api.groq.com/openai/v1/chat/completions", + "model": "openai/gpt-oss-120b"}, + {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", + "url": "https://api.cerebras.ai/v1/chat/completions", + "model": "gpt-oss-120b"}, + {"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai", + "url": "https://openrouter.ai/api/v1/chat/completions", + "model": "openai/gpt-4o-mini"}, + # ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as + # the backstop for a one-line classification, and this is the cheapest current Claude that + # does it well. A bigger model here would be spending the owner's money to pick between two + # labels it already has in front of it. + {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", + "url": "https://api.anthropic.com/v1/messages", + # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, + # and a deployment whose side rungs are out of credit can raise the tier without a release. + "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, +] +ANTHROPIC_VERSION = "2023-06-01" +TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) +MAX_FIELD_CHARS = 200 # per value handed to the model +MAX_FIELDS = 30 # columns handed to the model +MAX_REASON = 200 + + +def ladder(): + """The providers that are actually usable here, in order. Empty = the feature is off.""" + pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower() + live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()] + if pin: + live = [p for p in live if p["name"] == pin] + return live + + +def configured(): + return bool(ladder()) + + +def _record_text(row, fields): + """The record, as the model sees it. Values are truncated and the column set is bounded — + an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not + need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is + sitting at would be the model reading its own question back.""" + keys = [k for k in (fields or list((row or {}).keys())) + if not str(k).startswith("stage_")][:MAX_FIELDS] + lines = [] + for k in keys: + v = (row or {}).get(k) + if v is None or str(v).strip() == "": + continue + lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}") + return "\n".join(lines) or "(this record has no filled-in values)" + + +def _instruction(prompt, options, label): + return ( + f"You are deciding what happens to one record waiting at a review step called " + f"{label!r} in a workflow.\n\n" + f"The person who built this workflow told you: {prompt}\n\n" + f"Choose EXACTLY ONE of these next steps, by its exact name:\n" + + "\n".join(f"- {o}" for o in options) + + "\n\nAnswer with one line of JSON and nothing else:\n" + '{"choice": "", "reason": ""}\n' + "If the record does not give you enough to decide, answer " + '{"choice": "", "reason": "why not"} and a person will decide instead.' + ) + + +def _parse(text, options): + """The model's line → `(choice, reason)`. A choice that is not one of the offered stages is + DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss + resolved by string distance is how a card ends up somewhere nobody authorised.""" + raw = str(text or "").strip() + obj = None + m = re.search(r"\{.*\}", raw, re.S) + if m: + try: + obj = json.loads(m.group(0)) + except (ValueError, TypeError): + obj = None + if not isinstance(obj, dict): + return "", "" + choice = str(obj.get("choice") or "").strip() + reason = str(obj.get("reason") or "").strip()[:MAX_REASON] + for opt in options: + if choice.lower() == str(opt).lower(): + return str(opt), reason # the OFFERED spelling wins, never the model's + return "", reason + + +# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. +# +# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s +# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token +# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach +# this transport rather than be bolted onto its caller — the caller never sees the body. +# +# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The +# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), +# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers +# shape this repo keeps paying for. +def _call_openai(p, model, system, user, timeout): + r = requests.post(p["url"], timeout=timeout, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": model, "max_tokens": 300, "temperature": 0, + "messages": [{"role": "system", "content": system}, + {"role": "user", "content": user}]}) + if r.status_code >= 400: + # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused + # before running the model has nothing to bill. The call is still COUNTED by `decide`. + return "", f"{p['name']} answered {r.status_code}", None + body = r.json() + choices = body.get("choices") or [] + if not choices: + return "", f"{p['name']} returned no choices", body + return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body + + +#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. +#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say +#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. +LAST_ANTHROPIC_TRANSPORT = "" + + +def _call_anthropic(p, model, system, user, timeout): + """One classification turn on the Messages API, through `providers.anthropic_send`. + + ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which + prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is + absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises + both transports to `(status, body)`, so every line below this call is untouched. + """ + global LAST_ANTHROPIC_TRANSPORT + import providers as _prov # noqa: PLC0415 + # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is + # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. + req = _prov.anthropic_request( + model=model, key=os.environ[p["env"]].strip(), system=system, + messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) + status, body, transport = _prov.anthropic_send(req, timeout=timeout) + LAST_ANTHROPIC_TRANSPORT = transport + if status >= 400 or status == 0: + # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the + # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" + # (which is what this line used to say, and what the owner quoted back at us) cannot recur. + return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None + body = body or {} + # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so + # reading content[0] before this check turns a refusal into an IndexError inside a run. + # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. + if body.get("stop_reason") == "refusal": + return "", "anthropic declined to answer this record", body + parts = [b.get("text") or "" for b in (body.get("content") or []) + if isinstance(b, dict) and b.get("type") == "text"] + if not parts: + return "", "anthropic returned no text", body + return "".join(parts), "", body + + +def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): + """Pick this record's next stage. Returns `(choice, meta)`. + + `choice` is "" whenever a person should decide — which is every failure mode there is. + `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. + + ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because + this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it + HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as + unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for + why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI + dependency). + """ + import usage_ledger # noqa: PLC0415 + opts = [str(o) for o in (options or []) if str(o).strip()] + if not opts: + return "", {"problem": "the review offers no next stages"} + if not str(prompt or "").strip(): + return "", {"problem": "the review has no prompt for the model to follow"} + live = ladder() + if not live: + return "", {"problem": "no AI provider is configured on this deployment"} + system = _instruction(prompt, opts, label) + user = "Here is the record:\n\n" + _record_text(row, fields) + tmo = float(timeout or TIMEOUT_SECONDS) + override = (os.environ.get("AIOS_AI_MODEL") or "").strip() + problems = [] + for p in live: + model = override or p["model"] + body = None + try: + text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( + p, model, system, user, tmo) + except Exception as e: # noqa: BLE001 + text, err = "", f"{p['name']} failed: {type(e).__name__}" + # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. + # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an + # unusable answer. A meter that books only successes reports a cheap week for a run that + # spent its budget being declined — and a declined call is billed. The only path that does + # NOT record is a transport that never reached the vendor (`body is None`), which spent + # nothing. + if body is not None: + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("ai_review", p["name"], model, ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + if err: + problems.append(err) + continue # ladder: a dead provider degrades to the next one + choice, reason = _parse(text, opts) + if not choice: + # The provider ANSWERED and declined (or answered unusably). That is a decision about + # this record, not a fault in the provider, so it does NOT fall through to a more + # expensive one — the card goes to a human, which is what the model just asked for. + return "", {"provider": p["name"], "model": model, + "problem": reason or "the model did not choose one of the stages"} + return choice, {"provider": p["name"], "model": model, "reason": reason} + return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} + + +# ══════════════════════════════════════════════════════════════════ the FLOW WRITER +# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. +# +# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the +# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: +# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the +# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object +# in that vocabulary. +# +# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This +# is the one module in the API package that already holds an LLM ladder and its credentials, so a +# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs +# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 +# shipped three finished routers 404-dead behind green gates. The DOOR is a route on +# `routes_automation.py`, which is already mounted. +# +# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds +# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are +# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema +# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters +# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise +# and answers 200, so a draft accepted without the diff would show a person a flow that is not the +# flow the model described, with nothing anywhere saying so. +# +# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door +# can be proven end to end with no API key and no spend. A gate that can only run where a +# credential exists is a gate that never runs. + +#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: +#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you +#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, +#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. +FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") +MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 +MAX_PROMPT_CHARS = 2000 + + +def flow_providers(pin=None): + """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. + + ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and + the draft door used to read `prompt` off the body and nothing else — so the key the client sent + was accepted and dropped, and the picker was a control over nothing. + ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the + ladder stopped offering must not turn every later draft into an error; the caller is told which + rung actually answered, which is the honest half. + """ + by_name = {p["name"]: p for p in PROVIDERS} + live = [n for n in FLOW_PROVIDER_ORDER + if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] + wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() + if wanted and wanted in live: + return [by_name[wanted]] + return [by_name[n] for n in live] + + +def flow_schema(kinds, trigger_keys, table_keys): + """The tool schema the model fills — the catalog's OWN key lists as enums. + + ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the + transport rather than by a check afterwards. A model asked for "any action name" invents + `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a + 400 from `clean_actions` carrying a sentence about a word the person never typed. + ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a + schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a + transport failure (`routes_query._spec_schema` carries the same note, for the same measured + reason). + """ + return { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": ["flow", "refused"], + "description": "refused = this cannot be built from the steps available"}, + "refusal": {"type": "string", + "description": "when kind=refused: ONE plain sentence naming what is " + "missing, in the words a non-technical person would use"}, + "name": {"type": "string", "description": "a short title for the automation"}, + "trigger": {"type": "string", "enum": sorted(trigger_keys), + "description": "what starts this automation"}, + "table": {"type": "string", "enum": sorted(table_keys), + "description": "the database whose records this flow walks, if any"}, + "actions": { + "type": "array", + "maxItems": MAX_DRAFT_ACTIONS, + "items": { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": sorted(kinds)}, + "why": {"type": "string", + "description": "one short sentence: why this step is here"}, + "config": {"type": "object", "additionalProperties": True, + "description": "the step's settings, using ONLY the keys named " + "for that kind in the system message"}, + }, + "required": ["kind"], + }, + }, + }, + "required": ["kind"], + } + + +def flow_system_prompt(catalog, required, triggers, tables): + """What the model is told it may build with — DERIVED, never written down twice. + + Every list here is the server's own: the catalog rows the menu paints, the required-key table + the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases + with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to + the catalog is offered here on the same deploy and a kind removed stops being offered. + """ + lines = ["You build small automations for a business tool. You are given the EXACT set of " + "steps this tool can perform. You may use nothing else.", + "", "THE STEPS YOU MAY USE:"] + # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the + # PHRASE is the only human wording of that requirement anywhere — the one the runner's own + # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a + # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing + # what to call the setting. Handing it one and hoping is how a draft comes back configured + # against a key the validator drops. + req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} + for row in catalog or []: + if not row.get("ready"): + continue + kind = str(row.get("kind") or "") + need = req.get(kind) or [] + lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" + + (f" REQUIRED settings: {', '.join(need)}" if need else "")) + lines += ["", "WHAT CAN START AN AUTOMATION:"] + for t in triggers or []: + if t.get("planned") or not t.get("ready", True): + continue + lines.append(f"- {t.get('key')}: {t.get('label')}") + lines += ["", "THE DATABASES THIS PERSON HAS:"] + for t in (tables or [])[:40]: + cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) + lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") + if not tables: + lines.append("- (none - do not name a database)") + lines += [ + "", + "RULES:", + "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " + "answer kind=refused and say plainly which capability is missing.", + "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " + "rather than inventing a web address, a CSS selector or a column name.", + "3. A web address must start with http:// or https://.", + "4. To use a value from the record the flow is walking, write it as {{Column name}}.", + "5. Name a database only from the list above, by its key.", + "6. Keep it short. Fewer steps that work beat more steps that guess.", + ] + return "\n".join(lines) + + +def _flow_from_tool_call(obj): + """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" + if not isinstance(obj, dict): + return None, "the assistant's answer could not be read" + if str(obj.get("kind") or "") == "refused": + return None, (str(obj.get("refusal") or "").strip() + or "the assistant could not build this from the steps available") + acts = obj.get("actions") + if not isinstance(acts, list) or not acts: + return None, ("the assistant did not produce any steps - try describing what should " + "happen, one action at a time") + out = [] + for a in acts[:MAX_DRAFT_ACTIONS]: + if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): + continue + cfg = a.get("config") + out.append({"kind": str(a["kind"]).strip(), + "config": cfg if isinstance(cfg, dict) else {}, + "why": str(a.get("why") or "").strip()[:200]}) + if not out: + return None, "the assistant's steps could not be read" + return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", + "trigger": str(obj.get("trigger") or "").strip(), + "table": str(obj.get("table") or "").strip(), + # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, + # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need + # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the + # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past + # the twelfth and said nothing, so a model that answered with a twenty-step journey had + # eight steps deleted between the answer and the screen, with no key anywhere in the + # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this + # ceiling is the draft door's own, which makes reporting it the whole obligation. + # ⚠ Found by the verifier that checked this ticket, not by a gate. + "asked": len([a for a in acts if isinstance(a, dict)]), + "actions": out}, "" + + +def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, + st=None, user="", model=None): + """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. + + Exactly one of `draft` and `refusal_sentence` is truthy — the same contract + `web_agent.run_step` keeps, so a caller has no third case to get wrong. + + ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for + the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and + `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and + D asserts it; the two keywords are what D has to pass for the line to be attributable. + """ + # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. + # It parses either way; a reader scanning for the declaration does not look inside a `for`. + global LAST_ANTHROPIC_TRANSPORT + import usage_ledger # noqa: PLC0415 + text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] + if not text: + return None, "type what you want the automation to do", None + kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) + if not kinds: + return None, "this deployment offers no automation steps to build with", None + trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) + if t.get("key") and not t.get("planned")}) or ["manual"] + table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] + tools = [{"type": "function", "function": { + "name": "build_automation", + "description": "Emit the automation, or refuse.", + "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] + messages = [{"role": "system", + "content": flow_system_prompt(catalog, required, triggers, tables)}, + {"role": "user", "content": text}] + + if chat is not None: + # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal + # branches as a live call; only the transport is replaced. + draft, refusal = _flow_from_tool_call(chat(messages, tools)) + return draft, refusal, "injected" + + provs = flow_providers(model) + if not provs: + # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was + # never built [[flag-shipped-without-its-writer]]. + return None, ("the assistant is not configured on this deployment, so an automation cannot " + "be drafted from a description yet"), None + tmo = float(timeout or TIMEOUT_SECONDS) + problems = [] + import providers as _prov + for p in provs: + # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read + # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, + # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the + # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* + # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: + # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter + # 402), so without this branch the drafter cannot answer at all. + # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this + # ladder runs. The parameter is the caller's to send, which is why the wire takes it. + # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch + # below reads one thing. The anthropic side gets there through `anthropic_send` (official + # SDK when installed, the identical raw POST when not); the openai side still posts here + # because there is no shared SDK across three different vendors on that wire. + global LAST_ANTHROPIC_TRANSPORT + try: + if p["shape"] == "anthropic": + req = _prov.anthropic_request( + model=p["model"], key=os.environ[p["env"]].strip(), system=None, + messages=messages, tools=tools, max_tokens=1500, tool_choice="required") + status, body, transport = _prov.anthropic_send(req, timeout=tmo) + LAST_ANTHROPIC_TRANSPORT = transport + else: + r = requests.post(p["url"], timeout=tmo, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1500}) + status = r.status_code + body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: {type(e).__name__}") + continue + if status != 200: + # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off + # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and + # the memo makes the next turn SKIP the empty account instead of paying for it again. + # ⚠ The body is re-serialised for the substring test because the SDK path never had a + # `.text` — that is the one thing this normalisation costs, and it costs nothing else. + detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] + if _prov.is_credit_failure(status, detail): + _prov.mark_no_credit(p["name"]) + problems.append(_prov.refusal_sentence(p["name"], status, detail)) + continue + try: + body = body or {} + if p["shape"] == "anthropic": + _text, args, _refused = _prov.anthropic_read(body) + if _refused: + problems.append(f"{p['name']}: {_refused}") + continue + else: + calls = (((body.get("choices") or [{}])[0].get("message") or {}) + .get("tool_calls") or []) + args = json.loads(calls[0]["function"]["arguments"]) if calls else None + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") + continue + # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE + # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer + # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot + # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that + # branch by construction. The 200-with-junk case is rare and named here rather than + # silently rounded to zero. + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + draft, refusal = _flow_from_tool_call(args) + if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": + # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more + # expensive rung — the model has just said this cannot be built. `decide()` takes the + # same posture for the same reason. + return None, refusal, p["name"] + if draft is not None: + return draft, "", p["name"] + problems.append(f"{p['name']}: {refusal}") + return None, ("; ".join(problems)[:300] or "no provider answered"), None diff --git a/api/connectors_tt.py b/api/connectors_tt.py index 0603f4f23f7075252fdcef06ce19371ac1620523..60d96945a35f7fd8ca79443bf70cdec87e0a4c53 100644 --- a/api/connectors_tt.py +++ b/api/connectors_tt.py @@ -1,568 +1,568 @@ -"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). - -Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an -automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file -exists at all rather than another thousand lines inside the engine. - -⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 -post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read -live from `GET /datasets/{id}/metadata` for **$0.00** and written down in -`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at -close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a -name below reads through a candidate list it is because the vendor has two names for one fact -(`biography`/`signature`, `region`/`country`), never because the name is uncertain. - -⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — -public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The -vendor key is a key to a SUPPLIER. - -⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE -OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and -`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not -Instagram's — and re-implementing them here would be a second copy of the deferral handling, the -truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. -Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from -`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. -⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because -the sentence above is the kind that quietly stops being true. - -⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** - 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared - envelope keys (`timestamp`, `input`) that no metadata call mentions; - 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` - and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one - TikTok claim that matters most (`play_count`) is exactly a declaration. -""" -from __future__ import annotations - -import automation_engine as engine -# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an -# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were -# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever -# read them off this module (the engine imports them from the transport itself), so they were four -# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. -from connectors_bd import ( - _bd_first_url, - _bd_flag, - _bd_list, - _bd_source_payload, - _first, - _ig_int, - bd_scrape, -) - -# --------------------------------------------------------------------------------------------- -# THE DATASETS -# --------------------------------------------------------------------------------------------- -# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET -# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a -# full field list, and the two DISCOVERY halves answer **404 for our key**: -# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). -# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's -# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. -TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records -TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields -TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields - -#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` -#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. -TT_POST_TYPE_VIDEO = "video" -TT_POST_TYPE_CONTENT = "content" - -#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept -#: beside the dataset ids because it is the same class of vendor fact. -#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical -#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own -#: batch calls, and the gate asserts the two agree rather than trusting that they do. -TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" - - -def tt_profile_url(handle): - """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" - h = str(handle or "").strip().lstrip("@") - return TT_PROFILE_URL.format(handle=h) if h else "" - - -# --------------------------------------------------------------------------------------------- -# THE FIELD MAPS -# --------------------------------------------------------------------------------------------- -# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared -# in `automation_engine`. The rules they all obey, stated once: -# -# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, -# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` -# returns `None` (never 0) when nothing matches, which is what makes that possible. -# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` -# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) -# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the -# vendor's side is preserved rather than silently discarded while our column model catches up. -# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` -# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they -# ride in `source_payload` where they make no claim. - - -def _tt_str(node, *names): - """The first non-empty string among `names`, or None when the vendor sent nothing. - - ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — - an empty string written into a cell claims "we looked and it is empty". - """ - v = _first(node, *names) - if v is None: - return None - s = str(v).strip() - return s or None - - -def _tt_pct(node, *names): - """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. - - ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to - the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — - measured on the Instagram side, and TikTok sends the same shape on all three of its rates. - """ - v = _first(node, *names) - if v is None: - return None - out = engine._pct100(v) - return out or None - - -def _tt_day(node, *names): - """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" - v = _first(node, *names) - if v is None: - return None - return engine._day(v) or None - - -def _drop_blanks(row): - """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" - return {k: v for k, v in row.items() if v is not None and v != ""} - - -def normalize_profile(node, handle=""): - """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. - - ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: - * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` - populated on 85% of rows and they carry the same text; - * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented - two-letter-ISO description, `country` has no description at all. - ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its - `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the - only count of its kind the dataset offers. - """ - node = node if isinstance(node, dict) else {} - account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "handle": account, - "full_name": _tt_str(node, "nickname"), - "tt_id": _tt_str(node, "id"), - "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), - "bio": _tt_str(node, "biography", "signature"), - "external_url": _bd_first_url(_first(node, "bio_link")), - "verified": _bd_flag(node, "is_verified"), - "is_private": _bd_flag(node, "is_private"), - # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* - # on their own description. It is the closest thing TikTok has to Instagram's - # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so - # the approximation only ever fills a cell the vendor actually answered. - "is_business": _bd_flag(node, "is_commerce_user"), - "followers": _ig_int(_first(node, "followers")), - "following": _ig_int(_first(node, "following")), - "posts_count": _ig_int(_first(node, "videos_count")), - # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no - # nulls). It is not a post-level number and it is not our `likes` column, which is why it - # is stored under a different name. - "likes_received": _ig_int(_first(node, "likes")), - "avg_engagement": _tt_pct(node, "awg_engagement_rate"), - "like_engagement": _tt_pct(node, "like_engagement_rate"), - "comment_engagement": _tt_pct(node, "comment_engagement_rate"), - "country_code": _tt_str(node, "region", "country"), - "region": _tt_str(node, "region"), - "predicted_lang": _tt_str(node, "predicted_lang"), - # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT - # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, - # which is why the append law dates a snapshot by when WE read it. - "account_created_at": _tt_day(node, "create_time"), - "source_payload": _bd_source_payload(node), - }) - - -def tt_post_type(node): - """A TikTok post row → one of OUR three type options, or None. - - ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / - `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the - vendor's token would fail `_clean_field`'s option check on the way in and would put an - untranslated API word in front of a user on the way out. - - So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the - row carries more than one `carousel_images` entry, which is what a carousel IS on either - network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the - token cannot express it, so inferring `carousel` from the word would be inventing a fact. - ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and - that is exactly what would make the wrong default invisible). - """ - raw = str((node or {}).get("post_type") or "").strip().lower() - if raw == TT_POST_TYPE_VIDEO: - return "video" - if raw != TT_POST_TYPE_CONTENT: - return None - images = (node or {}).get("carousel_images") - return "carousel" if isinstance(images, list) and len(images) > 1 else "image" - - -def normalize_post(node): - """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. - - ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and - Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same - fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and - `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would - manufacture a second measurement that a rollup could average or double-count, which is a worse - outcome than the missing column it would paper over. - - ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and - the probe measured them as the same shape. That equality is what lets the comments→posts link - join with NO normaliser, which the Instagram side never had. - ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). - ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and - countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares - no equivalent, so `paid_partnership`/`partner` have no column on this family at all. - """ - node = node if isinstance(node, dict) else {} - shortcode = _tt_str(node, "shortcode", "post_id") - if not shortcode: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "shortcode": shortcode, - # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. - # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the - # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so - # one name means one thing across both corpora. Reading the display name silently broke the - # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched - # NOTHING, so a person could not filter posts by creator and a rollup would count zero. - # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a - # different fact, and filling a join key with it is worse than leaving it blank — a blank - # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. - # The URL is the honest second source: it carries the handle by construction. - "influencer_key": (_tt_str(node, "account_id") - or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), - "posted_at": _tt_day(node, "create_time"), - "type": tt_post_type(node), - "caption": _tt_str(node, "description"), - "url": _tt_str(node, "url"), - "hashtags": _bd_list(node, "hashtags"), - "tagged_location": _tt_str(node, "commerce_info"), - "views": _ig_int(_first(node, "play_count")), - "likes": _ig_int(_first(node, "digg_count")), - "comments": _ig_int(_first(node, "comment_count")), - "shares": _ig_int(_first(node, "num_share_count")), - "saves": _ig_int(_first(node, "collect_count")), - "video_duration": _ig_int(_first(node, "video_duration")), - "source_payload": _bd_source_payload(node), - }) - - -def normalize_comment(node): - """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. - - ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR - `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in - `source_payload`. Reading the array's length instead would be a second, disagreeing answer to - a question the vendor already answers — and it would disagree, because a page of replies is not - all of them. - ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is - text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor - that changes its mind cannot change ours. - ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged - PII) stay in `source_payload` and are promoted to no column, which is the same posture the - Instagram comment schema takes for the same D-24 reason. - """ - node = node if isinstance(node, dict) else {} - comment_key = _tt_str(node, "comment_id") - if not comment_key: - return None - return _drop_blanks({ - "platform": engine.PLATFORM_TIKTOK, - "comment_key": comment_key, - "shortcode": _tt_str(node, "post_id"), - # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the - # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); - # primary first, so a row carrying the rich form is not silently served the plain one. - # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. - "text": _tt_str(node, "comment_text", "comment_text_only"), - "commented_at": _tt_day(node, "date_created"), - "likes": _ig_int(_first(node, "num_likes")), - "replies": _ig_int(_first(node, "num_replies")), - "source_payload": _bd_source_payload(node), - }) - - -#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them -#: rather than naming three functions, and so adding a fourth dataset is one entry. -TT_NORMALIZERS = { - "tt_profile": normalize_profile, - "tt_post_metrics": normalize_post, - "tt_comments": normalize_comment, -} - - -# --------------------------------------------------------------------------------------------- -# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) -# --------------------------------------------------------------------------------------------- - -def tt_handle(url): - """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. - - Deliberately permissive about the input and strict about the output, because the two callers - hand it different things: an automation stores whatever a person typed in the profile column - (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean - `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot - resolve to two different handles. - """ - s = str(url or "").strip() - if not s: - return "" - if "tiktok.com" in s.lower(): - # Everything after the first `@`, up to the next path segment or query. - tail = s.split("@", 1)[1] if "@" in s else "" - s = tail.split("/")[0].split("?")[0].split("#")[0] - s = s.strip().lstrip("@").strip() - # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. - return s if s and all(c.isalnum() or c in "._" for c in s) else "" - - -def tt_post_urls(node, limit=0): - """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. - - ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row - we have already bought, so the posts read is a scrape of links we hold, never a search for them. - The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, - so a design that reached for either would not merely be dearer, it would not work. - - ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an - array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe - read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's - `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** - (measured below), so the reader built against the quoted sentence found nothing, forever, in - silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration - of X"* are different claims, and prose cannot be told apart by a reader downstream — which is - why the correction names the method, not just the value. - - ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, - and preferring whichever happened to be longer is how one creator's window silently differs - from another's. - ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, - validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is - never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, - not a number this function may invent. - """ - raw = (node or {}).get("top_videos") - out = [] - for item in raw if isinstance(raw, list) else []: - # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. - # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a - # live handle returns **19 DICTS**, keyed - # `video_url · video_id · playcount · diggcount · commentcount · share_count · - # favorites_count · create_date · cover_image`. - # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and - # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says - # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field - # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's - # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was - # the first. [[reachable-is-not-the-same-as-built]] - # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY - # list on every real profile — post capture could not have worked for anybody, and the - # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture - # written from a schema tests the schema. - # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept - # because it costs nothing and is what a future corpus revision would most likely use. The - # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents - # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as - # before, rather than to a URL built out of a guess. - # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring - # whichever array happened to be longer is how one creator's window silently differs from - # another's, and that reasoning is unchanged by this correction. - if isinstance(item, dict): - u = str(item.get("video_url") or item.get("url") or "").strip() - else: - u = str(item or "").strip() - if u.lower().startswith("http") and u not in out: - out.append(u) - return out[:limit] if limit and limit > 0 else out - - -def pull_posts_tt(post_urls, log=print, deferred=None): - """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. - - ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side - measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still - running at 67 minutes. Nothing here loops per URL. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] posts: {note}") - return [], note - out = [r for r in (normalize_post(n) for n in rows) if r] - return out, "" - - -def pull_comments_tt(post_urls, log=print, deferred=None): - """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. - - ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on - both networks: a comments scrape ingests identifiable third parties who never entered anybody's - list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes - none of them to a column; this function adds no new exposure, it just has to be asked for. - """ - urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] - if not urls: - return [], "" - rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) - if note: - log(f"[aios-tt] comments: {note}") - return [], note - out = [r for r in (normalize_comment(n) for n in rows) if r] - return out, "" - - -#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. -#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral -#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's -#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse -#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. -TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) - - -def _media_deferrals(deferred): - """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. - - ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the - handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one - kind here, so the id already carries everything a mapper choice depends on — and importing - Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). - """ - out = [] - for d in deferred or []: - if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: - out.append(dict(d)) - return out - - -def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, - max_posts=0, post_metrics=False, comment_metrics=False): - """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. - - `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, - so the engine's enrich branch treats every network identically and no caller learns a new - shape. - - ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's - do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — - no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` - then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction - rather than by a rule: their input IS a post permalink, so asking for comments with post capture - off is a request with no subject, and it returns none instead of quietly buying posts nobody - asked for. - - ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than - pessimistic. The Instagram contract reads `ok` only when identity AND media both landed - (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower - count and no posts must not paint green over a posts table that did not grow"*). So: posts not - ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → - `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", - never "did the function finish". - - ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the - paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, - with its own note explaining that a multi-provider chain is a promise something walks it and - that nothing walks Instagram's second name today either. So a refusal here is final, and it - says so instead of implying a retry somewhere. - """ - handle = tt_handle(url) - if not handle: - return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", - "note": f"{url!r} is not a TikTok profile URL or handle"} - - # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from - # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not - # happen; a miss falls through to the single-URL call below. - cached = prefetch.get(handle) if isinstance(prefetch, dict) else None - _deferred = [] - if isinstance(cached, dict) and cached: - rows, note = [cached], "" - else: - rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) - - node = rows[0] if rows else {} - profile = normalize_profile(node, handle) if node else {} - # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` - # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and - # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly - # this reason, and answering "0 followers" instead is the failure it exists to prevent. - unreadable = profile.get("followers") is None and profile.get("following") is None - if note or unreadable: - # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still - # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the - # same record. That was live on the Instagram profile path until 2026-08-09 — measured on - # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being - # reintroduced here by omission. - if isinstance(pending_profile, list): - for d in _deferred: - pending_profile.append({**d, "kind": "profile", "influencer": handle}) - why = note or ("the scrape answered, but no follower/following counts were readable in it " - "(the field names may have moved - see tiktok-capture.md)") - return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", - "deferredProfile": [d.get("snapshotId") for d in _deferred], - "note": why} - - # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ - if not post_metrics: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; post capture is off for this step"} - urls = tt_post_urls(node, limit=max_posts) - if not urls: - # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and - # saying so is what stops the next run paying to be told the same thing. - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", - "note": note or "profile read; this account's row carried no post links"} - posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) - comments, c_note = ([], "") - if comment_metrics and posts: - # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — - # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused - # is not silently asked about again one rung later. - comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], - log=log, deferred=_deferred) - # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is - # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to - # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE - # deferral is impossible — the profile branch above returns `blocked` on any note — so **every - # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose - # collector writes preset profile cells onto somebody's record from post rows. The engine keeps - # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). - # ⇒ So this returns them under their OWN key, filtered by dataset identity - # (`_media_deferrals`), and the engine files them in the metric queue with the handle it - # already holds. Returning rather than appending also keeps the queue's vocabulary out of a - # connector: this module knows which CORPUS deferred, never what the engine calls it. - # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that - # matters most — that is exactly the run where the vendor took too long, so a caller reading - # the ids only from the success path would lose every batch it actually paid for. - deferred_media = _media_deferrals(_deferred) - if not posts: - return {"state": "partial", "profile": profile, "posts": [], "comments": [], - "via": "brightdata", "deferredMedia": deferred_media, - "note": p_note or note or "profile read; the post source returned nothing"} - return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, - "via": "brightdata", "deferredMedia": deferred_media, - "note": c_note or note or ""} +"""connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2). + +Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an +automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file +exists at all rather than another thousand lines inside the engine. + +⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43 +post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read +live from `GET /datasets/{id}/metadata` for **$0.00** and written down in +`.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at +close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a +name below reads through a candidate list it is because the vendor has two names for one fact +(`biography`/`signature`, `region`/`country`), never because the name is uncertain. + +⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned — +public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The +vendor key is a key to a SUPPLIER. + +⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE +OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and +`bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not +Instagram's — and re-implementing them here would be a second copy of the deferral handling, the +truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug. +Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from +`connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier. +⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because +the sentence above is the kind that quietly stops being true. + +⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:** + 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared + envelope keys (`timestamp`, `input`) that no metadata call mentions; + 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number` + and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one + TikTok claim that matters most (`play_count`) is exactly a declaration. +""" +from __future__ import annotations + +import automation_engine as engine +# ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an +# import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were +# imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever +# read them off this module (the engine imports them from the transport itself), so they were four +# lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form. +from connectors_bd import ( + _bd_first_url, + _bd_flag, + _bd_list, + _bd_source_payload, + _first, + _ig_int, + bd_scrape, +) + +# --------------------------------------------------------------------------------------------- +# THE DATASETS +# --------------------------------------------------------------------------------------------- +# ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET +# /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a +# full field list, and the two DISCOVERY halves answer **404 for our key**: +# `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword). +# ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's +# does. A ticket that reaches for a by-keyword endpoint is reaching for a 404. +TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records +TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields +TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields + +#: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description` +#: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`. +TT_POST_TYPE_VIDEO = "video" +TT_POST_TYPE_CONTENT = "content" + +#: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept +#: beside the dataset ids because it is the same class of vendor fact. +#: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical +#: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own +#: batch calls, and the gate asserts the two agree rather than trusting that they do. +TT_PROFILE_URL = "https://www.tiktok.com/@{handle}" + + +def tt_profile_url(handle): + """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`.""" + h = str(handle or "").strip().lstrip("@") + return TT_PROFILE_URL.format(handle=h) if h else "" + + +# --------------------------------------------------------------------------------------------- +# THE FIELD MAPS +# --------------------------------------------------------------------------------------------- +# Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared +# in `automation_engine`. The rules they all obey, stated once: +# +# * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED, +# so a later, richer pull fills it instead of being overwritten by this one's silence. `_first` +# returns `None` (never 0) when nothing matches, which is what makes that possible. +# * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank` +# rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.) +# * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the +# vendor's side is preserved rather than silently discarded while our column model catches up. +# * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id` +# (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they +# ride in `source_payload` where they make no claim. + + +def _tt_str(node, *names): + """The first non-empty string among `names`, or None when the vendor sent nothing. + + ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest — + an empty string written into a cell claims "we looked and it is empty". + """ + v = _first(node, *names) + if v is None: + return None + s = str(v).strip() + return s or None + + +def _tt_pct(node, *names): + """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None. + + ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to + the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` — + measured on the Instagram side, and TikTok sends the same shape on all three of its rates. + """ + v = _first(node, *names) + if v is None: + return None + out = engine._pct100(v) + return out or None + + +def _tt_day(node, *names): + """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day.""" + v = _first(node, *names) + if v is None: + return None + return engine._day(v) or None + + +def _drop_blanks(row): + """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above.""" + return {k: v for k, v in row.items() if v is not None and v != ""} + + +def normalize_profile(node, handle=""): + """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape. + + ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess: + * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature` + populated on 85% of rows and they carry the same text; + * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented + two-letter-ISO description, `country` has no description at all. + ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its + `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the + only count of its kind the dataset offers. + """ + node = node if isinstance(node, dict) else {} + account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@") + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "handle": account, + "full_name": _tt_str(node, "nickname"), + "tt_id": _tt_str(node, "id"), + "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None), + "bio": _tt_str(node, "biography", "signature"), + "external_url": _bd_first_url(_first(node, "bio_link")), + "verified": _bd_flag(node, "is_verified"), + "is_private": _bd_flag(node, "is_private"), + # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"* + # on their own description. It is the closest thing TikTok has to Instagram's + # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so + # the approximation only ever fills a cell the vendor actually answered. + "is_business": _bd_flag(node, "is_commerce_user"), + "followers": _ig_int(_first(node, "followers")), + "following": _ig_int(_first(node, "following")), + "posts_count": _ig_int(_first(node, "videos_count")), + # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no + # nulls). It is not a post-level number and it is not our `likes` column, which is why it + # is stored under a different name. + "likes_received": _ig_int(_first(node, "likes")), + "avg_engagement": _tt_pct(node, "awg_engagement_rate"), + "like_engagement": _tt_pct(node, "like_engagement_rate"), + "comment_engagement": _tt_pct(node, "comment_engagement_rate"), + "country_code": _tt_str(node, "region", "country"), + "region": _tt_str(node, "region"), + "predicted_lang": _tt_str(node, "predicted_lang"), + # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT + # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram, + # which is why the append law dates a snapshot by when WE read it. + "account_created_at": _tt_day(node, "create_time"), + "source_payload": _bd_source_payload(node), + }) + + +def tt_post_type(node): + """A TikTok post row → one of OUR three type options, or None. + + ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` / + `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the + vendor's token would fail `_clean_field`'s option check on the way in and would put an + untranslated API word in front of a user on the way out. + + So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the + row carries more than one `carousel_images` entry, which is what a carousel IS on either + network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the + token cannot express it, so inferring `carousel` from the word would be inventing a fact. + ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and + that is exactly what would make the wrong default invisible). + """ + raw = str((node or {}).get("post_type") or "").strip().lower() + if raw == TT_POST_TYPE_VIDEO: + return "video" + if raw != TT_POST_TYPE_CONTENT: + return None + images = (node or {}).get("carousel_images") + return "carousel" if isinstance(images, list) and len(images) > 1 else "image" + + +def normalize_post(node): + """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity. + + ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and + Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same + fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and + `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would + manufacture a second measurement that a rollup could average or double-count, which is a worse + outcome than the missing column it would paper over. + + ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and + the probe measured them as the same shape. That equality is what lets the comments→posts link + join with NO normaliser, which the Instagram side never had. + ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor). + ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and + countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares + no equivalent, so `paid_partnership`/`partner` have no column on this family at all. + """ + node = node if isinstance(node, dict) else {} + shortcode = _tt_str(node, "shortcode", "post_id") + if not shortcode: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "shortcode": shortcode, + # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12. + # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the + # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so + # one name means one thing across both corpora. Reading the display name silently broke the + # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched + # NOTHING, so a person could not filter posts by creator and a rollup would count zero. + # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a + # different fact, and filling a join key with it is worse than leaving it blank — a blank + # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]]. + # The URL is the honest second source: it carries the handle by construction. + "influencer_key": (_tt_str(node, "account_id") + or tt_handle(_tt_str(node, "profile_url", "url") or "") or None), + "posted_at": _tt_day(node, "create_time"), + "type": tt_post_type(node), + "caption": _tt_str(node, "description"), + "url": _tt_str(node, "url"), + "hashtags": _bd_list(node, "hashtags"), + "tagged_location": _tt_str(node, "commerce_info"), + "views": _ig_int(_first(node, "play_count")), + "likes": _ig_int(_first(node, "digg_count")), + "comments": _ig_int(_first(node, "comment_count")), + "shares": _ig_int(_first(node, "num_share_count")), + "saves": _ig_int(_first(node, "collect_count")), + "video_duration": _ig_int(_first(node, "video_duration")), + "source_payload": _bd_source_payload(node), + }) + + +def normalize_comment(node): + """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id. + + ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR + `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in + `source_payload`. Reading the array's length instead would be a second, disagreeing answer to + a question the vendor already answers — and it would disagree, because a page of replies is not + all of them. + ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is + text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor + that changes its mind cannot change ours. + ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged + PII) stay in `source_payload` and are promoted to no column, which is the same posture the + Instagram comment schema takes for the same D-24 reason. + """ + node = node if isinstance(node, dict) else {} + comment_key = _tt_str(node, "comment_id") + if not comment_key: + return None + return _drop_blanks({ + "platform": engine.PLATFORM_TIKTOK, + "comment_key": comment_key, + "shortcode": _tt_str(node, "post_id"), + # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the + # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both); + # primary first, so a row carrying the rich form is not silently served the plain one. + # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`. + "text": _tt_str(node, "comment_text", "comment_text_only"), + "commented_at": _tt_day(node, "date_created"), + "likes": _ig_int(_first(node, "num_likes")), + "replies": _ig_int(_first(node, "num_replies")), + "source_payload": _bd_source_payload(node), + }) + + +#: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them +#: rather than naming three functions, and so adding a fourth dataset is one entry. +TT_NORMALIZERS = { + "tt_profile": normalize_profile, + "tt_post_metrics": normalize_post, + "tt_comments": normalize_comment, +} + + +# --------------------------------------------------------------------------------------------- +# THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05) +# --------------------------------------------------------------------------------------------- + +def tt_handle(url): + """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither. + + Deliberately permissive about the input and strict about the output, because the two callers + hand it different things: an automation stores whatever a person typed in the profile column + (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean + `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot + resolve to two different handles. + """ + s = str(url or "").strip() + if not s: + return "" + if "tiktok.com" in s.lower(): + # Everything after the first `@`, up to the next path segment or query. + tail = s.split("@", 1)[1] if "@" in s else "" + s = tail.split("/")[0].split("?")[0].split("#")[0] + s = s.strip().lstrip("@").strip() + # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores. + return s if s and all(c.isalnum() or c in "._" for c in s) else "" + + +def tt_post_urls(node, limit=0): + """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends. + + ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row + we have already bought, so the posts read is a scrape of links we hold, never a search for them. + The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**, + so a design that reached for either would not merely be dearer, it would not work. + + ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an + array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe + read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's + `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`** + (measured below), so the reader built against the quoted sentence found nothing, forever, in + silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration + of X"* are different claims, and prose cannot be told apart by a reader downstream — which is + why the correction names the method, not just the value. + + ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*, + and preferring whichever happened to be longer is how one creator's window silently differs + from another's. + ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`, + validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is + never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window, + not a number this function may invent. + """ + raw = (node or {}).get("top_videos") + out = [] + for item in raw if isinstance(raw, list) else []: + # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID. + # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a + # live handle returns **19 DICTS**, keyed + # `video_url · video_id · playcount · diggcount · commentcount · share_count · + # favorites_count · create_date · cover_image`. + # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and + # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says + # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field + # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's + # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was + # the first. [[reachable-is-not-the-same-as-built]] + # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY + # list on every real profile — post capture could not have worked for anybody, and the + # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture + # written from a schema tests the schema. + # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept + # because it costs nothing and is what a future corpus revision would most likely use. The + # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents + # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as + # before, rather than to a URL built out of a guess. + # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring + # whichever array happened to be longer is how one creator's window silently differs from + # another's, and that reasoning is unchanged by this correction. + if isinstance(item, dict): + u = str(item.get("video_url") or item.get("url") or "").strip() + else: + u = str(item or "").strip() + if u.lower().startswith("http") and u not in out: + out.append(u) + return out[:limit] if limit and limit > 0 else out + + +def pull_posts_tt(post_urls, log=print, deferred=None): + """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised. + + ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side + measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still + running at 67 minutes. Nothing here loops per URL. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] posts: {note}") + return [], note + out = [r for r in (normalize_post(n) for n in rows) if r] + return out, "" + + +def pull_comments_tt(post_urls, log=print, deferred=None): + """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised. + + ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on + both networks: a comments scrape ingests identifiable third parties who never entered anybody's + list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes + none of them to a column; this function adds no new exposure, it just has to be asked for. + """ + urls = [str(u) for u in (post_urls or []) if str(u or "").strip()] + if not urls: + return [], "" + rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred) + if note: + log(f"[aios-tt] comments: {note}") + return [], note + out = [r for r in (normalize_comment(n) for n in rows) if r] + return out, "" + + +#: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY. +#: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral +#: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's +#: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse +#: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can. +TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS) + + +def _media_deferrals(deferred): + """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's. + + ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the + handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one + kind here, so the id already carries everything a mapper choice depends on — and importing + Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines). + """ + out = [] + for d in deferred or []: + if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS: + out.append(dict(d)) + return out + + +def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None, + max_posts=0, post_metrics=False, comment_metrics=False): + """ONE TikTok profile from the vendor. Same return contract as `pull_profile`. + + `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`, + so the engine's enrich branch treats every network identically and no caller learns a new + shape. + + ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's + do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` — + no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics` + then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction + rather than by a rule: their input IS a post permalink, so asking for comments with post capture + off is a request with no subject, and it returns none instead of quietly buying posts nobody + asked for. + + ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than + pessimistic. The Instagram contract reads `ok` only when identity AND media both landed + (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower + count and no posts must not paint green over a posts table that did not grow"*). So: posts not + ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came → + `partial` with the vendor's reason. The state answers "did this pull deliver what it went for", + never "did the function finish". + + ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the + paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider, + with its own note explaining that a multi-provider chain is a promise something walks it and + that nothing walks Instagram's second name today either. So a refusal here is final, and it + says so instead of implying a retry somewhere. + """ + handle = tt_handle(url) + if not handle: + return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "", + "note": f"{url!r} is not a TikTok profile URL or handle"} + + # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from + # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not + # happen; a miss falls through to the single-URL call below. + cached = prefetch.get(handle) if isinstance(prefetch, dict) else None + _deferred = [] + if isinstance(cached, dict) and cached: + rows, note = [cached], "" + else: + rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred) + + node = rows[0] if rows else {} + profile = normalize_profile(node, handle) if node else {} + # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile` + # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and + # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly + # this reason, and answering "0 followers" instead is the failure it exists to prevent. + unreadable = profile.get("followers") is None and profile.get("following") is None + if note or unreadable: + # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still + # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the + # same record. That was live on the Instagram profile path until 2026-08-09 — measured on + # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being + # reintroduced here by omission. + if isinstance(pending_profile, list): + for d in _deferred: + pending_profile.append({**d, "kind": "profile", "influencer": handle}) + why = note or ("the scrape answered, but no follower/following counts were readable in it " + "(the field names may have moved - see tiktok-capture.md)") + return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata", + "deferredProfile": [d.get("snapshotId") for d in _deferred], + "note": why} + + # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------ + if not post_metrics: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; post capture is off for this step"} + urls = tt_post_urls(node, limit=max_posts) + if not urls: + # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and + # saying so is what stops the next run paying to be told the same thing. + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", + "note": note or "profile read; this account's row carried no post links"} + posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred) + comments, c_note = ([], "") + if comment_metrics and posts: + # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought — + # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused + # is not silently asked about again one rung later. + comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")], + log=log, deferred=_deferred) + # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is + # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to + # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE + # deferral is impossible — the profile branch above returns `blocked` on any note — so **every + # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose + # collector writes preset profile cells onto somebody's record from post rows. The engine keeps + # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`). + # ⇒ So this returns them under their OWN key, filtered by dataset identity + # (`_media_deferrals`), and the engine files them in the metric queue with the handle it + # already holds. Returning rather than appending also keeps the queue's vocabulary out of a + # connector: this module knows which CORPUS deferred, never what the engine calls it. + # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that + # matters most — that is exactly the run where the vendor took too long, so a caller reading + # the ids only from the success path would lose every batch it actually paid for. + deferred_media = _media_deferrals(_deferred) + if not posts: + return {"state": "partial", "profile": profile, "posts": [], "comments": [], + "via": "brightdata", "deferredMedia": deferred_media, + "note": p_note or note or "profile read; the post source returned nothing"} + return {"state": "ok", "profile": profile, "posts": posts, "comments": comments, + "via": "brightdata", "deferredMedia": deferred_media, + "note": c_note or note or ""} diff --git a/api/providers.py b/api/providers.py index d294400d33431e3b8c3bda9253357544cf451875..5425c862136396406714fecaa23eec0d3288ec52 100644 --- a/api/providers.py +++ b/api/providers.py @@ -1,897 +1,897 @@ -"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. - -⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if -anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize -when we scale to thousands+ customer … make sure our code is customizable and modular."* - -⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". -A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious -shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments -correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A -provider-level fallback would notice the missing views and re-buy the profile, the likes and the -comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that -is noise; at thousands of customers × twelve posts each it is the whole bill. - -So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: - - ig_profile -> brightdata (36 fields, works) - ig_post_metrics -> brightdata (likes + comments, works) - ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) - ig_comments -> brightdata - -Each capability has an ORDERED list of providers. The first one that is configured AND capable AND -succeeds wins; the rest are never called, so the common path costs exactly one vendor record. - -⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in -the one column you asked for has failed for our purposes, and a chain that only catches exceptions -would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying -answer exactly as it would on a 500. - -⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time -(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what -lets a tenant be moved off a vendor without a release. -""" -from __future__ import annotations - -import json -import os -import time -from dataclasses import dataclass, field -from typing import Callable - -# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in -# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the -# line that stops a container booting. That asymmetry is the whole reason the two are imported -# differently — see `anthropic_sdk()`. -import requests - -#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK -#: providers and to estimate a run's spend for the operator. They are not billing truth — the -#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is -#: why they are named and dated rather than guessed silently. -#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper -#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. -#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number -#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before -#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. -#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here -#: reads a live price — a modelled figure that says so is honest; one that pretends is not. -_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) -_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) - - -@dataclass(frozen=True) -class Capability: - """What ONE provider can do for ONE capability, and what it costs to ask.""" - #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never - #: called for this capability at all — it cannot be reached by a fallback, cannot be put first - #: by an env override, and cannot silently start being billed because somebody reordered a - #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route - #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four - #: accounts from 13K to 268M followers returned an account-grain constant or a null. - capable: bool = True - cost_per_record: float = 0.0 - #: Free-text, shown to an operator deciding where their money went. - note: str = "" - - -@dataclass -class Provider: - key: str - label: str - #: The env var holding this provider's credential. NEVER the credential itself — this module - #: is imported by surfaces that serialise their config. - key_env: str = "" - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True - - def cap(self, capability: str) -> Capability | None: - return self.caps.get(capability) - - def can(self, capability: str) -> bool: - c = self.cap(capability) - return bool(c and c.capable and self.configured()) - - -#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the -#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. -PROVIDERS: dict[str, Provider] = { - "brightdata": Provider( - key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", - caps={ - "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), - "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), - "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), - # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. - "ig_post_views": Capability( - False, _COST_BRIGHTDATA, - "MEASURED INCAPABLE: returns one account-grain number for every reel of a " - "creator (identical for two shortcodes in one call, ticking upward between " - "calls) and never populates video_play_count. Their own documented example " - "account no longer reproduces it."), - # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our - # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own - # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. - "tt_profile": Capability(True, _COST_BRIGHTDATA, - "40 fields incl. bio/engagement rates/region"), - # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. - # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says - # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post - # record and a separate view rung would be a second bill for a number we already have. - # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: - # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an - # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it - # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a - # quiet fallback bolted onto this one. - "tt_post_metrics": Capability( - True, _COST_BRIGHTDATA, - "likes + comments + shares + saves, and play_count inline (DECLARED " - "no-empties-or-zeros, UNPROVEN until one live pull)"), - "tt_comments": Capability(True, _COST_BRIGHTDATA, - "separate paid dataset, opt-in: 17 fields"), - }), - "apify": Provider( - key="apify", label="Apify", key_env="AIOS_APIFY_KEY", - caps={ - # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and - # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. - "ig_post_views": Capability(True, _COST_APIFY, - "videoPlayCount, matches Instagram's displayed views"), - "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), - "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), - }), -} - -#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. -DEFAULT_CHAINS: dict[str, tuple] = { - "ig_profile": ("brightdata", "apify"), - "ig_post_metrics": ("brightdata", "apify"), - # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here - # would be a lie that costs a wasted call on every single post. - "ig_post_views": ("apify",), - "ig_comments": ("brightdata",), - # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. - # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a - # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data - # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap - # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here - # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, - # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; - # it is not declared until a runner walks it. - "tt_profile": ("brightdata",), - "tt_post_metrics": ("brightdata",), - "tt_comments": ("brightdata",), -} - - -def chain(capability: str) -> list: - """The provider order for `capability` — env override first, then the default. - - `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the - owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a - vendor that is having a bad day. - ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — - `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore - resolves to EMPTY rather than to a provider that would return a wrong number, because a - plausible wrong number is worse than an honest refusal. - """ - raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() - names = None - for clause in raw.split(";"): - if "=" in clause: - cap_name, _, order = clause.partition("=") - if cap_name.strip() == capability: - names = [x.strip() for x in order.split(",") if x.strip()] - if names is None: - names = list(DEFAULT_CHAINS.get(capability) or ()) - return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] - - -def estimate(capability: str, records: int) -> dict: - """What the FIRST capable provider would cost for `records` — the number an operator plans on. - - Reported per capability rather than per run because that is the unit that scales: at a thousand - tenants the question is never "what did this run cost", it is "what does adding view counts to - every post cost per month". - """ - ch = chain(capability) - if not ch: - return {"capability": capability, "records": records, "provider": None, "usd": 0.0, - "note": "no configured, capable provider"} - p = ch[0] - c = p.cap(capability) - return {"capability": capability, "records": records, "provider": p.key, - "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), - "note": c.note} - - -@dataclass -class Attempt: - provider: str - ok: bool - note: str = "" - records: int = 0 - seconds: float = 0.0 - - -def run(capability, work, satisfied=None, log=None): - """Walk the chain for `capability` until one provider gives a SATISFYING answer. - - `work(provider) -> (result, note)`; a non-empty note means it did not answer. - `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any - truthy result. - - Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it - satisfied, and how long it took — because "where did this number come from and what did it - cost" is a question somebody asks about a bill, not about a stack trace. - - ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the - failure it must catch is not an error: a vendor answering 200 with the one field you needed - left blank. That is exactly how Bright Data behaves on view counts, and a chain that only - caught exceptions would have stopped there forever and never reached Apify. - """ - log = log or (lambda *_a: None) - ok = satisfied or (lambda r: bool(r)) - attempts, last = [], None - for provider in chain(capability): - started = time.time() - try: - result, note = work(provider) - except Exception as e: # noqa: BLE001 - # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is - # one refactor away from being a query param; this line must not be what leaks it. - result, note = None, f"{type(e).__name__}" - secs = round(time.time() - started, 2) - if note: - attempts.append(Attempt(provider.key, False, note, 0, secs)) - log(f" {provider.label}: {note}, falling through") - continue - if not ok(result): - attempts.append(Attempt(provider.key, False, "answered without the field asked for", - 0, secs)) - log(f" {provider.label}: answered, but not with what was asked for, falling through") - last = result if last is None else last - continue - attempts.append(Attempt(provider.key, True, "", _count(result), secs)) - log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") - return result, attempts - return last, attempts - - -def _count(result): - if isinstance(result, (list, tuple, set)): - return len(result) - if isinstance(result, dict): - return len(result) - return 1 if result else 0 - - -def wire(): - """What the Settings surface renders. Booleans and labels — NEVER a credential. - - Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a - provider is usable, never what the key is. - """ - return { - "providers": [ - {"key": p.key, "label": p.label, "configured": p.configured(), - "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} - for p in PROVIDERS.values() - ], - "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, - "incapable": { - f"{p.key}:{cap}": c.note - for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable - }, - } - - - - -# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ -# -# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the -# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: -# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this -# shape'."* -# -# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the -# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is -# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a -# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. -# -# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is -# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is -# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half -# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the -# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both -# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. -# -# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that -# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. -# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling -# turn at all, so the guess never has to be made. - -#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: -#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen -#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short -#: enough that a top-up is picked up without a restart. -CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) - -#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a -#: latency optimisation, not a billing record. A second container learns the same thing from its -#: own first 402, and neither one can be wrong for longer than the cooldown. -_NO_CREDIT: dict[str, float] = {} - - -@dataclass(frozen=True) -class LlmProvider: - """One chat-completions endpoint, and what it is DECLARED able to do.""" - name: str - label: str - env: str - url: str - model: str - #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, - #: which is a different body, a different auth header and a different result shape. - wire: str - caps: dict = field(default_factory=dict) - - def configured(self) -> bool: - return bool((os.environ.get(self.env) or "").strip()) - - def can(self, capability: str) -> bool: - cap = self.caps.get(capability) - return bool(cap and cap.capable and self.configured()) - - -#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put -#: cerebras first *"because this path needs tool calling and cerebras carries this account's -#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always -#: works, and the others are the cheap seats it falls through to. -LLM_PROVIDERS: dict[str, LlmProvider] = { - "anthropic": LlmProvider( - name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", - url="https://api.anthropic.com/v1/messages", - # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with - # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier - # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. - # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real - # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. - model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", - wire="anthropic", - caps={ - # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a - # typed `tool_use` content block carrying parsed `input`. There is nothing to recover - # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to - # apologise for on the other wire. - "llm_tool_calling": Capability(True, 0.0, - "typed tool_use content block; no text recovery path"), - "llm_chat": Capability(True, 0.0, "Messages API"), - "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), - }), - "cerebras": LlmProvider( - name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", - url="https://api.cerebras.ai/v1/chat/completions", - # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the - # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on - # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` - # already knows how to survive (R4: skipped, not tried). - model="gpt-oss-120b", wire="openai", - caps={ - # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not - # measure this rung: a 402 comes back before any tool is considered, so there is no - # observation to record. Leaving the old wording is the honest answer — upgrading it - # to MEASURED beside its two neighbours would be claiming an account balance as - # evidence about a capability. - "llm_tool_calling": Capability(True, 0.0, - "DECLARED, not measured: this account's tool-capable " - "model per the wave-32 ladder note. ⚠ 2026-08-19: the " - "account answers 402, so this stays unmeasured"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "groq": LlmProvider( - name="groq", label="Groq", env="GROQ_API_KEY", - # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq - # RETIRED it. Measured, not inferred — a real POST with the live key answered - # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with - # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A - # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the - # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). - # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. - # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while - # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY - # spelled differently and a sweep that "normalises" them breaks one of them. - url="https://api.groq.com/openai/v1/chat/completions", - model="openai/gpt-oss-120b", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old - # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME - # provider on this wire answers 400 with the call in `failed_generation`; the repo - # never recorded which. Flip this to False the day it is"*. It is recorded now: on - # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed - # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " - "with parsed arguments; no failed_generation recovery"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), - "openrouter": LlmProvider( - name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", - url="https://openrouter.ai/api/v1/chat/completions", - model="openai/gpt-4o-mini", wire="openai", - caps={ - # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed - # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and - # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. - "llm_tool_calling": Capability(True, 0.0, - "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), - "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), - "llm_json_mode": Capability(True, 0.0, "response_format json_object"), - }), -} - -LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") - - -def mark_no_credit(name, seconds=None): - """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). - - ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for - that to live: a stateless ladder re-tries the empty account on every single turn, pays its - round trip, and shows the reader a longer error each time. This is that memo. - """ - _NO_CREDIT[str(name)] = time.time() + float( - CREDIT_COOLDOWN_S if seconds is None else seconds) - return _NO_CREDIT[str(name)] - - -def no_credit(name): - """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" - until = _NO_CREDIT.get(str(name)) - if not until: - return False - if time.time() >= until: - _NO_CREDIT.pop(str(name), None) - return False - return True - - -def clear_credit_memo(name=None): - """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" - if name is None: - _NO_CREDIT.clear() - else: - _NO_CREDIT.pop(str(name), None) - - -def llm_chain(capability="llm_tool_calling"): - """The provider order for `capability` — declaration first, credit memo second. - - Three filters, in this order, and each removes a DIFFERENT kind of row: - 1. `can()` — declared capable AND configured. An incapable row is never offered, so a - turn cannot be spent discovering it (the `chain()` rule, one layer up). - 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over - until the memo expires. - 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment - can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` - carries for the scraping side. - """ - raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() - names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) - return [LLM_PROVIDERS[n] for n in names - if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] - - -#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND -#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner -#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. -_STATUS_WORDS = { - 401: "{label} would not accept our key", - 403: "{label} would not accept our key", - 402: "{label} is out of credit", - 404: "{label} does not offer the model we asked it for", - 408: "{label} took too long", - 413: "the question was too long for {label}", - 429: "{label} is rate limiting us right now", -} - -#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent -#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the -#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and -#: it only ever decides whether to SKIP a provider, never whether to trust one. -_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", - "quota exceeded", "billing", "payment required", "add credits") - - -def is_credit_failure(status, body=""): - """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" - if int(status or 0) == 402: - return True - if int(status or 0) not in (400, 403, 429): - return False - return any(word in str(body or "").lower() for word in _CREDIT_WORDS) - - -def refusal_sentence(name, status, body=""): - """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. - - ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key - prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a - credit problem", and what reaches the caller is this module's own wording. - """ - label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label - if is_credit_failure(status, body): - return f"{label} is out of credit" - code = int(status or 0) - if code in _STATUS_WORDS: - return _STATUS_WORDS[code].format(label=label) - if 500 <= code <= 599: - return f"{label} is having trouble at their end" - return f"{label} did not answer" - - -# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ -# -# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. -# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation -# drafter) both need it, and the owner quoted an error from EACH of them in one breath: -# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool -# calls are not wired for this shape"*. Two implementations of one wire is -# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the -# ladder that declares the rung. -# -# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: -# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message -# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` -# 3. `temperature` and friends are REMOVED on the current model family -# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string -# -# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on -# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits -# it, the assistant runs on the Opus tier and sends it. - -#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. -ANTHROPIC_VERSION = "2023-06-01" - - -def anthropic_request(*, model, key, system, messages, tools, max_tokens, - tool_choice="auto", effort=None): - """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. - - `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted - out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, - re-addressed rather than re-derived, so a schema change happens in one place. - """ - system_text = "\n\n".join(str(m.get("content") or "") for m in messages - if m.get("role") == "system") - if system: - system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) - turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), - "content": str(m.get("content") or "")} - for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] - wire_tools = [{"name": t["function"]["name"], - "description": t["function"]["description"], - "input_schema": t["function"]["parameters"]} for t in (tools or [])] - body = { - "model": str(model), - "max_tokens": int(max_tokens), - "system": system_text, - "messages": turns, - "tools": wire_tools, - } - # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that - # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, - # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no - # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: - # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. - # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and - # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. - if wire_tools: - body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} - if effort: - body["output_config"] = {"effort": str(effort)} - return {"url": LLM_PROVIDERS["anthropic"].url, - "headers": {"x-api-key": str(key), - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json"}, - "json": body} - - -#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. -#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every -#: call, and a failed import is not cheap. `False` says "asked and answered". -_SDK: object = None - - -def anthropic_sdk(): - """The official `anthropic` SDK, or `None` if this deployment does not carry it. - - ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. - `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's - fence this wave, so the pin is another session's edit. A hard `import anthropic` at module - scope would therefore turn a missing line in a manifest into a container that cannot boot at - all: the API imports this module on every request path. - - So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls - back to the raw `requests` POST that has always worked, and says which transport ran. When the - pin lands the SDK path takes over with no further change. - ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body - `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, - `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer - ([[one-question-two-normalizers]]). - """ - global _SDK - if _SDK is None: - try: - import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring - _SDK = _mod - except Exception: - _SDK = False - return _SDK or None - - -def anthropic_send(req, timeout=None, use_sdk=None): - """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. - - ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader - who cannot tell which path ran cannot tell whether the requirements pin reached the container - [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on - `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a - check keyed to it passes on a stale value written by an earlier check in the same process. - - `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — - the SDK raises where `requests` returns, and normalising that difference here is the only - reason this function exists rather than the caller branching on transport. - - ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports - without assigning to `_SDK`: a test that mutates a module global and then throws leaves every - later check in that process running on the wrong path and passing for the wrong reason. - """ - sdk = None if use_sdk is False else anthropic_sdk() - payload = dict(req.get("json") or {}) - if sdk is None: - r = requests.post(req["url"], headers=req["headers"], json=payload, - timeout=(timeout or 60)) - try: - return r.status_code, (r.json() if r.content else {}), "http" - except Exception: - return r.status_code, {"_text": (r.text or "")[:2000]}, "http" - - key = (req.get("headers") or {}).get("x-api-key") or "" - try: - client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) - msg = client.messages.create(**payload) - # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same - # wire-shaped dict the raw POST parses out of the response body. - return 200, msg.model_dump(), "sdk" - except Exception as exc: - # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by - # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out - # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. - status = int(getattr(exc, "status_code", 0) or 0) - body = getattr(exc, "body", None) - if not isinstance(body, dict): - body = {"_text": str(exc)[:2000]} - if not status: - # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one - # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. - status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 - return status, body, "sdk" - - -def anthropic_read(body): - """`(text, tool_input, refusal)` out of a Messages API answer. - - ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with - `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` - unconditionally breaks on exactly the turn a person most needs explained. - ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and - no regex recovering a call out of a 400, which is what the other wire needs. - """ - body = body if isinstance(body, dict) else {} - if str(body.get("stop_reason") or "") == "refusal": - return "", None, "the assistant declined to answer that one" - blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] - text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() - calls = [b for b in blocks if b.get("type") == "tool_use"] - got = calls[0].get("input") if calls else None - return text, (got if isinstance(got, dict) else None), None - - -def llm_status(capability="llm_tool_calling"): - """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. - - ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read - THIS, so a model offered in one place cannot be missing from the other, and neither can offer a - provider the ladder would refuse to call [[permitted-is-not-answerable]]. - """ - rows = [] - for name in LLM_DEFAULT_ORDER: - p = LLM_PROVIDERS[name] - cap = p.caps.get(capability) - rows.append({ - "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, - "configured": p.configured(), - "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), - "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), - "capable": bool(cap and cap.capable), - "outOfCredit": no_credit(name), - "note": (cap.note if cap else ""), - }) - return rows - - -# ============================================================================================= -# THE CANONICAL SCHEMA — owner ruling 2026-08-08: -# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set -# database even if the underlying engine changes"* -# -# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and -# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a -# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves -# with the vendor turns a routing change into a data migration. -# -# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. -# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so -# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column -# nobody declared. -# -# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded -# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a -# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is -# exactly the distinction that let an account-grain number wear the "Views" label for a month. -# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH -# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count -# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk -# fields AGREE with each other, which is precisely what makes picking by name so dangerous. -# canonical `views` <- apify `videoPlayCount` ✅ matches the grid -# canonical `views` <- apify `videoViewCount` ⛔ off by 13x -# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain -# ============================================================================================= - -#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider -#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier -#: paid run learned. -CANONICAL_POST_KEYS = ( - "shortcode", "url", "influencer_key", "posted_at", "type", "caption", - "likes", "comments", "views", "paid_partnership", "partner", "hashtags", - "alt_text", "tagged_location", "source_payload", - # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a - # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a - # value that normalises cleanly and is then dropped by the write door, silently. - "plays", "video_duration", "comments_disabled", -) - - -def _int_or_none(v): - """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like - count — a real state that is not a measurement. Writing -1 would render as a negative like - count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" - try: - n = int(v) - except (TypeError, ValueError): - return None - return None if n < 0 else n - - -def normalize_post_apify(row): - """One Apify `instagram-scraper` item -> the canonical post row.""" - if not isinstance(row, dict): - return None - code = str(row.get("shortCode") or "").strip() - if not code: - return None - out = { - "shortcode": code, - "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), - "influencer_key": str(row.get("ownerUsername") or ""), - "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], - "type": "video" if str(row.get("type") or "").lower() == "video" else - ("carousel" if row.get("childPosts") else "image"), - "caption": str(row.get("caption") or ""), - # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: - # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true - # count was ~137K, and `videoPlayCount` matched a browser read to the digit. - "views": _int_or_none(row.get("videoPlayCount")), - # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). - # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing - # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: - # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different - # real numbers, and we were storing only one of them. - # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two - # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: - # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured - # mean, and a column whose meaning changes halfway down is the one thing worse than a - # column with no data. Flagged rather than done. - "plays": _int_or_none(row.get("videoPlayCount")), - "likes": _int_or_none(row.get("likesCount")), - "comments": _int_or_none(row.get("commentsCount")), - # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in - # this same response (owner: *"whatever APIfy has more than BD pls use it"*). - "video_duration": _int_or_none(row.get("videoDuration")), - "comments_disabled": "1" if row.get("isCommentsDisabled") else "", - "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), - "alt_text": str(row.get("alt") or ""), - "paid_partnership": "1" if row.get("paidPartnership") else "", - "partner": ", ".join(str((s or {}).get("username") or "") - for s in (row.get("sponsors") or []) if isinstance(s, dict)), - "tagged_location": str((row.get("locationName") or "")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - return {k: v for k, v in out.items() if v not in (None, "")} - - -def normalize_profile_apify(row): - """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. - - ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly - in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and - `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran - that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify - was never asked, so a profile Bright Data cannot scrape came back `blocked` while a - configured, declared-capable provider sat unused. A registry entry with no runner is a - promise the product does not keep ([[flag-shipped-without-its-writer]]). - - ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module - already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the - snapshot writer cannot tell which vendor answered — which is what makes the fallback - invisible to every consumer instead of a second schema. - - ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` - drops a missing count: writing 0 followers would claim we measured an empty account, and the - caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would - stop the chain on a lie. - """ - if not isinstance(row, dict): - return None - # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle - # with a 200 and a well-formed item — `{"username": …, "error": "not_found", - # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" - # out of it: a username, a url and a source_payload, with every measured field absent. It - # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. - # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this - # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's - # ERROR shape is the one that must never become a row. - # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on - # `sriyynntt`), so `"error" in row` would refuse every good profile. - if row.get("error"): - return None - handle = str(row.get("username") or "").strip() - if not handle: - return None - out = { - "username": handle, - "full_name": str(row.get("fullName") or ""), - "bio": str(row.get("biography") or ""), - "followers": _int_or_none(row.get("followersCount")), - "following": _int_or_none(row.get("followsCount")), - "posts_count": _int_or_none(row.get("postsCount")), - "verified": "1" if row.get("verified") else "", - "external_url": str(row.get("externalUrl") or ""), - "ig_id": str(row.get("id") or ""), - "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), - "business_category": str(row.get("businessCategoryName") or ""), - "is_business": "1" if row.get("isBusinessAccount") else "", - "is_private": "1" if row.get("private") else "", - "highlights_count": _int_or_none(row.get("highlightReelCount")), - "source_payload": json.dumps(row, default=str)[:32_000_000], - } - # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter - # below must not treat 0 as absent the way the post normaliser can — an account really can - # have zero followers. Only None and "" are dropped. - return {k: v for k, v in out.items() if v is not None and v != ""} - - -def canonical_gaps(row, want=("views",)): - """Which requested canonical fields this row does NOT carry — the `satisfied` input. - - Named rather than inlined because "did the vendor actually answer the question" is the whole - fallback trigger, and a chain that asks it differently in two places will drift. - """ - row = row if isinstance(row, dict) else {} - return [k for k in want if row.get(k) in (None, "")] +"""THE PROVIDER LAYER — many vendors behind one capability, chosen per FIELD and per cost. + +⭐⭐ OWNER RULING 2026-08-08: *"Let's do a combination scraper. Do Bright Data first, and if +anything fails, we use APIfy or vice versa. Make this dynamic somehow. remember to cost optimize +when we scale to thousands+ customer … make sure our code is customizable and modular."* + +⛔ THE DESIGN DECISION THAT MATTERS, AND IT IS NOT "TRY A, THEN TRY B". +A provider-level failover ("if Bright Data errors, re-run the whole thing on Apify") is the obvious +shape and it is the expensive one. MEASURED: Bright Data answers profile, likes and comments +correctly and CANNOT answer view counts on any route; Apify answers view counts correctly. A +provider-level fallback would notice the missing views and re-buy the profile, the likes and the +comments from Apify as well — **paying twice for the 90% that already worked**. At one profile that +is noise; at thousands of customers × twelve posts each it is the whole bill. + +So routing is per CAPABILITY, and a capability is the smallest independently-billable thing: + + ig_profile -> brightdata (36 fields, works) + ig_post_metrics -> brightdata (likes + comments, works) + ig_post_views -> apify (Bright Data is INCAPABLE — see `capable=False` below) + ig_comments -> brightdata + +Each capability has an ORDERED list of providers. The first one that is configured AND capable AND +succeeds wins; the rest are never called, so the common path costs exactly one vendor record. + +⚠ "FAILURE" INCLUDES ANSWERING WITH THE FIELD BLANK. A vendor that returns HTTP 200 and a null in +the one column you asked for has failed for our purposes, and a chain that only catches exceptions +would stop at it forever. `run()` takes a `satisfied` predicate and falls through on an unsatisfying +answer exactly as it would on a 500. + +⚠ ORDER IS DATA, NOT CODE. `AIOS_PROVIDER_ORDER` overrides any chain at deploy time +(`ig_post_views=apify,brightdata;ig_profile=apify`), which is what "or vice versa" means and what +lets a tenant be moved off a vendor without a release. +""" +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Callable + +# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in +# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the +# line that stops a container booting. That asymmetry is the whole reason the two are imported +# differently — see `anthropic_sdk()`. +import requests + +#: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK +#: providers and to estimate a run's spend for the operator. They are not billing truth — the +#: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is +#: why they are named and dated rather than guessed silently. +#: Bright Data Instagram datasets ~ $0.0015/record (2026-08 list). Apify instagram-scraper +#: ~ $0.0027/result (2026-08 list). Both re-checked when a provider is added. +#: ⚠ WAVE 30 · T16 — APIFY CORRECTED 0.0023 -> 0.0027, and the direction matters: the old number +#: made the fallback look CHEAPER than it is, and `estimate()` is what a person is shown before +#: they authorise a run. An under-stated price is the one rounding error a cost guard cannot catch. +#: ⚠ Both are LIST rates used to rank and to estimate. They are not invoices, and nothing here +#: reads a live price — a modelled figure that says so is honest; one that pretends is not. +_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015) +_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027) + + +@dataclass(frozen=True) +class Capability: + """What ONE provider can do for ONE capability, and what it costs to ask.""" + #: ⛔ `False` means MEASURED INCAPABLE, not "untried". A provider declared incapable is never + #: called for this capability at all — it cannot be reached by a fallback, cannot be put first + #: by an env override, and cannot silently start being billed because somebody reordered a + #: list. Bright Data's `ig_post_views` is False on the strength of §4e-§4h: every route + #: (Posts x /p/, Posts x /reel/, Reels x /p/, Reels x /reel/, discover-by-profile) on four + #: accounts from 13K to 268M followers returned an account-grain constant or a null. + capable: bool = True + cost_per_record: float = 0.0 + #: Free-text, shown to an operator deciding where their money went. + note: str = "" + + +@dataclass +class Provider: + key: str + label: str + #: The env var holding this provider's credential. NEVER the credential itself — this module + #: is imported by surfaces that serialise their config. + key_env: str = "" + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.key_env) or "").strip()) if self.key_env else True + + def cap(self, capability: str) -> Capability | None: + return self.caps.get(capability) + + def can(self, capability: str) -> bool: + c = self.cap(capability) + return bool(c and c.capable and self.configured()) + + +#: ⚠ ADDING A PROVIDER IS A REGISTRY ENTRY PLUS A RUNNER — no change to any caller. That is the +#: "modular/customizable" half of the ruling, and the reason the chains below name STRINGS. +PROVIDERS: dict[str, Provider] = { + "brightdata": Provider( + key="brightdata", label="Bright Data", key_env="AIOS_BRIGHTDATA_KEY", + caps={ + "ig_profile": Capability(True, _COST_BRIGHTDATA, "36 fields incl. bio/email/category"), + "ig_post_metrics": Capability(True, _COST_BRIGHTDATA, "likes + comments + captions"), + "ig_comments": Capability(True, _COST_BRIGHTDATA, "separate paid dataset, opt-in"), + # ⛔ THE ONE THAT COSTS US NOTHING TO GET RIGHT AND EVERYTHING TO GET WRONG. + "ig_post_views": Capability( + False, _COST_BRIGHTDATA, + "MEASURED INCAPABLE: returns one account-grain number for every reel of a " + "creator (identical for two shortcodes in one call, ticking upward between " + "calls) and never populates video_play_count. Their own documented example " + "account no longer reproduces it."), + # ⭐⭐ WAVE 29 (item 7 / D-9 / R1) — TIKTOK. Three datasets, all reachable with our + # existing key, all schema-probed for $0.00 (40/43/17 fields with the vendor's own + # types): `gd_l1villgoiiidt09ci` · `gd_lu702nij2f790tmv9h` · `gd_lkf2st302ap89utw5k`. + "tt_profile": Capability(True, _COST_BRIGHTDATA, + "40 fields incl. bio/engagement rates/region"), + # ⛔ THERE IS NO `tt_post_views` CAPABILITY, AND ITS ABSENCE IS THE CLAIM. + # `TikTok - Posts` declares `play_count: number` and the vendor's own sample says + # "no empty values or zeros" — so on TikTok the view count arrives INSIDE the post + # record and a separate view rung would be a second bill for a number we already have. + # ⚠ DECLARED, NOT MEASURED, and that distinction is the whole scar tissue of §4e: + # Bright Data's Instagram Reels also DECLARES `views: number` and delivers an + # account-grain wrong one. One live pull settles it (W29-T06, spend-gated). If it + # fails, the honest repair is a `tt_post_views` capability routed elsewhere — never a + # quiet fallback bolted onto this one. + "tt_post_metrics": Capability( + True, _COST_BRIGHTDATA, + "likes + comments + shares + saves, and play_count inline (DECLARED " + "no-empties-or-zeros, UNPROVEN until one live pull)"), + "tt_comments": Capability(True, _COST_BRIGHTDATA, + "separate paid dataset, opt-in: 17 fields"), + }), + "apify": Provider( + key="apify", label="Apify", key_env="AIOS_APIFY_KEY", + caps={ + # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and + # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. + "ig_post_views": Capability(True, _COST_APIFY, + "videoPlayCount, matches Instagram's displayed views"), + "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), + "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), + }), +} + +#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`. +DEFAULT_CHAINS: dict[str, tuple] = { + "ig_profile": ("brightdata", "apify"), + "ig_post_metrics": ("brightdata", "apify"), + # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here + # would be a lie that costs a wasted call on every single post. + "ig_post_views": ("apify",), + "ig_comments": ("brightdata",), + # ⭐ WAVE 29 — TikTok, and every chain is deliberately SINGLE-PROVIDER. + # ⛔ A MULTI-PROVIDER CHAIN IS A PROMISE SOMETHING WALKS IT. `ig_post_metrics` has declared a + # two-provider fallback since wave 28 and NOTHING reaches the second name: if Bright Data + # answers with the likes blank, Apify is never asked. `verify_automation`'s E2b pins that gap + # BY NAME so a third unwalked chain turns it red — which is exactly what a second name here + # would be today. Apify does sell TikTok (`clockworks/tiktok-profile-scraper`, $0.003/result, + # 0.7% 30-day failure rate — measured from its own store record), so the fallback is buildable; + # it is not declared until a runner walks it. + "tt_profile": ("brightdata",), + "tt_post_metrics": ("brightdata",), + "tt_comments": ("brightdata",), +} + + +def chain(capability: str) -> list: + """The provider order for `capability` — env override first, then the default. + + `AIOS_PROVIDER_ORDER` is a `;`-separated list of `capability=p1,p2` clauses. This is how the + owner's "or vice versa" is expressed WITHOUT a release, and how one tenant can be moved off a + vendor that is having a bad day. + ⛔ An override may REORDER and may DROP, but it can never make an incapable provider capable — + `can()` still gates every name. A chain that reads `ig_post_views=brightdata` therefore + resolves to EMPTY rather than to a provider that would return a wrong number, because a + plausible wrong number is worse than an honest refusal. + """ + raw = (os.environ.get("AIOS_PROVIDER_ORDER") or "").strip() + names = None + for clause in raw.split(";"): + if "=" in clause: + cap_name, _, order = clause.partition("=") + if cap_name.strip() == capability: + names = [x.strip() for x in order.split(",") if x.strip()] + if names is None: + names = list(DEFAULT_CHAINS.get(capability) or ()) + return [PROVIDERS[n] for n in names if n in PROVIDERS and PROVIDERS[n].can(capability)] + + +def estimate(capability: str, records: int) -> dict: + """What the FIRST capable provider would cost for `records` — the number an operator plans on. + + Reported per capability rather than per run because that is the unit that scales: at a thousand + tenants the question is never "what did this run cost", it is "what does adding view counts to + every post cost per month". + """ + ch = chain(capability) + if not ch: + return {"capability": capability, "records": records, "provider": None, "usd": 0.0, + "note": "no configured, capable provider"} + p = ch[0] + c = p.cap(capability) + return {"capability": capability, "records": records, "provider": p.key, + "usd": round((c.cost_per_record or 0.0) * max(0, int(records)), 4), + "note": c.note} + + +@dataclass +class Attempt: + provider: str + ok: bool + note: str = "" + records: int = 0 + seconds: float = 0.0 + + +def run(capability, work, satisfied=None, log=None): + """Walk the chain for `capability` until one provider gives a SATISFYING answer. + + `work(provider) -> (result, note)`; a non-empty note means it did not answer. + `satisfied(result) -> bool` decides whether an answer is good enough to stop. Default: any + truthy result. + + Returns `(result, attempts)`. `attempts` is the audit trail — every provider tried, whether it + satisfied, and how long it took — because "where did this number come from and what did it + cost" is a question somebody asks about a bill, not about a stack trace. + + ⛔ THE `satisfied` HOOK IS THE WHOLE POINT. Without it this is an error-handler, and the + failure it must catch is not an error: a vendor answering 200 with the one field you needed + left blank. That is exactly how Bright Data behaves on view counts, and a chain that only + caught exceptions would have stopped there forever and never reached Apify. + """ + log = log or (lambda *_a: None) + ok = satisfied or (lambda r: bool(r)) + attempts, last = [], None + for provider in chain(capability): + started = time.time() + try: + result, note = work(provider) + except Exception as e: # noqa: BLE001 + # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is + # one refactor away from being a query param; this line must not be what leaks it. + result, note = None, f"{type(e).__name__}" + secs = round(time.time() - started, 2) + if note: + attempts.append(Attempt(provider.key, False, note, 0, secs)) + log(f" {provider.label}: {note}, falling through") + continue + if not ok(result): + attempts.append(Attempt(provider.key, False, "answered without the field asked for", + 0, secs)) + log(f" {provider.label}: answered, but not with what was asked for, falling through") + last = result if last is None else last + continue + attempts.append(Attempt(provider.key, True, "", _count(result), secs)) + log(f" {provider.label}: ok ({_count(result)} records, {secs}s)") + return result, attempts + return last, attempts + + +def _count(result): + if isinstance(result, (list, tuple, set)): + return len(result) + if isinstance(result, dict): + return len(result) + return 1 if result else 0 + + +def wire(): + """What the Settings surface renders. Booleans and labels — NEVER a credential. + + Mirrors the `hikerReady` rule the rest of the product follows: a surface is told WHETHER a + provider is usable, never what the key is. + """ + return { + "providers": [ + {"key": p.key, "label": p.label, "configured": p.configured(), + "capabilities": sorted(k for k, c in p.caps.items() if c.capable)} + for p in PROVIDERS.values() + ], + "chains": {cap: [p.key for p in chain(cap)] for cap in DEFAULT_CHAINS}, + "incapable": { + f"{p.key}:{cap}": c.note + for p in PROVIDERS.values() for cap, c in p.caps.items() if not c.capable + }, + } + + + + +# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ +# +# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the +# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: +# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this +# shape'."* +# +# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the +# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is +# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a +# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. +# +# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is +# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is +# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half +# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the +# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both +# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. +# +# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that +# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. +# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling +# turn at all, so the guess never has to be made. + +#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: +#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen +#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short +#: enough that a top-up is picked up without a restart. +CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) + +#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a +#: latency optimisation, not a billing record. A second container learns the same thing from its +#: own first 402, and neither one can be wrong for longer than the cooldown. +_NO_CREDIT: dict[str, float] = {} + + +@dataclass(frozen=True) +class LlmProvider: + """One chat-completions endpoint, and what it is DECLARED able to do.""" + name: str + label: str + env: str + url: str + model: str + #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, + #: which is a different body, a different auth header and a different result shape. + wire: str + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.env) or "").strip()) + + def can(self, capability: str) -> bool: + cap = self.caps.get(capability) + return bool(cap and cap.capable and self.configured()) + + +#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put +#: cerebras first *"because this path needs tool calling and cerebras carries this account's +#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always +#: works, and the others are the cheap seats it falls through to. +LLM_PROVIDERS: dict[str, LlmProvider] = { + "anthropic": LlmProvider( + name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", + url="https://api.anthropic.com/v1/messages", + # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with + # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier + # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real + # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. + model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", + wire="anthropic", + caps={ + # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a + # typed `tool_use` content block carrying parsed `input`. There is nothing to recover + # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to + # apologise for on the other wire. + "llm_tool_calling": Capability(True, 0.0, + "typed tool_use content block; no text recovery path"), + "llm_chat": Capability(True, 0.0, "Messages API"), + "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), + }), + "cerebras": LlmProvider( + name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", + url="https://api.cerebras.ai/v1/chat/completions", + # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the + # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on + # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` + # already knows how to survive (R4: skipped, not tried). + model="gpt-oss-120b", wire="openai", + caps={ + # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not + # measure this rung: a 402 comes back before any tool is considered, so there is no + # observation to record. Leaving the old wording is the honest answer — upgrading it + # to MEASURED beside its two neighbours would be claiming an account balance as + # evidence about a capability. + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured: this account's tool-capable " + "model per the wave-32 ladder note. ⚠ 2026-08-19: the " + "account answers 402, so this stays unmeasured"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "groq": LlmProvider( + name="groq", label="Groq", env="GROQ_API_KEY", + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq + # RETIRED it. Measured, not inferred — a real POST with the live key answered + # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with + # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A + # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the + # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). + # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. + # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while + # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY + # spelled differently and a sweep that "normalises" them breaks one of them. + url="https://api.groq.com/openai/v1/chat/completions", + model="openai/gpt-oss-120b", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old + # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME + # provider on this wire answers 400 with the call in `failed_generation`; the repo + # never recorded which. Flip this to False the day it is"*. It is recorded now: on + # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed + # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " + "with parsed arguments; no failed_generation recovery"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "openrouter": LlmProvider( + name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o-mini", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed + # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and + # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), +} + +LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") + + +def mark_no_credit(name, seconds=None): + """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). + + ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for + that to live: a stateless ladder re-tries the empty account on every single turn, pays its + round trip, and shows the reader a longer error each time. This is that memo. + """ + _NO_CREDIT[str(name)] = time.time() + float( + CREDIT_COOLDOWN_S if seconds is None else seconds) + return _NO_CREDIT[str(name)] + + +def no_credit(name): + """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" + until = _NO_CREDIT.get(str(name)) + if not until: + return False + if time.time() >= until: + _NO_CREDIT.pop(str(name), None) + return False + return True + + +def clear_credit_memo(name=None): + """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" + if name is None: + _NO_CREDIT.clear() + else: + _NO_CREDIT.pop(str(name), None) + + +def llm_chain(capability="llm_tool_calling"): + """The provider order for `capability` — declaration first, credit memo second. + + Three filters, in this order, and each removes a DIFFERENT kind of row: + 1. `can()` — declared capable AND configured. An incapable row is never offered, so a + turn cannot be spent discovering it (the `chain()` rule, one layer up). + 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over + until the memo expires. + 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment + can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` + carries for the scraping side. + """ + raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() + names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) + return [LLM_PROVIDERS[n] for n in names + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + + +#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND +#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner +#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. +_STATUS_WORDS = { + 401: "{label} would not accept our key", + 403: "{label} would not accept our key", + 402: "{label} is out of credit", + 404: "{label} does not offer the model we asked it for", + 408: "{label} took too long", + 413: "the question was too long for {label}", + 429: "{label} is rate limiting us right now", +} + +#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent +#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the +#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and +#: it only ever decides whether to SKIP a provider, never whether to trust one. +_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", + "quota exceeded", "billing", "payment required", "add credits") + + +def is_credit_failure(status, body=""): + """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" + if int(status or 0) == 402: + return True + if int(status or 0) not in (400, 403, 429): + return False + return any(word in str(body or "").lower() for word in _CREDIT_WORDS) + + +def refusal_sentence(name, status, body=""): + """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. + + ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key + prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a + credit problem", and what reaches the caller is this module's own wording. + """ + label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label + if is_credit_failure(status, body): + return f"{label} is out of credit" + code = int(status or 0) + if code in _STATUS_WORDS: + return _STATUS_WORDS[code].format(label=label) + if 500 <= code <= 599: + return f"{label} is having trouble at their end" + return f"{label} did not answer" + + +# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ +# +# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. +# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation +# drafter) both need it, and the owner quoted an error from EACH of them in one breath: +# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool +# calls are not wired for this shape"*. Two implementations of one wire is +# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the +# ladder that declares the rung. +# +# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: +# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message +# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` +# 3. `temperature` and friends are REMOVED on the current model family +# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string +# +# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on +# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits +# it, the assistant runs on the Opus tier and sends it. + +#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. +ANTHROPIC_VERSION = "2023-06-01" + + +def anthropic_request(*, model, key, system, messages, tools, max_tokens, + tool_choice="auto", effort=None): + """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. + + `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted + out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, + re-addressed rather than re-derived, so a schema change happens in one place. + """ + system_text = "\n\n".join(str(m.get("content") or "") for m in messages + if m.get("role") == "system") + if system: + system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) + turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), + "content": str(m.get("content") or "")} + for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + wire_tools = [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])] + body = { + "model": str(model), + "max_tokens": int(max_tokens), + "system": system_text, + "messages": turns, + "tools": wire_tools, + } + # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that + # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, + # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no + # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: + # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. + # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and + # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. + if wire_tools: + body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} + if effort: + body["output_config"] = {"effort": str(effort)} + return {"url": LLM_PROVIDERS["anthropic"].url, + "headers": {"x-api-key": str(key), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + "json": body} + + +#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. +#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every +#: call, and a failed import is not cheap. `False` says "asked and answered". +_SDK: object = None + + +def anthropic_sdk(): + """The official `anthropic` SDK, or `None` if this deployment does not carry it. + + ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. + `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's + fence this wave, so the pin is another session's edit. A hard `import anthropic` at module + scope would therefore turn a missing line in a manifest into a container that cannot boot at + all: the API imports this module on every request path. + + So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls + back to the raw `requests` POST that has always worked, and says which transport ran. When the + pin lands the SDK path takes over with no further change. + ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body + `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, + `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer + ([[one-question-two-normalizers]]). + """ + global _SDK + if _SDK is None: + try: + import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring + _SDK = _mod + except Exception: + _SDK = False + return _SDK or None + + +def anthropic_send(req, timeout=None, use_sdk=None): + """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. + + ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader + who cannot tell which path ran cannot tell whether the requirements pin reached the container + [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + check keyed to it passes on a stale value written by an earlier check in the same process. + + `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — + the SDK raises where `requests` returns, and normalising that difference here is the only + reason this function exists rather than the caller branching on transport. + + ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports + without assigning to `_SDK`: a test that mutates a module global and then throws leaves every + later check in that process running on the wrong path and passing for the wrong reason. + """ + sdk = None if use_sdk is False else anthropic_sdk() + payload = dict(req.get("json") or {}) + if sdk is None: + r = requests.post(req["url"], headers=req["headers"], json=payload, + timeout=(timeout or 60)) + try: + return r.status_code, (r.json() if r.content else {}), "http" + except Exception: + return r.status_code, {"_text": (r.text or "")[:2000]}, "http" + + key = (req.get("headers") or {}).get("x-api-key") or "" + try: + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + msg = client.messages.create(**payload) + # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same + # wire-shaped dict the raw POST parses out of the response body. + return 200, msg.model_dump(), "sdk" + except Exception as exc: + # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by + # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out + # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. + status = int(getattr(exc, "status_code", 0) or 0) + body = getattr(exc, "body", None) + if not isinstance(body, dict): + body = {"_text": str(exc)[:2000]} + if not status: + # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one + # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. + status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 + return status, body, "sdk" + + +def anthropic_read(body): + """`(text, tool_input, refusal)` out of a Messages API answer. + + ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with + `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` + unconditionally breaks on exactly the turn a person most needs explained. + ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and + no regex recovering a call out of a 400, which is what the other wire needs. + """ + body = body if isinstance(body, dict) else {} + if str(body.get("stop_reason") or "") == "refusal": + return "", None, "the assistant declined to answer that one" + blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] + text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() + calls = [b for b in blocks if b.get("type") == "tool_use"] + got = calls[0].get("input") if calls else None + return text, (got if isinstance(got, dict) else None), None + + +def llm_status(capability="llm_tool_calling"): + """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. + + ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read + THIS, so a model offered in one place cannot be missing from the other, and neither can offer a + provider the ladder would refuse to call [[permitted-is-not-answerable]]. + """ + rows = [] + for name in LLM_DEFAULT_ORDER: + p = LLM_PROVIDERS[name] + cap = p.caps.get(capability) + rows.append({ + "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, + "configured": p.configured(), + "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), + "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), + "capable": bool(cap and cap.capable), + "outOfCredit": no_credit(name), + "note": (cap.note if cap else ""), + }) + return rows + + +# ============================================================================================= +# THE CANONICAL SCHEMA — owner ruling 2026-08-08: +# *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set +# database even if the underlying engine changes"* +# +# ⛔ THE PRESET TABLES ARE THE CONTRACT; A VENDOR IS AN IMPLEMENTATION DETAIL. `ut_ig_posts` and +# `ut_ig_post_snapshots` must not gain, lose or rename a column because a chain was reordered — a +# tenant's saved views, filters, rollups and forms all bind to these keys, and a schema that moves +# with the vendor turns a routing change into a data migration. +# +# So every provider normalises INTO the keys below and nothing reads a vendor row downstream. +# `verify_automation` asserts both normalisers emit exactly `CANONICAL_POST_KEYS` on a fixture, so +# a third provider cannot ship with a near-miss key like `viewCount` and silently write a column +# nobody declared. +# +# ⚠ ONE NAME PER MEASUREMENT, AND `views` IS THE MEASUREMENT INSTAGRAM DISPLAYS. Meta folded +# Impressions/Plays/Video Views into a single **Views** metric on 2025-04-10, so carrying both a +# `views` and a `plays` column would be modelling a distinction the platform deleted — and it is +# exactly the distinction that let an account-grain number wear the "Views" label for a month. +# ⛔ THE VENDOR KEY THAT LOOKS RIGHT IS THE WRONG ONE, ON BOTH VENDORS. Apify ships BOTH +# `videoViewCount` (10,678) and `videoPlayCount` (137,684) for one reel whose true displayed count +# is ~137K — and Bright Data's useless `views` is 10,638 for that same reel. The two vendors' junk +# fields AGREE with each other, which is precisely what makes picking by name so dangerous. +# canonical `views` <- apify `videoPlayCount` ✅ matches the grid +# canonical `views` <- apify `videoViewCount` ⛔ off by 13x +# canonical `views` <- brightdata `views` ⛔ off by 13x AND account-grain +# ============================================================================================= + +#: Every key a normalised POST row may carry. Absent > blank: a key is omitted when the provider +#: did not answer, because `upsert_rows` merges and an empty string would ERASE what an earlier +#: paid run learned. +CANONICAL_POST_KEYS = ( + "shortcode", "url", "influencer_key", "posted_at", "type", "caption", + "likes", "comments", "views", "paid_partnership", "partner", "hashtags", + "alt_text", "tagged_location", "source_payload", + # ⭐ 2026-08-09 — the three the SECOND provider answers and the first does not. Each has a + # matching `field_def` in the engine's POST_FIELDS; a key here without a column there is a + # value that normalises cleanly and is then dropped by the write door, silently. + "plays", "video_duration", "comments_disabled", +) + + +def _int_or_none(v): + """`-1` IS NOT A COUNT. Apify returns `likesCount: -1` when the creator HIDES their like + count — a real state that is not a measurement. Writing -1 would render as a negative like + count; writing 0 would claim nobody liked it. Both are lies, so the key is omitted.""" + try: + n = int(v) + except (TypeError, ValueError): + return None + return None if n < 0 else n + + +def normalize_post_apify(row): + """One Apify `instagram-scraper` item -> the canonical post row.""" + if not isinstance(row, dict): + return None + code = str(row.get("shortCode") or "").strip() + if not code: + return None + out = { + "shortcode": code, + "url": str(row.get("url") or f"https://www.instagram.com/reel/{code}/"), + "influencer_key": str(row.get("ownerUsername") or ""), + "posted_at": str(row.get("timestamp") or "").replace("T", " ")[:16], + "type": "video" if str(row.get("type") or "").lower() == "video" else + ("carousel" if row.get("childPosts") else "image"), + "caption": str(row.get("caption") or ""), + # ⭐ THE FIELD THIS WHOLE PROVIDER EXISTS FOR. `videoPlayCount`, never `videoViewCount`: + # the wrong one agreed with Bright Data's junk (10,678 vs 10,638) on a reel whose true + # count was ~137K, and `videoPlayCount` matched a browser read to the digit. + "views": _int_or_none(row.get("videoPlayCount")), + # ⭐⭐ 2026-08-09 (owner: *"Video plays (# Plays) field isn't in APIfy? i believe it is"*). + # They were right, and the `plays` COLUMN had been empty on all 816 rows because nothing + # ever wrote it — the field existed with no writer. MEASURED on two of their own reels: + # `videoPlayCount` 216,904 / 95,331 and `videoViewCount` 115,929 / 30,871. Two different + # real numbers, and we were storing only one of them. + # ⚠ `plays` TAKES THE PLAY COUNT, which is also what `views` carries today — so the two + # columns will agree until somebody decides otherwise, and that decision is the OWNER'S: + # re-sourcing `views` to `videoViewCount` would change what the ~480 rows already captured + # mean, and a column whose meaning changes halfway down is the one thing worse than a + # column with no data. Flagged rather than done. + "plays": _int_or_none(row.get("videoPlayCount")), + "likes": _int_or_none(row.get("likesCount")), + "comments": _int_or_none(row.get("commentsCount")), + # ⭐ FIELDS BRIGHT DATA DOES NOT RETURN AT ALL, kept because they are already paid for in + # this same response (owner: *"whatever APIfy has more than BD pls use it"*). + "video_duration": _int_or_none(row.get("videoDuration")), + "comments_disabled": "1" if row.get("isCommentsDisabled") else "", + "hashtags": ", ".join(str(h) for h in (row.get("hashtags") or []) if h), + "alt_text": str(row.get("alt") or ""), + "paid_partnership": "1" if row.get("paidPartnership") else "", + "partner": ", ".join(str((s or {}).get("username") or "") + for s in (row.get("sponsors") or []) if isinstance(s, dict)), + "tagged_location": str((row.get("locationName") or "")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + return {k: v for k, v in out.items() if v not in (None, "")} + + +def normalize_profile_apify(row): + """One Apify `instagram-scraper` PROFILE item (`resultsType: "details"`) -> our snapshot shape. + + ⭐ WHY THIS EXISTS (owner report 2026-08-09: *"I need Bright Data and Apify to work correctly + in tandem"*). `PROVIDERS["apify"]` has declared `ig_profile` capable since 2026-08-08 and + `DEFAULT_CHAINS["ig_profile"]` has read `("brightdata", "apify")` — but **nothing ever ran + that chain.** `connectors_ig.pull_profile` went Bright Data -> anonymous HTML rungs and Apify + was never asked, so a profile Bright Data cannot scrape came back `blocked` while a + configured, declared-capable provider sat unused. A registry entry with no runner is a + promise the product does not keep ([[flag-shipped-without-its-writer]]). + + ⚠ THE KEYS ARE APIFY'S camelCase, and mapping them HERE is the boundary rule this module + already enforces for posts: nothing downstream may ever see a vendor-shaped key, so the + snapshot writer cannot tell which vendor answered — which is what makes the fallback + invisible to every consumer instead of a second schema. + + ⛔ OMIT, NEVER ZERO. A field Apify did not send is dropped, exactly as `normalize_post_apify` + drops a missing count: writing 0 followers would claim we measured an empty account, and the + caller's `satisfied` hook reads absence as "did not answer" and falls through. A zero would + stop the chain on a lie. + """ + if not isinstance(row, dict): + return None + # ⛔ AN ERROR ENVELOPE IS NOT A PROFILE (measured 2026-08-09). Apify answers a dead handle + # with a 200 and a well-formed item — `{"username": …, "error": "not_found", + # "errorDescription": "Post does not exist"}` — and this function used to build a "profile" + # out of it: a username, a url and a source_payload, with every measured field absent. It + # then read to the caller as a vendor that answered, so the reason was replaced by a shrug. + # Refused HERE as well as in `connectors_ig.apify_profile` on purpose: the boundary rule this + # module exists for is that nothing downstream ever sees a vendor-shaped key, and a vendor's + # ERROR shape is the one that must never become a row. + # ⚠ TRUTHINESS, NOT KEY PRESENCE — a successful item carries `error: null` (measured on + # `sriyynntt`), so `"error" in row` would refuse every good profile. + if row.get("error"): + return None + handle = str(row.get("username") or "").strip() + if not handle: + return None + out = { + "username": handle, + "full_name": str(row.get("fullName") or ""), + "bio": str(row.get("biography") or ""), + "followers": _int_or_none(row.get("followersCount")), + "following": _int_or_none(row.get("followsCount")), + "posts_count": _int_or_none(row.get("postsCount")), + "verified": "1" if row.get("verified") else "", + "external_url": str(row.get("externalUrl") or ""), + "ig_id": str(row.get("id") or ""), + "profile_url": str(row.get("url") or f"https://www.instagram.com/{handle}/"), + "business_category": str(row.get("businessCategoryName") or ""), + "is_business": "1" if row.get("isBusinessAccount") else "", + "is_private": "1" if row.get("private") else "", + "highlights_count": _int_or_none(row.get("highlightReelCount")), + "source_payload": json.dumps(row, default=str)[:32_000_000], + } + # ⚠ `followers`/`following` are ints and 0 is a LEGITIMATE value for them, so the filter + # below must not treat 0 as absent the way the post normaliser can — an account really can + # have zero followers. Only None and "" are dropped. + return {k: v for k, v in out.items() if v is not None and v != ""} + + +def canonical_gaps(row, want=("views",)): + """Which requested canonical fields this row does NOT carry — the `satisfied` input. + + Named rather than inlined because "did the vendor actually answer the question" is the whole + fallback trigger, and a chain that asks it differently in two places will drift. + """ + row = row if isinstance(row, dict) else {} + return [k for k in want if row.get(k) in (None, "")] diff --git a/api/routes_agent_harness.py b/api/routes_agent_harness.py index cfe1ffa6958ea00035eaaaf64d5c826e246fb209..ed34831c623af46ef28bd312848275948108c70b 100644 --- a/api/routes_agent_harness.py +++ b/api/routes_agent_harness.py @@ -1,455 +1,455 @@ -"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. - -Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file -pertaining to the agent skills, router etc. So we build a user can build a custom harness for each -agent through the chat interface."* - - GET /api/v1/agents/{id}/harness the file LIST (no bodies) - GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions - GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim - PUT /api/v1/agents/{id}/harness write a NEW version of one file - DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history - -⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write -appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the -history is a record of what happened rather than of what somebody last wanted it to look like. - -⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, -`routes_automation`); a harness file is not. Do not copy this file's posture over there or that -one's over here — the two rulings differ on purpose and they sit one screen apart in the product. - -⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every -write through this router is made BY a signed-in administrator, so `author` is stamped from the -session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by -a person or by the agent in the chat — a claim the client is entitled to make, because both are -permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side -door the automation engine uses when the agent writes with no session at all; that one stamps the -agent's own id as the author, which is the only case where `author` is not a username. - -⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of -one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in -`trimmed` on every payload that mentions the file, so a reader can see that the history is partial -rather than infer that the file was only ever saved twice. This is the tenant document, which is -already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is -a store-sized leak with a UI in front of it. -""" -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides -#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule -#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. -HARNESS_KEY = "agent_harness" - -#: One body. Generous for a skill or a router file and far under the point where a single write -#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a -#: truncation: a silently truncated skill file is a harness that does not do what its text says. -MAX_BODY_BYTES = 128 * 1024 - -#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving -#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. -MAX_FILES = 64 - -#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). -MAX_VERSIONS = 100 - -MAX_PATH = 200 - -#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — -#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is -#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a -#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a -#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. -_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") - -AUTHOR_KINDS = ("user", "agent") - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must - degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" - try: - found = rt.get(HARNESS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _files(rt, agent_id): - found = _all(rt).get(str(agent_id)) - return found if isinstance(found, dict) else {} - - -#: What kind of principal an id names. `None` = this tenant has no agent with that id. -#: -#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY -#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under -#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — -#: `Shell.tsx` mounts ``, and -#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds -#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that -#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival -#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on -#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO -#: occurrences of Canvas, Properties, Run history or panelTabs. -#: -#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent -#: skills, router" files in two places is the parallel code path item 13 exists to refuse. -AGENT_SLACK = "slack" -AGENT_AUTOMATION = "automation" - - -def agent_kind(rt, agent_id): - """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. - - ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at - read time from a column definition or a connector schedule; they have no stored home, so a - harness file hung off one is an orphan the moment the column changes. `patch_automation` - already refuses those ids for the neighbouring reason, and this refuses them by simply not - finding them — `all_definitions` holds stored automations only. - """ - import routes_slack - aid = str(agent_id or "") - if isinstance(routes_slack._agents(rt).get(aid), dict): - return AGENT_SLACK - import automation_engine as engine - try: - known = engine.all_definitions(rt) or {} - except Exception: # noqa: BLE001 - return None - return AGENT_AUTOMATION if aid in known else None - - -def _known_agent(rt, agent_id): - """Does this tenant have an agent with this id, in EITHER registry?""" - return agent_kind(rt, agent_id) is not None - - -def agent_wall(session, agent_id): - """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. - - ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a - THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door - in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in - `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the - agent it hangs off, so it is reached by whoever may already configure that agent. Picking one - of the two walls for both would either lock the Agents module's own users out of a tab the - owner asked for, or hand the Slack permission wall to anyone with an automation grant. - """ - kind = agent_kind(session.runtime, agent_id) - if kind is None: - raise err(404, "no_agent", "there is no agent with that id in this workspace") - if kind == AGENT_SLACK: - import core.perms as perms - if not perms.is_admin(session.user): - raise err(403, "forbidden", "administrators only") - else: - session.require("automation") - return kind - - -def normalize_path(raw): - """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. - - ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There - are two ways into this store — the HTTP route (which must answer 400) and `record_version()` - (which must raise `ValueError`, having no response to put a status into). Written twice, the - two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one - copy is invisible because the other still refuses, so nothing goes red and the wall is now - half there. Written once, a change to the rule is felt at both doors and by the gate. - """ - path = str(raw or "").strip().strip("/") - if not path or len(path) > MAX_PATH: - return None - if set(path) - _PATH_OK or ".." in path or "//" in path: - return None - return path - - -def _clean_path(raw): - """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" - path = normalize_path(raw) - if path is None: - if not str(raw or "").strip().strip("/"): - raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") - if len(str(raw)) > MAX_PATH: - raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") - raise err(400, "bad_path", - "a harness path may use letters, digits, dot, dash, underscore and / only") - return path - - -def _blank(path, author, author_kind): - return {"path": path, "versions": [], "trimmed": 0, - "created": _now(), "createdBy": author, "createdKind": author_kind} - - -def _append(record, body, author, author_kind, restored_from=None): - """Append ONE version to a file record, in place, and report what was trimmed. - - The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. - A version list whose numbers restart at 1 after a trim would make two different bodies share a - name, and `restoredFrom` would then point at whichever one happened to be in the window. - """ - # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback - # the live document and may run it more than once; appending in place would then stack two - # copies of the same version into the history on a retry. - prior = record.get("versions") if isinstance(record.get("versions"), list) else [] - last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) - entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, - "created": _now(), "bytes": len(body.encode("utf-8"))} - if restored_from: - entry["restoredFrom"] = int(restored_from) - versions = [*prior, entry] - dropped = max(0, len(versions) - MAX_VERSIONS) - if dropped: - versions = versions[dropped:] - record["versions"] = versions - record["trimmed"] = int(record.get("trimmed") or 0) + dropped - return entry - - -def _head(record): - """The newest version of a file record, or `None` for a record with no versions at all.""" - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - return versions[-1] if versions else None - - -def _row(record): - """One file, as the LIST door reports it: everything except the bodies. - - ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would - ship the whole harness on every pane render; the Harness tab lists first and opens one file - second, which is exactly the shape this answers. - """ - head = _head(record) or {} - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - return {"path": record.get("path") or "", - "version": int(head.get("version") or 0), - "bytes": int(head.get("bytes") or 0), - "author": head.get("author") or record.get("createdBy") or "", - "authorKind": head.get("authorKind") or record.get("createdKind") or "user", - "updated": head.get("created") or record.get("created") or "", - "created": record.get("created") or "", - "versions": len(versions), - "trimmed": int(record.get("trimmed") or 0)} - - -def _version_rows(record): - """The history of one file, newest first, WITHOUT the bodies. - - A body per version is what makes a diff possible, and it is also what makes this payload big: - 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing - (`?path=…&version=N`), which is two round trips for a diff and none for a history list. - """ - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - out = [] - for entry in reversed(versions): - if not isinstance(entry, dict): - continue - row = {"version": int(entry.get("version") or 0), - "author": entry.get("author") or "", - "authorKind": entry.get("authorKind") or "user", - "created": entry.get("created") or "", - "bytes": int(entry.get("bytes") or 0)} - if entry.get("restoredFrom"): - row["restoredFrom"] = int(entry["restoredFrom"]) - out.append(row) - return out - - -def _limits(): - """The caps, IN the payload, so a client can say "this file is full" before a write fails. - - ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the - one cap that acts without refusing; these report the three that refuse. - """ - return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, - "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} - - -# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── -def record_version(runtime, agent_id, path, body, author="", author_kind="agent", - restored_from=None): - """Write one version from INSIDE the server — the agent's own half of R8. - - ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" - true. An agent acting inside an automation run holds no session and no cookie; if its only way - to write were the HTTP door it would have to borrow a person's identity, and the authorship - column would then be a record of who was logged in rather than of what wrote the file. - - Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a - `ValueError`, because a server-side caller has no HTTP response to put a 404 into. - """ - agent_id = str(agent_id or "") - if not _known_agent(runtime, agent_id): - raise ValueError(f"no agent {agent_id!r} in this workspace") - path = normalize_path(path) - if path is None: - raise ValueError("that is not an acceptable harness path") - body = str(body or "") - if len(body.encode("utf-8")) > MAX_BODY_BYTES: - raise ValueError("harness body is over the size limit") - kind = author_kind if author_kind in AUTHOR_KINDS else "agent" - author = str(author or agent_id) - # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a - # store `update` callback propagates out of a half-run read-modify-write, and the one thing a - # refusal must never do is leave the caller unsure whether the write happened. - existing = _files(runtime, agent_id) - if path not in existing and len(existing) >= MAX_FILES: - raise ValueError(f"this agent already has {MAX_FILES} harness files") - appended = {} - - def _set(cur): - cur = dict(cur or {}) - files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} - record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) - appended.clear() - appended.update(_append(record, body, author, kind, restored_from)) - files[path] = record - cur[agent_id] = files - return cur - - runtime.update(HARNESS_KEY, _set, flush="sync") - return appended - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/agents/{agent_id}/harness") -def get_harness(agent_id: str, path: str = "", version: int = 0, - session: Session = Depends(require_session)): - """The list, one file, or one older body — decided by the query string (C4). - - ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than - consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to - editing a permission than to editing a document. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - files = _files(session.runtime, agent_id) - if not path: - rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] - return {"agent": agent_id, "files": rows, "limits": _limits()} - - wanted = _clean_path(path) - record = files.get(wanted) - if not isinstance(record, dict): - raise err(404, "no_file", f"this agent has no harness file at {wanted}") - versions = record.get("versions") if isinstance(record.get("versions"), list) else [] - if version: - for entry in versions: - if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): - return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", - "atVersion": int(version), "history": _version_rows(record), - "limits": _limits()} - # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked - # for something that existed and no longer does, and telling it apart from a bad number is - # the difference between "roll back to v3" failing loudly and failing as if v3 never was. - trimmed = int(record.get("trimmed") or 0) - if trimmed and int(version) <= trimmed: - raise err(410, "version_trimmed", - f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " - f"for this file, and its body is gone") - raise err(404, "no_version", f"this file has no version {int(version)}") - head = _head(record) or {} - return {"agent": agent_id, **_row(record), "body": head.get("body") or "", - "atVersion": int(head.get("version") or 0), "history": _version_rows(record), - "limits": _limits()} - - -@router.put("/agents/{agent_id}/harness") -def put_harness(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). - - ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a - UI affordance; it is this function refusing to have a code path that replaces a body. A - roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that - somebody went back rather than pretending the intervening versions never happened. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - body = body if isinstance(body, dict) else {} - path = _clean_path(body.get("path")) - text = body.get("body") - if not isinstance(text, str): - raise err(400, "no_body", "a harness file needs a body, even an empty one") - if len(text.encode("utf-8")) > MAX_BODY_BYTES: - raise err(413, "body_too_long", - f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") - - # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a - # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the - # session and is never read off the request. Nothing is bought by forging the first — both - # principals may write (R8) — and everything would be bought by forging the second. - kind = str(body.get("authorKind") or "user").strip().lower() - if kind not in AUTHOR_KINDS: - raise err(400, "bad_author_kind", "authorKind is either user or agent") - restored = body.get("restoredFrom") - try: - restored = int(restored) if restored else None - except (TypeError, ValueError): - raise err(400, "bad_version", "restoredFrom must be a version number") - - files = _files(session.runtime, agent_id) - if path not in files and len(files) >= MAX_FILES: - raise err(409, "too_many_files", - f"this agent already has {MAX_FILES} harness files; delete one to add another") - try: - record_version(session.runtime, agent_id, path, text, - author=session.uname, author_kind=kind, restored_from=restored) - except ValueError as exc: - raise err(400, "refused", str(exc)) - - fresh = _files(session.runtime, agent_id).get(path) - if not isinstance(fresh, dict) or not _head(fresh): - # The store took the write and did not record it. A 200 here would tell an administrator - # their skill file was saved when it was not — the shape `routes_slack` refuses too. - raise err(503, "store_unavailable", "the harness file was NOT saved") - head = _head(fresh) - return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", - "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), - "limits": _limits()} - - -@router.delete("/agents/{agent_id}/harness") -def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): - """Drop one harness file AND its history. - - ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing - versions as a side effect of an edit. Deleting a file is a person deciding the file should not - exist, which is a different act with a different button, and a store with no way to remove a - file is one where a typo'd path is permanent. - """ - agent_id = str(agent_id or "") - agent_wall(session, agent_id) - wanted = _clean_path(path) - if wanted not in _files(session.runtime, agent_id): - raise err(404, "no_file", f"this agent has no harness file at {wanted}") - - def _set(cur): - cur = dict(cur or {}) - files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} - files.pop(wanted, None) - # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a - # bucket accumulates a row for every agent anybody ever opened the tab on. - if files: - cur[agent_id] = files - else: - cur.pop(agent_id, None) - return cur - - session.runtime.update(HARNESS_KEY, _set, flush="sync") - return {"agent": agent_id, "deleted": wanted, - "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) - if isinstance(rec, dict)], - "limits": _limits()} +"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. + +Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file +pertaining to the agent skills, router etc. So we build a user can build a custom harness for each +agent through the chat interface."* + + GET /api/v1/agents/{id}/harness the file LIST (no bodies) + GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions + GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim + PUT /api/v1/agents/{id}/harness write a NEW version of one file + DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history + +⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write +appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the +history is a record of what happened rather than of what somebody last wanted it to look like. + +⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, +`routes_automation`); a harness file is not. Do not copy this file's posture over there or that +one's over here — the two rulings differ on purpose and they sit one screen apart in the product. + +⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every +write through this router is made BY a signed-in administrator, so `author` is stamped from the +session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by +a person or by the agent in the chat — a claim the client is entitled to make, because both are +permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side +door the automation engine uses when the agent writes with no session at all; that one stamps the +agent's own id as the author, which is the only case where `author` is not a username. + +⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of +one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in +`trimmed` on every payload that mentions the file, so a reader can see that the history is partial +rather than infer that the file was only ever saved twice. This is the tenant document, which is +already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is +a store-sized leak with a UI in front of it. +""" +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule +#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. +HARNESS_KEY = "agent_harness" + +#: One body. Generous for a skill or a router file and far under the point where a single write +#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a +#: truncation: a silently truncated skill file is a harness that does not do what its text says. +MAX_BODY_BYTES = 128 * 1024 + +#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving +#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. +MAX_FILES = 64 + +#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). +MAX_VERSIONS = 100 + +MAX_PATH = 200 + +#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — +#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is +#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a +#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a +#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. +_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") + +AUTHOR_KINDS = ("user", "agent") + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must + degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" + try: + found = rt.get(HARNESS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _files(rt, agent_id): + found = _all(rt).get(str(agent_id)) + return found if isinstance(found, dict) else {} + + +#: What kind of principal an id names. `None` = this tenant has no agent with that id. +#: +#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY +#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under +#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — +#: `Shell.tsx` mounts ``, and +#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds +#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that +#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival +#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on +#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO +#: occurrences of Canvas, Properties, Run history or panelTabs. +#: +#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent +#: skills, router" files in two places is the parallel code path item 13 exists to refuse. +AGENT_SLACK = "slack" +AGENT_AUTOMATION = "automation" + + +def agent_kind(rt, agent_id): + """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. + + ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at + read time from a column definition or a connector schedule; they have no stored home, so a + harness file hung off one is an orphan the moment the column changes. `patch_automation` + already refuses those ids for the neighbouring reason, and this refuses them by simply not + finding them — `all_definitions` holds stored automations only. + """ + import routes_slack + aid = str(agent_id or "") + if isinstance(routes_slack._agents(rt).get(aid), dict): + return AGENT_SLACK + import automation_engine as engine + try: + known = engine.all_definitions(rt) or {} + except Exception: # noqa: BLE001 + return None + return AGENT_AUTOMATION if aid in known else None + + +def _known_agent(rt, agent_id): + """Does this tenant have an agent with this id, in EITHER registry?""" + return agent_kind(rt, agent_id) is not None + + +def agent_wall(session, agent_id): + """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. + + ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a + THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door + in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in + `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the + agent it hangs off, so it is reached by whoever may already configure that agent. Picking one + of the two walls for both would either lock the Agents module's own users out of a tab the + owner asked for, or hand the Slack permission wall to anyone with an automation grant. + """ + kind = agent_kind(session.runtime, agent_id) + if kind is None: + raise err(404, "no_agent", "there is no agent with that id in this workspace") + if kind == AGENT_SLACK: + import core.perms as perms + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + else: + session.require("automation") + return kind + + +def normalize_path(raw): + """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. + + ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There + are two ways into this store — the HTTP route (which must answer 400) and `record_version()` + (which must raise `ValueError`, having no response to put a status into). Written twice, the + two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one + copy is invisible because the other still refuses, so nothing goes red and the wall is now + half there. Written once, a change to the rule is felt at both doors and by the gate. + """ + path = str(raw or "").strip().strip("/") + if not path or len(path) > MAX_PATH: + return None + if set(path) - _PATH_OK or ".." in path or "//" in path: + return None + return path + + +def _clean_path(raw): + """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" + path = normalize_path(raw) + if path is None: + if not str(raw or "").strip().strip("/"): + raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") + if len(str(raw)) > MAX_PATH: + raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") + raise err(400, "bad_path", + "a harness path may use letters, digits, dot, dash, underscore and / only") + return path + + +def _blank(path, author, author_kind): + return {"path": path, "versions": [], "trimmed": 0, + "created": _now(), "createdBy": author, "createdKind": author_kind} + + +def _append(record, body, author, author_kind, restored_from=None): + """Append ONE version to a file record, in place, and report what was trimmed. + + The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. + A version list whose numbers restart at 1 after a trim would make two different bodies share a + name, and `restoredFrom` would then point at whichever one happened to be in the window. + """ + # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback + # the live document and may run it more than once; appending in place would then stack two + # copies of the same version into the history on a retry. + prior = record.get("versions") if isinstance(record.get("versions"), list) else [] + last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) + entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, + "created": _now(), "bytes": len(body.encode("utf-8"))} + if restored_from: + entry["restoredFrom"] = int(restored_from) + versions = [*prior, entry] + dropped = max(0, len(versions) - MAX_VERSIONS) + if dropped: + versions = versions[dropped:] + record["versions"] = versions + record["trimmed"] = int(record.get("trimmed") or 0) + dropped + return entry + + +def _head(record): + """The newest version of a file record, or `None` for a record with no versions at all.""" + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return versions[-1] if versions else None + + +def _row(record): + """One file, as the LIST door reports it: everything except the bodies. + + ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would + ship the whole harness on every pane render; the Harness tab lists first and opens one file + second, which is exactly the shape this answers. + """ + head = _head(record) or {} + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return {"path": record.get("path") or "", + "version": int(head.get("version") or 0), + "bytes": int(head.get("bytes") or 0), + "author": head.get("author") or record.get("createdBy") or "", + "authorKind": head.get("authorKind") or record.get("createdKind") or "user", + "updated": head.get("created") or record.get("created") or "", + "created": record.get("created") or "", + "versions": len(versions), + "trimmed": int(record.get("trimmed") or 0)} + + +def _version_rows(record): + """The history of one file, newest first, WITHOUT the bodies. + + A body per version is what makes a diff possible, and it is also what makes this payload big: + 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing + (`?path=…&version=N`), which is two round trips for a diff and none for a history list. + """ + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + out = [] + for entry in reversed(versions): + if not isinstance(entry, dict): + continue + row = {"version": int(entry.get("version") or 0), + "author": entry.get("author") or "", + "authorKind": entry.get("authorKind") or "user", + "created": entry.get("created") or "", + "bytes": int(entry.get("bytes") or 0)} + if entry.get("restoredFrom"): + row["restoredFrom"] = int(entry["restoredFrom"]) + out.append(row) + return out + + +def _limits(): + """The caps, IN the payload, so a client can say "this file is full" before a write fails. + + ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the + one cap that acts without refusing; these report the three that refuse. + """ + return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, + "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} + + +# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── +def record_version(runtime, agent_id, path, body, author="", author_kind="agent", + restored_from=None): + """Write one version from INSIDE the server — the agent's own half of R8. + + ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" + true. An agent acting inside an automation run holds no session and no cookie; if its only way + to write were the HTTP door it would have to borrow a person's identity, and the authorship + column would then be a record of who was logged in rather than of what wrote the file. + + Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a + `ValueError`, because a server-side caller has no HTTP response to put a 404 into. + """ + agent_id = str(agent_id or "") + if not _known_agent(runtime, agent_id): + raise ValueError(f"no agent {agent_id!r} in this workspace") + path = normalize_path(path) + if path is None: + raise ValueError("that is not an acceptable harness path") + body = str(body or "") + if len(body.encode("utf-8")) > MAX_BODY_BYTES: + raise ValueError("harness body is over the size limit") + kind = author_kind if author_kind in AUTHOR_KINDS else "agent" + author = str(author or agent_id) + # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a + # store `update` callback propagates out of a half-run read-modify-write, and the one thing a + # refusal must never do is leave the caller unsure whether the write happened. + existing = _files(runtime, agent_id) + if path not in existing and len(existing) >= MAX_FILES: + raise ValueError(f"this agent already has {MAX_FILES} harness files") + appended = {} + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) + appended.clear() + appended.update(_append(record, body, author, kind, restored_from)) + files[path] = record + cur[agent_id] = files + return cur + + runtime.update(HARNESS_KEY, _set, flush="sync") + return appended + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/agents/{agent_id}/harness") +def get_harness(agent_id: str, path: str = "", version: int = 0, + session: Session = Depends(require_session)): + """The list, one file, or one older body — decided by the query string (C4). + + ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than + consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to + editing a permission than to editing a document. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + files = _files(session.runtime, agent_id) + if not path: + rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] + return {"agent": agent_id, "files": rows, "limits": _limits()} + + wanted = _clean_path(path) + record = files.get(wanted) + if not isinstance(record, dict): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + if version: + for entry in versions: + if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): + return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", + "atVersion": int(version), "history": _version_rows(record), + "limits": _limits()} + # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked + # for something that existed and no longer does, and telling it apart from a bad number is + # the difference between "roll back to v3" failing loudly and failing as if v3 never was. + trimmed = int(record.get("trimmed") or 0) + if trimmed and int(version) <= trimmed: + raise err(410, "version_trimmed", + f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " + f"for this file, and its body is gone") + raise err(404, "no_version", f"this file has no version {int(version)}") + head = _head(record) or {} + return {"agent": agent_id, **_row(record), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(record), + "limits": _limits()} + + +@router.put("/agents/{agent_id}/harness") +def put_harness(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). + + ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a + UI affordance; it is this function refusing to have a code path that replaces a body. A + roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that + somebody went back rather than pretending the intervening versions never happened. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + body = body if isinstance(body, dict) else {} + path = _clean_path(body.get("path")) + text = body.get("body") + if not isinstance(text, str): + raise err(400, "no_body", "a harness file needs a body, even an empty one") + if len(text.encode("utf-8")) > MAX_BODY_BYTES: + raise err(413, "body_too_long", + f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") + + # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a + # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the + # session and is never read off the request. Nothing is bought by forging the first — both + # principals may write (R8) — and everything would be bought by forging the second. + kind = str(body.get("authorKind") or "user").strip().lower() + if kind not in AUTHOR_KINDS: + raise err(400, "bad_author_kind", "authorKind is either user or agent") + restored = body.get("restoredFrom") + try: + restored = int(restored) if restored else None + except (TypeError, ValueError): + raise err(400, "bad_version", "restoredFrom must be a version number") + + files = _files(session.runtime, agent_id) + if path not in files and len(files) >= MAX_FILES: + raise err(409, "too_many_files", + f"this agent already has {MAX_FILES} harness files; delete one to add another") + try: + record_version(session.runtime, agent_id, path, text, + author=session.uname, author_kind=kind, restored_from=restored) + except ValueError as exc: + raise err(400, "refused", str(exc)) + + fresh = _files(session.runtime, agent_id).get(path) + if not isinstance(fresh, dict) or not _head(fresh): + # The store took the write and did not record it. A 200 here would tell an administrator + # their skill file was saved when it was not — the shape `routes_slack` refuses too. + raise err(503, "store_unavailable", "the harness file was NOT saved") + head = _head(fresh) + return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), + "limits": _limits()} + + +@router.delete("/agents/{agent_id}/harness") +def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): + """Drop one harness file AND its history. + + ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing + versions as a side effect of an edit. Deleting a file is a person deciding the file should not + exist, which is a different act with a different button, and a store with no way to remove a + file is one where a typo'd path is permanent. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + wanted = _clean_path(path) + if wanted not in _files(session.runtime, agent_id): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + files.pop(wanted, None) + # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a + # bucket accumulates a row for every agent anybody ever opened the tab on. + if files: + cur[agent_id] = files + else: + cur.pop(agent_id, None) + return cur + + session.runtime.update(HARNESS_KEY, _set, flush="sync") + return {"agent": agent_id, "deleted": wanted, + "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) + if isinstance(rec, dict)], + "limits": _limits()} diff --git a/api/routes_keychain.py b/api/routes_keychain.py index 4ef61ca06030bf7abf2238ae1421cbde6686560d..0d8534c02e4acf8e5318379694ff5a39602887f5 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -1,1073 +1,1073 @@ -"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). - -Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER -return a decrypted field — list rows carry a masked preview, and the decrypt function is a -connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's -env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** -count (rows holding overlay data whose pids the current pool no longer serves; counted and -drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly -in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with -the keychain cutover wave R3 staged. -""" -import os - -from fastapi import Body, Depends -from fastapi import APIRouter - -from deps import Session, err, require_session -# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door -# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall -# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per -# entry instead of per endpoint. `session.admin` is still what business-wide requires. - -router = APIRouter(prefix="/api/v1") - -#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by -#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another -#: freezes nothing while reporting success, which is the shape of the bug D-10 books. -from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 - ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) -#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED -#: odoo connector is paused; read by the pool path while paused; survives a Space restart. -SNAPSHOT_KEY = "connector_snapshots" - -#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. -#: -#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves -#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and -#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's -#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's -#: connection, and handing it to another tenant is the leak this method exists to prevent."* -#: -#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and -#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value -#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, -#: the gate reds instead of the product quietly disagreeing with itself. -ENV_ODOO_TENANT = "royal-imports" - - -def env_odoo_available(rt): - """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) - - ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. - `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant - guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was - `connected` and offered "Manage keys" for a credential belonging to another company. This - module asked the same question correctly two functions below, which is the whole shape of - [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in - as the second tenant. - - ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which - source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, - wrong for "should the environment row be listed at all", which is what the connectors pane - needs in order to show an inactive env source beside an active keychain one. - - Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is - how another tenant's environment got reported as this tenant's connection in the first place. - """ - try: - if not (os.environ.get("ODOO_URL") or "").strip(): - return False - return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT - except Exception: # noqa: BLE001 - return False - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. -# Every keychain entry and connector carries a scope, selectable per connection; business-wide is -# admin-only and applies to every user in the tenant."* -# -# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file -# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is -# an access rule, not a secret, so it lives in the layer that already decides who may ask. -# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves -# cannot drift into two vocabularies. -SCOPES = ("business", "personal") -#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before -#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. -#: Materialising it would be a migration nobody authorised and would rewrite the store on the next -#: read [[a-migration-that-runs-on-the-next-write]]. -DEFAULT_SCOPE = "business" -#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. -#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean -#: the same thing and there is only one way to spell the common case. -SCOPE_KEY = "keychain_scopes" - -#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. -#: -#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* -#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is -#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they -#: would silently become the credential the entire workspace's databases are built from, which is -#: a credential elevation wearing a scope picker. -#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes -#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is -#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door -#: from the only side B owns, and the resolver-side guard is booked for A. -TENANT_WIDE_TYPES = ("odoo", "meta_ads") - - -def clean_scope(raw, default=DEFAULT_SCOPE): - """A scope word from the wire, or None when the caller said something we do not speak. - - Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole - reason this returns None rather than falling back: a typo'd `"personel"` silently becoming - business-wide is exactly the failure a scope picker exists to prevent. - """ - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default - got = str(raw).strip().lower() - return got if got in SCOPES else None - - -def _scope_rows(rt): - try: - return dict(rt.get(SCOPE_KEY) or {}) - except Exception: # noqa: BLE001 - return {} - - -def entry_scope(rt, entry_id, rows=None): - """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a - census does not re-read the store once per entry.""" - r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) - if not isinstance(r, dict): - return DEFAULT_SCOPE, "" - return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) - - -def may_see(scope, owner, uname): - """R4's visibility rule: business-wide is everyone's, personal is its owner's. - - ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the - per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token - is identity, not infrastructure"*). An admin who could read every member's personal credential - would make "personal" a label rather than a boundary. - """ - return scope != "personal" or str(owner) == str(uname) - - -def visible_entries(rt, uname, is_admin=False): - """This USER's view of the keychain: every business entry plus their own personal ones, each - row carrying its `scope` and `owner` so no caller has to ask a second time. - - ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can - hold a connection of their own; it does not hand them four characters of the workspace's Odoo - key. So a business row a member did not create arrives WITHOUT `preview` — they can see that - the connection exists and is theirs to use, which is the whole of what R4 grants. The default - is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. - """ - rows = _scope_rows(rt) - out = [] - for e in _kc().list_entries(rt): - scope, owner = entry_scope(rt, e["id"], rows) - if not owner: - owner = str(e.get("createdBy") or "") - if not may_see(scope, owner, uname): - continue - row = {**e, "scope": scope, "owner": owner} - if not (is_admin or str(owner) == str(uname)): - row["preview"] = "" - out.append(row) - return out - - -def _write_scope(rt, entry_id, scope, owner): - """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the - default, so the store holds one spelling of the common case.""" - def _up(cur): - if scope == DEFAULT_SCOPE: - cur.pop(str(entry_id), None) - else: - cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} - return cur - - rt.update(SCOPE_KEY, _up, flush="sync") - return True - - -def _may_touch(session, row): - """May this session change or delete `row`? An admin owns the business-wide ones; a member - owns their own personal ones. Anything else is not theirs to move.""" - if row.get("scope") == "personal": - return str(row.get("owner") or "") == str(session.uname) - return bool(session.admin) - - -def _kc(): - import core.keychain as keychain - return keychain - - -def _resolved_odoo_key(rt): - """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the - shapes this module needs: the display string (`env` / `keychain:`) and the key the pause - flag is stored under. - - ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, - beside the `odoo_source()` it must agree with. This function had its own copy of the same - three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and - a second copy is exactly how a pause flag comes to be written against one resolution and read - against another, freezing nothing while the UI reports success. The two SHAPES stay here - because they are this module's presentation concern; the DECISION does not. - """ - flag_key = rt.odoo_flag_key() - if not flag_key: - return None, None - return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key - - -def odoo_paused(rt): - """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry - that is not the resolved source freezes nothing — it serves nothing. - - ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the - measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question - the customer pool does. This name stays because `routes_customers`, `routes_products` and - `verify_api` all call it — moving the logic without moving the door keeps one answer and - costs no caller a change. - """ - return bool(rt.odoo_paused()) - - -def _snap_scope_key(team_id, agent): - return f"t={team_id}|a={agent}" - - -def load_pool_snapshot(rt, team_id, agent): - """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider - scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" - try: - snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} - e = snap.get(_snap_scope_key(team_id, agent)) - if isinstance(e, dict) and isinstance(e.get("rows"), list): - return float(e.get("ts") or 0), e["rows"] - except Exception: - pass - return None - - -def save_pool_snapshots(rt, taken_by=""): - """Persist every currently-cached pool scope as the pause-time snapshot ('the last - successful sync', made concrete). Ensures the consolidated default scope exists first so - a pause on a cold process still captures something to serve.""" - import time as _time - import routes_customers as _rc - try: - _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on - except Exception: - pass # cold + Odoo down: persist whatever IS cached - pools = {} - for key, entry in list(rt.pool_cache.items()): - if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" - and isinstance(entry, tuple) and len(entry) == 2 - and isinstance(entry[1], list)): - pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} - if not pools: - return 0 - - def _up(cur): - cur["odoo_pool"] = pools - cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") - cur["takenBy"] = str(taken_by or "") - return cur - - rt.update(SNAPSHOT_KEY, _up, flush="sync") - return len(pools) - - -def _rel_reconnect(rt): - """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot - disagree about what "resume" means.""" - import odoo_relational as rel - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = False - cur.pop("frozenAt", None) - cur.pop("frozenBy", None) - return cur - - rt.update(rel.CONFIG_KEY, _up, flush="sync") - return True - - -# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION -# -# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as -# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so -# the risk is what this block is mostly about. -# -# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env -# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already -# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, -# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build -# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks -# empty while its Odoo grids work. -# -# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns -# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — -# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under -# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector -# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That -# is D-259's exact shape (a pause written under one key and read under another) and -# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the -# SAME pass, and the carry is asserted. - - -def _env_odoo_fields(): - """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. - - ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. - Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a - PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo - offline, on a deployment where it had been working. The env fallback cannot save it, because the - entry's existence is what turns the fallback off. - ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the - field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two - vocabularies meet here; nowhere else. - """ - want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), - ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) - got = {field: (os.environ.get(env) or "").strip() for field, env in want} - missing = sorted(env for field, env in want if not got[field]) - if missing: - return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " - f"would take this tenant's Odoo offline rather than migrate it") - return got, "" - - -def migrate_env_odoo(rt): - """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. - - `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every - path including the skips, because "already migrated", "no keychain key on this deployment" and - "the env is incomplete" are three different operator actions. - - ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, - measured three times: a developer's CLI write to the tenant store is reverted by the running - Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success - every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here - would be a dry run that lies, and the thing it would lie about is a credential. - - ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a - third-party failure must not take the boot down. - """ - out = {"done": False, "entry": "", "carried_pause": False, "why": ""} - if not env_odoo_available(rt): - # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. - out["why"] = "this tenant has no environment Odoo credential to migrate" - return out - fields, why = _env_odoo_fields() - if not fields: - out["why"] = why - return out - # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the - # NEW key and the old one is unreachable through the resolver — so the only moment this fact can - # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. - try: - was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) - .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) - except Exception: # noqa: BLE001 - was_paused = False - row, why = _kc().ensure_entry_of_type( - rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") - if row is None: - out["why"] = why - return out - out["done"], out["entry"] = True, row["id"] - if was_paused: - # The freeze followed the credential. Without this the connector silently RESUMES pulling - # live Odoo at the first boot after the migration. - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - entry = dict(cur.get(row["id"]) or {}) - entry["paused"] = True - entry["pausedBy"] = "system" - entry["pausedNote"] = ("carried over from the environment source when the credential " - "was migrated onto the keychain") - cur[row["id"]] = entry - return cur - - try: - rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") - out["carried_pause"] = True - except Exception as exc: # noqa: BLE001 - # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse - # than one that did not run, so this is the one failure that must never be silent. - out["why"] = (f"the credential migrated but the PAUSE could not be carried over " - f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") - return out - - -def _own_row(session, entry_id): - """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the - caller cannot see: telling a member that somebody else's personal credential EXISTS is the - disclosure the scope is for.""" - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(entry_id)), None) - if row is None: - raise err(404, "no_entry", "no such key") - return row - - -@router.get("/admin/keychain") -def list_keychain(session: Session = Depends(require_session)): - """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. - R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would - ship the feature and no door to it. The wall moved INTO the payload: a member sees the - business-wide entries and their own, never anybody else's personal one.""" - kc = _kc() - return {"entries": visible_entries(session.runtime, session.uname, - bool(session.admin)), - "locked": not kc.unlocked(), - #: the vocabulary and the permission, so the client renders a picker it can honour - #: rather than offering an option the server will refuse (R4: business is admin-only). - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - "tenantWideTypes": list(TENANT_WIDE_TYPES)} - - -@router.post("/admin/keychain", status_code=201) -def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): - kc = _kc() - body = body or {} - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - scope = clean_scope(body.get("scope")) - if scope is None: - raise err(400, "bad_scope", - f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can create one. You can add it as a personal connection instead.") - etype = str(body.get("type") or "").strip().lower() - if scope == "personal" and etype in TENANT_WIDE_TYPES: - # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this - # credential is what the WHOLE workspace's databases are built from, so "personal" would - # be a label on a tenant-wide key rather than a boundary around it. - raise err(400, "scope_not_available", - f"a {etype} connection is what this whole workspace's databases are read " - f"through, so it is always business-wide — it cannot be a personal connection. " - f"An administrator can add it for everyone.") - try: - row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), - body.get("fields"), session.uname) - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", - f"the keychain is locked — {e}. A secret is never stored unencrypted.") - except ValueError as e: - raise err(400, "bad_entry", str(e)) - except Exception: - raise err(503, "store_unavailable", "the entry was not saved — try again") - # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode - # of a side bucket is to publish a credential, not to hide one. So the second write is not - # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was - # stored. `business` needs no row at all, so this branch is the only one that can be partial. - # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing - # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin - # find a second switch. The tables were never dropped, so "resume" is one flag. - if etype == "odoo": - try: - _rel_reconnect(session.runtime) - except Exception: # noqa: BLE001 - pass - if scope != DEFAULT_SCOPE: - try: - _write_scope(session.runtime, row["id"], scope, session.uname) - except Exception: - try: - kc.delete_entry(session.runtime, row["id"]) - except Exception: # noqa: BLE001 - pass - raise err(503, "store_unavailable", - "the key was not saved — its sharing setting could not be stored, so " - "nothing was kept. Try again.") - return {"entry": {**row, "scope": scope, "owner": session.uname}} - - -@router.put("/admin/keychain/{entry_id}") -def update_key(entry_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so - "edit this key" means "replace it" and that is `DELETE` + `POST`.""" - row = _own_row(session, entry_id) - scope = clean_scope((body or {}).get("scope"), default=None) - if scope is None: - raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") - if scope == "business" and not session.admin: - raise err(403, "not_admin", - "a business-wide connection applies to everyone in this workspace, so only an " - "administrator can make one business-wide.") - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to change") - if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: - raise err(400, "scope_not_available", - f"a {row.get('type')} connection is what this whole workspace's databases are " - f"read through, so it is always business-wide.") - owner = row.get("owner") or session.uname - try: - _write_scope(session.runtime, entry_id, scope, owner) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} - - -@router.delete("/admin/keychain/{entry_id}") -def delete_key(entry_id: str, session: Session = Depends(require_session)): - row = _own_row(session, entry_id) - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to delete") - try: - _kc().delete_entry(session.runtime, entry_id) - _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it - except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") - return {"ok": True} - - -@router.post("/admin/keychain/{entry_id}/test") -def test_key(entry_id: str, session: Session = Depends(require_session)): - _own_row(session, entry_id) # 404 for an entry this caller may not see - return _kc().test_entry(session.runtime, entry_id) - - -def _unsynced_customer_records(session): - """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no - longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a - tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the - answer is `known: False`, never a fabricated zero.""" - try: - import core.table_store as table_store - bucket = session.runtime.get("customer_table_workspace") or {} - overlay_pids = {} - for uname, ws in bucket.items(): - if uname == table_store.SHARED_KEY or not isinstance(ws, dict): - continue - for pid, cells in (ws.get("overlays") or {}).items(): - if isinstance(cells, dict) and cells: - overlay_pids.setdefault(str(pid), cells) - if not overlay_pids: - return {"known": True, "count": 0, "rows": []} - from routes_customers import allowed_pids - pool = {str(p) for p in allowed_pids(session)} - orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) - if str(x).isdigit() else 0) - rows = [] - for p in orphans[:50]: - cells = overlay_pids[p] - hint = next((str(v) for v in cells.values() if str(v).strip()), "") - rows.append({"pid": int(p) if str(p).isdigit() else p, - "fields": len(cells), "hint": hint[:80]}) - return {"known": True, "count": len(orphans), "rows": rows, - "shown": min(len(orphans), 50)} - except Exception as e: - return {"known": False, "count": None, "rows": [], - "note": f"pool unavailable — {type(e).__name__}"} - - -@router.get("/admin/connectors") -def connectors(session: Session = Depends(require_session)): - kc = _kc() - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same - # facts with a status column. Reading `kc.list_entries` here instead would have shown a member - # every colleague's personal connection on the screen next door to the one that hides them. - entries = visible_entries(session.runtime, session.uname, bool(session.admin)) - # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — - # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env - # for tenant #0 only, else nothing (fail closed — never another tenant's environment). - # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source - # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would - # make the answer depend on who opened the pane. Personal entries are excluded for the same - # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. - _scopes = _scope_rows(session.runtime) - first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) - if e["type"] == "odoo" - and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) - # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the - # connectors DIRECTORY answer "does the environment serve this tenant?" with one function - # instead of two spellings that agreed until a second tenant signed in. - if first_odoo and kc.unlocked(): - resolved = f"keychain:{first_odoo}" - elif env_odoo_available(session.runtime): - resolved = "env" - else: - resolved = None - rows = [] - if env_odoo_available(session.runtime): - rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", - "source": "env", "active": resolved == "env", - # the deployment's own credential — business-wide by construction, and it - # has no owner to be personal to. - "scope": DEFAULT_SCOPE, "owner": "", - "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) - for e in entries: - rows.append({"key": e["id"], "label": e["label"], "type": e["type"], - "source": "keychain", "preview": e["preview"], - "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", - "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), - "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) - out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, - "scopes": list(SCOPES), "canBusiness": bool(session.admin), - # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. - # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the - # hole it left: "measures not already computed may still reach the source". D-10 - # closed that hole — `harness/datastore.py` (the mirror every measure column is - # answered from) refuses to sync while paused, and `routes_products._pool_for` got the - # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather - # than left standing, because a warning that outlives its defect teaches the reader to - # ignore warnings. - # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely - # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and - # collapsing them into "everything freezes" would be the kind of tidy summary that - # stops being true the first time one of them changes. - # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the - # correction is recorded here rather than silently applied. D-62 said this note - # "promises a behaviour on a dashboard measure path that has been dead since W16". - # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from - # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while - # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is - # `/api/v1/pages/{key}` (D-52), which this note never mentioned. - # - # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures - # stop moving rather than going blank". BOTH pool paths answer **503** when they have - # no copy to serve — the customer path for a scope with no snapshot - # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because - # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says - # so in as many words). So a paused connector plus a restarted server is exactly the - # blank screen this sentence promised could not happen. A warning that over-promises - # is worse than none: it is the sentence somebody quotes when the screen disagrees. - "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " - "views stay, and nothing reaches the source while it is paused. " - "Anything this server has already read keeps showing: the customer " - "workspace serves its pause-time snapshot, the product list serves " - "the last copy read since startup, and measure columns keep answering " - "from the mirror as it stood when you paused. What has NOT been read " - "cannot be shown — a scope with no snapshot, or the product list after " - "a restart, reports that the source is paused instead of showing " - "figures. Resume to start reading live again.")} - # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to - # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on - # the payload that is NOT about connectors: it unions overlays across EVERY user of the - # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user - # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member - # would have read colleagues' typed notes off the Connectors pane. It also filters against - # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and - # the number itself becomes wrong for them as well as private. - # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already - # returned, and the audit has to walk the payload, not the entry list I was thinking about. - if session.tenant == "royal-imports" and session.admin: - out["unsynced"] = _unsynced_customer_records(session) - return out - - -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS -# ═════════════════════════════════════════════════════════════════════════════════════════════ -# -# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database -# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to -# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a -# switch flipped here is a switch the sync path obeys — a config the route knows and the sync -# path does not is a control that does nothing and reports success. -def _rel(): - import odoo_relational as rel - return rel - - -def _odoo_entry(session): - """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing - is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" - kc = _kc() - scopes = _scope_rows(session.runtime) - for e in kc.list_entries(session.runtime): - if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": - return e - return None - - -def _odoo_source_fields(session, entry): - """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. - - ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no - route returns its output; this returns the three fields that identify WHICH server, and the - api key is not among them. The masked preview is the entry's own and was computed at write. - ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by - the process, and an admin editing it from a tenant screen would be editing the container. - """ - if entry is None: - return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), - os.environ.get("ODOO_USER", ""), False) - try: - f = _kc().read_fields(session.runtime, entry["id"]) or {} - except Exception: # noqa: BLE001 - return ("", "", "", True) # locked keychain: honest blanks, still editable - return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) - - -def _odoo_admin(session): - """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the - whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide - type cannot be personal in the first place.""" - if not session.admin: - raise err(403, "not_admin", - "the Odoo connection serves this whole workspace, so only an administrator can " - "configure it") - - -@router.get("/admin/connectors/odoo/config") -def odoo_config(session: Session = Depends(require_session)): - """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a - person needs to see before changing any of it.""" - _odoo_admin(session) - rel = _rel() - entry = _odoo_entry(session) - server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) - cfg = rel.read_config(session.runtime) - return { - "applicable": bool(rel.is_royal(session.tenant)), - "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) - else "none"), - "entryId": (entry or {}).get("id", ""), - "label": (entry or {}).get("label", "Odoo (environment)"), - "preview": (entry or {}).get("preview", ""), - "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, - "serverDbEditable": editable, - "grids": rel.grid_choices(session.runtime), - "syncEvery": cfg["syncEvery"], - "syncOptions": list(rel.SYNC_PRESETS), - "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, - "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], - # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: - # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders - # no button rather than one that 400s. - "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) - and not cfg["frozen"]), - } - - -@router.put("/admin/connectors/odoo/config") -def odoo_config_put(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED - back rather than silently applied.""" - _odoo_admin(session) - rel = _rel() - body = body or {} - notes = [] - - grids = body.get("grids") - known = {c["key"] for c in rel.grid_choices(session.runtime)} - clean_grids = None - if isinstance(grids, dict): - unknown = sorted(str(k) for k in grids if str(k) not in known) - if unknown: - # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid - # this connector does not have, and swallowing it makes the two disagree quietly. - raise err(400, "unknown_grid", - f"this connector has no grid called {', '.join(unknown)}") - clean_grids = {str(k): bool(v) for k, v in grids.items()} - if clean_grids and not any(clean_grids.get(k, True) for k in known): - notes.append("every grid is switched off — nothing will be materialised on the next " - "sync, and the databases you already have are left untouched") - - every = body.get("syncEvery") - clean_every = None - if every is not None: - clean_every = str(every).strip().lower() - if clean_every not in rel.SYNC_PRESETS: - # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. - # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, - # and never silently ignored either, because a control that discards your answer - # without a word is how a limit becomes invisible. - clean_every = rel.DEFAULT_SYNC - notes.append(f"{every!r} is not an interval this connector offers, and anything under " - f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " - f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") - - server_db = body.get("serverDb") - if server_db is not None: - server_db = " ".join(str(server_db).split())[:80] - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - if clean_grids is not None: - cur.setdefault("grids", {}).update(clean_grids) - if clean_every is not None: - cur["syncEvery"] = clean_every - return cur - - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - - if server_db: - notes.append(_rewrite_server_db(session, server_db)) - - # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE - # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: - # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( - # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So - # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value - # is stored, clamped, displayed and never read. - # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a - # default 1800 s and syncs anyway. "Only when I ask" asks all the same. - # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not - # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it - # is REPORTED here, at the moment of the change, with what it really controls. Delete these - # notes when the loop becomes per-tenant, not before. - if clean_every is not None: - if not rel.is_royal(session.tenant): - notes.append("this interval is saved, but the sync loop currently reads its schedule " - "from one workspace for the whole deployment — so it will not change how " - "often YOUR data refreshes until per-workspace scheduling ships") - else: - notes.append("this interval is saved and it is the one the deployment's sync loop " - "uses — it changes the refresh rate for every workspace on this " - "deployment, not only this one") - if clean_every == "manual": - notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " - "manual-only branch, so data still refreshes on the default interval") - - out = odoo_config(session) - return {**out, "notes": [n for n in notes if n]} - - -def _rewrite_server_db(session, server_db): - """Point the stored Odoo credential at a different server database (R9's first reading). - - ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of - how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. - So this is add-then-delete through the module's own doors, with the side rows (pause flag, - scope) carried across because they are keyed by ENTRY ID. - ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the - delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a - resync landing inside that window could read the OLD database. The alternative order can - leave the workspace with no credential at all, which is worse than one stale read. Milliseconds - of ambiguity beats a lost key. - """ - kc = _kc() - entry = _odoo_entry(session) - if entry is None: - return ("the server database is set on this deployment's environment, not in the " - "keychain, so it was not changed here") - try: - fields = kc.read_fields(session.runtime, entry["id"]) or {} - except kc.KeychainLocked as e: - raise err(503, "keychain_locked", f"the keychain is locked — {e}") - if not fields: - raise err(400, "bad_entry", "this credential could not be read back to be changed") - if str(fields.get("db") or "") == server_db: - return "" - fields["db"] = server_db - try: - new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) - except Exception: - raise err(503, "store_unavailable", - "the server database was not changed — the existing connection is untouched") - # carry the side rows across, then retire the old entry - try: - flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} - if entry["id"] in flags: - def _mv(cur): - cur[new["id"]] = cur.pop(entry["id"], {}) - return cur - session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) - kc.delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - except Exception: # noqa: BLE001 - return (f"the connection now points at {server_db}, but the previous credential could " - f"not be removed — delete it under Keychains") - return f"the connection now points at the {server_db} database" - - -@router.post("/admin/connectors/odoo/disconnect") -def odoo_disconnect(session: Session = Depends(require_session)): - """R10 — remove the credential and FREEZE the grids as static data. - - ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the - key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — - including the boot rebuild and the resync loop, which for tenant #0 would otherwise - re-materialise from the process ENVIRONMENT and quietly undo the disconnect. - ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and - every FIELD DEFINITION stays, user-added columns included, because a field a person added is - the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes - one flag in a different bucket, which is what makes that guarantee structural rather than - careful. - """ - _odoo_admin(session) - rel = _rel() - import datetime as _dt - entry = _odoo_entry(session) - if not entry and not env_odoo_available(session.runtime): - raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") - - def _up(cur): - cur = cur if isinstance(cur, dict) else {} - cur["frozen"] = True - cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - cur["frozenBy"] = str(session.uname) - return cur - - # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT - # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which - # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, - # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read - # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, - # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected - # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's - # own sentence promised *"nothing is being refreshed"*. - # - # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause - # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` - # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed - # from `pause_connector` because it learned them the hard way: - # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no - # resolved source left to name, and the flag would land under a key nothing reads (D-10). - # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to - # serve; a failed capture leaves the connector live rather than paused-with-nothing. - _, flag_key = _resolved_odoo_key(session.runtime) - snapshots = 0 - if flag_key: - try: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - except Exception: # noqa: BLE001 - # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. - snapshots = 0 - - # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and - # the connector is still live; if the delete failed AFTER the flag landed, the tenant is - # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order - # can leave a workspace with no key and a connector that still tries to sync. - try: - session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") - except Exception: - raise err(503, "store_unavailable", "nothing was disconnected — try again") - - paused_mirror = False - if flag_key: - def _pause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": True} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") - paused_mirror = True - except Exception: # noqa: BLE001 - paused_mirror = False - - removed = "" - if entry is not None: - try: - _kc().delete_entry(session.runtime, entry["id"]) - _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") - removed = entry["id"] - except Exception: - raise err(503, "store_unavailable", - "the databases are frozen but the stored credential was not removed — " - "delete it under Keychains") - - # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the - # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT - # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a - # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: - # the limit that cannot be removed is REPORTED, with what to do instead. - note = ("Your Odoo databases are frozen: every row and every column you had is still there and " - "still readable, and nothing is being refreshed.") - if not paused_mirror and flag_key: - note += (" ⚠ The live mirror could not be paused, so the two read-through databases " - "(order lines and GL lines) may keep advancing — pause the Odoo connector under " - "Keychains to stop them.") - note += (" Reconnecting adds the key back and resumes into the same databases." if entry - else " This workspace's Odoo credential comes from the deployment environment, so " - "there was no stored key to remove — the databases are frozen and Reconnect " - "resumes them into the same tables.") - return {"frozen": True, "removedEntry": removed, - # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean - # nobody returns is a guarantee nobody can check. - "pausedMirror": paused_mirror, "snapshots": snapshots, - "source": "keychain" if entry else "env", - "note": note} - - -@router.post("/admin/connectors/odoo/reconnect") -def odoo_reconnect(session: Session = Depends(require_session)): - """R10's second sentence — *"Reconnecting resumes into the same tables."* - - It clears the freeze and nothing else: the tables were never dropped, so there is nothing to - recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this - is the switch that lets the sync path see it again. - """ - _odoo_admin(session) - try: - _rel_reconnect(session.runtime) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only - # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the - # two read-through grids would sit at the disconnect date while the panel said "connected - # again" — the same disagreement between the halves of "frozen", pointing the other way. - # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the - # resolved source only exists again by this point. - _, flag_key = _resolved_odoo_key(session.runtime) - resumed = False - if flag_key: - def _unpause(cur): - cur = cur if isinstance(cur, dict) else {} - cur[str(flag_key)] = {"paused": False} - return cur - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") - resumed = True - except Exception: # noqa: BLE001 - resumed = False - connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) - return {"frozen": False, "connected": connected, "resumedMirror": resumed, - "note": ("Odoo is connected again and the databases you already had will refresh in " - "place." if connected else - "The freeze is lifted, but there is no Odoo credential yet — add one under " - "Keychains and the databases resume into the same tables.")} - - -@router.post("/admin/connectors/{key}/pause") -def pause_connector(key: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - paused = bool((body or {}).get("paused")) - if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable") - # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an - # admin act; pausing your own personal one is yours. The env source has no keychain row and - # is business-wide by construction, hence the admin fallthrough. - row = next((e for e in visible_entries(session.runtime, session.uname, - bool(session.admin)) - if e["id"] == str(key)), None) - if row is not None: - if not _may_touch(session, row): - raise err(403, "not_yours", "this connection is not yours to pause") - elif not session.admin: - raise err(403, "forbidden", "administrators only") - - # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a - # "last successful sync" to serve before the freeze takes effect. Capturing before the - # flag flips means a failed capture leaves the connector live (never paused-with-nothing). - snapshots = 0 - _, flag_key = _resolved_odoo_key(session.runtime) - if paused and flag_key and str(key) == flag_key: - snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) - - def _up(cur): - cur[str(key)] = {"paused": paused} - return cur - - try: - session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) - except Exception: - raise err(503, "store_unavailable", "the change was not saved — try again") - return {"key": key, "paused": paused, "snapshots": snapshots} +"""routes_keychain.py — Keychains + Connectors admin surfaces (wave 18, C7 / R3). + +Keychain: encrypted per-tenant credential entries (`core/keychain.py`). The routes NEVER +return a decrypted field — list rows carry a masked preview, and the decrypt function is a +connector-layer internal. Connectors: the tenant's data sources as STATUS rows — Royal's +env-configured Odoo, keychain-held sources — plus R3's guardrail: the **Unsynced records** +count (rows holding overlay data whose pids the current pool no longer serves; counted and +drillable, never silently dropped) and a pause toggle whose v1 semantics are stated honestly +in the payload (`pausedNote`): pausing marks intent and warns; the source cutover ships with +the keychain cutover wave R3 staged. +""" +import os + +from fastapi import Body, Depends +from fastapi import APIRouter + +from deps import Session, err, require_session +# ⚠ W32-T11 / R4: `admin_gate` is GONE from this module, and its absence is the ruling. Every door +# here was admin-only, which made "a member may hold a personal connection" unbuildable — the wall +# moved from the ROUTE into the ROW (`may_see` / `_may_touch`), where a scope can be enforced per +# entry instead of per endpoint. `session.admin` is still what business-wide requires. + +router = APIRouter(prefix="/api/v1") + +#: ⭐ D-10 (wave 24): ONE literal, owned by the harness. It was spelled here AND implied by +#: `harness/runtime.py`'s reader; a pause flag written under one spelling and read under another +#: freezes nothing while reporting success, which is the shape of the bug D-10 books. +from harness.runtime import (CONNECTOR_FLAGS_KEY as _CONNECTOR_FLAGS_KEY, # noqa: E402 + ENV_ODOO_FLAG_KEY as _ENV_ODOO_FLAG_KEY) +#: DEBT-2 (2026-08-04): the last-successful-sync snapshot bucket. Written when the RESOLVED +#: odoo connector is paused; read by the pool path while paused; survives a Space restart. +SNAPSHOT_KEY = "connector_snapshots" + +#: ⛔⛔ W32-T10 / OWNER ITEM 6 — THE TENANT THAT OWNS THE PROCESS ENVIRONMENT. +#: +#: `ODOO_URL` and its siblings are tenant #0's `.env` / Space secrets, and ONE Space process serves +#: EVERY tenant. So "the environment has Odoo credentials" is a fact about this DEPLOYMENT, and +#: turning it into "your workspace is connected to Odoo" is only true for one slug. That is R3's +#: rule, stated in `harness/runtime.py::odoo_source` as *"NEVER the environment: env is tenant #0's +#: connection, and handing it to another tenant is the leak this method exists to prevent."* +#: +#: ⚠ IT IS SPELLED HERE BECAUSE THE RUNTIME EXPORTS NO CONSTANT FOR IT — `odoo_source` and +#: `odoo_flag_key` both carry the literal. `verify_meta`'s W32-T10 section asserts this value +#: against `runtime.py`'s own text, so the day the runtime's answer changes and this one does not, +#: the gate reds instead of the product quietly disagreeing with itself. +ENV_ODOO_TENANT = "royal-imports" + + +def env_odoo_available(rt): + """Does the PROCESS ENVIRONMENT offer an Odoo connection to THIS tenant? (W32-T10.) + + ⛔ THE ONE NORMALIZER FOR ONE QUESTION, and it exists because there were two answers to it. + `routes_connectors.directory._odoo` read a bare `os.environ.get("ODOO_URL")` with **no tenant + guard** — one process, every tenant — so a nurilab admin opening Connectors was told Odoo was + `connected` and offered "Manage keys" for a credential belonging to another company. This + module asked the same question correctly two functions below, which is the whole shape of + [[one-question-two-normalizers]]: the correct copy hides the wrong one until somebody signs in + as the second tenant. + + ⚠ NOT the same question as `rt.odoo_flag_key() == ENV_ODOO_FLAG_KEY`. That one answers *which + source WINS*, so it goes False the moment a keychain entry exists — correct for a pause flag, + wrong for "should the environment row be listed at all", which is what the connectors pane + needs in order to show an inactive env source beside an active keychain one. + + Never raises: a runtime that cannot answer is not connected. Fail closed — a missing guard is + how another tenant's environment got reported as this tenant's connection in the first place. + """ + try: + if not (os.environ.get("ODOO_URL") or "").strip(): + return False + return str(getattr(rt, "key", "") or "") == ENV_ODOO_TENANT + except Exception: # noqa: BLE001 + return False + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T11 / CONTRACT C1 / OWNER RULING R4 — BUSINESS-WIDE vs PERSONAL, ON EVERY CONNECTION +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# R4, verbatim: *"The business-wide vs personal split lands on ALL connections, not just Odoo. +# Every keychain entry and connector carries a scope, selectable per connection; business-wide is +# admin-only and applies to every user in the tenant."* +# +# ⛔ THE VOCABULARY IS DECLARED HERE AND NOWHERE ELSE (contract C1, and it names this file +# explicitly). `core/keychain.py` is the ENCRYPTED STORE and belongs to the integrator; a scope is +# an access rule, not a secret, so it lives in the layer that already decides who may ask. +# `verify_meta` asserts this tuple against `connectors/ConnectorsPage.tsx`, so the two halves +# cannot drift into two vocabularies. +SCOPES = ("business", "personal") +#: ⚠ THE READ DEFAULT, AND IT IS A READ DEFAULT — never a write (C1). Every entry stored before +#: this wave was, by construction, an admin's tenant-wide credential, so `business` is not a guess. +#: Materialising it would be a migration nobody authorised and would rewrite the store on the next +#: read [[a-migration-that-runs-on-the-next-write]]. +DEFAULT_SCOPE = "business" +#: The side bucket: `{entry_id: {"scope": "personal", "owner": ""}}`. +#: ⛔ A BUSINESS ENTRY WRITES NO ROW — it IS the default, so an absent row and a `business` row mean +#: the same thing and there is only one way to spell the common case. +SCOPE_KEY = "keychain_scopes" + +#: ⛔⛔ THE TYPES A WHOLE WORKSPACE READS THROUGH, WHICH THEREFORE CANNOT BE PERSONAL. +#: +#: `keychain.odoo_creds` and `keychain.meta_creds` both resolve to *the first entry of that type* +#: for the TENANT — that is what spawns `ut_odoo_*` / `ut_meta_*` and what every measure column is +#: answered from. So a member storing a personal Odoo key would not get "their own Odoo": they +#: would silently become the credential the entire workspace's databases are built from, which is +#: a credential elevation wearing a scope picker. +#: ⚠ REFUSED WITH A REASON, NEVER SILENTLY COERCED TO `business` — a picker that quietly changes +#: your answer is worse than one that says no (W30/R6's second sentence). The resolver itself is +#: `harness/runtime.py` / `core/keychain.py`, the integrator's files; this refusal closes the door +#: from the only side B owns, and the resolver-side guard is booked for A. +TENANT_WIDE_TYPES = ("odoo", "meta_ads") + + +def clean_scope(raw, default=DEFAULT_SCOPE): + """A scope word from the wire, or None when the caller said something we do not speak. + + Distinguishing "said nothing" (⇒ the default) from "said nonsense" (⇒ 400) is the whole + reason this returns None rather than falling back: a typo'd `"personel"` silently becoming + business-wide is exactly the failure a scope picker exists to prevent. + """ + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + got = str(raw).strip().lower() + return got if got in SCOPES else None + + +def _scope_rows(rt): + try: + return dict(rt.get(SCOPE_KEY) or {}) + except Exception: # noqa: BLE001 + return {} + + +def entry_scope(rt, entry_id, rows=None): + """`(scope, owner)` for one entry. `rows` is the bucket, passed in when walking a list so a + census does not re-read the store once per entry.""" + r = (rows if rows is not None else _scope_rows(rt)).get(str(entry_id)) + if not isinstance(r, dict): + return DEFAULT_SCOPE, "" + return (str(r.get("scope") or DEFAULT_SCOPE), str(r.get("owner") or "")) + + +def may_see(scope, owner, uname): + """R4's visibility rule: business-wide is everyone's, personal is its owner's. + + ⛔ AND AN ADMIN IS NOT AN EXCEPTION. R4 says a personal entry belongs to a person; the + per-user OAuth slots one module down have made the same call since wave 22 (*"a refresh token + is identity, not infrastructure"*). An admin who could read every member's personal credential + would make "personal" a label rather than a boundary. + """ + return scope != "personal" or str(owner) == str(uname) + + +def visible_entries(rt, uname, is_admin=False): + """This USER's view of the keychain: every business entry plus their own personal ones, each + row carrying its `scope` and `owner` so no caller has to ask a second time. + + ⛔ THE MASKED PREVIEW IS NOT PART OF "VISIBLE". R4 opens this room to members so they can + hold a connection of their own; it does not hand them four characters of the workspace's Odoo + key. So a business row a member did not create arrives WITHOUT `preview` — they can see that + the connection exists and is theirs to use, which is the whole of what R4 grants. The default + is the RESTRICTED one deliberately: a caller that forgets the argument leaks nothing. + """ + rows = _scope_rows(rt) + out = [] + for e in _kc().list_entries(rt): + scope, owner = entry_scope(rt, e["id"], rows) + if not owner: + owner = str(e.get("createdBy") or "") + if not may_see(scope, owner, uname): + continue + row = {**e, "scope": scope, "owner": owner} + if not (is_admin or str(owner) == str(uname)): + row["preview"] = "" + out.append(row) + return out + + +def _write_scope(rt, entry_id, scope, owner): + """Persist one entry's scope. A `business` entry CLEARS its row rather than writing the + default, so the store holds one spelling of the common case.""" + def _up(cur): + if scope == DEFAULT_SCOPE: + cur.pop(str(entry_id), None) + else: + cur[str(entry_id)] = {"scope": scope, "owner": str(owner or "")} + return cur + + rt.update(SCOPE_KEY, _up, flush="sync") + return True + + +def _may_touch(session, row): + """May this session change or delete `row`? An admin owns the business-wide ones; a member + owns their own personal ones. Anything else is not theirs to move.""" + if row.get("scope") == "personal": + return str(row.get("owner") or "") == str(session.uname) + return bool(session.admin) + + +def _kc(): + import core.keychain as keychain + return keychain + + +def _resolved_odoo_key(rt): + """`(source, flag_key)` — which source would serve this tenant's Odoo queries, in both the + shapes this module needs: the display string (`env` / `keychain:`) and the key the pause + flag is stored under. + + ⭐ D-10 (wave 24): THE RESOLUTION ITSELF NOW LIVES IN ONE PLACE, `TenantRuntime.odoo_flag_key`, + beside the `odoo_source()` it must agree with. This function had its own copy of the same + three rules — first unlocked keychain odoo entry, else env for tenant #0, else nothing — and + a second copy is exactly how a pause flag comes to be written against one resolution and read + against another, freezing nothing while the UI reports success. The two SHAPES stay here + because they are this module's presentation concern; the DECISION does not. + """ + flag_key = rt.odoo_flag_key() + if not flag_key: + return None, None + return ("env" if flag_key == _ENV_ODOO_FLAG_KEY else f"keychain:{flag_key}"), flag_key + + +def odoo_paused(rt): + """True when the tenant's RESOLVED Odoo source carries the pause flag. Pausing an entry + that is not the resolved source freezes nothing — it serves nothing. + + ⭐ D-10: a thin delegate now. The implementation moved to `TenantRuntime.odoo_paused` so the + measure mirror (`harness/datastore.py`, which cannot import this layer) asks the SAME question + the customer pool does. This name stays because `routes_customers`, `routes_products` and + `verify_api` all call it — moving the logic without moving the door keeps one answer and + costs no caller a change. + """ + return bool(rt.odoo_paused()) + + +def _snap_scope_key(team_id, agent): + return f"t={team_id}|a={agent}" + + +def load_pool_snapshot(rt, team_id, agent): + """(ts, rows) from the persisted snapshot for this exact scope, or None. NEVER a wider + scope's rows — serving the consolidated snapshot to a scoped user would widen their book.""" + try: + snap = (rt.get(SNAPSHOT_KEY) or {}).get("odoo_pool") or {} + e = snap.get(_snap_scope_key(team_id, agent)) + if isinstance(e, dict) and isinstance(e.get("rows"), list): + return float(e.get("ts") or 0), e["rows"] + except Exception: + pass + return None + + +def save_pool_snapshots(rt, taken_by=""): + """Persist every currently-cached pool scope as the pause-time snapshot ('the last + successful sync', made concrete). Ensures the consolidated default scope exists first so + a pause on a cold process still captures something to serve.""" + import time as _time + import routes_customers as _rc + try: + _rc._pool_for(rt, None, None) # the scope every admin/all-BU account lands on + except Exception: + pass # cold + Odoo down: persist whatever IS cached + pools = {} + for key, entry in list(rt.pool_cache.items()): + if (isinstance(key, tuple) and len(key) == 3 and key[0] == "pool" + and isinstance(entry, tuple) and len(entry) == 2 + and isinstance(entry[1], list)): + pools[_snap_scope_key(key[1], key[2])] = {"ts": entry[0], "rows": entry[1]} + if not pools: + return 0 + + def _up(cur): + cur["odoo_pool"] = pools + cur["taken"] = _time.strftime("%Y-%m-%dT%H:%M:%S") + cur["takenBy"] = str(taken_by or "") + return cur + + rt.update(SNAPSHOT_KEY, _up, flush="sync") + return len(pools) + + +def _rel_reconnect(rt): + """Lift the W32-T16 freeze. Its own function so `add_key` and the reconnect route cannot + disagree about what "resume" means.""" + import odoo_relational as rel + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = False + cur.pop("frozenAt", None) + cur.pop("frozenBy", None) + return cur + + rt.update(rel.CONFIG_KEY, _up, flush="sync") + return True + + +# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION +# +# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as +# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so +# the risk is what this block is mostly about. +# +# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env +# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already +# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, +# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build +# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks +# empty while its Odoo grids work. +# +# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns +# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — +# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under +# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector +# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That +# is D-259's exact shape (a pause written under one key and read under another) and +# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the +# SAME pass, and the carry is asserted. + + +def _env_odoo_fields(): + """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. + + ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. + Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a + PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo + offline, on a deployment where it had been working. The env fallback cannot save it, because the + entry's existence is what turns the fallback off. + ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the + field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two + vocabularies meet here; nowhere else. + """ + want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), + ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) + got = {field: (os.environ.get(env) or "").strip() for field, env in want} + missing = sorted(env for field, env in want if not got[field]) + if missing: + return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " + f"would take this tenant's Odoo offline rather than migrate it") + return got, "" + + +def migrate_env_odoo(rt): + """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. + + `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every + path including the skips, because "already migrated", "no keychain key on this deployment" and + "the env is incomplete" are three different operator actions. + + ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success + every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here + would be a dry run that lies, and the thing it would lie about is a credential. + + ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a + third-party failure must not take the boot down. + """ + out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + if not env_odoo_available(rt): + # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. + out["why"] = "this tenant has no environment Odoo credential to migrate" + return out + fields, why = _env_odoo_fields() + if not fields: + out["why"] = why + return out + # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the + # NEW key and the old one is unreachable through the resolver — so the only moment this fact can + # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. + try: + was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) + .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) + except Exception: # noqa: BLE001 + was_paused = False + row, why = _kc().ensure_entry_of_type( + rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") + if row is None: + out["why"] = why + return out + out["done"], out["entry"] = True, row["id"] + if was_paused: + # The freeze followed the credential. Without this the connector silently RESUMES pulling + # live Odoo at the first boot after the migration. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = dict(cur.get(row["id"]) or {}) + entry["paused"] = True + entry["pausedBy"] = "system" + entry["pausedNote"] = ("carried over from the environment source when the credential " + "was migrated onto the keychain") + cur[row["id"]] = entry + return cur + + try: + rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + out["carried_pause"] = True + except Exception as exc: # noqa: BLE001 + # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse + # than one that did not run, so this is the one failure that must never be silent. + out["why"] = (f"the credential migrated but the PAUSE could not be carried over " + f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + return out + + +def _own_row(session, entry_id): + """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the + caller cannot see: telling a member that somebody else's personal credential EXISTS is the + disclosure the scope is for.""" + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(entry_id)), None) + if row is None: + raise err(404, "no_entry", "no such key") + return row + + +@router.get("/admin/keychain") +def list_keychain(session: Session = Depends(require_session)): + """⭐ W32-T11 / R4 — SESSION-GATED, NOT ADMIN-GATED, and that is the ruling not a relaxation. + R4 puts a PERSONAL connection in every member's hands, so a room only an admin can open would + ship the feature and no door to it. The wall moved INTO the payload: a member sees the + business-wide entries and their own, never anybody else's personal one.""" + kc = _kc() + return {"entries": visible_entries(session.runtime, session.uname, + bool(session.admin)), + "locked": not kc.unlocked(), + #: the vocabulary and the permission, so the client renders a picker it can honour + #: rather than offering an option the server will refuse (R4: business is admin-only). + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + "tenantWideTypes": list(TENANT_WIDE_TYPES)} + + +@router.post("/admin/keychain", status_code=201) +def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)): + kc = _kc() + body = body or {} + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + scope = clean_scope(body.get("scope")) + if scope is None: + raise err(400, "bad_scope", + f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can create one. You can add it as a personal connection instead.") + etype = str(body.get("type") or "").strip().lower() + if scope == "personal" and etype in TENANT_WIDE_TYPES: + # ⛔ REPORTED, NOT COERCED (W30/R6's second sentence). See TENANT_WIDE_TYPES above: this + # credential is what the WHOLE workspace's databases are built from, so "personal" would + # be a label on a tenant-wide key rather than a boundary around it. + raise err(400, "scope_not_available", + f"a {etype} connection is what this whole workspace's databases are read " + f"through, so it is always business-wide — it cannot be a personal connection. " + f"An administrator can add it for everyone.") + try: + row = kc.add_entry(session.runtime, body.get("label"), body.get("type"), + body.get("fields"), session.uname) + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", + f"the keychain is locked — {e}. A secret is never stored unencrypted.") + except ValueError as e: + raise err(400, "bad_entry", str(e)) + except Exception: + raise err(503, "store_unavailable", "the entry was not saved — try again") + # ⛔⛔ A PERSONAL ENTRY THAT LOSES ITS SCOPE ROW READS AS BUSINESS-WIDE — i.e. the failure mode + # of a side bucket is to publish a credential, not to hide one. So the second write is not + # best-effort: if it does not land, the entry is REMOVED and the caller is told nothing was + # stored. `business` needs no row at all, so this branch is the only one that can be partial. + # ⭐ W32-T16 / R10's SECOND SENTENCE — *"Reconnecting resumes into the same tables."* Storing + # an Odoo credential IS reconnecting, so it lifts the freeze here rather than making the admin + # find a second switch. The tables were never dropped, so "resume" is one flag. + if etype == "odoo": + try: + _rel_reconnect(session.runtime) + except Exception: # noqa: BLE001 + pass + if scope != DEFAULT_SCOPE: + try: + _write_scope(session.runtime, row["id"], scope, session.uname) + except Exception: + try: + kc.delete_entry(session.runtime, row["id"]) + except Exception: # noqa: BLE001 + pass + raise err(503, "store_unavailable", + "the key was not saved — its sharing setting could not be stored, so " + "nothing was kept. Try again.") + return {"entry": {**row, "scope": scope, "owner": session.uname}} + + +@router.put("/admin/keychain/{entry_id}") +def update_key(entry_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C1's scope door. Only the scope moves — a stored secret is never re-openable, so + "edit this key" means "replace it" and that is `DELETE` + `POST`.""" + row = _own_row(session, entry_id) + scope = clean_scope((body or {}).get("scope"), default=None) + if scope is None: + raise err(400, "bad_scope", f"scope must be one of {', '.join(SCOPES)}") + if scope == "business" and not session.admin: + raise err(403, "not_admin", + "a business-wide connection applies to everyone in this workspace, so only an " + "administrator can make one business-wide.") + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to change") + if scope == "personal" and str(row.get("type") or "") in TENANT_WIDE_TYPES: + raise err(400, "scope_not_available", + f"a {row.get('type')} connection is what this whole workspace's databases are " + f"read through, so it is always business-wide.") + owner = row.get("owner") or session.uname + try: + _write_scope(session.runtime, entry_id, scope, owner) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"entry": {**row, "scope": scope, "owner": owner if scope == "personal" else ""}} + + +@router.delete("/admin/keychain/{entry_id}") +def delete_key(entry_id: str, session: Session = Depends(require_session)): + row = _own_row(session, entry_id) + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to delete") + try: + _kc().delete_entry(session.runtime, entry_id) + _write_scope(session.runtime, entry_id, DEFAULT_SCOPE, "") # drop the side row with it + except Exception: + raise err(503, "store_unavailable", "the delete did not land — try again") + return {"ok": True} + + +@router.post("/admin/keychain/{entry_id}/test") +def test_key(entry_id: str, session: Session = Depends(require_session)): + _own_row(session, entry_id) # 404 for an entry this caller may not see + return _kc().test_entry(session.runtime, entry_id) + + +def _unsynced_customer_records(session): + """R3's guardrail, tenant #0's customer topic: overlay-holding pids the CURRENT pool no + longer serves. Overlays are unioned across EVERY user of the table (the guardrail is a + tenant fact, not a per-user one). Honest degradation: when the pool cannot be built the + answer is `known: False`, never a fabricated zero.""" + try: + import core.table_store as table_store + bucket = session.runtime.get("customer_table_workspace") or {} + overlay_pids = {} + for uname, ws in bucket.items(): + if uname == table_store.SHARED_KEY or not isinstance(ws, dict): + continue + for pid, cells in (ws.get("overlays") or {}).items(): + if isinstance(cells, dict) and cells: + overlay_pids.setdefault(str(pid), cells) + if not overlay_pids: + return {"known": True, "count": 0, "rows": []} + from routes_customers import allowed_pids + pool = {str(p) for p in allowed_pids(session)} + orphans = sorted((p for p in overlay_pids if p not in pool), key=lambda x: int(x) + if str(x).isdigit() else 0) + rows = [] + for p in orphans[:50]: + cells = overlay_pids[p] + hint = next((str(v) for v in cells.values() if str(v).strip()), "") + rows.append({"pid": int(p) if str(p).isdigit() else p, + "fields": len(cells), "hint": hint[:80]}) + return {"known": True, "count": len(orphans), "rows": rows, + "shown": min(len(orphans), 50)} + except Exception as e: + return {"known": False, "count": None, "rows": [], + "note": f"pool unavailable — {type(e).__name__}"} + + +@router.get("/admin/connectors") +def connectors(session: Session = Depends(require_session)): + kc = _kc() + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + # ⭐ W32-T11 / R4 — THE SAME VISIBILITY RULE AS THE KEYCHAIN, because this pane is the same + # facts with a status column. Reading `kc.list_entries` here instead would have shown a member + # every colleague's personal connection on the screen next door to the one that hides them. + entries = visible_entries(session.runtime, session.uname, bool(session.admin)) + # R3 cutover (2026-08-04): which source would actually serve this tenant's Odoo queries — + # mirrors TenantRuntime.odoo_source() exactly: first unlocked keychain odoo entry, else env + # for tenant #0 only, else nothing (fail closed — never another tenant's environment). + # ⚠ W32-T11: computed over the TENANT's entries, not over `entries` above. "Which source + # serves this workspace" is one fact for everybody, and deriving it from a per-USER list would + # make the answer depend on who opened the pane. Personal entries are excluded for the same + # reason `TENANT_WIDE_TYPES` refuses them: they must never become the workspace's source. + _scopes = _scope_rows(session.runtime) + first_odoo = next((e["id"] for e in kc.list_entries(session.runtime) + if e["type"] == "odoo" + and entry_scope(session.runtime, e["id"], _scopes)[0] != "personal"), None) + # ⛔ W32-T10 — the env leg goes through `env_odoo_available` now, so this route and the + # connectors DIRECTORY answer "does the environment serve this tenant?" with one function + # instead of two spellings that agreed until a second tenant signed in. + if first_odoo and kc.unlocked(): + resolved = f"keychain:{first_odoo}" + elif env_odoo_available(session.runtime): + resolved = "env" + else: + resolved = None + rows = [] + if env_odoo_available(session.runtime): + rows.append({"key": _ENV_ODOO_FLAG_KEY, "label": "Odoo (environment)", "type": "odoo", + "source": "env", "active": resolved == "env", + # the deployment's own credential — business-wide by construction, and it + # has no owner to be personal to. + "scope": DEFAULT_SCOPE, "owner": "", + "paused": bool((flags.get(_ENV_ODOO_FLAG_KEY) or {}).get("paused"))}) + for e in entries: + rows.append({"key": e["id"], "label": e["label"], "type": e["type"], + "source": "keychain", "preview": e["preview"], + "scope": e.get("scope") or DEFAULT_SCOPE, "owner": e.get("owner") or "", + "active": (e["type"] == "odoo" and resolved == f"keychain:{e['id']}"), + "paused": bool((flags.get(e["id"]) or {}).get("paused"))}) + out = {"connectors": rows, "locked": not kc.unlocked(), "resolved": resolved, + "scopes": list(SCOPES), "canBusiness": bool(session.admin), + # ⭐ D-10 (wave 24) — THIS SENTENCE IS NOW TRUE OF EVERY PATH, which it was not before. + # DEBT-2 (2026-08-04) froze the CUSTOMER pool and this note honestly disclosed the + # hole it left: "measures not already computed may still reach the source". D-10 + # closed that hole — `harness/datastore.py` (the mirror every measure column is + # answered from) refuses to sync while paused, and `routes_products._pool_for` got the + # guard its customer sibling has had since DEBT-2. So the caveat is deleted rather + # than left standing, because a warning that outlives its defect teaches the reader to + # ignore warnings. + # ⚠ THE THREE BEHAVIOURS ARE NAMED SEPARATELY on purpose: they are genuinely + # different answers (a persisted snapshot, an in-process cache, a frozen mirror), and + # collapsing them into "everything freezes" would be the kind of tidy summary that + # stops being true the first time one of them changes. + # ⭐ D-62 CLOSED (wave 27) — AND THE REGISTER'S DIAGNOSIS OF IT WAS WRONG, so the + # correction is recorded here rather than silently applied. D-62 said this note + # "promises a behaviour on a dashboard measure path that has been dead since W16". + # MEASURED 2026-08-08, and it is not: measure COLUMNS are grid columns answered from + # the DuckDB mirror, and `harness/datastore.py` genuinely refuses to sync while + # paused (`source_paused()` at four sites), so that clause was TRUE. The dead path is + # `/api/v1/pages/{key}` (D-52), which this note never mentioned. + # + # ⛔ THE REAL DEFECT WAS THE OPPOSITE ONE, and it was the last sentence: "so figures + # stop moving rather than going blank". BOTH pool paths answer **503** when they have + # no copy to serve — the customer path for a scope with no snapshot + # (`routes_customers.py:88`) and the product path ALWAYS after a restart, because + # there is no product snapshot bucket at all (`routes_products.py:60-73`, which says + # so in as many words). So a paused connector plus a restarted server is exactly the + # blank screen this sentence promised could not happen. A warning that over-promises + # is worse than none: it is the sentence somebody quotes when the screen disagrees. + "pausedNote": ("Pausing a connector never deletes data — notes, custom fields and " + "views stay, and nothing reaches the source while it is paused. " + "Anything this server has already read keeps showing: the customer " + "workspace serves its pause-time snapshot, the product list serves " + "the last copy read since startup, and measure columns keep answering " + "from the mirror as it stood when you paused. What has NOT been read " + "cannot be shown — a scope with no snapshot, or the product list after " + "a restart, reports that the source is paused instead of showing " + "figures. Resume to start reading live again.")} + # ⛔⛔ W32-T11 — ADMIN-GATED, AND THIS IS A DISCLOSURE FIX, NOT TIDINESS. Opening this route to + # members (R4) opened this block with it, and `_unsynced_customer_records` is the one thing on + # the payload that is NOT about connectors: it unions overlays across EVERY user of the + # customer table — its own docstring says so, *"the guardrail is a tenant fact, not a per-user + # one"* — and returns `hint`, the first non-empty cell of somebody else's overlay. So a member + # would have read colleagues' typed notes off the Connectors pane. It also filters against + # `allowed_pids(session)`, so a BU-scoped member's narrower pool inflates the orphan count and + # the number itself becomes wrong for them as well as private. + # ⚠ The lesson generalises past this line: opening a route widens EVERY field it already + # returned, and the audit has to walk the payload, not the entry list I was thinking about. + if session.tenant == "royal-imports" and session.admin: + out["unsynced"] = _unsynced_customer_records(session) + return out + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W32-T15/T16/T17 / CONTRACT C2 / RULINGS R9, R10, R11 — THE ODOO CONNECTOR ACTUALLY OPENS +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner clicked "Manage keys" on Odoo and found a credential list. Not: which server database +# this workspace reads, which of the ten mirrored grids it wants, how often they sync, or how to +# stop. Every decision below lives in `odoo_relational` (the module `refresh()` reads) so that a +# switch flipped here is a switch the sync path obeys — a config the route knows and the sync +# path does not is a control that does nothing and reports success. +def _rel(): + import odoo_relational as rel + return rel + + +def _odoo_entry(session): + """The keychain entry SERVING this tenant's Odoo, or None when the environment is (or nothing + is). Business-scoped by construction — `TENANT_WIDE_TYPES` refuses a personal one.""" + kc = _kc() + scopes = _scope_rows(session.runtime) + for e in kc.list_entries(session.runtime): + if e["type"] == "odoo" and entry_scope(session.runtime, e["id"], scopes)[0] != "personal": + return e + return None + + +def _odoo_source_fields(session, entry): + """`(serverDb, serverUrl, apiUser, editable)` — what the panel may SHOW about the connection. + + ⛔ NEVER THE SECRET. `read_fields` is documented as the connector layer's internal and no + route returns its output; this returns the three fields that identify WHICH server, and the + api key is not among them. The masked preview is the entry's own and was computed at write. + ⚠ The ENVIRONMENT source is not editable and says so: it is the deployment's `.env`, shared by + the process, and an admin editing it from a tenant screen would be editing the container. + """ + if entry is None: + return (os.environ.get("ODOO_DB", ""), os.environ.get("ODOO_URL", ""), + os.environ.get("ODOO_USER", ""), False) + try: + f = _kc().read_fields(session.runtime, entry["id"]) or {} + except Exception: # noqa: BLE001 + return ("", "", "", True) # locked keychain: honest blanks, still editable + return (str(f.get("db") or ""), str(f.get("url") or ""), str(f.get("user") or ""), True) + + +def _odoo_admin(session): + """C2's doors are admin doors: they show the credential that serves EVERYONE and can turn the + whole workspace's databases off. R4's personal scope has nothing to say here — a tenant-wide + type cannot be personal in the first place.""" + if not session.admin: + raise err(403, "not_admin", + "the Odoo connection serves this whole workspace, so only an administrator can " + "configure it") + + +@router.get("/admin/connectors/odoo/config") +def odoo_config(session: Session = Depends(require_session)): + """Contract C2's read: `{serverDb, grids, syncEvery, canDisconnect}` and the rest of what a + person needs to see before changing any of it.""" + _odoo_admin(session) + rel = _rel() + entry = _odoo_entry(session) + server_db, server_url, api_user, editable = _odoo_source_fields(session, entry) + cfg = rel.read_config(session.runtime) + return { + "applicable": bool(rel.is_royal(session.tenant)), + "source": "keychain" if entry else ("env" if env_odoo_available(session.runtime) + else "none"), + "entryId": (entry or {}).get("id", ""), + "label": (entry or {}).get("label", "Odoo (environment)"), + "preview": (entry or {}).get("preview", ""), + "serverDb": server_db, "serverUrl": server_url, "apiUser": api_user, + "serverDbEditable": editable, + "grids": rel.grid_choices(session.runtime), + "syncEvery": cfg["syncEvery"], + "syncOptions": list(rel.SYNC_PRESETS), + "syncFloorSeconds": rel.SYNC_FLOOR_SECONDS, + "frozen": cfg["frozen"], "frozenAt": cfg["frozenAt"], + # ⚠ There is nothing to disconnect FROM when the source is the deployment environment: + # tenant #0's `.env` is not this tenant's to remove. Said as a field so the client renders + # no button rather than one that 400s. + "canDisconnect": bool(entry) or (env_odoo_available(session.runtime) + and not cfg["frozen"]), + } + + +@router.put("/admin/connectors/odoo/config") +def odoo_config_put(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Contract C2's write. Grids, cadence and the server database — each optional, each REPORTED + back rather than silently applied.""" + _odoo_admin(session) + rel = _rel() + body = body or {} + notes = [] + + grids = body.get("grids") + known = {c["key"] for c in rel.grid_choices(session.runtime)} + clean_grids = None + if isinstance(grids, dict): + unknown = sorted(str(k) for k in grids if str(k) not in known) + if unknown: + # ⛔ NAMED, NOT DROPPED. A key we do not serve is a client that believes in a grid + # this connector does not have, and swallowing it makes the two disagree quietly. + raise err(400, "unknown_grid", + f"this connector has no grid called {', '.join(unknown)}") + clean_grids = {str(k): bool(v) for k, v in grids.items()} + if clean_grids and not any(clean_grids.get(k, True) for k in known): + notes.append("every grid is switched off — nothing will be materialised on the next " + "sync, and the databases you already have are left untouched") + + every = body.get("syncEvery") + clean_every = None + if every is not None: + clean_every = str(every).strip().lower() + if clean_every not in rel.SYNC_PRESETS: + # ⛔⛔ R11 + W30/R6's SECOND SENTENCE: the floor is enforced AND the caller is told. + # A crafted `"5m"` is CLAMPED to the floor and the response says so — never applied, + # and never silently ignored either, because a control that discards your answer + # without a word is how a limit becomes invisible. + clean_every = rel.DEFAULT_SYNC + notes.append(f"{every!r} is not an interval this connector offers, and anything under " + f"{rel.SYNC_FLOOR_SECONDS // 60} minutes is not available at all — the " + f"sync interval was set to the {rel.DEFAULT_SYNC} floor instead") + + server_db = body.get("serverDb") + if server_db is not None: + server_db = " ".join(str(server_db).split())[:80] + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + if clean_grids is not None: + cur.setdefault("grids", {}).update(clean_grids) + if clean_every is not None: + cur["syncEvery"] = clean_every + return cur + + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + + if server_db: + notes.append(_rewrite_server_db(session, server_db)) + + # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE + # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: + # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( + # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So + # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value + # is stored, clamped, displayed and never read. + # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a + # default 1800 s and syncs anyway. "Only when I ask" asks all the same. + # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not + # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it + # is REPORTED here, at the moment of the change, with what it really controls. Delete these + # notes when the loop becomes per-tenant, not before. + if clean_every is not None: + if not rel.is_royal(session.tenant): + notes.append("this interval is saved, but the sync loop currently reads its schedule " + "from one workspace for the whole deployment — so it will not change how " + "often YOUR data refreshes until per-workspace scheduling ships") + else: + notes.append("this interval is saved and it is the one the deployment's sync loop " + "uses — it changes the refresh rate for every workspace on this " + "deployment, not only this one") + if clean_every == "manual": + notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " + "manual-only branch, so data still refreshes on the default interval") + + out = odoo_config(session) + return {**out, "notes": [n for n in notes if n]} + + +def _rewrite_server_db(session, server_db): + """Point the stored Odoo credential at a different server database (R9's first reading). + + ⛔ THERE IS NO "UPDATE ENTRY" IN THE KEYCHAIN, and writing one here would be a SECOND copy of + how a secret is encrypted and previewed — the thing `core/keychain.py` exists to hold alone. + So this is add-then-delete through the module's own doors, with the side rows (pause flag, + scope) carried across because they are keyed by ENTRY ID. + ⚠ THE ORDER IS DELIBERATE AND THE WINDOW IS REAL: for the moment between the add and the + delete this tenant has TWO odoo entries, and `odoo_creds` takes the first by id sort — so a + resync landing inside that window could read the OLD database. The alternative order can + leave the workspace with no credential at all, which is worse than one stale read. Milliseconds + of ambiguity beats a lost key. + """ + kc = _kc() + entry = _odoo_entry(session) + if entry is None: + return ("the server database is set on this deployment's environment, not in the " + "keychain, so it was not changed here") + try: + fields = kc.read_fields(session.runtime, entry["id"]) or {} + except kc.KeychainLocked as e: + raise err(503, "keychain_locked", f"the keychain is locked — {e}") + if not fields: + raise err(400, "bad_entry", "this credential could not be read back to be changed") + if str(fields.get("db") or "") == server_db: + return "" + fields["db"] = server_db + try: + new = kc.add_entry(session.runtime, entry["label"], "odoo", fields, session.uname) + except Exception: + raise err(503, "store_unavailable", + "the server database was not changed — the existing connection is untouched") + # carry the side rows across, then retire the old entry + try: + flags = session.runtime.get(_CONNECTOR_FLAGS_KEY) or {} + if entry["id"] in flags: + def _mv(cur): + cur[new["id"]] = cur.pop(entry["id"], {}) + return cur + session.runtime.update(_CONNECTOR_FLAGS_KEY, _mv) + kc.delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + except Exception: # noqa: BLE001 + return (f"the connection now points at {server_db}, but the previous credential could " + f"not be removed — delete it under Keychains") + return f"the connection now points at the {server_db} database" + + +@router.post("/admin/connectors/odoo/disconnect") +def odoo_disconnect(session: Session = Depends(require_session)): + """R10 — remove the credential and FREEZE the grids as static data. + + ⛔ DISTINCT FROM PAUSE, and the difference is the credential. Pause is temporary and keeps the + key; disconnect deletes it and marks the databases frozen so nothing refreshes them again — + including the boot rebuild and the resync loop, which for tenant #0 would otherwise + re-materialise from the process ENVIRONMENT and quietly undo the disconnect. + ⛔⛔ AND IT DELETES NOTHING ELSE. The owner's words are *"so we don't fuck up"*: every row and + every FIELD DEFINITION stays, user-added columns included, because a field a person added is + the thing a naive freeze drops first. This route never touches `fields` or `rows` — it writes + one flag in a different bucket, which is what makes that guarantee structural rather than + careful. + """ + _odoo_admin(session) + rel = _rel() + import datetime as _dt + entry = _odoo_entry(session) + if not entry and not env_odoo_available(session.runtime): + raise err(400, "not_connected", "this workspace has no Odoo connection to disconnect") + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + cur["frozen"] = True + cur["frozenAt"] = _dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + cur["frozenBy"] = str(session.uname) + return cur + + # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT + # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which + # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, + # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read + # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, + # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected + # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's + # own sentence promised *"nothing is being refreshed"*. + # + # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause + # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` + # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed + # from `pause_connector` because it learned them the hard way: + # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no + # resolved source left to name, and the flag would land under a key nothing reads (D-10). + # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to + # serve; a failed capture leaves the connector live rather than paused-with-nothing. + _, flag_key = _resolved_odoo_key(session.runtime) + snapshots = 0 + if flag_key: + try: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + except Exception: # noqa: BLE001 + # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. + snapshots = 0 + + # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and + # the connector is still live; if the delete failed AFTER the flag landed, the tenant is + # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order + # can leave a workspace with no key and a connector that still tries to sync. + try: + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + except Exception: + raise err(503, "store_unavailable", "nothing was disconnected — try again") + + paused_mirror = False + if flag_key: + def _pause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": True} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") + paused_mirror = True + except Exception: # noqa: BLE001 + paused_mirror = False + + removed = "" + if entry is not None: + try: + _kc().delete_entry(session.runtime, entry["id"]) + _write_scope(session.runtime, entry["id"], DEFAULT_SCOPE, "") + removed = entry["id"] + except Exception: + raise err(503, "store_unavailable", + "the databases are frozen but the stored credential was not removed — " + "delete it under Keychains") + + # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the + # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT + # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a + # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: + # the limit that cannot be removed is REPORTED, with what to do instead. + note = ("Your Odoo databases are frozen: every row and every column you had is still there and " + "still readable, and nothing is being refreshed.") + if not paused_mirror and flag_key: + note += (" ⚠ The live mirror could not be paused, so the two read-through databases " + "(order lines and GL lines) may keep advancing — pause the Odoo connector under " + "Keychains to stop them.") + note += (" Reconnecting adds the key back and resumes into the same databases." if entry + else " This workspace's Odoo credential comes from the deployment environment, so " + "there was no stored key to remove — the databases are frozen and Reconnect " + "resumes them into the same tables.") + return {"frozen": True, "removedEntry": removed, + # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean + # nobody returns is a guarantee nobody can check. + "pausedMirror": paused_mirror, "snapshots": snapshots, + "source": "keychain" if entry else "env", + "note": note} + + +@router.post("/admin/connectors/odoo/reconnect") +def odoo_reconnect(session: Session = Depends(require_session)): + """R10's second sentence — *"Reconnecting resumes into the same tables."* + + It clears the freeze and nothing else: the tables were never dropped, so there is nothing to + recreate. A tenant whose source was a keychain entry adds it back under Keychains first; this + is the switch that lets the sync path see it again. + """ + _odoo_admin(session) + try: + _rel_reconnect(session.runtime) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only + # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the + # two read-through grids would sit at the disconnect date while the panel said "connected + # again" — the same disagreement between the halves of "frozen", pointing the other way. + # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the + # resolved source only exists again by this point. + _, flag_key = _resolved_odoo_key(session.runtime) + resumed = False + if flag_key: + def _unpause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": False} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") + resumed = True + except Exception: # noqa: BLE001 + resumed = False + connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) + return {"frozen": False, "connected": connected, "resumedMirror": resumed, + "note": ("Odoo is connected again and the databases you already had will refresh in " + "place." if connected else + "The freeze is lifted, but there is no Odoo credential yet — add one under " + "Keychains and the databases resume into the same tables.")} + + +@router.post("/admin/connectors/{key}/pause") +def pause_connector(key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + paused = bool((body or {}).get("paused")) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable") + # ⭐ W32-T11 / R4 — pausing a BUSINESS-WIDE connector stops it for everyone, so it stays an + # admin act; pausing your own personal one is yours. The env source has no keychain row and + # is business-wide by construction, hence the admin fallthrough. + row = next((e for e in visible_entries(session.runtime, session.uname, + bool(session.admin)) + if e["id"] == str(key)), None) + if row is not None: + if not _may_touch(session, row): + raise err(403, "not_yours", "this connection is not yours to pause") + elif not session.admin: + raise err(403, "forbidden", "administrators only") + + # DEBT-2: pausing the RESOLVED Odoo source captures the snapshot FIRST, so there is a + # "last successful sync" to serve before the freeze takes effect. Capturing before the + # flag flips means a failed capture leaves the connector live (never paused-with-nothing). + snapshots = 0 + _, flag_key = _resolved_odoo_key(session.runtime) + if paused and flag_key and str(key) == flag_key: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + + def _up(cur): + cur[str(key)] = {"paused": paused} + return cur + + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _up) + except Exception: + raise err(503, "store_unavailable", "the change was not saved — try again") + return {"key": key, "paused": paused, "snapshots": snapshots} diff --git a/api/routes_oauth.py b/api/routes_oauth.py index 8e49c41134aafde8a9738e53d63775c2d76fea88..7795caf496a8598b3c52b4cf5b0663939012d5c8 100644 --- a/api/routes_oauth.py +++ b/api/routes_oauth.py @@ -1,114 +1,114 @@ -"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). - -Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, -shapes and status codes here; every decision that could be wrong lives in the module a gate -can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, -so the day a second provider lands here is the day nothing in this file changes. - -MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session -`main.py`, and `routes_automation` is already included there — so this router rides inside it -(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that -alters no path. - -⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent -screen** — it is a top-level navigation the client reaches by ``, never JSON. The -callback 302s BACK to the return path the `state` carried (relative-only, sanitised by -`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever -went wrong rides in the query string; a dead-end error page where the app used to be reads as -"the product broke", not "the connect failed". -""" -import os - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import RedirectResponse - -import oauth_connect -from deps import Session, err, require_session - -router = APIRouter(prefix="/oauth") - - -def _redirect_uri(request: Request, provider: str) -> str: - """The redirect URI this deployment registers at the provider — env-pinned when the - container sits behind a proxy that rewrites the scheme (the HF Space), else derived from - the request. MUST match a console-registered URI verbatim, so it is computed in exactly - one place. - - ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to - the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and - the request-derived fallback below is effectively dev-only. - ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host - this returns is where the provider sends the user BACK, and the session cookie is host-only - (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host - the user actually browsed plants the session on the wrong hostname and they return logged - out. Moving the app to a new hostname means moving this value AND re-registering the - resulting URI in the provider console; one without the other fails closed. - Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" - base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") - if not base: - base = f"{request.url.scheme}://{request.url.netloc}" - return f"{base}/api/v1/oauth/{provider}/callback" - - -@router.get("/status") -def oauth_status(session: Session = Depends(require_session)): - """C5's status shape for the SESSION user, one entry per registry provider: - `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's - `ready` reads through.""" - return oauth_connect.status(session.runtime, session.uname) - - -def _offered_or_404(provider: str): - """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. - - The owner pasted the failure this replaces: clicking Connect on the Google card answered - `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is - not offered at all, so the honest status is the one for a URL that does not exist. `404` - rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not - coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). - - ⚠ Every door in this router goes through it, START included, because the JSON the owner saw - came from the start route and a guard on one leg is a guard on one leg. - """ - if not oauth_connect.offered(provider): - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - - -@router.get("/{provider}/start") -def oauth_start(provider: str, request: Request, next: str = "", - session: Session = Depends(require_session)): - """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the - RELATIVE path the callback returns the browser to; it rides inside the single-use state, - sanitised, so the round trip cannot be steered off-origin.""" - _offered_or_404(provider) - url, problem = oauth_connect.start(provider, session.uname, - _redirect_uri(request, provider), next_path=next) - if problem: - raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) - return RedirectResponse(url, status_code=302) - - -@router.get("/{provider}/callback") -def oauth_callback(provider: str, request: Request, - session: Session = Depends(require_session), - state: str = "", code: str = "", error: str = ""): - """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends - the browser back to the state's return path — connected or not (see module header).""" - _offered_or_404(provider) - if error: - home = "/#/" - return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) - email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) - sep = "&" if "?" in home else "?" - if problem: - return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) - return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) - - -@router.post("/{provider}/disconnect") -def oauth_disconnect(provider: str, session: Session = Depends(require_session)): - _offered_or_404(provider) - if oauth_connect.provider_def(provider) is None: - raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") - oauth_connect.disconnect(session.runtime, session.uname, provider) - return {"disconnected": provider} +"""routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12). + +Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions, +shapes and status codes here; every decision that could be wrong lives in the module a gate +can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry, +so the day a second provider lands here is the day nothing in this file changes. + +MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session +`main.py`, and `routes_automation` is already included there — so this router rides inside it +(`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that +alters no path. + +⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent +screen** — it is a top-level navigation the client reaches by ``, never JSON. The +callback 302s BACK to the return path the `state` carried (relative-only, sanitised by +`oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever +went wrong rides in the query string; a dead-end error page where the app used to be reads as +"the product broke", not "the connect failed". +""" +import os + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import RedirectResponse + +import oauth_connect +from deps import Session, err, require_session + +router = APIRouter(prefix="/oauth") + + +def _redirect_uri(request: Request, provider: str) -> str: + """The redirect URI this deployment registers at the provider — env-pinned when the + container sits behind a proxy that rewrites the scheme (the HF Space), else derived from + the request. MUST match a console-registered URI verbatim, so it is computed in exactly + one place. + + ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to + the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and + the request-derived fallback below is effectively dev-only. + ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host + this returns is where the provider sends the user BACK, and the session cookie is host-only + (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host + the user actually browsed plants the session on the wrong hostname and they return logged + out. Moving the app to a new hostname means moving this value AND re-registering the + resulting URI in the provider console; one without the other fails closed. + Runbook: `.claude/wiki/research/loopable-domain-runbook.md`.""" + base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/") + if not base: + base = f"{request.url.scheme}://{request.url.netloc}" + return f"{base}/api/v1/oauth/{provider}/callback" + + +@router.get("/status") +def oauth_status(session: Session = Depends(require_session)): + """C5's status shape for the SESSION user, one entry per registry provider: + `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's + `ready` reads through.""" + return oauth_connect.status(session.runtime, session.uname) + + +def _offered_or_404(provider: str): + """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR. + + The owner pasted the failure this replaces: clicking Connect on the Google card answered + `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is + not offered at all, so the honest status is the one for a URL that does not exist. `404` + rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not + coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr). + + ⚠ Every door in this router goes through it, START included, because the JSON the owner saw + came from the start route and a guard on one leg is a guard on one leg. + """ + if not oauth_connect.offered(provider): + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + + +@router.get("/{provider}/start") +def oauth_start(provider: str, request: Request, next: str = "", + session: Session = Depends(require_session)): + """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the + RELATIVE path the callback returns the browser to; it rides inside the single-use state, + sanitised, so the round trip cannot be steered off-origin.""" + _offered_or_404(provider) + url, problem = oauth_connect.start(provider, session.uname, + _redirect_uri(request, provider), next_path=next) + if problem: + raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem) + return RedirectResponse(url, status_code=302) + + +@router.get("/{provider}/callback") +def oauth_callback(provider: str, request: Request, + session: Session = Depends(require_session), + state: str = "", code: str = "", error: str = ""): + """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends + the browser back to the state's return path — connected or not (see module header).""" + _offered_or_404(provider) + if error: + home = "/#/" + return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302) + email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code) + sep = "&" if "?" in home else "?" + if problem: + return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302) + return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302) + + +@router.post("/{provider}/disconnect") +def oauth_disconnect(provider: str, session: Session = Depends(require_session)): + _offered_or_404(provider) + if oauth_connect.provider_def(provider) is None: + raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider") + oauth_connect.disconnect(session.runtime, session.uname, provider) + return {"disconnected": provider} diff --git a/api/routes_records.py b/api/routes_records.py index ef0bb2c8153db1e40b2656a715f545d80b1fe22b..c37f74b261d63bb390db45660363e8d94ff796d4 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -1,211 +1,211 @@ -"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. - -⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a -customer-shaped wall bolted to the module import line: `_in_book` asked -`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and -typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the -panel answered "that customer is not in your book" — the owner's report. The dangerous half is -the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment -is filed against somebody's customer where the whole team can read it. - -THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the -events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall -per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving -one here: - - customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant - product `routes_products.scoped_pool` behind the `product_data` grant - ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall - (404 unknown / 403 not yours — a user table has no module grant). - ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every - ROW to derive a pid set this module discards, and RAISES 409 on a - read-through grid past one window, which is why the record drawer painted - an error page on `ut_odoo_gl_lines`. - -⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a -section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning -another session's gate mid-wave — the WALL is the query parameter, not the word. - -⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every -shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a -400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for -the same reason: a typo served as `customer` answers a question nobody asked). -""" -from fastapi import APIRouter, Body, Depends, Query - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table -#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. -_CUSTOMER_SCOPES = ("", "customer", "cohort") - - -def _scope_or_400(raw): - scope = str(raw or "customer").strip().lower() - if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): - return "customer" if scope in _CUSTOMER_SCOPES else scope - raise err(400, "bad_scope", - "scope must be customer, cohort, product or a ut_ database — refusing to guess") - - -def _pool_or_refuse(session: Session, scope: str): - """The pids this session may attach comments to ON THIS DATABASE — grant wall included. - - Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, - so a caller who may not open the surface never learns anything about the row they asked about. - - ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past - one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns - `frozenset()` for both — see the branch below. - ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that - do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` - that REPLACES this function with its own lambda. A gate's test double is a caller - ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a - bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no - failing name. Change the shape here and that double changes with it. - """ - if scope == "product": - from routes_products import MODULE as PRODUCT_MODULE, scoped_pool - - session.require(PRODUCT_MODULE) - pids, _team, _rows, _fields = scoped_pool(session) - # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes - # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised - # a `TypeError` — a 500 on every product comment — while both other branches worked. A - # return shape is a contract even when the function is private. - return (pids, False) - if scope.startswith("ut_"): - # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS - # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the - # rows routes do. - # - # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. - # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on - # a read-through grid larger than one window it RAISES `409 window_required`. So opening the - # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel - # that renders comments about ONE row it already has. `scoped_pids` answers the identical - # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes - # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` - # used to become openable on those two grids. - from routes_tables import scoped_pids - - limits = [] - pids, _fields, _defn = scoped_pids(session, scope, limits=limits) - # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is - # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a - # database with no rows and for a read-through grid too big to enumerate — it distinguishes - # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move - # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which - # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). - # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database - # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the - # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside - # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. - return (pids, bool(limits)) - from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids - - session.require(CUSTOMER_MODULE) - return (frozenset(allowed_pids(session)), False) - - -def _in_book(pid, session, scope): - pids, unbounded = _pool_or_refuse(session, scope) - if unbounded: - # The table wall passed and the row set is larger than this process will enumerate. Said - # out loud rather than silently admitting: R6's second sentence is that a limit which - # cannot be removed gets REPORTED, and this is the one place the report has no envelope to - # ride in. - print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " - f"admitting on the table wall alone, per D-72") - return - if pid not in pids: - # 403, not 404: the record may exist, but this session may not inspect it. - raise err(403, "out_of_scope", "that record is not in your book") - - -def _unavailable(): - return err( - 503, - "store_unavailable", - "record comments are temporarily unavailable — no change was saved", - ) - - -# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD -# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the -# wall was always the query param). The old path stays as an ALIAS because the shipped client -# still calls it; verify_api pins the canonical path AND that the alias answers, so removing -# the alias later is a decision, never an accident. -@router.get("/records/{pid}/comments") -@router.get("/customers/{pid}/comments") -def comments(pid: int, scope: str = Query(default="customer"), - session: Session = Depends(require_session)): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - rows = record_comments.list_comments(session.runtime, pid, scope=scope) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comments": rows} - - -@router.post("/records/{pid}/comments", status_code=201) -@router.post("/customers/{pid}/comments", status_code=201) -def create_comment( - pid: int, - body: dict = Body(default=None), - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - comment = record_comments.add_comment( - session.runtime, - pid, - (body or {}).get("body"), - session.uname, - session.user.get("name") or session.uname, - scope=scope, - ) - except ValueError as exc: - raise err(400, "bad_comment", str(exc)) - except record_comments.CommentsUnavailable: - raise _unavailable() - return {"comment": comment} - - -@router.delete("/records/{pid}/comments/{comment_id}") -@router.delete("/customers/{pid}/comments/{comment_id}") -def remove_comment( - pid: int, - comment_id: str, - scope: str = Query(default="customer"), - session: Session = Depends(require_session), -): - from core import record_comments - - scope = _scope_or_400(scope) - _in_book(pid, session, scope) - try: - deleted = record_comments.delete_comment( - session.runtime, - pid, - comment_id, - session.uname, - admin=session.admin, - scope=scope, - ) - except record_comments.CommentForbidden: - raise err(403, "comment_forbidden", "only the author may delete this comment") - except record_comments.CommentsUnavailable: - raise _unavailable() - if not deleted: - raise err(404, "comment_not_found", "that comment no longer exists") - return {"ok": True, "id": comment_id} +"""Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE. + +⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a +customer-shaped wall bolted to the module import line: `_in_book` asked +`routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and +typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the +panel answered "that customer is not in your book" — the owner's report. The dangerous half is +the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment +is filed against somebody's customer where the whole team can read it. + +THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the +events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall +per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving +one here: + + customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant + product `routes_products.scoped_pool` behind the `product_data` grant + ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall + (404 unknown / 403 not yours — a user table has no module grant). + ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every + ROW to derive a pid set this module discards, and RAISES 409 on a + read-through grid past one window, which is why the record drawer painted + an error page on `ut_odoo_gl_lines`. + +⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a +section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning +another session's gate mid-wave — the WALL is the query parameter, not the word. + +⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every +shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a +400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for +the same reason: a typo served as `customer` answers a question nobody asked). +""" +from fastapi import APIRouter, Body, Depends, Query + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table +#: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`. +_CUSTOMER_SCOPES = ("", "customer", "cohort") + + +def _scope_or_400(raw): + scope = str(raw or "customer").strip().lower() + if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"): + return "customer" if scope in _CUSTOMER_SCOPES else scope + raise err(400, "bad_scope", + "scope must be customer, cohort, product or a ut_ database — refusing to guess") + + +def _pool_or_refuse(session: Session, scope: str): + """The pids this session may attach comments to ON THIS DATABASE — grant wall included. + + Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, + so a caller who may not open the surface never learns anything about the row they asked about. + + ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past + one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns + `frozenset()` for both — see the branch below. + ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that + do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` + that REPLACES this function with its own lambda. A gate's test double is a caller + ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a + bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no + failing name. Change the shape here and that double changes with it. + """ + if scope == "product": + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + session.require(PRODUCT_MODULE) + pids, _team, _rows, _fields = scoped_pool(session) + # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes + # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised + # a `TypeError` — a 500 on every product comment — while both other branches worked. A + # return shape is a contract even when the function is private. + return (pids, False) + if scope.startswith("ut_"): + # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS + # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the + # rows routes do. + # + # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. + # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on + # a read-through grid larger than one window it RAISES `409 window_required`. So opening the + # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel + # that renders comments about ONE row it already has. `scoped_pids` answers the identical + # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes + # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` + # used to become openable on those two grids. + from routes_tables import scoped_pids + + limits = [] + pids, _fields, _defn = scoped_pids(session, scope, limits=limits) + # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is + # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a + # database with no rows and for a read-through grid too big to enumerate — it distinguishes + # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move + # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which + # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). + # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database + # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the + # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside + # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. + return (pids, bool(limits)) + from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids + + session.require(CUSTOMER_MODULE) + return (frozenset(allowed_pids(session)), False) + + +def _in_book(pid, session, scope): + pids, unbounded = _pool_or_refuse(session, scope) + if unbounded: + # The table wall passed and the row set is larger than this process will enumerate. Said + # out loud rather than silently admitting: R6's second sentence is that a limit which + # cannot be removed gets REPORTED, and this is the one place the report has no envelope to + # ride in. + print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " + f"admitting on the table wall alone, per D-72") + return + if pid not in pids: + # 403, not 404: the record may exist, but this session may not inspect it. + raise err(403, "out_of_scope", "that record is not in your book") + + +def _unavailable(): + return err( + 503, + "store_unavailable", + "record comments are temporarily unavailable — no change was saved", + ) + + +# ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD +# in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the +# wall was always the query param). The old path stays as an ALIAS because the shipped client +# still calls it; verify_api pins the canonical path AND that the alias answers, so removing +# the alias later is a decision, never an accident. +@router.get("/records/{pid}/comments") +@router.get("/customers/{pid}/comments") +def comments(pid: int, scope: str = Query(default="customer"), + session: Session = Depends(require_session)): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + rows = record_comments.list_comments(session.runtime, pid, scope=scope) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comments": rows} + + +@router.post("/records/{pid}/comments", status_code=201) +@router.post("/customers/{pid}/comments", status_code=201) +def create_comment( + pid: int, + body: dict = Body(default=None), + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + comment = record_comments.add_comment( + session.runtime, + pid, + (body or {}).get("body"), + session.uname, + session.user.get("name") or session.uname, + scope=scope, + ) + except ValueError as exc: + raise err(400, "bad_comment", str(exc)) + except record_comments.CommentsUnavailable: + raise _unavailable() + return {"comment": comment} + + +@router.delete("/records/{pid}/comments/{comment_id}") +@router.delete("/customers/{pid}/comments/{comment_id}") +def remove_comment( + pid: int, + comment_id: str, + scope: str = Query(default="customer"), + session: Session = Depends(require_session), +): + from core import record_comments + + scope = _scope_or_400(scope) + _in_book(pid, session, scope) + try: + deleted = record_comments.delete_comment( + session.runtime, + pid, + comment_id, + session.uname, + admin=session.admin, + scope=scope, + ) + except record_comments.CommentForbidden: + raise err(403, "comment_forbidden", "only the author may delete this comment") + except record_comments.CommentsUnavailable: + raise _unavailable() + if not deleted: + raise err(404, "comment_not_found", "that comment no longer exists") + return {"ok": True, "id": comment_id} diff --git a/api/routes_script_views.py b/api/routes_script_views.py index a4feec967331ab1c46fa8a1d6a789eda61b6f63a..8db6fc5871a3e0b8f16bd0f077af8552c663cab6 100644 --- a/api/routes_script_views.py +++ b/api/routes_script_views.py @@ -1,439 +1,439 @@ -"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). - -Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user -can build whatever they want through the Agent chat interface. be able to create any dashboard -they want. User should have the ability to see the code AND the dashboard output of course… Limit -the code script View per database… Any agent can add into more AI script, so we can see different -versions or different things the AI code for us."* - - GET /api/v1/script-views?database=K the views bound to ONE database - POST /api/v1/script-views create one {database, name?, source} - GET /api/v1/script-views/{id} one view, its source and its history - PUT /api/v1/script-views/{id} a NEW VERSION of the source - DELETE /api/v1/script-views/{id} drop it - POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} - POST /api/v1/script-views/{id}/revert go back to an earlier version {version} - -⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may -read ONLY the database it lives in, and a script that names another database is REFUSED with a -message naming both. *Not capped*: there is **no limit on how many script views a database may -carry**, because that is how an agent offers three attempts and the owner picks one. So nothing -below counts views. What IS bounded is what makes them big — one source is capped, and one view's -edit history is capped and REPORTS what it dropped. - -⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed -`session.user`, so a script written by an administrator and opened by a scoped analyst reads the -ANALYST's rows. The author decides what the code does; the reader decides what it can see. -⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script -that exfiltrates anything, because the only thing a script can return is a render spec drawn on -the screen of the person who ran it. There is no network, no file and no second reader. - -⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a -coroutine that would block the event loop for every other request in the container. FastAPI runs a -plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. -""" -import threading -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends - -from deps import Session, err, require_session - -router = APIRouter(prefix="/api/v1") - -#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. -VIEWS_KEY = "script_views" - -MAX_NAME = 80 -MAX_SOURCE_BYTES = 128 * 1024 - -#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far -#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader -#: can see the history is partial instead of concluding the view was only ever saved twice. -MAX_HISTORY = 40 - -#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal -#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, -#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what -#: gives up. A 429 that says so is honest; an unbounded fork is not. -MAX_CONCURRENT_RUNS = 4 -_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) - - -def _now(): - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -def _all(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this database has no script views", never to a 500 on the view rail.""" - try: - found = rt.get(VIEWS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _database_ok(session, database): - """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. - - ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True - for an ADMIN on any key at all, including one no database answers to. So a create validated - with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing - can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it - RAISES `UnknownTable`, which is exactly the question being asked. - """ - import core.perm_scope as perm_scope - try: - perm_scope.scoped_fields(session.user, database, st=session.runtime) - except perm_scope.UnknownTable: - raise err(404, "no_database", f"there is no database '{database}' in this workspace") - except perm_scope.Denied: - raise err(403, "forbidden", f"your account may not read '{database}'") - except perm_scope.Unresolvable as exc: - # The database is real and cannot be served under this call's constraints. Standing rule - # 1's second sentence: report the cause and the recommendation, never a bare refusal. - raise err(409, "unresolvable", str(exc)) from None - - -def _clean_source(raw, *, allow_empty=False): - """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. - - ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. - Picking "Custom View" in the mode picker mints the view immediately, before a line of code - exists, so that `mode === 'script'` always implies a real id and no reader has to carry a - "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). - A view that has been created and not yet written is therefore a REAL, legible state: the editor - shows its starter placeholder and the Run control is right there. - A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a - working script by saving nothing over it is not an edit anybody means to make. So the guard - stays exactly where it was on that door, and a person who wants the view gone deletes it. - - ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns - None, so "an empty script is storable" looked true and was written into a mailbox answer another - lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox - knows nothing about. Two validators for one question, disagreeing - [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no - unit of this feature was asserting. - """ - source = str(raw or "") - if not source.strip() and not allow_empty: - raise err(400, "no_source", "a script view needs some code") - if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - raise err(413, "source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") - return source - - -def _row(rec, *, source=False): - """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" - out = {"id": rec.get("id") or "", "database": rec.get("database") or "", - "name": rec.get("name") or "", "author": rec.get("author") or "", - "version": int(rec.get("version") or 1), - "created": rec.get("created") or "", "updated": rec.get("updated") or "", - "versions": len(rec.get("history") or []) + 1, - "trimmed": int(rec.get("trimmed") or 0), - # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the - # row (not only the history) because the editor's header is where a person reads "what - # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back - # legible as an event rather than as a coincidence of matching code. - "restoredFrom": rec.get("restoredFrom")} - if source: - out["source"] = rec.get("source") or "" - out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", - "created": h.get("created") or "", - "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} - for h in reversed(rec.get("history") or []) if isinstance(h, dict)] - return out - - -def _limits(): - return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, - "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, - # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. - "maxViewsPerDatabase": None} - - -def _put(session, view_id, mutate): - """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" - def _set(cur): - cur = dict(cur or {}) - nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) - if nxt is None: - cur.pop(view_id, None) - else: - cur[view_id] = nxt - return cur - - session.runtime.update(VIEWS_KEY, _set, flush="sync") - - -def _mine_or_admin(session, rec): - """Who may EDIT or DELETE a view: its author, or an administrator. - - ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database - may run any view on it, under their OWN scope. That is the whole of "so we can see different - versions or different things the AI code for us": a colleague's attempt is worth nothing if - only its author can open it. - """ - if session.admin or str(rec.get("author") or "") == session.uname: - return - raise err(403, "forbidden", "only the author or an administrator can change this script view") - - -# ── the routes ──────────────────────────────────────────────────────────────────────────────── -@router.get("/script-views") -def list_script_views(database: str = "", session: Session = Depends(require_session)): - """Every script view bound to ONE database, newest first. `database` is required.""" - key = str(database or "").strip() - if not key: - raise err(400, "no_database", "name the database whose script views you want") - _database_ok(session, key) - rows = [_row(rec) for rec in _all(session.runtime).values() - if isinstance(rec, dict) and str(rec.get("database") or "") == key] - rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) - return {"database": key, "views": rows, "limits": _limits()} - - -@router.post("/script-views") -def create_script_view(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. - - A script view that cannot be run is a broken feature the reader discovers by pressing a button, - and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, - so the refusal arrives while the author still has the code in front of them. - """ - import secrets # noqa: PLC0415 - import core.script_sandbox as sandbox # noqa: PLC0415 - - body = body if isinstance(body, dict) else {} - database = str(body.get("database") or "").strip() - if not database: - raise err(400, "no_database", "a script view is bound to one database") - _database_ok(session, database) - # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the - # moment a person chooses the mode, so the empty source is the normal first state, not a slip. - source = _clean_source(body.get("source"), allow_empty=True) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - view_id = "sv_" + secrets.token_urlsafe(9) - rec = {"id": view_id, "database": database, - "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], - "source": source, "author": session.uname, "version": 1, - "created": _now(), "updated": _now(), "history": [], "trimmed": 0} - _put(session, view_id, lambda _prior: rec) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell the author their - # script was saved when it was not. - raise err(503, "store_unavailable", "the script view was NOT created") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.get("/script-views/{view_id}") -def get_script_view(view_id: str, session: Session = Depends(require_session)): - """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _database_ok(session, str(rec.get("database") or "")) - return {"view": _row(rec, source=True), "limits": _limits()} - - -@router.put("/script-views/{view_id}") -def update_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - source = _clean_source(body.get("source")) - refusal = sandbox.check_source(source) - if refusal is not None: - raise err(400, refusal.code, refusal.message) - - def _mutate(prior): - prior = dict(prior or rec) - history = list(prior.get("history") or []) - history.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", "created": prior.get("updated") or ""}) - dropped = max(0, len(history) - MAX_HISTORY) - prior["history"] = history[dropped:] if dropped else history - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - if body.get("name"): - prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the new version was NOT saved") - return {"view": _row(fresh, source=True), "limits": _limits()} - - -@router.post("/script-views/{view_id}/revert") -def revert_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """Put one view back to an earlier version. D-338 / W37-T46. - - ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole - point of the ticket rather than an implementation detail. The history is what lets a person - trust an agent with their code: if reverting destroyed the versions it stepped over, one - mistaken revert would cost exactly what the history existed to protect, and there would be no - way back from the way back. So this appends, and `restoredFrom` records where the text came - from — the same posture the agent-harness roll-back already takes, so a reader who has seen one - is not surprised by the other. - - ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history - entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old - version's text to re-PUT it. Handing the source out just so the client could send it straight - back would widen a payload for a round trip that does not need to exist, and would make the - revert non-atomic: two calls, and a failure between them leaves the person looking at code that - is not what is stored. - ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was - first saved, and a source that a later, stricter rule would now reject is exactly the source a - person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is - about what the code would DO and that must never be bypassed by a route. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - body = body if isinstance(body, dict) else {} - try: - want = int(body.get("version")) - except (TypeError, ValueError): - raise err(400, "no_version", "say which version to go back to") from None - - history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] - match = next((h for h in history if int(h.get("version") or 0) == want), None) - if match is None: - # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never - # written; the other was TRIMMED off the 40-deep history and the record even counts how - # many. Saying "that version is gone" for the first would be a lie a person could act on. - trimmed = int(rec.get("trimmed") or 0) - if trimmed and want <= trimmed: - raise err(410, "version_trimmed", - f"version {want} is older than this view's history keeps " - f"({trimmed} earlier versions have been dropped)") - raise err(404, "no_version", f"this view has no version {want}") - - source = str(match.get("source") or "") - refusal = sandbox.check_source(source) - if refusal is not None: - # An older version that today's sandbox refuses. The person is told which version and why, - # rather than being handed a 400 about code they did not just type. - raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") - - def _mutate(prior): - prior = dict(prior or rec) - history_now = list(prior.get("history") or []) - history_now.append({"version": int(prior.get("version") or 1), - "source": prior.get("source") or "", - "author": prior.get("author") or "", - "created": prior.get("updated") or ""}) - dropped = max(0, len(history_now) - MAX_HISTORY) - prior["history"] = history_now[dropped:] if dropped else history_now - prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped - prior["source"] = source - prior["version"] = int(prior.get("version") or 1) + 1 - prior["updated"] = _now() - # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that - # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical - # code and no way to tell a revert from a coincidence. - prior["restoredFrom"] = want - return prior - - _put(session, view_id, _mutate) - fresh = _all(session.runtime).get(view_id) - if not isinstance(fresh, dict): - raise err(503, "store_unavailable", "the roll-back was NOT saved") - return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} - - -@router.delete("/script-views/{view_id}") -def delete_script_view(view_id: str, session: Session = Depends(require_session)): - view_id = str(view_id) - rec = _all(session.runtime).get(view_id) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - _mine_or_admin(session, rec) - _put(session, view_id, lambda _prior: None) - return {"deleted": view_id} - - -@router.post("/script-views/{view_id}/run") -def run_script_view(view_id: str, body: dict = Body(default=None), - session: Session = Depends(require_session)): - """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. - - ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser - executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key - before this function ever sees it, so a renderer cannot be talked into running something by a - script that was itself perfectly well behaved. - - ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess - wait on the event loop is a ten-second outage for the whole container. - """ - import core.script_sandbox as sandbox # noqa: PLC0415 - - rec = _all(session.runtime).get(str(view_id)) - if not isinstance(rec, dict): - raise err(404, "no_view", "there is no script view with that id") - database = str(rec.get("database") or "") - # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. - # It is checked exactly as a stored one is, because "unsaved" is not a permission. - draft = (body or {}).get("source") if isinstance(body, dict) else None - source = _clean_source(draft) if draft else str(rec.get("source") or "") - # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and - # asking twice builds a registry topic's pool twice. The three refusals are translated below - # instead, which is the same wall reached through the same door. - - if not _RUN_SLOTS.acquire(blocking=False): - raise err(429, "busy", - f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " - f"Try again in a moment") - try: - out = sandbox.run_view(session.user, database, source, st=session.runtime) - finally: - _RUN_SLOTS.release() - - # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" - # answered 200 would be a permission decision the client has to go looking for, and every - # other door in this app answers 403 for it. Everything BELOW this line is a well-formed - # request whose ANSWER is that the script did not produce a view — that is a 200 with - # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. - if out.get("code") == "unknown_table": - raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") - if out.get("code") == "denied": - raise err(403, "forbidden", out.get("message") or "your account may not read that database") - if out.get("code") == "unresolvable": - refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") - refusal.detail["error"]["limit"] = out.get("limit") or {} - raise refusal - - answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), - "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), - "ms": int(out.get("ms") or 0), "code": out.get("code") or "", - # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits - # were applied; on a Windows host the memory and CPU ones were not, and a screen - # that claims an enforcement which did not happen is the failure the rule is about. - "caps": out.get("caps") or {}} - if not answer["ok"]: - answer["error"] = out.get("message") or "the script view did not produce a view" - return answer +"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). + +Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user +can build whatever they want through the Agent chat interface. be able to create any dashboard +they want. User should have the ability to see the code AND the dashboard output of course… Limit +the code script View per database… Any agent can add into more AI script, so we can see different +versions or different things the AI code for us."* + + GET /api/v1/script-views?database=K the views bound to ONE database + POST /api/v1/script-views create one {database, name?, source} + GET /api/v1/script-views/{id} one view, its source and its history + PUT /api/v1/script-views/{id} a NEW VERSION of the source + DELETE /api/v1/script-views/{id} drop it + POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} + POST /api/v1/script-views/{id}/revert go back to an earlier version {version} + +⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may +read ONLY the database it lives in, and a script that names another database is REFUSED with a +message naming both. *Not capped*: there is **no limit on how many script views a database may +carry**, because that is how an agent offers three attempts and the owner picks one. So nothing +below counts views. What IS bounded is what makes them big — one source is capped, and one view's +edit history is capped and REPORTS what it dropped. + +⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed +`session.user`, so a script written by an administrator and opened by a scoped analyst reads the +ANALYST's rows. The author decides what the code does; the reader decides what it can see. +⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script +that exfiltrates anything, because the only thing a script can return is a render spec drawn on +the screen of the person who ran it. There is no network, no file and no second reader. + +⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a +coroutine that would block the event loop for every other request in the container. FastAPI runs a +plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. +""" +import threading +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. +VIEWS_KEY = "script_views" + +MAX_NAME = 80 +MAX_SOURCE_BYTES = 128 * 1024 + +#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far +#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader +#: can see the history is partial instead of concluding the view was only ever saved twice. +MAX_HISTORY = 40 + +#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal +#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, +#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what +#: gives up. A 429 that says so is honest; an unbounded fork is not. +MAX_CONCURRENT_RUNS = 4 +_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this database has no script views", never to a 500 on the view rail.""" + try: + found = rt.get(VIEWS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _database_ok(session, database): + """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. + + ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True + for an ADMIN on any key at all, including one no database answers to. So a create validated + with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing + can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it + RAISES `UnknownTable`, which is exactly the question being asked. + """ + import core.perm_scope as perm_scope + try: + perm_scope.scoped_fields(session.user, database, st=session.runtime) + except perm_scope.UnknownTable: + raise err(404, "no_database", f"there is no database '{database}' in this workspace") + except perm_scope.Denied: + raise err(403, "forbidden", f"your account may not read '{database}'") + except perm_scope.Unresolvable as exc: + # The database is real and cannot be served under this call's constraints. Standing rule + # 1's second sentence: report the cause and the recommendation, never a bare refusal. + raise err(409, "unresolvable", str(exc)) from None + + +def _clean_source(raw, *, allow_empty=False): + """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. + + ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. + Picking "Custom View" in the mode picker mints the view immediately, before a line of code + exists, so that `mode === 'script'` always implies a real id and no reader has to carry a + "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). + A view that has been created and not yet written is therefore a REAL, legible state: the editor + shows its starter placeholder and the Run control is right there. + A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a + working script by saving nothing over it is not an edit anybody means to make. So the guard + stays exactly where it was on that door, and a person who wants the view gone deletes it. + + ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns + None, so "an empty script is storable" looked true and was written into a mailbox answer another + lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox + knows nothing about. Two validators for one question, disagreeing + [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no + unit of this feature was asserting. + """ + source = str(raw or "") + if not source.strip() and not allow_empty: + raise err(400, "no_source", "a script view needs some code") + if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + raise err(413, "source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") + return source + + +def _row(rec, *, source=False): + """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" + out = {"id": rec.get("id") or "", "database": rec.get("database") or "", + "name": rec.get("name") or "", "author": rec.get("author") or "", + "version": int(rec.get("version") or 1), + "created": rec.get("created") or "", "updated": rec.get("updated") or "", + "versions": len(rec.get("history") or []) + 1, + "trimmed": int(rec.get("trimmed") or 0), + # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the + # row (not only the history) because the editor's header is where a person reads "what + # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back + # legible as an event rather than as a coincidence of matching code. + "restoredFrom": rec.get("restoredFrom")} + if source: + out["source"] = rec.get("source") or "" + out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", + "created": h.get("created") or "", + "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} + for h in reversed(rec.get("history") or []) if isinstance(h, dict)] + return out + + +def _limits(): + return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, + "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, + # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. + "maxViewsPerDatabase": None} + + +def _put(session, view_id, mutate): + """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" + def _set(cur): + cur = dict(cur or {}) + nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) + if nxt is None: + cur.pop(view_id, None) + else: + cur[view_id] = nxt + return cur + + session.runtime.update(VIEWS_KEY, _set, flush="sync") + + +def _mine_or_admin(session, rec): + """Who may EDIT or DELETE a view: its author, or an administrator. + + ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database + may run any view on it, under their OWN scope. That is the whole of "so we can see different + versions or different things the AI code for us": a colleague's attempt is worth nothing if + only its author can open it. + """ + if session.admin or str(rec.get("author") or "") == session.uname: + return + raise err(403, "forbidden", "only the author or an administrator can change this script view") + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/script-views") +def list_script_views(database: str = "", session: Session = Depends(require_session)): + """Every script view bound to ONE database, newest first. `database` is required.""" + key = str(database or "").strip() + if not key: + raise err(400, "no_database", "name the database whose script views you want") + _database_ok(session, key) + rows = [_row(rec) for rec in _all(session.runtime).values() + if isinstance(rec, dict) and str(rec.get("database") or "") == key] + rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) + return {"database": key, "views": rows, "limits": _limits()} + + +@router.post("/script-views") +def create_script_view(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. + + A script view that cannot be run is a broken feature the reader discovers by pressing a button, + and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, + so the refusal arrives while the author still has the code in front of them. + """ + import secrets # noqa: PLC0415 + import core.script_sandbox as sandbox # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "no_database", "a script view is bound to one database") + _database_ok(session, database) + # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the + # moment a person chooses the mode, so the empty source is the normal first state, not a slip. + source = _clean_source(body.get("source"), allow_empty=True) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + view_id = "sv_" + secrets.token_urlsafe(9) + rec = {"id": view_id, "database": database, + "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], + "source": source, "author": session.uname, "version": 1, + "created": _now(), "updated": _now(), "history": [], "trimmed": 0} + _put(session, view_id, lambda _prior: rec) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell the author their + # script was saved when it was not. + raise err(503, "store_unavailable", "the script view was NOT created") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.get("/script-views/{view_id}") +def get_script_view(view_id: str, session: Session = Depends(require_session)): + """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _database_ok(session, str(rec.get("database") or "")) + return {"view": _row(rec, source=True), "limits": _limits()} + + +@router.put("/script-views/{view_id}") +def update_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + def _mutate(prior): + prior = dict(prior or rec) + history = list(prior.get("history") or []) + history.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", "created": prior.get("updated") or ""}) + dropped = max(0, len(history) - MAX_HISTORY) + prior["history"] = history[dropped:] if dropped else history + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + if body.get("name"): + prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the new version was NOT saved") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.post("/script-views/{view_id}/revert") +def revert_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Put one view back to an earlier version. D-338 / W37-T46. + + ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole + point of the ticket rather than an implementation detail. The history is what lets a person + trust an agent with their code: if reverting destroyed the versions it stepped over, one + mistaken revert would cost exactly what the history existed to protect, and there would be no + way back from the way back. So this appends, and `restoredFrom` records where the text came + from — the same posture the agent-harness roll-back already takes, so a reader who has seen one + is not surprised by the other. + + ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history + entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old + version's text to re-PUT it. Handing the source out just so the client could send it straight + back would widen a payload for a round trip that does not need to exist, and would make the + revert non-atomic: two calls, and a failure between them leaves the person looking at code that + is not what is stored. + ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was + first saved, and a source that a later, stricter rule would now reject is exactly the source a + person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is + about what the code would DO and that must never be bypassed by a route. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + try: + want = int(body.get("version")) + except (TypeError, ValueError): + raise err(400, "no_version", "say which version to go back to") from None + + history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] + match = next((h for h in history if int(h.get("version") or 0) == want), None) + if match is None: + # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never + # written; the other was TRIMMED off the 40-deep history and the record even counts how + # many. Saying "that version is gone" for the first would be a lie a person could act on. + trimmed = int(rec.get("trimmed") or 0) + if trimmed and want <= trimmed: + raise err(410, "version_trimmed", + f"version {want} is older than this view's history keeps " + f"({trimmed} earlier versions have been dropped)") + raise err(404, "no_version", f"this view has no version {want}") + + source = str(match.get("source") or "") + refusal = sandbox.check_source(source) + if refusal is not None: + # An older version that today's sandbox refuses. The person is told which version and why, + # rather than being handed a 400 about code they did not just type. + raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") + + def _mutate(prior): + prior = dict(prior or rec) + history_now = list(prior.get("history") or []) + history_now.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", + "created": prior.get("updated") or ""}) + dropped = max(0, len(history_now) - MAX_HISTORY) + prior["history"] = history_now[dropped:] if dropped else history_now + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that + # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical + # code and no way to tell a revert from a coincidence. + prior["restoredFrom"] = want + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the roll-back was NOT saved") + return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} + + +@router.delete("/script-views/{view_id}") +def delete_script_view(view_id: str, session: Session = Depends(require_session)): + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + _put(session, view_id, lambda _prior: None) + return {"deleted": view_id} + + +@router.post("/script-views/{view_id}/run") +def run_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. + + ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser + executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key + before this function ever sees it, so a renderer cannot be talked into running something by a + script that was itself perfectly well behaved. + + ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess + wait on the event loop is a ten-second outage for the whole container. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + database = str(rec.get("database") or "") + # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. + # It is checked exactly as a stored one is, because "unsaved" is not a permission. + draft = (body or {}).get("source") if isinstance(body, dict) else None + source = _clean_source(draft) if draft else str(rec.get("source") or "") + # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and + # asking twice builds a registry topic's pool twice. The three refusals are translated below + # instead, which is the same wall reached through the same door. + + if not _RUN_SLOTS.acquire(blocking=False): + raise err(429, "busy", + f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " + f"Try again in a moment") + try: + out = sandbox.run_view(session.user, database, source, st=session.runtime) + finally: + _RUN_SLOTS.release() + + # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" + # answered 200 would be a permission decision the client has to go looking for, and every + # other door in this app answers 403 for it. Everything BELOW this line is a well-formed + # request whose ANSWER is that the script did not produce a view — that is a 200 with + # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. + if out.get("code") == "unknown_table": + raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") + if out.get("code") == "denied": + raise err(403, "forbidden", out.get("message") or "your account may not read that database") + if out.get("code") == "unresolvable": + refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") + refusal.detail["error"]["limit"] = out.get("limit") or {} + raise refusal + + answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), + "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), + "ms": int(out.get("ms") or 0), "code": out.get("code") or "", + # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits + # were applied; on a Windows host the memory and CPU ones were not, and a screen + # that claims an enforcement which did not happen is the failure the rule is about. + "caps": out.get("caps") or {}} + if not answer["ok"]: + answer["error"] = out.get("message") or "the script view did not produce a view" + return answer diff --git a/api/routes_slack.py b/api/routes_slack.py index a2d7853942165a006f557e79ec53c035def5796a..50d382c1fae50935e6148dda677d9a8772dd110d 100644 --- a/api/routes_slack.py +++ b/api/routes_slack.py @@ -1,789 +1,789 @@ -"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a -person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) - -Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel -whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle -permissioning per user."* - -──────────────────────────────────────────────────────────────────────────────────────────────── -⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). - -`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since -2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is -`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own -`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its -authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a -live specimen of identity that LOOKS like a permission and is not. - -`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per -principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` -below, which is the whole projection and is nine lines. Study: -`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). - -What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` -below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a -second Node process, a persistent outbound socket and each tenant's Slack credentials held by a -third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven -unauthenticated-public shape plus Slack's signing-secret check. - -──────────────────────────────────────────────────────────────────────────────────────────────── -⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every -human in that channel reads through the same rules. That is the honest reading of "a bot per -channel" and the pane says it out loud, because the alternative is an admin assuming per-person -scoping from a screen that looks exactly like the per-person one. - -⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. -A wall bound to `#sales` silently re-points the day somebody renames the channel. -""" -import hmac -import json -import os -import secrets -import time -from datetime import datetime, timezone - -from fastapi import APIRouter, Body, Depends, Request - -from deps import Session, err, require_session -from routes_admin import admin_gate - -router = APIRouter(prefix="/api/v1") - -#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides -#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. -AGENTS_KEY = "slack_agents" - -#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. -PENDING_KEY = "slack_pending_writes" - -#: How long an approval card stays clickable. An approval is a statement about the world as it was -#: when the card was rendered; an hour later the values it names may no longer be the values it -#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. -APPROVAL_TTL_S = 15 * 60 - -#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. -SLACK_MAX_SKEW_S = 60 * 5 - -MAX_BODY_BYTES = 64 * 1024 - -#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied -#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: -#: past a certain length Slack collapses a message behind "Show more", and a human clicking -#: Approve on a card whose tail they never saw is worse than no card at all. -APPROVAL_MAX_ROWS = 12 - - -# ── the tenant's agent records ─────────────────────────────────────────────────────────────── -def _agents(rt): - """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to - "this tenant has no agents", never to a 500 on the settings pane.""" - try: - found = rt.get(AGENTS_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def _clean_agent(raw, prior=None): - """One stored record, built KEY BY KEY. - - ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; - a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule - `routes_forms._public_form` follows on the way out, applied here on the way in. - """ - prior = prior if isinstance(prior, dict) else {} - out = { - "id": str(prior.get("id") or raw.get("id") or ""), - "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], - "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], - "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], - "active": bool(raw.get("active", prior.get("active", True))), - # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this - # key existed must read as "approval required": the one direction a default here is - # allowed to be wrong in is the safe one, because an unattended write cannot be undone by - # revoking the bot afterwards [[default-must-pass-its-own-guard]]. - "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, - # The permission block. Same shape, same validator, same engine as a user's. - "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, - "perms_v": int(prior.get("perms_v") or 0), - "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), - "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), - } - return out - - -def _put_agent(session, agent_id, mutate): - """Read-modify-write ONE agent under the tenant's bucket, synchronously. - - `flush="sync"` because the client re-reads the list immediately after every write here, and an - async flush would let the refetch win the race and paint the pre-write state — the - [[refetch-eats-its-own-write]] shape. - """ - def _set(cur): - cur = dict(cur or {}) - rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None - nxt = mutate(rec) - if nxt is None: - cur.pop(agent_id, None) - else: - cur[agent_id] = nxt - return cur - - session.runtime.update(AGENTS_KEY, _set, flush="sync") - - -# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── -def agent_principal(agent): - """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. - - ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user - RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. - So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, - `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second - engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about - a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means - when it is true in code rather than in appearance. - - ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True - unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a - locked store. A bot has no such emergency and no hands: an agent record that could carry - `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is - not copied from the record; it is written here, every time. - - ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY - (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy - grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with - access to everything. Fail-closed from the first byte. - """ - import core.perm_scope as perm_scope - return { - "username": f"slack:{(agent or {}).get('channel') or '?'}", - "role": "agent", - "perms": (agent or {}).get("perms") or {}, - "perms_v": perm_scope.PERMS_VERSION, - # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent - # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction - # this must fail in. - } - - -def agent_may_read(agent, module): - """May this channel open `module` at all?""" - import core.perm_scope as perm_scope - if not (agent or {}).get("active", True): - return False - return bool(perm_scope.may_access(agent_principal(agent), module)) - - -def agent_visible_fields(agent, fields, module): - """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, - so a formula over a hidden column cannot leak it back through arithmetic.""" - import core.perm_scope as perm_scope - return perm_scope.visible_fields(fields, agent_principal(agent), module) - - -def agent_rows(agent, rows, module, fields, ctx=None, st=None): - """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES - rather than being ignored. - - ⭐⭐ `st` IS THE TENANT HANDLE AND IT IS HERE BECAUSE OF THE SENTENCE IN `agent_principal`: - a channel agent IS a principal of the same wall, so a permanent filter naming a - user-generated column has to mean the SAME thing here as it does on the grid (owner I16). - A door that lends the handle and a door that does not are one stored rule with two meanings. - - ⚠ PASS THE TENANT'S OWN RUNTIME — every other store read in this file takes - `session.runtime`, and a channel is not a second tenancy. It is a keyword with a `None` - default for the same reason `perm_scope.apply_row_scope`'s is: without one this is - byte-identical to the wall that shipped before the argument existed. - - ⚠ `hidden_keys` DELIBERATELY DOES NOT TAKE IT IN THIS CHANGE. Its field-grant leg - under-hides without a handle (see `perm_scope.visible_overlays`), which is a real and - separate gap on this surface — but it is the FIELD wall, it moves what a channel receives - rather than what I16 asked for, and widening two walls in one ticket is how a permission - change stops being reviewable. - """ - import core.perm_scope as perm_scope - p = agent_principal(agent) - hide = perm_scope.hidden_keys(p, module, fields) - kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st) - # Both wires, never one: the field list and the row payload are separate, and stripping only - # the first leaves the value sitting in the second where anyone can read it. - return [perm_scope.strip_row(r, hide) for r in kept] - - -# ── the credential ─────────────────────────────────────────────────────────────────────────── -def slack_creds(rt): - """This TENANT's Slack credentials from its own keychain, or None. - - ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT - HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because - `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a - tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells - out the rule: handing the environment's credential to a tenant whose admin has not stored one - is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane - SAYS so. - - ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` - (private, and in a file outside this lane's fence). Same deterministic rule: entries are - id-sorted, so "first" is stable across reads rather than dict-order luck. - """ - try: - import core.keychain as keychain - for row in keychain.list_entries(rt): - if str(row.get("type") or "") != "slack": - continue - fields = keychain.read_fields(rt, row.get("id")) - if isinstance(fields, dict) and fields.get("signing_secret"): - return fields - except Exception: # noqa: BLE001 - # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; - # the signature check below fails closed regardless, so this cannot widen anything. - return None - return None - - -# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── -def _summary(agent, modules): - """One sentence per agent for the list row — computed here, because the alternative is one - round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" - # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to - # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and - # summarised every agent as "" (no databases at all). - governed = list(modules) - perms = agent.get("perms") or {} - open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) - if not governed: - return "" - if open_n == 0: - return "No access" - restricted = sum(1 for m in governed - if (perms.get(m["key"]) or {}).get("access") - and ((perms.get(m["key"]) or {}).get("filter") - or (perms.get(m["key"]) or {}).get("hiddenFields"))) - head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) - else f"{open_n} of {len(governed)} databases") - return f"{head}, {restricted} restricted" if restricted else head - - -@router.get("/agents") -def list_channel_agents(session: Session = Depends(admin_gate)): - """This tenant's channel agents, plus whether Slack is reachable at all.""" - import routes_admin - modules = routes_admin._perm_modules(session) - rows = [] - for aid, a in sorted(_agents(session.runtime).items()): - if not isinstance(a, dict): - continue - rows.append({"id": aid, "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", - "label": a.get("label") or "", "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "summary": _summary(a, modules)}) - return {"agents": rows, "configured": slack_creds(session.runtime) is not None} - - -@router.post("/agents") -def create_channel_agent(body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - body = body if isinstance(body, dict) else {} - channel = str(body.get("channel") or "").strip() - if not channel: - raise err(400, "no_channel", "a Slack channel ID is required") - existing = _agents(session.runtime) - # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing - # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one - # place where the two answers are "may read" and "may not". - for a in existing.values(): - if isinstance(a, dict) and str(a.get("channel") or "") == channel: - raise err(409, "channel_taken", - "that channel already has an agent — edit it instead of adding a second") - aid = secrets.token_urlsafe(9) - rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, - createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) - _put_agent(session, aid, lambda _prior: rec) - fresh = _agents(session.runtime).get(aid) - if not isinstance(fresh, dict): - # The store took the write and did not record it. A 200 here would tell an administrator - # the agent exists when it does not. - raise err(503, "store_unavailable", "the agent was NOT created") - return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], - "label": rec["label"], "active": rec["active"], - "writeApproval": rec["writeApproval"], "summary": "No access"}} - - -@router.get("/agents/{agent_id}") -def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): - """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. - - ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and - `settings/ModulePermsList` renders it, because they are literally the same client code. A - second envelope here would mean a second parser and a second set of defaults within a wave. - """ - import routes_admin - a = _agents(session.runtime).get(str(agent_id)) - if not isinstance(a, dict): - raise err(404, "no_such_agent", "no agent with that id") - modules = routes_admin._perm_modules(session) - # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S - # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the - # key is None, so this list would have been EMPTY and the channel perms editor would have - # governed ZERO databases while looking entirely correct. A Slack channel is a principal of - # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. - governed = [m["key"] for m in modules] - stored = a.get("perms") or {} - principal = agent_principal(a) - import core.perm_scope as perm_scope - perms_out = {} - for k in governed: - e = stored.get(k) - # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled - # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a - # channel agent is not a person with a share, so a `ut_*` key it has not been granted - # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the - # table routes would give it. - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), - "filter": None, "hiddenFields": []} - return {"id": str(agent_id), "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", "label": a.get("label") or "", - "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "perms": perms_out, - # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must - # render "undeclared = denied", not the legacy-grant reading it would show at 0. - "perms_v": perm_scope.PERMS_VERSION, - # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints - # the "Everything (admin)" state for a principal that cannot have it. - "is_admin": False, - "modules": modules, - "fields_by_module": {k: routes_admin._module_fields(k, session=session) - for k in governed}} - - -@router.put("/agents/{agent_id}/perms") -def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - """Replace this agent's permission block wholesale — the same validator a user's block gets.""" - import routes_admin - body = body if isinstance(body, dict) else {} - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - if "perms" not in body: - raise err(400, "empty_patch", "no perms to save") - mods = routes_admin._perm_modules(session) - # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot - # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the - # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, - # and a second validator here is a second place for one of them to go missing. - # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal - # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left - # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives - # it ([[two-gates-can-assert-opposite-things]]). - cleaned = routes_admin._clean_perms( - body.get("perms"), - governed_keys={m["key"] for m in mods}, session=session) or {} - - import core.perm_scope as perm_scope - - def _mut(prior): - if not isinstance(prior, dict): - return None - nxt = dict(prior) - nxt["perms"] = cleaned - nxt["perms_v"] = perm_scope.PERMS_VERSION - return nxt - - _put_agent(session, str(agent_id), _mut) - fresh = _agents(session.runtime).get(str(agent_id)) or {} - if (fresh.get("perms") or {}) != cleaned: - # The one direction this must never fail in: telling an administrator the wall is up when - # it is not. - raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") - return {"id": str(agent_id), "perms": fresh.get("perms") or {}, - "perms_v": int(fresh.get("perms_v") or 0)} - - -@router.patch("/agents/{agent_id}") -def patch_channel_agent(agent_id: str, body: dict = Body(default=None), - session: Session = Depends(admin_gate)): - body = body if isinstance(body, dict) else {} - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it - #: on the metadata route would be a second, unvalidated way to write the wall. - allowed = {k: v for k, v in body.items() - if k in ("label", "active", "writeApproval", "channelName")} - if not allowed: - raise err(400, "empty_patch", "nothing to change") - - def _mut(prior): - return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None - - _put_agent(session, str(agent_id), _mut) - import routes_admin - a = _agents(session.runtime).get(str(agent_id)) or {} - return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", - "channelName": a.get("channelName") or "", "label": a.get("label") or "", - "active": a.get("active", True) is not False, - "writeApproval": a.get("writeApproval", True) is not False, - "summary": _summary(a, routes_admin._perm_modules(session))}} - - -@router.delete("/agents/{agent_id}") -def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): - if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(404, "no_such_agent", "no agent with that id") - _put_agent(session, str(agent_id), lambda _prior: None) - if isinstance(_agents(session.runtime).get(str(agent_id)), dict): - raise err(503, "store_unavailable", "the agent was NOT removed") - return {"ok": True} - - -# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── -def _is_empty(value): - """Is this value ABSENT, as opposed to falsy? - - ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: - `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to - False is precisely the kind of write somebody needs to approve, and a naive `if not value` - drops both rows silently — the approver then reads a card that does not describe the write - they are approving. The MIT licence gate for this borrowing is recorded in - `.claude/wiki/research/opentag-adoption.md`. - """ - return value is None or (isinstance(value, str) and value.strip() == "") - - -def approval_card(action, values, agent=None): - """The card a human reads before a bot writes. `{action, rows, truncated, note}`. - - ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain - length Slack collapses a message behind "Show more", and **an approver who cannot read the - card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody - was told about is the violation, not the truncation (wave-30 R6's second sentence). - """ - rows = [{"field": str(k), "value": v} - for k, v in (values or {}).items() if not _is_empty(v)] - rows.sort(key=lambda r: r["field"]) - shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) - return { - "action": str(action or "")[:80], - "rows": shown, - "truncated": dropped, - "note": (f"{dropped} more field(s) not shown — open the record to review them before " - f"approving." if dropped else ""), - "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", - } - - -def intercept_write(agent, action, values, stash): - """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. - - ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card - naming the action and its values, and committed only on an explicit human accept. Everything - else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). - - ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` - existed requires approval. The unsafe direction here is not recoverable — revoking a bot does - not un-write what it wrote. - """ - if (agent or {}).get("writeApproval", True) is not False: - card = approval_card(action, values, agent) - stash(card) - return "await_approval", card - return "commit", None - - -def _pending(rt): - try: - found = rt.get(PENDING_KEY) or {} - except Exception: # noqa: BLE001 - return {} - return found if isinstance(found, dict) else {} - - -def stash_pending(rt, agent, action, values, now=None): - """Store one awaiting-approval mutation and return its token. - - The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the - action's id: a guessable value on an unauthenticated door is a way to approve somebody else's - write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. - """ - token = secrets.token_urlsafe(18) - row = {"at": float(now if now is not None else time.time()), - "channel": str((agent or {}).get("channel") or ""), - "action": str(action or "")[:80], - "values": values if isinstance(values, dict) else {}} - rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") - return token - - -#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. -#: -#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE -#: place an approved write may be performed, so it is the place a negative control can bite — but -#: a gate is only as honest as what it guards. Until something registers here, an approved card -#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a -#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. -_COMMITTERS = {} - - -def register_committer(kind, fn): - """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" - _COMMITTERS[str(kind)] = fn - - -def commit_pending(rt, token, approved, now=None): - """Resolve ONE approval card. Returns one of - `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. - - ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's - `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the - route could not bite while the route had no write in it at all — it would stay green with the - guard deleted, because there would be nothing for the guard to be protecting anything from - ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's - first draft). Putting the commit behind ONE named seam gives the control a real subject: the - gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it - exactly once. - - ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that - survives its own click is a replayable write, and "approve twice" must never mean "write - twice". The delete is `flush="sync"` for the same reason. - """ - now = float(now if now is not None else time.time()) - row = _pending(rt).get(str(token)) - if not isinstance(row, dict): - return "unknown" - # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. - rt.update(PENDING_KEY, - lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, - flush="sync") - if now - float(row.get("at") or 0) > APPROVAL_TTL_S: - return "expired" - if not approved: - # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. - return "rejected" - fn = _COMMITTERS.get(str(row.get("action") or "")) - if fn is None: - return "no_committer" - try: - fn(rt, row) - except Exception: # noqa: BLE001 - # A committer that raises must not read as a commit. The card is already consumed, so the - # honest report is that it failed, and the human asks again. - return "failed" - return "committed" - - -# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── -#: Sliding window, keyed on the SLACK TEAM, never the client IP. -#: -#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own -#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one -#: busy tenant would spend the allowance for all of them, and the symptom would be another -#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. -#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER -#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature -#: check is a single HMAC). -RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 -_HITS: dict = {} - - -def _rate_ok(key, now): - seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] - seen.append(now) - _HITS[key] = seen - if len(_HITS) > 4096: - for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: - _HITS.pop(k, None) - return len(seen) <= RATE_PER_WINDOW - - -async def _raw_body(request): - """The body as RAW BYTES, capped. - - ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string - `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is - otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim - copy of it could not verify a Slack signature at all: re-serialising the dict changes - whitespace and key order and the HMAC no longer matches. The bug would look like "Slack - signatures are always invalid", which is indistinguishable from a wrong secret. - - ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered - the whole body before the handler's first line, so a content-length check inside the handler - caps nothing (`routes_forms`' own scar, recorded in its docstring). - """ - size, chunks = 0, [] - async for chunk in request.stream(): - size += len(chunk) - if size > MAX_BODY_BYTES: - raise err(413, "body_too_large", "that request is too large") - chunks.append(chunk) - return b"".join(chunks) - - -def _refuse(): - """ONE refusal for every reason. No oracle. - - A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel - with no agent all answer identically — otherwise the door tells an unauthenticated caller - which tenants exist and which channels are configured, which is the question it was built to - not answer. `routes_forms._refuse` takes the same posture for the same reason. - """ - return err(403, "bad_request", "that request could not be verified") - - -def _verify(secret, timestamp, raw, signature, now): - """Slack's v0 signature, with the replay window checked FIRST.""" - try: - ts = int(str(timestamp or "0")) - except ValueError: - return False - if abs(now - ts) > SLACK_MAX_SKEW_S: - return False - base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") - import hashlib - expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() - return hmac.compare_digest(expected, str(signature or "")) - - -def _resolve_tenant(raw, timestamp, signature, now): - """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. - - ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` - would let an unauthenticated caller name the tenant it wants to be — the exact widening - `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured - tenant's secret is tried and the one that VERIFIES names the tenant. - - ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not - depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the - same reason — a public door has no session to ask. - """ - from harness import runtime as _rt - hit = None - try: - slugs = list(_rt.known_tenants()) - except Exception: # noqa: BLE001 - return None, None, None - for slug in slugs: - try: - rt = _rt.get_runtime(slug) - except Exception: # noqa: BLE001 - continue - creds = slack_creds(rt) - if not creds: - continue - if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: - hit = (slug, rt, creds) - return hit if hit else (None, None, None) - - -@router.post("/slack/events") -async def slack_events(request: Request): - """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. - - ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, - no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves - this deployment (amendment A1). - """ - now = time.time() - raw = await _raw_body(request) - ts = request.headers.get("x-slack-request-timestamp") - sig = request.headers.get("x-slack-signature") - slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) - if not rt: - raise _refuse() - - try: - payload = json.loads(raw or b"{}") - except ValueError: - raise _refuse() - if not isinstance(payload, dict): - raise _refuse() - - # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an - # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. - if payload.get("type") == "url_verification": - return {"challenge": str(payload.get("challenge") or "")[:512]} - - team = str(payload.get("team_id") or slug) - if not _rate_ok(f"{slug}:{team}", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - - event = payload.get("event") if isinstance(payload.get("event"), dict) else {} - channel = str(event.get("channel") or "") - agent = None - for a in _agents(rt).values(): - if isinstance(a, dict) and str(a.get("channel") or "") == channel: - agent = a - break - # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries - # a non-2xx up to three times, so refusing here would turn "this channel is not configured" - # into three refusals per message, and the retry storm would be the only visible symptom. - # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which - # channels this workspace has configured. - if not isinstance(agent, dict) or agent.get("active", True) is False: - return {"ok": True} - - # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the - # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a - # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing - # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built - # answerer that returns something plausible is worse than one that returns nothing. - # Booked rather than faked [[flag-shipped-without-its-writer]]. - return {"ok": True} - - -@router.post("/slack/interact") -async def slack_interact(request: Request): - """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" - now = time.time() - raw = await _raw_body(request) - slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), - request.headers.get("x-slack-signature"), now) - if not rt: - raise _refuse() - # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. - from urllib.parse import parse_qs - try: - form = parse_qs(raw.decode("utf-8")) - payload = json.loads((form.get("payload") or ["{}"])[0]) - except Exception: # noqa: BLE001 - raise _refuse() - if not isinstance(payload, dict): - raise _refuse() - if not _rate_ok(f"{slug}:interact", now): - raise err(429, "too_many_requests", "too many requests — wait a moment and try again") - - actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] - choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) - token = str(choice.get("value") or "") - # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, - # truncated or unknown action id: the direction that can be wrong here without anyone - # noticing is the one that writes. - approved = str(choice.get("action_id") or "") == "approve" - # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative - # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all - # live there; this handler only turns its verdict into a sentence. - verdict = commit_pending(rt, token, approved, now=now) - return {"text": { - # An unknown, already-used or expired token is not an error: a person clicking a stale - # card should be told it is stale, not shown a failure. - "unknown": "That approval is no longer available. Ask the bot again.", - "expired": "That approval expired before it was answered. Ask the bot again.", - "rejected": "Rejected. Nothing was changed.", - # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, - # so an approved card today performs nothing and says exactly that rather than "Approved." - "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", - "failed": "That change could not be completed. Nothing was saved; ask the bot again.", - "committed": "Approved.", - }.get(verdict, "That approval is no longer available. Ask the bot again.")} - - -@router.get("/slack/health") -def slack_health(session: Session = Depends(require_session)): - """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — - enumerating the others would answer a question about our customer list.""" - creds = slack_creds(session.runtime) - return {"configured": creds is not None, - "agents": len(_agents(session.runtime)), - "hasBotToken": bool((creds or {}).get("bot_token"))} +"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a +person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) + +Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel +whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle +permissioning per user."* + +──────────────────────────────────────────────────────────────────────────────────────────────── +⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). + +`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since +2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is +`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own +`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its +authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a +live specimen of identity that LOOKS like a permission and is not. + +`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per +principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` +below, which is the whole projection and is nine lines. Study: +`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). + +What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` +below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a +second Node process, a persistent outbound socket and each tenant's Slack credentials held by a +third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven +unauthenticated-public shape plus Slack's signing-secret check. + +──────────────────────────────────────────────────────────────────────────────────────────────── +⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every +human in that channel reads through the same rules. That is the honest reading of "a bot per +channel" and the pane says it out loud, because the alternative is an admin assuming per-person +scoping from a screen that looks exactly like the per-person one. + +⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. +A wall bound to `#sales` silently re-points the day somebody renames the channel. +""" +import hmac +import json +import os +import secrets +import time +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends, Request + +from deps import Session, err, require_session +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. +AGENTS_KEY = "slack_agents" + +#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. +PENDING_KEY = "slack_pending_writes" + +#: How long an approval card stays clickable. An approval is a statement about the world as it was +#: when the card was rendered; an hour later the values it names may no longer be the values it +#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. +APPROVAL_TTL_S = 15 * 60 + +#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. +SLACK_MAX_SKEW_S = 60 * 5 + +MAX_BODY_BYTES = 64 * 1024 + +#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied +#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: +#: past a certain length Slack collapses a message behind "Show more", and a human clicking +#: Approve on a card whose tail they never saw is worse than no card at all. +APPROVAL_MAX_ROWS = 12 + + +# ── the tenant's agent records ─────────────────────────────────────────────────────────────── +def _agents(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this tenant has no agents", never to a 500 on the settings pane.""" + try: + found = rt.get(AGENTS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _clean_agent(raw, prior=None): + """One stored record, built KEY BY KEY. + + ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; + a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule + `routes_forms._public_form` follows on the way out, applied here on the way in. + """ + prior = prior if isinstance(prior, dict) else {} + out = { + "id": str(prior.get("id") or raw.get("id") or ""), + "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], + "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], + "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], + "active": bool(raw.get("active", prior.get("active", True))), + # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this + # key existed must read as "approval required": the one direction a default here is + # allowed to be wrong in is the safe one, because an unattended write cannot be undone by + # revoking the bot afterwards [[default-must-pass-its-own-guard]]. + "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, + # The permission block. Same shape, same validator, same engine as a user's. + "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, + "perms_v": int(prior.get("perms_v") or 0), + "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), + "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), + } + return out + + +def _put_agent(session, agent_id, mutate): + """Read-modify-write ONE agent under the tenant's bucket, synchronously. + + `flush="sync"` because the client re-reads the list immediately after every write here, and an + async flush would let the refetch win the race and paint the pre-write state — the + [[refetch-eats-its-own-write]] shape. + """ + def _set(cur): + cur = dict(cur or {}) + rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None + nxt = mutate(rec) + if nxt is None: + cur.pop(agent_id, None) + else: + cur[agent_id] = nxt + return cur + + session.runtime.update(AGENTS_KEY, _set, flush="sync") + + +# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── +def agent_principal(agent): + """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. + + ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user + RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. + So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, + `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second + engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about + a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means + when it is true in code rather than in appearance. + + ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True + unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a + locked store. A bot has no such emergency and no hands: an agent record that could carry + `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is + not copied from the record; it is written here, every time. + + ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY + (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy + grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with + access to everything. Fail-closed from the first byte. + """ + import core.perm_scope as perm_scope + return { + "username": f"slack:{(agent or {}).get('channel') or '?'}", + "role": "agent", + "perms": (agent or {}).get("perms") or {}, + "perms_v": perm_scope.PERMS_VERSION, + # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent + # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction + # this must fail in. + } + + +def agent_may_read(agent, module): + """May this channel open `module` at all?""" + import core.perm_scope as perm_scope + if not (agent or {}).get("active", True): + return False + return bool(perm_scope.may_access(agent_principal(agent), module)) + + +def agent_visible_fields(agent, fields, module): + """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, + so a formula over a hidden column cannot leak it back through arithmetic.""" + import core.perm_scope as perm_scope + return perm_scope.visible_fields(fields, agent_principal(agent), module) + + +def agent_rows(agent, rows, module, fields, ctx=None, st=None): + """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES + rather than being ignored. + + ⭐⭐ `st` IS THE TENANT HANDLE AND IT IS HERE BECAUSE OF THE SENTENCE IN `agent_principal`: + a channel agent IS a principal of the same wall, so a permanent filter naming a + user-generated column has to mean the SAME thing here as it does on the grid (owner I16). + A door that lends the handle and a door that does not are one stored rule with two meanings. + + ⚠ PASS THE TENANT'S OWN RUNTIME — every other store read in this file takes + `session.runtime`, and a channel is not a second tenancy. It is a keyword with a `None` + default for the same reason `perm_scope.apply_row_scope`'s is: without one this is + byte-identical to the wall that shipped before the argument existed. + + ⚠ `hidden_keys` DELIBERATELY DOES NOT TAKE IT IN THIS CHANGE. Its field-grant leg + under-hides without a handle (see `perm_scope.visible_overlays`), which is a real and + separate gap on this surface — but it is the FIELD wall, it moves what a channel receives + rather than what I16 asked for, and widening two walls in one ticket is how a permission + change stops being reviewable. + """ + import core.perm_scope as perm_scope + p = agent_principal(agent) + hide = perm_scope.hidden_keys(p, module, fields) + kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st) + # Both wires, never one: the field list and the row payload are separate, and stripping only + # the first leaves the value sitting in the second where anyone can read it. + return [perm_scope.strip_row(r, hide) for r in kept] + + +# ── the credential ─────────────────────────────────────────────────────────────────────────── +def slack_creds(rt): + """This TENANT's Slack credentials from its own keychain, or None. + + ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT + HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because + `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a + tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells + out the rule: handing the environment's credential to a tenant whose admin has not stored one + is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane + SAYS so. + + ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` + (private, and in a file outside this lane's fence). Same deterministic rule: entries are + id-sorted, so "first" is stable across reads rather than dict-order luck. + """ + try: + import core.keychain as keychain + for row in keychain.list_entries(rt): + if str(row.get("type") or "") != "slack": + continue + fields = keychain.read_fields(rt, row.get("id")) + if isinstance(fields, dict) and fields.get("signing_secret"): + return fields + except Exception: # noqa: BLE001 + # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; + # the signature check below fails closed regardless, so this cannot widen anything. + return None + return None + + +# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── +def _summary(agent, modules): + """One sentence per agent for the list row — computed here, because the alternative is one + round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" + # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to + # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and + # summarised every agent as "" (no databases at all). + governed = list(modules) + perms = agent.get("perms") or {} + open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) + if not governed: + return "" + if open_n == 0: + return "No access" + restricted = sum(1 for m in governed + if (perms.get(m["key"]) or {}).get("access") + and ((perms.get(m["key"]) or {}).get("filter") + or (perms.get(m["key"]) or {}).get("hiddenFields"))) + head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) + else f"{open_n} of {len(governed)} databases") + return f"{head}, {restricted} restricted" if restricted else head + + +@router.get("/agents") +def list_channel_agents(session: Session = Depends(admin_gate)): + """This tenant's channel agents, plus whether Slack is reachable at all.""" + import routes_admin + modules = routes_admin._perm_modules(session) + rows = [] + for aid, a in sorted(_agents(session.runtime).items()): + if not isinstance(a, dict): + continue + rows.append({"id": aid, "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", + "label": a.get("label") or "", "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, modules)}) + return {"agents": rows, "configured": slack_creds(session.runtime) is not None} + + +@router.post("/agents") +def create_channel_agent(body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + channel = str(body.get("channel") or "").strip() + if not channel: + raise err(400, "no_channel", "a Slack channel ID is required") + existing = _agents(session.runtime) + # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing + # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one + # place where the two answers are "may read" and "may not". + for a in existing.values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + raise err(409, "channel_taken", + "that channel already has an agent — edit it instead of adding a second") + aid = secrets.token_urlsafe(9) + rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, + createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) + _put_agent(session, aid, lambda _prior: rec) + fresh = _agents(session.runtime).get(aid) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell an administrator + # the agent exists when it does not. + raise err(503, "store_unavailable", "the agent was NOT created") + return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], + "label": rec["label"], "active": rec["active"], + "writeApproval": rec["writeApproval"], "summary": "No access"}} + + +@router.get("/agents/{agent_id}") +def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. + + ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and + `settings/ModulePermsList` renders it, because they are literally the same client code. A + second envelope here would mean a second parser and a second set of defaults within a wave. + """ + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) + if not isinstance(a, dict): + raise err(404, "no_such_agent", "no agent with that id") + modules = routes_admin._perm_modules(session) + # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S + # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the + # key is None, so this list would have been EMPTY and the channel perms editor would have + # governed ZERO databases while looking entirely correct. A Slack channel is a principal of + # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. + governed = [m["key"] for m in modules] + stored = a.get("perms") or {} + principal = agent_principal(a) + import core.perm_scope as perm_scope + perms_out = {} + for k in governed: + e = stored.get(k) + # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled + # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a + # channel agent is not a person with a share, so a `ut_*` key it has not been granted + # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the + # table routes would give it. + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), + "filter": None, "hiddenFields": []} + return {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "perms": perms_out, + # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must + # render "undeclared = denied", not the legacy-grant reading it would show at 0. + "perms_v": perm_scope.PERMS_VERSION, + # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints + # the "Everything (admin)" state for a principal that cannot have it. + "is_admin": False, + "modules": modules, + "fields_by_module": {k: routes_admin._module_fields(k, session=session) + for k in governed}} + + +@router.put("/agents/{agent_id}/perms") +def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this agent's permission block wholesale — the same validator a user's block gets.""" + import routes_admin + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + mods = routes_admin._perm_modules(session) + # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot + # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the + # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, + # and a second validator here is a second place for one of them to go missing. + # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal + # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left + # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives + # it ([[two-gates-can-assert-opposite-things]]). + cleaned = routes_admin._clean_perms( + body.get("perms"), + governed_keys={m["key"] for m in mods}, session=session) or {} + + import core.perm_scope as perm_scope + + def _mut(prior): + if not isinstance(prior, dict): + return None + nxt = dict(prior) + nxt["perms"] = cleaned + nxt["perms_v"] = perm_scope.PERMS_VERSION + return nxt + + _put_agent(session, str(agent_id), _mut) + fresh = _agents(session.runtime).get(str(agent_id)) or {} + if (fresh.get("perms") or {}) != cleaned: + # The one direction this must never fail in: telling an administrator the wall is up when + # it is not. + raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") + return {"id": str(agent_id), "perms": fresh.get("perms") or {}, + "perms_v": int(fresh.get("perms_v") or 0)} + + +@router.patch("/agents/{agent_id}") +def patch_channel_agent(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it + #: on the metadata route would be a second, unvalidated way to write the wall. + allowed = {k: v for k, v in body.items() + if k in ("label", "active", "writeApproval", "channelName")} + if not allowed: + raise err(400, "empty_patch", "nothing to change") + + def _mut(prior): + return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None + + _put_agent(session, str(agent_id), _mut) + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) or {} + return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, routes_admin._perm_modules(session))}} + + +@router.delete("/agents/{agent_id}") +def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + _put_agent(session, str(agent_id), lambda _prior: None) + if isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(503, "store_unavailable", "the agent was NOT removed") + return {"ok": True} + + +# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── +def _is_empty(value): + """Is this value ABSENT, as opposed to falsy? + + ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: + `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to + False is precisely the kind of write somebody needs to approve, and a naive `if not value` + drops both rows silently — the approver then reads a card that does not describe the write + they are approving. The MIT licence gate for this borrowing is recorded in + `.claude/wiki/research/opentag-adoption.md`. + """ + return value is None or (isinstance(value, str) and value.strip() == "") + + +def approval_card(action, values, agent=None): + """The card a human reads before a bot writes. `{action, rows, truncated, note}`. + + ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain + length Slack collapses a message behind "Show more", and **an approver who cannot read the + card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody + was told about is the violation, not the truncation (wave-30 R6's second sentence). + """ + rows = [{"field": str(k), "value": v} + for k, v in (values or {}).items() if not _is_empty(v)] + rows.sort(key=lambda r: r["field"]) + shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) + return { + "action": str(action or "")[:80], + "rows": shown, + "truncated": dropped, + "note": (f"{dropped} more field(s) not shown — open the record to review them before " + f"approving." if dropped else ""), + "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", + } + + +def intercept_write(agent, action, values, stash): + """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. + + ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card + naming the action and its values, and committed only on an explicit human accept. Everything + else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). + + ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` + existed requires approval. The unsafe direction here is not recoverable — revoking a bot does + not un-write what it wrote. + """ + if (agent or {}).get("writeApproval", True) is not False: + card = approval_card(action, values, agent) + stash(card) + return "await_approval", card + return "commit", None + + +def _pending(rt): + try: + found = rt.get(PENDING_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def stash_pending(rt, agent, action, values, now=None): + """Store one awaiting-approval mutation and return its token. + + The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the + action's id: a guessable value on an unauthenticated door is a way to approve somebody else's + write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. + """ + token = secrets.token_urlsafe(18) + row = {"at": float(now if now is not None else time.time()), + "channel": str((agent or {}).get("channel") or ""), + "action": str(action or "")[:80], + "values": values if isinstance(values, dict) else {}} + rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") + return token + + +#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. +#: +#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE +#: place an approved write may be performed, so it is the place a negative control can bite — but +#: a gate is only as honest as what it guards. Until something registers here, an approved card +#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a +#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. +_COMMITTERS = {} + + +def register_committer(kind, fn): + """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" + _COMMITTERS[str(kind)] = fn + + +def commit_pending(rt, token, approved, now=None): + """Resolve ONE approval card. Returns one of + `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. + + ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's + `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the + route could not bite while the route had no write in it at all — it would stay green with the + guard deleted, because there would be nothing for the guard to be protecting anything from + ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's + first draft). Putting the commit behind ONE named seam gives the control a real subject: the + gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it + exactly once. + + ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that + survives its own click is a replayable write, and "approve twice" must never mean "write + twice". The delete is `flush="sync"` for the same reason. + """ + now = float(now if now is not None else time.time()) + row = _pending(rt).get(str(token)) + if not isinstance(row, dict): + return "unknown" + # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. + rt.update(PENDING_KEY, + lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, + flush="sync") + if now - float(row.get("at") or 0) > APPROVAL_TTL_S: + return "expired" + if not approved: + # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. + return "rejected" + fn = _COMMITTERS.get(str(row.get("action") or "")) + if fn is None: + return "no_committer" + try: + fn(rt, row) + except Exception: # noqa: BLE001 + # A committer that raises must not read as a commit. The card is already consumed, so the + # honest report is that it failed, and the human asks again. + return "failed" + return "committed" + + +# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── +#: Sliding window, keyed on the SLACK TEAM, never the client IP. +#: +#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own +#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one +#: busy tenant would spend the allowance for all of them, and the symptom would be another +#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. +#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER +#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature +#: check is a single HMAC). +RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 +_HITS: dict = {} + + +def _rate_ok(key, now): + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + return len(seen) <= RATE_PER_WINDOW + + +async def _raw_body(request): + """The body as RAW BYTES, capped. + + ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string + `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is + otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim + copy of it could not verify a Slack signature at all: re-serialising the dict changes + whitespace and key order and the HMAC no longer matches. The bug would look like "Slack + signatures are always invalid", which is indistinguishable from a wrong secret. + + ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered + the whole body before the handler's first line, so a content-length check inside the handler + caps nothing (`routes_forms`' own scar, recorded in its docstring). + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + return b"".join(chunks) + + +def _refuse(): + """ONE refusal for every reason. No oracle. + + A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel + with no agent all answer identically — otherwise the door tells an unauthenticated caller + which tenants exist and which channels are configured, which is the question it was built to + not answer. `routes_forms._refuse` takes the same posture for the same reason. + """ + return err(403, "bad_request", "that request could not be verified") + + +def _verify(secret, timestamp, raw, signature, now): + """Slack's v0 signature, with the replay window checked FIRST.""" + try: + ts = int(str(timestamp or "0")) + except ValueError: + return False + if abs(now - ts) > SLACK_MAX_SKEW_S: + return False + base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") + import hashlib + expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, str(signature or "")) + + +def _resolve_tenant(raw, timestamp, signature, now): + """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. + + ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` + would let an unauthenticated caller name the tenant it wants to be — the exact widening + `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured + tenant's secret is tried and the one that VERIFIES names the tenant. + + ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not + depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the + same reason — a public door has no session to ask. + """ + from harness import runtime as _rt + hit = None + try: + slugs = list(_rt.known_tenants()) + except Exception: # noqa: BLE001 + return None, None, None + for slug in slugs: + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + creds = slack_creds(rt) + if not creds: + continue + if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: + hit = (slug, rt, creds) + return hit if hit else (None, None, None) + + +@router.post("/slack/events") +async def slack_events(request: Request): + """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. + + ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, + no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves + this deployment (amendment A1). + """ + now = time.time() + raw = await _raw_body(request) + ts = request.headers.get("x-slack-request-timestamp") + sig = request.headers.get("x-slack-signature") + slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) + if not rt: + raise _refuse() + + try: + payload = json.loads(raw or b"{}") + except ValueError: + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + + # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an + # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. + if payload.get("type") == "url_verification": + return {"challenge": str(payload.get("challenge") or "")[:512]} + + team = str(payload.get("team_id") or slug) + if not _rate_ok(f"{slug}:{team}", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + event = payload.get("event") if isinstance(payload.get("event"), dict) else {} + channel = str(event.get("channel") or "") + agent = None + for a in _agents(rt).values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + agent = a + break + # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries + # a non-2xx up to three times, so refusing here would turn "this channel is not configured" + # into three refusals per message, and the retry storm would be the only visible symptom. + # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which + # channels this workspace has configured. + if not isinstance(agent, dict) or agent.get("active", True) is False: + return {"ok": True} + + # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the + # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a + # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing + # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built + # answerer that returns something plausible is worse than one that returns nothing. + # Booked rather than faked [[flag-shipped-without-its-writer]]. + return {"ok": True} + + +@router.post("/slack/interact") +async def slack_interact(request: Request): + """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" + now = time.time() + raw = await _raw_body(request) + slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), + request.headers.get("x-slack-signature"), now) + if not rt: + raise _refuse() + # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. + from urllib.parse import parse_qs + try: + form = parse_qs(raw.decode("utf-8")) + payload = json.loads((form.get("payload") or ["{}"])[0]) + except Exception: # noqa: BLE001 + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + if not _rate_ok(f"{slug}:interact", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] + choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) + token = str(choice.get("value") or "") + # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, + # truncated or unknown action id: the direction that can be wrong here without anyone + # noticing is the one that writes. + approved = str(choice.get("action_id") or "") == "approve" + # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative + # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all + # live there; this handler only turns its verdict into a sentence. + verdict = commit_pending(rt, token, approved, now=now) + return {"text": { + # An unknown, already-used or expired token is not an error: a person clicking a stale + # card should be told it is stale, not shown a failure. + "unknown": "That approval is no longer available. Ask the bot again.", + "expired": "That approval expired before it was answered. Ask the bot again.", + "rejected": "Rejected. Nothing was changed.", + # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, + # so an approved card today performs nothing and says exactly that rather than "Approved." + "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", + "failed": "That change could not be completed. Nothing was saved; ask the bot again.", + "committed": "Approved.", + }.get(verdict, "That approval is no longer available. Ask the bot again.")} + + +@router.get("/slack/health") +def slack_health(session: Session = Depends(require_session)): + """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — + enumerating the others would answer a question about our customer list.""" + creds = slack_creds(session.runtime) + return {"configured": creds is not None, + "agents": len(_agents(session.runtime)), + "hasBotToken": bool((creds or {}).get("bot_token"))} diff --git a/api/routes_statements.py b/api/routes_statements.py index c2cbedf8619c07e441bc5a6f1f80ffb5982b0772..c39730724d527120c094a5afc94652703bb1de87 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -1,298 +1,298 @@ -"""routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6). - -⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is -read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists -exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and -add no write of their own. - -WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the -Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from -the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so -when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than -dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same -preview, same test-send, same two-step confirm. - -THE THREE GUARDRAILS, and where each is enforced: - 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route, - payload or UI can bypass it. These routes only REPORT it; they never re-implement the check. - 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`. - 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()` - reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to - TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab - admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the - single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall - rather than copy one. - -NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no -"send all" parameter, deliberately — the confirm step is a product requirement, not a formality. -""" -from fastapi import APIRouter, Body, Depends - -from deps import Session, err -from routes_admin import admin_gate - -# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this -# way round. `main.py` already imports both modules, so this adds no load. -import automation_engine as engine - -router = APIRouter(prefix="/api/v1") - -#: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the -#: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read. -_TTL = 1800 -_cache = {"at": 0.0, "rows": None} - - -def _cs(): - import modules.collections_send as cs - return cs - - -def _royal_only(session: Session) -> Session: - """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is - tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. - - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. - - ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an - agent step this wave, so the automation engine has to answer the same question at two more - doors (which tenants see the action, which tenants may store it). C8's words are "the predicate - is imported, never re-expressed", and this is the direction that import can run: the engine - imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and - `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. - ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the - surface is invisible rather than forbidden. The engine must not know about status codes.""" - if not engine.is_statement_tenant(session.runtime): - raise err(404, "not_found", "statements are not configured for this workspace") - return session - - -def _gate(session: Session = Depends(admin_gate)) -> Session: - return _royal_only(session) - - -def _rows(force=False): - import time - cs = _cs() - if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL: - _cache["rows"] = cs.load_collection_list(cs.Odoo()) - _cache["at"] = time.time() - return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"])) - - -def _public(row): - """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because - the client needs a stable row identity that is not the display name).""" - return {k: v for k, v in row.items() if not str(k).startswith("_")} - - -@router.get("/admin/statements") -def statements(refresh: int = 0, session: Session = Depends(_gate)): - """The worklist + everything the sender UI needs to render itself honestly.""" - cs = _cs() - try: - rows, loaded_at = _rows(force=bool(refresh)) - except Exception as e: - raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}") - return { - "rows": [_public(r) for r in rows], - "loadedAt": loaded_at, - # The guardrail is reported, never decided, here — the data layer owns it. - "safeMode": bool(cs.SAFE_MODE), - "safeRecipients": sorted(cs.SAFE_RECIPIENTS), - "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, - "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, - "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, - "footer": cs.DEFAULT_FOOTER}, - # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements - # agent's tier filter cannot come to mean different things. - "tiers": list(engine.STATEMENT_TIERS), - } - - -def _find(rows, customer): - return next((r for r in rows if r.get("Customer") == customer), None) - - -@router.post("/admin/statements/preview") -def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): - """Render ONE customer's statement exactly as the send path would.""" - cs = _cs() - body = body or {} - rows, _ = _rows() - row = _find(rows, body.get("customer")) - if row is None: - raise err(404, "not_found", "no such customer on the collection list") - t = body.get("templates") or {} - import datetime as dt - month = dt.date.today().strftime("%B %Y") - subject = (t.get("subject") or cs.DEFAULT_SUBJECT) - try: - subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) - except Exception: # noqa: BLE001 - # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they - # can see what they typed rather than getting an opaque error. - # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. - # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a - # TypeError, and every one of them is the same user mistake this branch was written for. - # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the - # one screen somebody opens to avoid mailing 200 people the wrong thing. - pass - return { - "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, - t.get("footer") or cs.DEFAULT_FOOTER), - "to": row.get("Email") or "", - "subject": subject, - } - - -@router.post("/admin/statements/send") -def send(body: dict = Body(default=None), session: Session = Depends(_gate)): - """Queue statements. Returns per-customer outcomes — NEVER a bare count. - - `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the - data layer refuses an address outside the allow-list), which is why the route does not check it. - """ - cs = _cs() - body = body or {} - names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] - if not names: - raise err(400, "bad_request", "name at least one customer") - # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A - # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent - # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) - # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On - # the one route in this product that is allowed to write to Odoo, the difference between a - # rehearsal and mailing a live customer was one empty string. - # - # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` - # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature - # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key - # being present, or `test: true`, is that statement, and an empty address beside it is the - # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` - # asserted with no key at all. - # - # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. - # `automationApi.testSendStatement` refuses an empty address before the request is built, and - # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client - # wall is bypassable by construction (anything can POST); this one is not. - declared_test = ("overrideTo" in body) or bool(body.get("test")) - override = str(body.get("overrideTo") or "").strip() - if declared_test and not override: - raise err(400, "no_override_address", - "a test send needs the address to send the test to. Without one this would " - "mail the customer's own address, which is the opposite of a test") - override = override or None - if override and len(names) != 1: - raise err(400, "bad_request", "a test send takes exactly one customer") - t = body.get("templates") or {} - rows, _ = _rows() - - sent, failed, skipped = [], [], [] - for name in names: - row = _find(rows, name) - if row is None: - failed.append({"customer": name, "error": "not on the current collection list"}) - continue - if not override and not row.get("Email"): - # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures - # keeps "we could not" distinct from "there was nowhere to send". - skipped.append({"customer": name, "reason": "no email address on the customer record"}) - continue - try: - mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT, - t.get("intro") or cs.DEFAULT_INTRO, - t.get("footer") or cs.DEFAULT_FOOTER, override_to=override) - sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid}) - except Exception as e: - # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal - # and an Odoo error are both "this one did not go", each with its own honest message. - failed.append({"customer": name, "error": str(e)[:200]}) - return {"sent": sent, "failed": failed, "skipped": skipped, - "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} - - -# --------------------------------------------------------------------------------------------- -# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. -# -# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch -# and parks it on the automation definition, and it imports no mail path at all. THIS is the only -# door that releases one, and it is deliberately built out of the parts already here: -# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three -# endpoints use. Not a copy of the checks: the same object. -# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal -# happens in the DATA LAYER, where no route, payload or UI can reach around it. -# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. -# -# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked -# `html` is what a person APPROVED and is what the review screen shows; but a balance can move -# between the parking and the click, and mailing a figure we know to be stale is worse than mailing -# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the -# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely -# (they paid) is reported as skipped rather than invoiced. - -@router.get("/admin/statements/agent/{auto_id}") -def agent_batch(auto_id: str, session: Session = Depends(_gate)): - """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - pending = defn.get("pendingStatements") - if not isinstance(pending, dict): - return {"pending": None} - # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is - # megabytes, and the review screen needs the count and the names to decide. `preview` already - # renders ONE on demand. - return {"pending": { - "ts": pending.get("ts"), "count": int(pending.get("count") or 0), - "notes": list(pending.get("notes") or []), - "items": [{k: v for k, v in it.items() if k != "html"} - for it in (pending.get("items") or [])], - "safeMode": bool(_cs().SAFE_MODE)}} - - -@router.post("/admin/statements/agent/{auto_id}/send") -def agent_send(auto_id: str, body: dict = Body(default=None), - session: Session = Depends(_gate)): - """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. - - ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send - that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) - arriving on the one route in this system that talks to a mail server. - """ - cs = _cs() - defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} - pending = defn.get("pendingStatements") - items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] - if not items: - raise err(404, "no_batch", "there is nothing parked for this agent to send") - # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it - # may only NARROW the parked batch. A name that was never parked cannot be introduced by the - # payload, or the review stops being what authorised the send. - want = {str(n) for n in (body or {}).get("customers") or []} - if want: - items = [it for it in items if str(it.get("customer")) in want] - if not items: - raise err(400, "bad_request", "none of those customers are in the parked batch") - rows, _ = _rows() - sent, failed, skipped = [], [], [] - for it in items: - name = str(it.get("customer") or "") - row = _find(rows, name) - if row is None: - skipped.append({"customer": name, - "reason": "no longer on the collection list, so nothing is owed"}) - continue - try: - # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE - # `queue_statement`, so the guardrail cannot be argued with from here. - mid = cs.queue_statement(cs.Odoo(), row, - str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, - cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) - sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) - except Exception as e: # noqa: BLE001 - failed.append({"customer": name, "error": str(e)[:200]}) - # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would - # lose the list of who still needs a statement, and nobody would know to look. - if sent and not failed: - engine.clear_statements(session.runtime, auto_id) - return {"sent": sent, "failed": failed, "skipped": skipped, - "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} +"""routes_statements.py — the statement-of-account sender, ported off Streamlit (EXIT-6). + +⛔ THIS IS THE ONE SANCTIONED ODOO WRITER IN THE ENTIRE SYSTEM. Everything else in AIOS is +read-only on Odoo by hard block. `modules/collections_send.py` owns a narrow client that whitelists +exactly `mail.mail create` (queueing an outbound email) and nothing else; these routes call it and +add no write of their own. + +WHY IT EXISTS SEPARATELY FROM THE COLLECTIONS PAGE. Owner ruling wave-17 item 15 retired the +Collections *dashboard* — the worklist is the shared "Collections" view on the Customer grid, from +the same reconciled blocks. What that ruling explicitly left "untouched" is this send workflow, so +when `app.py` is deleted it is the ONLY live Streamlit-only feature, and it moves here rather than +dying with the host. Ported faithfully: same tiers, same filters, same template placeholders, same +preview, same test-send, same two-step confirm. + +THE THREE GUARDRAILS, and where each is enforced: + 1. SAFE_MODE (default ON) — in the DATA LAYER (`collections_send.queue_statement`), so no route, + payload or UI can bypass it. These routes only REPORT it; they never re-implement the check. + 2. Admin only — `admin_gate` (role, fail-closed), mirroring the Streamlit `if not is_admin()`. + 3. ⭐ TENANT — NEW HERE, and it did not exist in Streamlit because it could not. `cs_mod.Odoo()` + reads Odoo credentials from the ENVIRONMENT, which after the keychain cutover belongs to + TENANT #0 ALONE. On the multi-tenant API an unguarded route would let a nurilab or gtmlab + admin queue mail from Royal Imports' Odoo, as Royal Imports. `_royal_only` closes that; the + single-tenant Streamlit host never had the exposure, so this is a port that must ADD a wall + rather than copy one. + +NOTHING SENDS ON A GET. The send route requires an explicit customer list in the body; there is no +"send all" parameter, deliberately — the confirm step is a product requirement, not a formality. +""" +from fastapi import APIRouter, Body, Depends + +from deps import Session, err +from routes_admin import admin_gate + +# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this +# way round. `main.py` already imports both modules, so this adds no load. +import automation_engine as engine + +router = APIRouter(prefix="/api/v1") + +#: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the +#: list moves slowly (it is a dunning worklist, not a live feed) and the pull is a multi-model read. +_TTL = 1800 +_cache = {"at": 0.0, "rows": None} + + +def _cs(): + import modules.collections_send as cs + return cs + + +def _royal_only(session: Session) -> Session: + """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is + tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. + + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. + + ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an + agent step this wave, so the automation engine has to answer the same question at two more + doors (which tenants see the action, which tenants may store it). C8's words are "the predicate + is imported, never re-expressed", and this is the direction that import can run: the engine + imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and + `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. + ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the + surface is invisible rather than forbidden. The engine must not know about status codes.""" + if not engine.is_statement_tenant(session.runtime): + raise err(404, "not_found", "statements are not configured for this workspace") + return session + + +def _gate(session: Session = Depends(admin_gate)) -> Session: + return _royal_only(session) + + +def _rows(force=False): + import time + cs = _cs() + if force or _cache["rows"] is None or (time.time() - _cache["at"]) > _TTL: + _cache["rows"] = cs.load_collection_list(cs.Odoo()) + _cache["at"] = time.time() + return _cache["rows"], time.strftime("%Y-%m-%d %H:%M", time.localtime(_cache["at"])) + + +def _public(row): + """Strip the internals the Streamlit grid also hid (`_`-prefixed + partner_id is kept, because + the client needs a stable row identity that is not the display name).""" + return {k: v for k, v in row.items() if not str(k).startswith("_")} + + +@router.get("/admin/statements") +def statements(refresh: int = 0, session: Session = Depends(_gate)): + """The worklist + everything the sender UI needs to render itself honestly.""" + cs = _cs() + try: + rows, loaded_at = _rows(force=bool(refresh)) + except Exception as e: + raise err(502, "odoo_unavailable", f"could not load the collection list: {str(e)[:200]}") + return { + "rows": [_public(r) for r in rows], + "loadedAt": loaded_at, + # The guardrail is reported, never decided, here — the data layer owns it. + "safeMode": bool(cs.SAFE_MODE), + "safeRecipients": sorted(cs.SAFE_RECIPIENTS), + "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, + "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, + "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, + "footer": cs.DEFAULT_FOOTER}, + # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements + # agent's tier filter cannot come to mean different things. + "tiers": list(engine.STATEMENT_TIERS), + } + + +def _find(rows, customer): + return next((r for r in rows if r.get("Customer") == customer), None) + + +@router.post("/admin/statements/preview") +def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): + """Render ONE customer's statement exactly as the send path would.""" + cs = _cs() + body = body or {} + rows, _ = _rows() + row = _find(rows, body.get("customer")) + if row is None: + raise err(404, "not_found", "no such customer on the collection list") + t = body.get("templates") or {} + import datetime as dt + month = dt.date.today().strftime("%B %Y") + subject = (t.get("subject") or cs.DEFAULT_SUBJECT) + try: + subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) + except Exception: # noqa: BLE001 + # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they + # can see what they typed rather than getting an opaque error. + # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. + # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a + # TypeError, and every one of them is the same user mistake this branch was written for. + # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the + # one screen somebody opens to avoid mailing 200 people the wrong thing. + pass + return { + "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, + t.get("footer") or cs.DEFAULT_FOOTER), + "to": row.get("Email") or "", + "subject": subject, + } + + +@router.post("/admin/statements/send") +def send(body: dict = Body(default=None), session: Session = Depends(_gate)): + """Queue statements. Returns per-customer outcomes — NEVER a bare count. + + `overrideTo` is the test-send path: one customer, one address. SAFE_MODE still applies (the + data layer refuses an address outside the allow-list), which is why the route does not check it. + """ + cs = _cs() + body = body or {} + names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] + if not names: + raise err(400, "bad_request", "name at least one customer") + # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A + # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent + # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) + # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On + # the one route in this product that is allowed to write to Odoo, the difference between a + # rehearsal and mailing a live customer was one empty string. + # + # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` + # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature + # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key + # being present, or `test: true`, is that statement, and an empty address beside it is the + # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` + # asserted with no key at all. + # + # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. + # `automationApi.testSendStatement` refuses an empty address before the request is built, and + # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client + # wall is bypassable by construction (anything can POST); this one is not. + declared_test = ("overrideTo" in body) or bool(body.get("test")) + override = str(body.get("overrideTo") or "").strip() + if declared_test and not override: + raise err(400, "no_override_address", + "a test send needs the address to send the test to. Without one this would " + "mail the customer's own address, which is the opposite of a test") + override = override or None + if override and len(names) != 1: + raise err(400, "bad_request", "a test send takes exactly one customer") + t = body.get("templates") or {} + rows, _ = _rows() + + sent, failed, skipped = [], [], [] + for name in names: + row = _find(rows, name) + if row is None: + failed.append({"customer": name, "error": "not on the current collection list"}) + continue + if not override and not row.get("Email"): + # The Streamlit page warned and skipped these. Reporting them SEPARATELY from failures + # keeps "we could not" distinct from "there was nowhere to send". + skipped.append({"customer": name, "reason": "no email address on the customer record"}) + continue + try: + mid = cs.queue_statement(cs.Odoo(), row, t.get("subject") or cs.DEFAULT_SUBJECT, + t.get("intro") or cs.DEFAULT_INTRO, + t.get("footer") or cs.DEFAULT_FOOTER, override_to=override) + sent.append({"customer": name, "to": override or row.get("Email"), "mailId": mid}) + except Exception as e: + # SafeModeBlocked lands here too, and that is correct: to the caller a guardrail refusal + # and an Odoo error are both "this one did not go", each with its own honest message. + failed.append({"customer": name, "error": str(e)[:200]}) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. +# +# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch +# and parks it on the automation definition, and it imports no mail path at all. THIS is the only +# door that releases one, and it is deliberately built out of the parts already here: +# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three +# endpoints use. Not a copy of the checks: the same object. +# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal +# happens in the DATA LAYER, where no route, payload or UI can reach around it. +# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. +# +# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked +# `html` is what a person APPROVED and is what the review screen shows; but a balance can move +# between the parking and the click, and mailing a figure we know to be stale is worse than mailing +# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the +# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely +# (they paid) is reported as skipped rather than invoiced. + +@router.get("/admin/statements/agent/{auto_id}") +def agent_batch(auto_id: str, session: Session = Depends(_gate)): + """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + if not isinstance(pending, dict): + return {"pending": None} + # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is + # megabytes, and the review screen needs the count and the names to decide. `preview` already + # renders ONE on demand. + return {"pending": { + "ts": pending.get("ts"), "count": int(pending.get("count") or 0), + "notes": list(pending.get("notes") or []), + "items": [{k: v for k, v in it.items() if k != "html"} + for it in (pending.get("items") or [])], + "safeMode": bool(_cs().SAFE_MODE)}} + + +@router.post("/admin/statements/agent/{auto_id}/send") +def agent_send(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_gate)): + """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. + + ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send + that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) + arriving on the one route in this system that talks to a mail server. + """ + cs = _cs() + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] + if not items: + raise err(404, "no_batch", "there is nothing parked for this agent to send") + # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it + # may only NARROW the parked batch. A name that was never parked cannot be introduced by the + # payload, or the review stops being what authorised the send. + want = {str(n) for n in (body or {}).get("customers") or []} + if want: + items = [it for it in items if str(it.get("customer")) in want] + if not items: + raise err(400, "bad_request", "none of those customers are in the parked batch") + rows, _ = _rows() + sent, failed, skipped = [], [], [] + for it in items: + name = str(it.get("customer") or "") + row = _find(rows, name) + if row is None: + skipped.append({"customer": name, + "reason": "no longer on the collection list, so nothing is owed"}) + continue + try: + # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE + # `queue_statement`, so the guardrail cannot be argued with from here. + mid = cs.queue_statement(cs.Odoo(), row, + str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, + cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) + sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) + except Exception as e: # noqa: BLE001 + failed.append({"customer": name, "error": str(e)[:200]}) + # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would + # lose the list of who still needs a statement, and nobody would know to look. + if sent and not failed: + engine.clear_statements(session.runtime, auto_id) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} diff --git a/api/routes_web_agent.py b/api/routes_web_agent.py index 57453ded998810e7046aa5d14acae4d739fd235b..12364aa7911fe7e0042054e3a7f07e050fae7700 100644 --- a/api/routes_web_agent.py +++ b/api/routes_web_agent.py @@ -1,84 +1,84 @@ -"""routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51). - -The owner's words are the whole reason this file exists: *"we already laid the foundation of this -but never test anything."* The capability is otherwise reachable only from inside an automation -run, which means the first person to discover it is broken is a customer at 3am. - - GET /api/v1/web-agent/capability any session — can this deployment run a web step, and - if not, the SENTENCE saying why - POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what - came back, or the sentence - -⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract -C5); these are an operator surface over the same function, so a green test here and a red run -there cannot disagree about anything except the input. - -⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`, -so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody. - -⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's -network, never from this server, so this is not a door into our own infrastructure. It is still -admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems -the tenant is authorised to use, at their instruction") is not something an ordinary member -should be able to commit the tenant to. - -⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and -`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting -either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which -is the same class of stale claim as a green gate on a router nobody wired: three finished routers -once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. -""" -from fastapi import APIRouter, Body, Depends - -import web_agent -from deps import Session, require_session, err -from routes_admin import admin_gate - -router = APIRouter(prefix="/api/v1") - -MAX_URL = 2000 -MAX_SELECTOR = 400 - - -@router.get("/web-agent/capability") -def web_agent_capability(session: Session = Depends(require_session)): - """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`. - - Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say - if not") and withholds the deployment detail (namespace, which token key, the image) that - only an operator has any use for. Nothing here is ever a credential. - """ - cap = web_agent.capability() - return {"ready": bool(cap["ready"]), "reason": cap["reason"], - "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]} - - -@router.post("/web-agent/test") -def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)): - """Run ONE real `web_read` and report exactly what the seam returned. - - The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`, - or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something - went wrong" would hide the one thing it exists to show. - """ - url = str(body.get("url") or "").strip() - selector = str(body.get("selector") or "").strip() - if len(url) > MAX_URL or len(selector) > MAX_SELECTOR: - raise err(400, "too_long", "the URL or selector is longer than this door accepts") - - step = {"kind": "web_read", "id": "test", "url": url, "selector": selector, - "attr": (body.get("attr") or "text"), "all": bool(body.get("all")), - "timeoutMs": int(body.get("timeoutMs") or 20000)} - if body.get("waitFor"): - step["waitFor"] = str(body["waitFor"]) - - notes = [] - result, error = web_agent.run_step( - step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}", - "log": notes.append}) - if error: - # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the - # web step did not succeed. A 500 here would make an ordinary "the selector matched - # nothing" look like a server fault. - return {"ok": False, "error": error, "log": notes[-6:]} - return {"ok": True, "result": result, "log": notes[-6:]} +"""routes_web_agent.py — the door that lets a person TEST the web agent (wave 31, R10 / D-51). + +The owner's words are the whole reason this file exists: *"we already laid the foundation of this +but never test anything."* The capability is otherwise reachable only from inside an automation +run, which means the first person to discover it is broken is a customer at 3am. + + GET /api/v1/web-agent/capability any session — can this deployment run a web step, and + if not, the SENTENCE saying why + POST /api/v1/web-agent/test ADMIN — run one real `web_read` and show what + came back, or the sentence + +⛔ NEITHER ROUTE IS THE SEAM. `automation_engine` calls `web_agent.run_step` directly (contract +C5); these are an operator surface over the same function, so a green test here and a red run +there cannot disagree about anything except the input. + +⚠ `POST /test` BLOCKS FOR ~10-30 s and COSTS A FRACTION OF A CENT. It is `def`, not `async def`, +so FastAPI runs it in the threadpool and one test cannot stall the event loop for everybody. + +⚠ ON THE URL IT WILL FETCH: the fetch happens inside an ephemeral HF Job on Hugging Face's +network, never from this server, so this is not a door into our own infrastructure. It is still +admin-gated, because it spends money and because D-51 §5's authorisation posture ("only systems +the tenant is authorised to use, at their instruction") is not something an ordinary member +should be able to commit the tenant to. + +⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and +`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting +either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which +is the same class of stale claim as a green gate on a router nobody wired: three finished routers +once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. +""" +from fastapi import APIRouter, Body, Depends + +import web_agent +from deps import Session, require_session, err +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +MAX_URL = 2000 +MAX_SELECTOR = 400 + + +@router.get("/web-agent/capability") +def web_agent_capability(session: Session = Depends(require_session)): + """Can a web step run here at all? Configuration, not liveness — see `web_agent.capability`. + + Deliberately NARROW: it answers the question a UI needs ("may I offer this, and what do I say + if not") and withholds the deployment detail (namespace, which token key, the image) that + only an operator has any use for. Nothing here is ever a credential. + """ + cap = web_agent.capability() + return {"ready": bool(cap["ready"]), "reason": cap["reason"], + "runnableKinds": cap["runnableKinds"], "profile": cap["profile"]} + + +@router.post("/web-agent/test") +def web_agent_test(session: Session = Depends(admin_gate), body: dict = Body(...)): + """Run ONE real `web_read` and report exactly what the seam returned. + + The response mirrors the seam's own contract rather than flattening it: `ok` plus a `value`, + or `ok:false` plus the SENTENCE. A test surface that turns a named failure into "something + went wrong" would hide the one thing it exists to show. + """ + url = str(body.get("url") or "").strip() + selector = str(body.get("selector") or "").strip() + if len(url) > MAX_URL or len(selector) > MAX_SELECTOR: + raise err(400, "too_long", "the URL or selector is longer than this door accepts") + + step = {"kind": "web_read", "id": "test", "url": url, "selector": selector, + "attr": (body.get("attr") or "text"), "all": bool(body.get("all")), + "timeoutMs": int(body.get("timeoutMs") or 20000)} + if body.get("waitFor"): + step["waitFor"] = str(body["waitFor"]) + + notes = [] + result, error = web_agent.run_step( + step, {"tenant": session.tenant, "runId": f"test-{session.tenant}-{session.uname}", + "log": notes.append}) + if error: + # 200 with `ok:false`, not a 4xx: the request was well-formed and the ANSWER is that the + # web step did not succeed. A 500 here would make an ordinary "the selector matched + # nothing" look like a server fault. + return {"ok": False, "error": error, "log": notes[-6:]} + return {"ok": True, "result": result, "log": notes[-6:]} diff --git a/platform/core/script_sandbox.py b/platform/core/script_sandbox.py index babf60594d17d540e348efa836e40056260a1f96..231ce07dd952f2d67caf31e11e9f1525a6a5a38f 100644 --- a/platform/core/script_sandbox.py +++ b/platform/core/script_sandbox.py @@ -1,519 +1,519 @@ -"""core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python. - -Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they -want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this -script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was -stated, so this file is the guardrail, and one engine serves both items. - -════════════════════════════════════════════════════════════════════════════════════════════════ -⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE. - -In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated -namespace stops import, file, network and environment access — but it **cannot cap memory and -cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow -request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**: -`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the -allow-list inside. Neither half is sufficient; both are load-bearing. - -⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under -the CALLING user's record and serialises the result; the child imports nothing from this repo and -holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store -path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child -reports its own `sys.modules`, and no `core.*` name may appear in it. - -⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute -names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and -the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute -lookup inside `format`, so there is no `ast.Attribute` node to refuse). -════════════════════════════════════════════════════════════════════════════════════════════════ - -The two layers, and they refuse DIFFERENT things on purpose: - - 1. `check_source()` — a pure function over source text. Refuses a construct the language offers - and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every - attribute name outside `ALLOWED_ATTRS`. - 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`, - `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets - past layer 1 still finds nothing to call. - -⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is -refused twice, so a negative control that drops ONE layer sees the other refuse and reports -green — the shape that already cost this wave one missed control in `routes_agent_harness`. So -each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings, -and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching -boundary goes red. -""" -import ast -import json -import os -import subprocess -import sys -import tempfile -import time -from pathlib import Path - -#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform. -DEFAULT_TIMEOUT_S = 10.0 - -#: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report. -DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024 - -#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the -#: wall-clock kill is the primary control and this is the backstop for a child that stops being -#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal. -DEFAULT_CPU_SECONDS = 15 - -#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer, -#: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything -#: that escaped far enough to write to fd 1 has nowhere for it to land. -MAX_STDOUT_BYTES = 64 * 1024 - -#: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule -#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()` -#: returns a named limit carrying its cause and a recommendation. -MAX_PAYLOAD_BYTES = 32 * 1024 * 1024 - -#: The emitted spec. A render spec is a description of a picture; one larger than this is data -#: pretending to be a description. -MAX_SPEC_BYTES = 2 * 1024 * 1024 - -MAX_SOURCE_BYTES = 128 * 1024 - - -# ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════ -#: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` / -#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its -#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__` -#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every -#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never -#: awaited is a silent no-op that looks like a working script). -ALLOWED_NODES = frozenset(""" -Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise -If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword -BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name -Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension -Load Store Del -And Or Not Invert UAdd USub -Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult -Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn -""".split()) - -#: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is -#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are -#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace, -#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__` -#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this -#: sandbox has not been asked for. -#: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup -#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are -#: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused. -ALLOWED_ATTRS = frozenset(""" -append extend insert pop remove clear sort reverse copy count index -keys values items get setdefault update -add discard union intersection difference issubset issuperset -join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold -replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix -isdigit isalpha isalnum isspace isupper islower isnumeric -real imag numerator denominator -""".split()) - - -class Refused(Exception): - """A named refusal: `code` for a caller to branch on, `message` for a person to read.""" - - def __init__(self, code, message): - self.code, self.message = code, message - super().__init__(f"{code}: {message}") - - -def _attr_ok(name): - """An attribute name passes only if it is on the list AND is not private. - - ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every - private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by - accident, which is the one edit a future reader is most likely to make in a hurry. - """ - return name in ALLOWED_ATTRS and not name.startswith("_") - - -def check_source(source): - """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`. - - ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no - process and touches no store, so a gate can hand it a hundred hostile strings for free and an - NC can drop one entry from one frozenset and watch exactly this function change its answer. - """ - text = str(source or "") - if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: - return Refused("source_too_long", - f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source") - try: - tree = ast.parse(text) - except SyntaxError as exc: - return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}") - - for node in ast.walk(tree): - kind = type(node).__name__ - if kind not in ALLOWED_NODES: - return Refused("refused_construct", - f"line {getattr(node, 'lineno', 0)}: this sandbox does not run " - f"{_english(kind)}") - if isinstance(node, ast.Attribute) and not _attr_ok(node.attr): - return Refused("refused_attribute", - f"line {getattr(node, 'lineno', 0)}: the attribute " - f"'{node.attr}' is not available inside a script view") - # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a - # script that could bind one could shadow the machinery it runs on top of. - if isinstance(node, ast.Name) and node.id.startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: names starting with an " - f"underscore are reserved by the sandbox") - if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str( - getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: names starting with an " - f"underscore are reserved by the sandbox") - if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"): - return Refused("reserved_name", - f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with " - f"an underscore are reserved by the sandbox") - return None - - -_ENGLISH = { - "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition", - "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop", - "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield", - "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement", - "Match": "a match statement", -} - - -def _english(kind): - return _ENGLISH.get(kind, f"a {kind} expression") - - -# ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════ -#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child. -#: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input` -#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help` -#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step -#: on a published escape, and none has ever been asked for by a script that shapes rows. -#: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause -#: cannot name what it catches is a construct that reads as supported and is not. They are safe -#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this -#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway. -SANDBOX_BUILTIN_NAMES = ( - "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex " - "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed " - "round set slice sorted str sum tuple zip True False None " - "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError " - "AttributeError StopIteration OverflowError" -).split() - -#: The literal program the child runs. It is TEXT rather than a module because the child must -#: import nothing from this repo: a module would be found on `sys.path` and would drag `core` -#: with it, which is exactly the second store path W1 forbids. -#: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so -#: the runner's own machinery cannot be shadowed by the source it executes. -_RUNNER = r''' -import json as _json, os as _os, sys as _sys - -_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read()) -_out = {"ok": False, "code": "not_run", "message": "the script did not run", - "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}} - -# ── the caps this platform can actually apply, reported either way (standing rule 1) ────────── -try: - import resource as _res - _mem = int(_pay["memoryBytes"]) - _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem)) - _out["caps"]["memory"] = True - _cpu = int(_pay["cpuSeconds"]) - _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu)) - _out["caps"]["cpu"] = True -except Exception: - # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says - # which of the three held, never a silent partial. - pass - -_printed = [] -_spent = [0] -_LIMIT = int(_pay["maxStdout"]) - - -def _print(*_a, **_k): - _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n") - _room = _LIMIT - _spent[0] - if _room > 0: - _printed.append(_text[:_room]) - _spent[0] += len(_text) - - -class _Refusal(Exception): - """The SANDBOX refusing, as distinct from the SCRIPT failing. - - Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the - script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's - own code. Two different facts, two different codes. - """ - - -_emitted = [] - - -def _emit(_spec): - if not isinstance(_spec, dict): - raise _Refusal("emit() takes a view spec, which is a dictionary") - if _emitted: - raise _Refusal("emit() was already called; a script view emits exactly one view") - _emitted.append(_spec) - - -_rows = _pay["rows"] -_fields = _pay["fields"] -_bound = _pay["table"] - - -def _scoped_table(_table=None): - if _table is not None and str(_table) != _bound: - raise _Refusal( - "this script view is bound to the database '" + _bound + "' and asked for '" - + str(_table) + "'. A script view reads its own database only") - return [dict(_r) for _r in _rows] - - -def _scoped_fields(): - return [dict(_f) for _f in _fields] - - -_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict) - else getattr(__builtins__, _n) - for _n in _pay["builtins"]}} -_ns["__builtins__"]["print"] = _print -_ns["print"] = _print -_ns["emit"] = _emit -_ns["scoped_table"] = _scoped_table -_ns["scoped_fields"] = _scoped_fields -_ns["table"] = _bound - -try: - exec(compile(_pay["source"], " - - + + + + + + + Loopable + + + +
+
+ + + diff --git a/web/src/App.tsx b/web/src/App.tsx index 0dcffc3f4790d9b8a0842285b64f4ff08b9d5da4..8e740b30aa739af89fe5978458176d8cc964a6e6 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,41 +1,41 @@ -import { Suspense, lazy } from "react"; -// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 -// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the -// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build -// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, -// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. -// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is -// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is -// indistinguishable from a broken one. -const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); -import { OverlayProvider } from "./customer-grid/OverlaySurface"; -import { isStreamlitComponent } from "./customer-grid/hostBridge"; -import Shell from "./shell/Shell"; - -// HOSTED = either Streamlit path (the component iframe, or the legacy injected -// payload). There the surrounding app owns all chrome and this tree renders the -// BARE grid, byte-for-byte the behavior the embed always had. Only the -// standalone build gets the shell (wave 4, the strangler frame). Computed once -// at module level, like useCustomerData's own mode constants — the host mode -// cannot change within a page lifetime. -// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to -// install the standalone event sink, and two copies of a mode predicate is how -// the two halves of "standalone-only" end up disagreeing. -export const HOSTED = - isStreamlitComponent() || - typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; - -export default function App() { - if (HOSTED) { - return ( -
- - - -
- ); - } - return ; -} +import { Suspense, lazy } from "react"; +// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 +// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the +// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build +// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, +// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. +// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is +// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is +// indistinguishable from a broken one. +const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); +import { OverlayProvider } from "./customer-grid/OverlaySurface"; +import { isStreamlitComponent } from "./customer-grid/hostBridge"; +import Shell from "./shell/Shell"; + +// HOSTED = either Streamlit path (the component iframe, or the legacy injected +// payload). There the surrounding app owns all chrome and this tree renders the +// BARE grid, byte-for-byte the behavior the embed always had. Only the +// standalone build gets the shell (wave 4, the strangler frame). Computed once +// at module level, like useCustomerData's own mode constants — the host mode +// cannot change within a page lifetime. +// EXPORTED (EXIT wave 1): main.tsx needs the same answer to decide whether to +// install the standalone event sink, and two copies of a mode predicate is how +// the two halves of "standalone-only" end up disagreeing. +export const HOSTED = + isStreamlitComponent() || + typeof (window as unknown as { __AIOS_GRID__?: unknown }).__AIOS_GRID__ !== "undefined"; + +export default function App() { + if (HOSTED) { + return ( +
+ + + +
+ ); + } + return ; +} diff --git a/web/src/account/SubscriptionPage.tsx b/web/src/account/SubscriptionPage.tsx index 2cbc71a05dca231d01b3badbb396d5bd70452472..59f00a708e4ab698bd6e9863238898e970f0a56c 100644 --- a/web/src/account/SubscriptionPage.tsx +++ b/web/src/account/SubscriptionPage.tsx @@ -1,49 +1,49 @@ -// --------------------------------------------------------------------------- -// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). -// -// Owner item 13, verbatim: *"add below settings the following button: Feedback …, -// Usage credits …, and a Subscription module … Specialized for the Royal Imports -// tenant, do not show it."* -// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the -// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. -// -// This is the third of those three, and the only one with nothing behind it yet. -// So it says so, in the words this app already uses for a boarded door -// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later -// release."). One voice for one state — DESIGN.md 1's "variety is a defect". -// -// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. -// C6 puts the Royal Imports rule in TWO places that are both the frame's: the -// account MENU does not draw the row, and the ROUTE refuses. A third opinion here -// would be a third thing to keep in step, and the day they disagree the reader -// gets a menu row that opens a page which says it does not exist. The page renders -// the same for every tenant that can reach it, because reaching it IS the decision. -// -// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. -// The wave's rule is that a prop must be REQUIRED, never optional — an optional one -// degrades to "the page does not exist", which is indistinguishable from never -// built. That rule constrains the props that EXIST; it does not ask for a prop that -// carries nothing. There is nothing this page needs from the frame: it holds no -// state, makes no call, and takes no decision the frame has already taken. A -// `tenant` prop here would be exactly the second opinion the paragraph above -// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. -// --------------------------------------------------------------------------- - -import "./account.css"; - -export default function SubscriptionPage() { - return ( -
-

Subscription

-
- {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not - narrate itself). The reader arrived here on purpose and wants one fact: is there - anything to do? There is not, and every further word would be spent on someone - who is already leaving. No feature list, no roadmap, no "in the meantime". */} -

- Under construction. Plans, billing and invoices will open here in a later release. -

-
-
- ); -} +// --------------------------------------------------------------------------- +// account/SubscriptionPage.tsx — WAVE 35 · W35-T20 (owner item 13, contract C6). +// +// Owner item 13, verbatim: *"add below settings the following button: Feedback …, +// Usage credits …, and a Subscription module … Specialized for the Royal Imports +// tenant, do not show it."* +// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the +// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. +// +// This is the third of those three, and the only one with nothing behind it yet. +// So it says so, in the words this app already uses for a boarded door +// (`Shell.tsx`'s templates note: "Under construction. … will open here in a later +// release."). One voice for one state — DESIGN.md 1's "variety is a defect". +// +// ⛔ THIS PAGE DOES NOT HIDE ITSELF, AND THAT IS THE TICKET'S OWN TRAP. +// C6 puts the Royal Imports rule in TWO places that are both the frame's: the +// account MENU does not draw the row, and the ROUTE refuses. A third opinion here +// would be a third thing to keep in step, and the day they disagree the reader +// gets a menu row that opens a page which says it does not exist. The page renders +// the same for every tenant that can reach it, because reaching it IS the decision. +// +// ⛔ AND IT TAKES NO PROPS, WHICH IS ALSO A DECISION rather than an omission. +// The wave's rule is that a prop must be REQUIRED, never optional — an optional one +// degrades to "the page does not exist", which is indistinguishable from never +// built. That rule constrains the props that EXIST; it does not ask for a prop that +// carries nothing. There is nothing this page needs from the frame: it holds no +// state, makes no call, and takes no decision the frame has already taken. A +// `tenant` prop here would be exactly the second opinion the paragraph above +// refuses. The mount is proven by A's `verify_wiring` row (W2), not by a signature. +// --------------------------------------------------------------------------- + +import "./account.css"; + +export default function SubscriptionPage() { + return ( +
+

Subscription

+
+ {/* ⚠ TWO SHORT SENTENCES AND NO TOUR (DESIGN.md 4 / wave-22 R13: the app does not + narrate itself). The reader arrived here on purpose and wants one fact: is there + anything to do? There is not, and every further word would be spent on someone + who is already leaving. No feature list, no roadmap, no "in the meantime". */} +

+ Under construction. Plans, billing and invoices will open here in a later release. +

+
+
+ ); +} diff --git a/web/src/account/UsagePage.tsx b/web/src/account/UsagePage.tsx index f735d26732cc9caca5ada0eb71c190c9584ec1b4..6ab2ceb32d8c3e1213c3df4e9667466d4384d916 100644 --- a/web/src/account/UsagePage.tsx +++ b/web/src/account/UsagePage.tsx @@ -1,297 +1,297 @@ -// --------------------------------------------------------------------------- -// account/UsagePage.tsx — WAVE 35 · W35-T19 (owner item 13, ruling R9, C6/C7). -// -// R9: ONE meter for every AI surface. It REPORTS weekly usage against an -// allowance and does not cut anyone off this wave; over the allowance the -// product still works and the bar is red. -// -// ⛔⛔ THIS PAGE MAY NOT INVENT A FIGURE, AND THAT IS THE WHOLE TICKET. -// Every number here comes off `GET /usage` — the week, the reset date, the -// allowance, each surface's tokens and calls. Nothing is derived from a clock, -// nothing is estimated, and a surface with no ledger lines shows **0**, not a -// blank and not an omission (E-9 guarantees all four surfaces are always -// present, so there is no missing-row branch to get wrong). -// [[no-unverifiable-aggregates]] -// -// ⛔ AND `unmeasured` IS RENDERED, NOT SWALLOWED. Those are real calls whose -// provider declined to report a token count, so the total beside them is a -// FLOOR. A page that hides them presents the floor as a complete figure, which -// is the cost-surprise failure one step removed — the same reason -// `usage_ledger` books an unmeasured call as unknown rather than as zero. -// -// ⚠ SURFACE/READ SPLIT, as on Home and Starred: `renderToStaticMarkup` runs no -// effect, so a shot of the fetching component photographs only its spinner. -// --------------------------------------------------------------------------- - -import { useCallback, useEffect, useRef, useState } from "react"; - -import { fmt } from "../ui/fmt"; -import { loadUsage } from "./accountApi"; -import type { Usage, UsageSurface } from "./accountApi"; -import "./account.css"; - -/** - * The four surfaces C7 names, in the order R9 lists them. - * - * ⚠ AN UNKNOWN KEY IS HUMANISED, NEVER DROPPED. Dropping it would hide real usage from a meter - * whose whole job is to account for it, and printing the raw key would put an internal identifier - * on screen. The day a fifth LLM entry point lands (this product added two in one wave) it shows - * up here as a readable name with the right number beside it, and somebody can then decide what - * to call it. - */ -const SURFACE_LABEL: Record = { - assistant: "Assistant", - field_agent: "Field agents", - ai_review: "Reviews", - automation_draft: "Drafting", -}; -const ORDER = ["assistant", "field_agent", "ai_review", "automation_draft"]; - -export function surfaceLabel(key: string): string { - const known = SURFACE_LABEL[key]; - if (known) return known; - const words = key.replace(/[_-]+/g, " ").trim(); - return words ? words.charAt(0).toUpperCase() + words.slice(1) : key; -} - -/** C7's order first, then anything the server added, alphabetically. */ -export function orderSurfaces(surfaces: UsageSurface[]): UsageSurface[] { - const rank = (s: UsageSurface) => { - const i = ORDER.indexOf(s.surface); - return i === -1 ? ORDER.length : i; - }; - return [...surfaces].sort( - (a, b) => rank(a) - rank(b) || surfaceLabel(a.surface).localeCompare(surfaceLabel(b.surface)) - ); -} - -/** - * The bar's fill, as a percentage, CLAMPED to 100. - * - * ⚠ The clamp is what keeps "over the allowance" a COLOUR rather than a bar that overflows its - * own track and paints across the page. The number above the bar is unclamped and is where the - * overage is actually read. - * ⚠ An allowance of 0 (unset) yields 0 rather than a division by zero: no allowance means the - * meter has nothing to measure against, and a full red bar would be an assertion nobody made. - */ -export function barPct(value: number, of: number): number { - if (!(of > 0)) return 0; - return Math.max(0, Math.min(100, (value / of) * 100)); -} - -export type UsageLoad = - | { phase: "loading" } - | { phase: "ready"; usage: Usage } - | { phase: "failed"; message: string }; - -function Bar({ pct, over }: { pct: number; over: boolean }) { - return ( -