runloopable / api /ai_review.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
ce9586e verified
Raw History Blame Contribute Delete
44 kB
"""AI REVIEW β€” let a model decide which stage a card moves to (wave 23, owner ruling R4/R14).
⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made
that somebody optionally a MODEL: the engine hands over the record's own values, the review's
prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of
those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit
log a human click writes to, tagged `by: "ai"` with the provider and model that made it.
β›” FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a
network failure, a slow answer, a malformed answer, or an answer naming a stage the review does
not offer β€” all return `("", {...})`, and the caller leaves the card exactly where a human would
have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing
the work. Nothing here can move a card somewhere the review does not already permit.
⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is
`groq β†’ cerebras β†’ openrouter β†’ anthropic`; the first CONFIGURED provider wins, and Anthropic is
last rather than absent β€” it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER`
pins one; `AIOS_AI_MODEL` overrides the model.
⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). β›” THIS PARAGRAPH
USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting
the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it.
The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the
integrator can overturn it deliberately rather than by drift"*. This is that overturn.
Its two reasons were real and are both answered rather than waved away:
1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is
not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers
the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no
Anthropic knowledge β€” it lost some.
2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the
import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the
same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH
manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave.
⚠ The other three legs stay OpenAI-chat-shaped over `requests` β€” they have no shared SDK, and that
half of the original reasoning never expired.
The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and
`stop_reason: "refusal"` is checked BEFORE reading `content` β€” a refusal answers HTTP 200 with an
empty content list, so code that indexes `content[0]` unconditionally breaks on it.
"""
from __future__ import annotations
import json
import os
import re
import requests
#: The ladder. Order IS the policy (R14) β€” cheapest capable first, Anthropic last.
PROVIDERS = [
# β›”β›” D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED
# it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent
# from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo
# for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`.
{"name": "groq", "env": "GROQ_API_KEY", "shape": "openai",
"url": "https://api.groq.com/openai/v1/chat/completions",
"model": "openai/gpt-oss-120b"},
{"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai",
"url": "https://api.cerebras.ai/v1/chat/completions",
"model": "gpt-oss-120b"},
{"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai",
"url": "https://openrouter.ai/api/v1/chat/completions",
"model": "openai/gpt-4o-mini"},
# ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as
# the backstop for a one-line classification, and this is the cheapest current Claude that
# does it well. A bigger model here would be spending the owner's money to pick between two
# labels it already has in front of it.
{"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
"url": "https://api.anthropic.com/v1/messages",
# ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above,
# and a deployment whose side rungs are out of credit can raise the tier without a release.
"model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"},
]
ANTHROPIC_VERSION = "2023-06-01"
TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
MAX_FIELD_CHARS = 200 # per value handed to the model
MAX_FIELDS = 30 # columns handed to the model
MAX_REASON = 200
MAX_HARNESS_PROMPT_CHARS = 4000
MAX_HARNESS_SUMMARY_CHARS = 240
def ladder():
"""The providers that are actually usable here, in order. Empty = the feature is off."""
pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower()
live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()]
if pin:
live = [p for p in live if p["name"] == pin]
return live
def configured():
return bool(ladder())
def _record_text(row, fields):
"""The record, as the model sees it. Values are truncated and the column set is bounded β€”
an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not
need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is
sitting at would be the model reading its own question back."""
keys = [k for k in (fields or list((row or {}).keys()))
if not str(k).startswith("stage_")][:MAX_FIELDS]
lines = []
for k in keys:
v = (row or {}).get(k)
if v is None or str(v).strip() == "":
continue
lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}")
return "\n".join(lines) or "(this record has no filled-in values)"
def _instruction(prompt, options, label):
return (
f"You are deciding what happens to one record waiting at a review step called "
f"{label!r} in a workflow.\n\n"
f"The person who built this workflow told you: {prompt}\n\n"
f"Choose EXACTLY ONE of these next steps, by its exact name:\n"
+ "\n".join(f"- {o}" for o in options)
+ "\n\nAnswer with one line of JSON and nothing else:\n"
'{"choice": "<one name from the list above>", "reason": "<one short sentence>"}\n'
"If the record does not give you enough to decide, answer "
'{"choice": "", "reason": "why not"} and a person will decide instead.'
)
def _parse(text, options):
"""The model's line β†’ `(choice, reason)`. A choice that is not one of the offered stages is
DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss
resolved by string distance is how a card ends up somewhere nobody authorised."""
raw = str(text or "").strip()
obj = None
m = re.search(r"\{.*\}", raw, re.S)
if m:
try:
obj = json.loads(m.group(0))
except (ValueError, TypeError):
obj = None
if not isinstance(obj, dict):
return "", ""
choice = str(obj.get("choice") or "").strip()
reason = str(obj.get("reason") or "").strip()[:MAX_REASON]
for opt in options:
if choice.lower() == str(opt).lower():
return str(opt), reason # the OFFERED spelling wins, never the model's
return "", reason
# ⭐⭐ WAVE 35 Β· W35-T41 / C7 β€” THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`.
#
# β›” THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s
# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token
# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach
# this transport rather than be bolted onto its caller β€” the caller never sees the body.
#
# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The
# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None),
# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers
# shape this repo keeps paying for.
def _call_openai(p, model, system, user, timeout):
r = requests.post(p["url"], timeout=timeout,
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
"Content-Type": "application/json"},
json={"model": model, "max_tokens": 300, "temperature": 0,
"messages": [{"role": "system", "content": system},
{"role": "user", "content": user}]})
if r.status_code >= 400:
# ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused
# before running the model has nothing to bill. The call is still COUNTED by `decide`.
return "", f"{p['name']} answered {r.status_code}", None
body = r.json()
choices = body.get("choices") or []
if not choices:
return "", f"{p['name']} returned no choices", body
return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body
#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so an administrator can ask WHICH path ran
#: rather than infer it from a requirements file: `GET /api/v1/automations/ai/transport`
#: (`routes_automation.ai_transport`, D-384) serves it beside the installed SDK version.
LAST_ANTHROPIC_TRANSPORT = ""
def _call_anthropic(p, model, system, user, timeout):
"""One classification turn on the Messages API, through `providers.anthropic_send`.
⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which
prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is
absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises
both transports to `(status, body)`, so every line below this call is untouched.
"""
global LAST_ANTHROPIC_TRANSPORT
import providers as _prov # noqa: PLC0415
# ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is
# nothing to strip here β€” the rule lives in `anthropic_request`, beside the thing it constrains.
req = _prov.anthropic_request(
model=model, key=os.environ[p["env"]].strip(), system=system,
messages=[{"role": "user", "content": user}], tools=[], max_tokens=300)
status, body, transport = _prov.anthropic_send(req, timeout=timeout)
LAST_ANTHROPIC_TRANSPORT = transport
if status >= 400 or status == 0:
# ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the
# vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402"
# (which is what this line used to say, and what the owner quoted back at us) cannot recur.
return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None
body = body or {}
# β›” stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so
# reading content[0] before this check turns a refusal into an IndexError inside a run.
# ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too.
if body.get("stop_reason") == "refusal":
return "", "anthropic declined to answer this record", body
parts = [b.get("text") or "" for b in (body.get("content") or [])
if isinstance(b, dict) and b.get("type") == "text"]
if not parts:
return "", "anthropic returned no text", body
return "".join(parts), "", body
def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""):
"""Pick this record's next stage. Returns `(choice, meta)`.
`choice` is "" whenever a person should decide β€” which is every failure mode there is.
`meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer.
⭐ W35-T41 / C7 β€” `st` and `user` are the USAGE LEDGER's target. Both default to absent because
this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it
HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as
unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for
why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI
dependency).
"""
import usage_ledger # noqa: PLC0415
opts = [str(o) for o in (options or []) if str(o).strip()]
if not opts:
return "", {"problem": "the review offers no next stages"}
if not str(prompt or "").strip():
return "", {"problem": "the review has no prompt for the model to follow"}
live = ladder()
if not live:
return "", {"problem": "no AI provider is configured on this deployment"}
system = _instruction(prompt, opts, label)
user = "Here is the record:\n\n" + _record_text(row, fields)
tmo = float(timeout or TIMEOUT_SECONDS)
override = (os.environ.get("AIOS_AI_MODEL") or "").strip()
problems = []
for p in live:
model = override or p["model"]
body = None
try:
text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)(
p, model, system, user, tmo)
except Exception as e: # noqa: BLE001
text, err = "", f"{p['name']} failed: {type(e).__name__}"
# ⭐⭐ C7 β€” THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER.
# β›” IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an
# unusable answer. A meter that books only successes reports a cheap week for a run that
# spent its budget being declined β€” and a declined call is billed. The only path that does
# NOT record is a transport that never reached the vendor (`body is None`), which spent
# nothing.
if body is not None:
ins, outs = usage_ledger.tokens_from(body)
usage_ledger.record("ai_review", p["name"], model, ins, outs,
total=usage_ledger.total_from(body), st=st, user=user)
if err:
problems.append(err)
continue # ladder: a dead provider degrades to the next one
choice, reason = _parse(text, opts)
if not choice:
# The provider ANSWERED and declined (or answered unusably). That is a decision about
# this record, not a fault in the provider, so it does NOT fall through to a more
# expensive one β€” the card goes to a human, which is what the model just asked for.
return "", {"provider": p["name"], "model": model,
"problem": reason or "the model did not choose one of the stages"}
return choice, {"provider": p["name"], "model": model, "reason": reason}
return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
# ══════════════════════════════════════════════════════════════════ the FLOW WRITER
# ⭐⭐ WAVE 33 Β· W33-T54/T55 (owner item 7, ruling R3) β€” A PROMPT BECOMES A DRAFT AUTOMATION.
#
# Owner, verbatim: *"lay out the foundation with custom tools we provide"* β€” and the tools are the
# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary:
# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the
# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object
# in that vocabulary.
#
# β›” WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This
# is the one module in the API package that already holds an LLM ladder and its credentials, so a
# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs
# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23
# shipped three finished routers 404-dead behind green gates. The DOOR is a route on
# `routes_automation.py`, which is already mounted.
#
# β›”β›” CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds
# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are
# enforced mechanically rather than asked for politely β€” the kind list is an `enum` in the schema
# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters
# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise
# and answers 200, so a draft accepted without the diff would show a person a flow that is not the
# flow the model described, with nothing anywhere saying so.
#
# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door
# can be proven end to end with no API key and no spend. A gate that can only run where a
# credential exists is a gate that never runs.
#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it:
#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you
#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow,
#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`.
FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic")
MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25
MAX_PROMPT_CHARS = 2000
def flow_providers(pin=None):
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.
⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and
the draft door used to read `prompt` off the body and nothing else β€” so the key the client sent
was accepted and dropped, and the picker was a control over nothing.
⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the
ladder stopped offering must not turn every later draft into an error; the caller is told which
rung actually answered, which is the honest half.
"""
by_name = {p["name"]: p for p in PROVIDERS}
live = [n for n in FLOW_PROVIDER_ORDER
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
if wanted and wanted in live:
return [by_name[wanted]]
return [by_name[n] for n in live]
def flow_schema(kinds, trigger_keys, table_keys):
"""The tool schema the model fills β€” the catalog's OWN key lists as enums.
β›” `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the
transport rather than by a check afterwards. A model asked for "any action name" invents
`send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a
400 from `clean_actions` carrying a sentence about a word the person never typed.
⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a
schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a
transport failure (`routes_query._spec_schema` carries the same note, for the same measured
reason).
"""
return {
"type": "object",
"properties": {
"kind": {"type": "string", "enum": ["flow", "refused"],
"description": "refused = this cannot be built from the steps available"},
"refusal": {"type": "string",
"description": "when kind=refused: ONE plain sentence naming what is "
"missing, in the words a non-technical person would use"},
"name": {"type": "string", "description": "a short title for the automation"},
"trigger": {"type": "string", "enum": sorted(trigger_keys),
"description": "what starts this automation"},
"table": {"type": "string", "enum": sorted(table_keys),
"description": "the database whose records this flow walks, if any"},
"actions": {
"type": "array",
"maxItems": MAX_DRAFT_ACTIONS,
"items": {
"type": "object",
"properties": {
"kind": {"type": "string", "enum": sorted(kinds)},
"why": {"type": "string",
"description": "one short sentence: why this step is here"},
"config": {"type": "object", "additionalProperties": True,
"description": "the step's settings, using ONLY the keys named "
"for that kind in the system message"},
},
"required": ["kind"],
},
},
},
"required": ["kind"],
}
def flow_system_prompt(catalog, required, triggers, tables):
"""What the model is told it may build with β€” DERIVED, never written down twice.
Every list here is the server's own: the catalog rows the menu paints, the required-key table
the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases
with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to
the catalog is offered here on the same deploy and a kind removed stops being offered.
"""
lines = ["You build small automations for a business tool. You are given the EXACT set of "
"steps this tool can perform. You may use nothing else.",
"", "THE STEPS YOU MAY USE:"]
# ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the
# PHRASE is the only human wording of that requirement anywhere β€” the one the runner's own
# refusal sentence is built from. Keys alone leave the model guessing what `field` means on a
# `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing
# what to call the setting. Handing it one and hoping is how a draft comes back configured
# against a key the validator drops.
req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()}
for row in catalog or []:
if not row.get("ready"):
continue
kind = str(row.get("kind") or "")
need = req.get(kind) or []
lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}"
+ (f" REQUIRED settings: {', '.join(need)}" if need else ""))
lines += ["", "WHAT CAN START AN AUTOMATION:"]
for t in triggers or []:
if t.get("planned") or not t.get("ready", True):
continue
lines.append(f"- {t.get('key')}: {t.get('label')}")
lines += ["", "THE DATABASES THIS PERSON HAS:"]
for t in (tables or [])[:40]:
cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25])
lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}")
if not tables:
lines.append("- (none - do not name a database)")
lines += [
"",
"RULES:",
"1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, "
"answer kind=refused and say plainly which capability is missing.",
"2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank "
"rather than inventing a web address, a CSS selector or a column name.",
"3. A web address must start with http:// or https://.",
"4. To use a value from the record the flow is walking, write it as {{Column name}}.",
"5. Name a database only from the list above, by its key.",
"6. Keep it short. Fewer steps that work beat more steps that guess.",
]
return "\n".join(lines)
def _flow_from_tool_call(obj):
"""The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash."""
if not isinstance(obj, dict):
return None, "the assistant's answer could not be read"
if str(obj.get("kind") or "") == "refused":
return None, (str(obj.get("refusal") or "").strip()
or "the assistant could not build this from the steps available")
acts = obj.get("actions")
if not isinstance(acts, list) or not acts:
return None, ("the assistant did not produce any steps - try describing what should "
"happen, one action at a time")
out = []
for a in acts[:MAX_DRAFT_ACTIONS]:
if not isinstance(a, dict) or not str(a.get("kind") or "").strip():
continue
cfg = a.get("config")
out.append({"kind": str(a["kind"]).strip(),
"config": cfg if isinstance(cfg, dict) else {},
"why": str(a.get("why") or "").strip()[:200]})
if not out:
return None, "the assistant's steps could not be read"
return {"name": str(obj.get("name") or "").strip()[:80] or "New automation",
"trigger": str(obj.get("trigger") or "").strip(),
"table": str(obj.get("table") or "").strip(),
# β›”β›” THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner,
# 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need
# to explicitly tell me why and recommend a fix"* β€” **a silent truncation IS the
# violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past
# the twelfth and said nothing, so a model that answered with a twenty-step journey had
# eight steps deleted between the answer and the screen, with no key anywhere in the
# response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE β€” this
# ceiling is the draft door's own, which makes reporting it the whole obligation.
# ⚠ Found by the verifier that checked this ticket, not by a gate.
"asked": len([a for a in acts if isinstance(a, dict)]),
"actions": out}, ""
def harness_proposal_schema(max_body_chars):
"""One typed answer for an agent-authored harness proposal.
``kind`` alone is required so a refusal remains representable. The caller validates the
proposed body in bytes against the store's real cap; JSON Schema's ``maxLength`` is a character
count and cannot safely stand in for that UTF-8 boundary.
"""
return {
"type": "object",
"properties": {
"kind": {"type": "string", "enum": ["proposal", "refused"]},
"refusal": {"type": "string",
"description": "when refused: one plain sentence saying what is missing"},
"summary": {"type": "string",
"description": "one short sentence describing the proposed change"},
"body": {"type": "string", "maxLength": int(max_body_chars),
"description": "the complete replacement body, never a patch or excerpt"},
},
"required": ["kind"],
}
def harness_system_prompt(index, loaded):
"""Build the bounded metadata index + ONE exact body the proposal model receives."""
rows = []
for row in index or []:
rows.append(
f"- {row.get('path')} | {row.get('name')} | v{int(row.get('version') or 0)} | "
f"{row.get('description')}"
)
source = loaded.get("source") if isinstance(loaded.get("source"), dict) else {}
return "\n".join([
"You maintain one stored AIOS Agent's harness. The harness index below contains metadata "
"only. Exactly one file body has been loaded because the person opened that file and asked "
"for a proposal. Do not claim to have read any other file.",
"",
"Return a complete replacement body for the loaded file, not a diff. Preserve instructions "
"the request does not ask to change. Never widen data access, invent credentials, or turn "
"a fact into an instruction. This is a proposal only; a person must approve it before it "
"becomes a harness version.",
"",
"BOUNDED HARNESS INDEX:",
*(rows or ["- (no other readable harness files)"]),
"",
"LOADED EXACT FILE:",
f"path: {loaded.get('path')}",
f"version: {int(loaded.get('version') or 0)}",
f"author: {source.get('author') or ''}",
f"author kind: {source.get('authorKind') or ''}",
f"updated: {source.get('updated') or ''}",
"--- BEGIN EXACT BODY ---",
str(loaded.get("body") or ""),
"--- END EXACT BODY ---",
])
def _harness_from_tool_call(obj, max_body_bytes):
if not isinstance(obj, dict):
return None, "the agent's harness proposal could not be read"
if str(obj.get("kind") or "") == "refused":
return None, (str(obj.get("refusal") or "").strip()
or "the agent could not propose a safe harness change")
if str(obj.get("kind") or "") != "proposal" or not isinstance(obj.get("body"), str):
return None, "the agent did not return a complete harness file"
body = obj["body"]
if len(body.encode("utf-8")) > int(max_body_bytes):
return None, "the proposed harness file is over the size limit"
return {"body": body,
"summary": " ".join(str(obj.get("summary") or "").split())[
:MAX_HARNESS_SUMMARY_CHARS]}, ""
def propose_harness_version(*, prompt, index, loaded, max_body_bytes, chat=None, timeout=None,
st=None, user="", model=None):
"""Return ``(proposal, refusal, provider)`` without writing the proposal or the harness.
This is the production model turn behind the Harness panel. The host supplies a bounded
metadata index and one exact, provenance-bearing body; this function has no store access and
therefore cannot silently widen that scope or approve its own output.
"""
global LAST_ANTHROPIC_TRANSPORT
text = str(prompt or "").strip()[:MAX_HARNESS_PROMPT_CHARS]
if not text:
return None, "say what the agent should improve in this file", None
loaded = loaded if isinstance(loaded, dict) else {}
if not isinstance(loaded.get("body"), str) or not loaded.get("path"):
return None, "the exact harness file could not be loaded", None
schema = harness_proposal_schema(max(1, int(max_body_bytes)))
tools = [{"type": "function", "function": {
"name": "propose_harness_version",
"description": "Propose one complete replacement body, or refuse.",
"parameters": schema,
}}]
messages = [{"role": "system", "content": harness_system_prompt(index, loaded)},
{"role": "user", "content": text}]
if chat is not None:
proposal, refusal = _harness_from_tool_call(chat(messages, tools), max_body_bytes)
return proposal, refusal, "injected"
provs = flow_providers(model)
if not provs:
return None, ("the assistant is not configured on this deployment, so it cannot propose "
"a harness version yet"), None
tmo = float(timeout or TIMEOUT_SECONDS)
problems = []
import providers as _prov # noqa: PLC0415
import usage_ledger # noqa: PLC0415
for p in provs:
try:
if p["shape"] == "anthropic":
req = _prov.anthropic_request(
model=p["model"], key=os.environ[p["env"]].strip(), system=None,
messages=messages, tools=tools, max_tokens=6000, tool_choice="required")
status, response, transport = _prov.anthropic_send(req, timeout=tmo)
LAST_ANTHROPIC_TRANSPORT = transport
else:
wire = requests.post(
p["url"], timeout=tmo,
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
"Content-Type": "application/json"},
json={"model": p["model"], "messages": messages, "tools": tools,
"tool_choice": "required", "temperature": 0.1,
"max_tokens": 6000})
status = wire.status_code
response = (wire.json() if wire.content and wire.status_code == 200
else {"_text": wire.text})
except Exception as exc: # noqa: BLE001
problems.append(f"{p['name']}: {type(exc).__name__}")
continue
if status != 200:
detail = json.dumps(response)[:600] if isinstance(response, dict) else str(response)[:600]
if _prov.is_credit_failure(status, detail):
_prov.mark_no_credit(p["name"])
problems.append(_prov.refusal_sentence(p["name"], status, detail))
continue
try:
response = response or {}
if p["shape"] == "anthropic":
_text, args, declined = _prov.anthropic_read(response)
if declined:
problems.append(f"{p['name']}: {declined}")
continue
else:
calls = (((response.get("choices") or [{}])[0].get("message") or {})
.get("tool_calls") or [])
args = json.loads(calls[0]["function"]["arguments"]) if calls else None
except Exception as exc: # noqa: BLE001
problems.append(f"{p['name']}: unreadable answer ({type(exc).__name__})")
continue
ins, outs = usage_ledger.tokens_from(response)
usage_ledger.record("agent_harness_proposal", p["name"], p["model"], ins, outs,
total=usage_ledger.total_from(response), st=st, user=user)
proposal, refusal = _harness_from_tool_call(args, max_body_bytes)
if proposal is not None:
return proposal, "", p["name"]
if isinstance(args, dict) and str(args.get("kind")) == "refused":
return None, refusal, p["name"]
problems.append(f"{p['name']}: {refusal}")
return None, ("; ".join(problems)[:300] or "no provider answered"), None
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None,
st=None, user="", model=None):
"""A sentence -> `(draft, refusal_sentence, provider)`. β›” NOTHING IS SAVED HERE.
Exactly one of `draft` and `refusal_sentence` is truthy β€” the same contract
`web_agent.run_step` keeps, so a caller has no third case to get wrong.
⭐ W35-T41 / C7 β€” `st`/`user` are the usage ledger's target, exactly as on `decide` above and for
the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and
`routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and
D asserts it; the two keywords are what D has to pass for the line to be attributable.
"""
# ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop.
# It parses either way; a reader scanning for the declaration does not look inside a `for`.
global LAST_ANTHROPIC_TRANSPORT
import usage_ledger # noqa: PLC0415
text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
if not text:
return None, "type what you want the automation to do", None
kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")})
if not kinds:
return None, "this deployment offers no automation steps to build with", None
trigger_keys = sorted({str(t.get("key")) for t in (triggers or [])
if t.get("key") and not t.get("planned")}) or ["manual"]
table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""]
tools = [{"type": "function", "function": {
"name": "build_automation",
"description": "Emit the automation, or refuse.",
"parameters": flow_schema(kinds, trigger_keys, table_keys)}}]
messages = [{"role": "system",
"content": flow_system_prompt(catalog, required, triggers, tables)},
{"role": "user", "content": text}]
if chat is not None:
# ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal
# branches as a live call; only the transport is replaced.
draft, refusal = _flow_from_tool_call(chat(messages, tools))
return draft, refusal, "injected"
provs = flow_providers(model)
if not provs:
# β›” SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
# never built [[flag-shipped-without-its-writer]].
return None, ("the assistant is not configured on this deployment, so an automation cannot "
"be drafted from a description yet"), None
tmo = float(timeout or TIMEOUT_SECONDS)
problems = []
import providers as _prov
for p in provs:
# ⭐⭐ W36-T35 / R4 β€” THE OWNER QUOTED THIS LINE BACK AT US. It used to read
# `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`,
# so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the
# Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."*
# It is wired now, through the ONE wire in `providers`, and it matters more than it looks:
# every side rung on this account is refusing today (cerebras 402, groq 404, openrouter
# 402), so without this branch the drafter cannot answer at all.
# ⚠ NO `effort` ON THIS DOOR β€” `output_config.effort` errors on Haiku 4.5, the tier this
# ladder runs. The parameter is the caller's to send, which is why the wire takes it.
# ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch
# below reads one thing. The anthropic side gets there through `anthropic_send` (official
# SDK when installed, the identical raw POST when not); the openai side still posts here
# because there is no shared SDK across three different vendors on that wire.
global LAST_ANTHROPIC_TRANSPORT
try:
if p["shape"] == "anthropic":
req = _prov.anthropic_request(
model=p["model"], key=os.environ[p["env"]].strip(), system=None,
messages=messages, tools=tools, max_tokens=1500, tool_choice="required")
status, body, transport = _prov.anthropic_send(req, timeout=tmo)
LAST_ANTHROPIC_TRANSPORT = transport
else:
r = requests.post(p["url"], timeout=tmo,
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
"Content-Type": "application/json"},
json={"model": p["model"], "messages": messages, "tools": tools,
"tool_choice": "required", "temperature": 0.1,
"max_tokens": 1500})
status = r.status_code
body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text}
except Exception as e: # noqa: BLE001
problems.append(f"{p['name']}: {type(e).__name__}")
continue
if status != 200:
# β›” A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off
# this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and
# the memo makes the next turn SKIP the empty account instead of paying for it again.
# ⚠ The body is re-serialised for the substring test because the SDK path never had a
# `.text` β€” that is the one thing this normalisation costs, and it costs nothing else.
detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600]
if _prov.is_credit_failure(status, detail):
_prov.mark_no_credit(p["name"])
problems.append(_prov.refusal_sentence(p["name"], status, detail))
continue
try:
body = body or {}
if p["shape"] == "anthropic":
_text, args, _refused = _prov.anthropic_read(body)
if _refused:
problems.append(f"{p['name']}: {_refused}")
continue
else:
calls = (((body.get("choices") or [{}])[0].get("message") or {})
.get("tool_calls") or [])
args = json.loads(calls[0]["function"]["arguments"]) if calls else None
except Exception as e: # noqa: BLE001
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
continue
# ⭐⭐ C7 β€” THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE
# answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer
# is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot
# parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that
# branch by construction. The 200-with-junk case is rare and named here rather than
# silently rounded to zero.
ins, outs = usage_ledger.tokens_from(body)
usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs,
total=usage_ledger.total_from(body), st=st, user=user)
draft, refusal = _flow_from_tool_call(args)
if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused":
# β›” A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more
# expensive rung β€” the model has just said this cannot be built. `decide()` takes the
# same posture for the same reason.
return None, refusal, p["name"]
if draft is not None:
return draft, "", p["name"]
problems.append(f"{p['name']}: {refusal}")
return None, ("; ".join(problems)[:300] or "no provider answered"), None