cfo-os / api /ai_review.py
fsanyoto's picture
Deploy AIOS web (React glide grid + FastAPI slice)
6c9f944 verified
Raw History Blame Contribute Delete
25.9 kB
"""AI REVIEW β€” let a model decide which stage a card moves to (wave 23, owner ruling R4/R14).
⭐ WHAT THIS IS. A review stage holds a record until somebody decides where it goes next. R4 made
that somebody optionally a MODEL: the engine hands over the record's own values, the review's
prompt, and the list of stages the review is allowed to send a card to, and gets back ONE of
those stage labels plus a one-line reason. Every decision is written to the same `reviews` audit
log a human click writes to, tagged `by: "ai"` with the provider and model that made it.
β›” FAIL-CLOSED IN EVERY DIRECTION, and this is the whole safety story. No key configured, a
network failure, a slow answer, a malformed answer, or an answer naming a stage the review does
not offer β€” all return `("", {...})`, and the caller leaves the card exactly where a human would
have found it. The feature can be absent, broken or wrong and the worst outcome is a person doing
the work. Nothing here can move a card somewhere the review does not already permit.
⭐ CHEAP FIRST (owner R14, verbatim: *"Claude is a bit too expensive"*). The ladder is
`groq β†’ cerebras β†’ openrouter β†’ anthropic`; the first CONFIGURED provider wins, and Anthropic is
last rather than absent β€” it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER`
pins one; `AIOS_AI_MODEL` overrides the model.
⚠ WHY RAW HTTP RATHER THAN THE `anthropic` SDK, stated because it is a deliberate deviation from
the /claude-api skill's default and not an oversight. Three of the four legs are OpenAI-chat-shaped
endpoints with no shared SDK, so a ladder built on the SDK would be one SDK leg beside three
hand-rolled ones β€” two implementations of the same call, the seam this repo keeps closing. And
`aios-web/api/requirements.txt` is PINNED to what the verify battery proves ([[pin-deps-space-
rebuilds]]): adding a dependency there rebuilds the container, which is a real deploy risk to take
for one leg of an optional feature. `requests` is already a dependency and `harness/analyst.py`
established per-provider raw HTTP as the house pattern. Booked as a DEBT line so the integrator
can overturn it deliberately rather than by drift.
The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and
`stop_reason: "refusal"` is checked BEFORE reading `content` β€” a refusal answers HTTP 200 with an
empty content list, so code that indexes `content[0]` unconditionally breaks on it.
"""
from __future__ import annotations
import json
import os
import re
import requests
#: The ladder. Order IS the policy (R14) β€” cheapest capable first, Anthropic last.
PROVIDERS = [
{"name": "groq", "env": "GROQ_API_KEY", "shape": "openai",
"url": "https://api.groq.com/openai/v1/chat/completions",
"model": "llama-3.3-70b-versatile"},
{"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai",
"url": "https://api.cerebras.ai/v1/chat/completions",
"model": "gpt-oss-120b"},
{"name": "openrouter", "env": "OPENROUTER_API_KEY", "shape": "openai",
"url": "https://openrouter.ai/api/v1/chat/completions",
"model": "openai/gpt-4o-mini"},
# ⚠ haiku-class DELIBERATELY, not the default Opus tier: R14 put Anthropic on this ladder as
# the backstop for a one-line classification, and this is the cheapest current Claude that
# does it well. A bigger model here would be spending the owner's money to pick between two
# labels it already has in front of it.
{"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic",
"url": "https://api.anthropic.com/v1/messages",
"model": "claude-haiku-4-5"},
]
ANTHROPIC_VERSION = "2023-06-01"
TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20)
MAX_FIELD_CHARS = 200 # per value handed to the model
MAX_FIELDS = 30 # columns handed to the model
MAX_REASON = 200
def ladder():
"""The providers that are actually usable here, in order. Empty = the feature is off."""
pin = (os.environ.get("AIOS_AI_REVIEW_PROVIDER") or "").strip().lower()
live = [p for p in PROVIDERS if (os.environ.get(p["env"]) or "").strip()]
if pin:
live = [p for p in live if p["name"] == pin]
return live
def configured():
return bool(ladder())
def _record_text(row, fields):
"""The record, as the model sees it. Values are truncated and the column set is bounded β€”
an automation table can carry a 32 KB JSON blob per row (C7) and a review decision does not
need it. Machine bookkeeping columns are dropped: a stage cell naming the stage the card is
sitting at would be the model reading its own question back."""
keys = [k for k in (fields or list((row or {}).keys()))
if not str(k).startswith("stage_")][:MAX_FIELDS]
lines = []
for k in keys:
v = (row or {}).get(k)
if v is None or str(v).strip() == "":
continue
lines.append(f"{k}: {str(v)[:MAX_FIELD_CHARS]}")
return "\n".join(lines) or "(this record has no filled-in values)"
def _instruction(prompt, options, label):
return (
f"You are deciding what happens to one record waiting at a review step called "
f"{label!r} in a workflow.\n\n"
f"The person who built this workflow told you: {prompt}\n\n"
f"Choose EXACTLY ONE of these next steps, by its exact name:\n"
+ "\n".join(f"- {o}" for o in options)
+ "\n\nAnswer with one line of JSON and nothing else:\n"
'{"choice": "<one name from the list above>", "reason": "<one short sentence>"}\n'
"If the record does not give you enough to decide, answer "
'{"choice": "", "reason": "why not"} and a person will decide instead.'
)
def _parse(text, options):
"""The model's line β†’ `(choice, reason)`. A choice that is not one of the offered stages is
DISCARDED, not fuzzy-matched: the offered list is a permission boundary, and a near-miss
resolved by string distance is how a card ends up somewhere nobody authorised."""
raw = str(text or "").strip()
obj = None
m = re.search(r"\{.*\}", raw, re.S)
if m:
try:
obj = json.loads(m.group(0))
except (ValueError, TypeError):
obj = None
if not isinstance(obj, dict):
return "", ""
choice = str(obj.get("choice") or "").strip()
reason = str(obj.get("reason") or "").strip()[:MAX_REASON]
for opt in options:
if choice.lower() == str(opt).lower():
return str(opt), reason # the OFFERED spelling wins, never the model's
return "", reason
def _call_openai(p, model, system, user, timeout):
r = requests.post(p["url"], timeout=timeout,
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
"Content-Type": "application/json"},
json={"model": model, "max_tokens": 300, "temperature": 0,
"messages": [{"role": "system", "content": system},
{"role": "user", "content": user}]})
if r.status_code >= 400:
return "", f"{p['name']} answered {r.status_code}"
body = r.json()
choices = body.get("choices") or []
if not choices:
return "", f"{p['name']} returned no choices"
return str(((choices[0] or {}).get("message") or {}).get("content") or ""), ""
def _call_anthropic(p, model, system, user, timeout):
r = requests.post(p["url"], timeout=timeout,
headers={"x-api-key": os.environ[p["env"]].strip(),
"anthropic-version": ANTHROPIC_VERSION,
"content-type": "application/json"},
json={"model": model, "max_tokens": 300, "system": system,
"messages": [{"role": "user", "content": user}]})
if r.status_code >= 400:
return "", f"anthropic answered {r.status_code}"
body = r.json()
# β›” stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so
# reading content[0] before this check turns a refusal into an IndexError inside a run.
if body.get("stop_reason") == "refusal":
return "", "anthropic declined to answer this record"
parts = [b.get("text") or "" for b in (body.get("content") or [])
if isinstance(b, dict) and b.get("type") == "text"]
if not parts:
return "", "anthropic returned no text"
return "".join(parts), ""
def decide(*, prompt, options, row, fields=(), label="Review", timeout=None):
"""Pick this record's next stage. Returns `(choice, meta)`.
`choice` is "" whenever a person should decide β€” which is every failure mode there is.
`meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer.
"""
opts = [str(o) for o in (options or []) if str(o).strip()]
if not opts:
return "", {"problem": "the review offers no next stages"}
if not str(prompt or "").strip():
return "", {"problem": "the review has no prompt for the model to follow"}
live = ladder()
if not live:
return "", {"problem": "no AI provider is configured on this deployment"}
system = _instruction(prompt, opts, label)
user = "Here is the record:\n\n" + _record_text(row, fields)
tmo = float(timeout or TIMEOUT_SECONDS)
override = (os.environ.get("AIOS_AI_MODEL") or "").strip()
problems = []
for p in live:
model = override or p["model"]
try:
text, err = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)(
p, model, system, user, tmo)
except Exception as e: # noqa: BLE001
text, err = "", f"{p['name']} failed: {type(e).__name__}"
if err:
problems.append(err)
continue # ladder: a dead provider degrades to the next one
choice, reason = _parse(text, opts)
if not choice:
# The provider ANSWERED and declined (or answered unusably). That is a decision about
# this record, not a fault in the provider, so it does NOT fall through to a more
# expensive one β€” the card goes to a human, which is what the model just asked for.
return "", {"provider": p["name"], "model": model,
"problem": reason or "the model did not choose one of the stages"}
return choice, {"provider": p["name"], "model": model, "reason": reason}
return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
# ══════════════════════════════════════════════════════════════════ the FLOW WRITER
# ⭐⭐ WAVE 33 Β· W33-T54/T55 (owner item 7, ruling R3) β€” A PROMPT BECOMES A DRAFT AUTOMATION.
#
# Owner, verbatim: *"lay out the foundation with custom tools we provide"* β€” and the tools are the
# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary:
# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the
# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object
# in that vocabulary.
#
# β›” WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This
# is the one module in the API package that already holds an LLM ladder and its credentials, so a
# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs
# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23
# shipped three finished routers 404-dead behind green gates. The DOOR is a route on
# `routes_automation.py`, which is already mounted.
#
# β›”β›” CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds
# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are
# enforced mechanically rather than asked for politely β€” the kind list is an `enum` in the schema
# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters
# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise
# and answers 200, so a draft accepted without the diff would show a person a flow that is not the
# flow the model described, with nothing anywhere saying so.
#
# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door
# can be proven end to end with no API key and no spend. A gate that can only run where a
# credential exists is a gate that never runs.
#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it:
#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you
#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow,
#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`.
FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic")
MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25
MAX_PROMPT_CHARS = 2000
def flow_providers():
"""The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off."""
pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
by_name = {p["name"]: p for p in PROVIDERS}
order = [pin] if pin else list(FLOW_PROVIDER_ORDER)
return [by_name[n] for n in order
if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
def flow_schema(kinds, trigger_keys, table_keys):
"""The tool schema the model fills β€” the catalog's OWN key lists as enums.
β›” `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the
transport rather than by a check afterwards. A model asked for "any action name" invents
`send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a
400 from `clean_actions` carrying a sentence about a word the person never typed.
⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a
schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a
transport failure (`routes_query._spec_schema` carries the same note, for the same measured
reason).
"""
return {
"type": "object",
"properties": {
"kind": {"type": "string", "enum": ["flow", "refused"],
"description": "refused = this cannot be built from the steps available"},
"refusal": {"type": "string",
"description": "when kind=refused: ONE plain sentence naming what is "
"missing, in the words a non-technical person would use"},
"name": {"type": "string", "description": "a short title for the automation"},
"trigger": {"type": "string", "enum": sorted(trigger_keys),
"description": "what starts this automation"},
"table": {"type": "string", "enum": sorted(table_keys),
"description": "the database whose records this flow walks, if any"},
"actions": {
"type": "array",
"maxItems": MAX_DRAFT_ACTIONS,
"items": {
"type": "object",
"properties": {
"kind": {"type": "string", "enum": sorted(kinds)},
"why": {"type": "string",
"description": "one short sentence: why this step is here"},
"config": {"type": "object", "additionalProperties": True,
"description": "the step's settings, using ONLY the keys named "
"for that kind in the system message"},
},
"required": ["kind"],
},
},
},
"required": ["kind"],
}
def flow_system_prompt(catalog, required, triggers, tables):
"""What the model is told it may build with β€” DERIVED, never written down twice.
Every list here is the server's own: the catalog rows the menu paints, the required-key table
the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases
with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to
the catalog is offered here on the same deploy and a kind removed stops being offered.
"""
lines = ["You build small automations for a business tool. You are given the EXACT set of "
"steps this tool can perform. You may use nothing else.",
"", "THE STEPS YOU MAY USE:"]
# ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the
# PHRASE is the only human wording of that requirement anywhere β€” the one the runner's own
# refusal sentence is built from. Keys alone leave the model guessing what `field` means on a
# `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing
# what to call the setting. Handing it one and hoping is how a draft comes back configured
# against a key the validator drops.
req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()}
for row in catalog or []:
if not row.get("ready"):
continue
kind = str(row.get("kind") or "")
need = req.get(kind) or []
lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}"
+ (f" REQUIRED settings: {', '.join(need)}" if need else ""))
lines += ["", "WHAT CAN START AN AUTOMATION:"]
for t in triggers or []:
if t.get("planned") or not t.get("ready", True):
continue
lines.append(f"- {t.get('key')}: {t.get('label')}")
lines += ["", "THE DATABASES THIS PERSON HAS:"]
for t in (tables or [])[:40]:
cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25])
lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}")
if not tables:
lines.append("- (none - do not name a database)")
lines += [
"",
"RULES:",
"1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, "
"answer kind=refused and say plainly which capability is missing.",
"2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank "
"rather than inventing a web address, a CSS selector or a column name.",
"3. A web address must start with http:// or https://.",
"4. To use a value from the record the flow is walking, write it as {{Column name}}.",
"5. Name a database only from the list above, by its key.",
"6. Keep it short. Fewer steps that work beat more steps that guess.",
]
return "\n".join(lines)
def _flow_from_tool_call(obj):
"""The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash."""
if not isinstance(obj, dict):
return None, "the assistant's answer could not be read"
if str(obj.get("kind") or "") == "refused":
return None, (str(obj.get("refusal") or "").strip()
or "the assistant could not build this from the steps available")
acts = obj.get("actions")
if not isinstance(acts, list) or not acts:
return None, ("the assistant did not produce any steps - try describing what should "
"happen, one action at a time")
out = []
for a in acts[:MAX_DRAFT_ACTIONS]:
if not isinstance(a, dict) or not str(a.get("kind") or "").strip():
continue
cfg = a.get("config")
out.append({"kind": str(a["kind"]).strip(),
"config": cfg if isinstance(cfg, dict) else {},
"why": str(a.get("why") or "").strip()[:200]})
if not out:
return None, "the assistant's steps could not be read"
return {"name": str(obj.get("name") or "").strip()[:80] or "New automation",
"trigger": str(obj.get("trigger") or "").strip(),
"table": str(obj.get("table") or "").strip(),
# β›”β›” THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner,
# 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need
# to explicitly tell me why and recommend a fix"* β€” **a silent truncation IS the
# violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past
# the twelfth and said nothing, so a model that answered with a twenty-step journey had
# eight steps deleted between the answer and the screen, with no key anywhere in the
# response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE β€” this
# ceiling is the draft door's own, which makes reporting it the whole obligation.
# ⚠ Found by the verifier that checked this ticket, not by a gate.
"asked": len([a for a in acts if isinstance(a, dict)]),
"actions": out}, ""
def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None):
"""A sentence -> `(draft, refusal_sentence, provider)`. β›” NOTHING IS SAVED HERE.
Exactly one of `draft` and `refusal_sentence` is truthy β€” the same contract
`web_agent.run_step` keeps, so a caller has no third case to get wrong.
"""
text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
if not text:
return None, "type what you want the automation to do", None
kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")})
if not kinds:
return None, "this deployment offers no automation steps to build with", None
trigger_keys = sorted({str(t.get("key")) for t in (triggers or [])
if t.get("key") and not t.get("planned")}) or ["manual"]
table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""]
tools = [{"type": "function", "function": {
"name": "build_automation",
"description": "Emit the automation, or refuse.",
"parameters": flow_schema(kinds, trigger_keys, table_keys)}}]
messages = [{"role": "system",
"content": flow_system_prompt(catalog, required, triggers, tables)},
{"role": "user", "content": text}]
if chat is not None:
# ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal
# branches as a live call; only the transport is replaced.
draft, refusal = _flow_from_tool_call(chat(messages, tools))
return draft, refusal, "injected"
provs = flow_providers()
if not provs:
# β›” SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
# never built [[flag-shipped-without-its-writer]].
return None, ("the assistant is not configured on this deployment, so an automation cannot "
"be drafted from a description yet"), None
tmo = float(timeout or TIMEOUT_SECONDS)
problems = []
for p in provs:
if p["shape"] != "openai":
# Anthropic's tool transport differs; it stays on the ladder for `decide()` and is
# SKIPPED here rather than half-supported. Named in `problems` so it is not invisible.
problems.append(f"{p['name']}: tool calls are not wired for this shape")
continue
try:
r = requests.post(p["url"], timeout=tmo,
headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
"Content-Type": "application/json"},
json={"model": p["model"], "messages": messages, "tools": tools,
"tool_choice": "required", "temperature": 0.1,
"max_tokens": 1500})
except Exception as e: # noqa: BLE001
problems.append(f"{p['name']}: {type(e).__name__}")
continue
if r.status_code != 200:
problems.append(f"{p['name']}: HTTP {r.status_code}")
continue
try:
calls = (((r.json().get("choices") or [{}])[0].get("message") or {})
.get("tool_calls") or [])
args = json.loads(calls[0]["function"]["arguments"]) if calls else None
except Exception as e: # noqa: BLE001
problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
continue
draft, refusal = _flow_from_tool_call(args)
if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused":
# β›” A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more
# expensive rung β€” the model has just said this cannot be built. `decide()` takes the
# same posture for the same reason.
return None, refusal, p["name"]
if draft is not None:
return draft, "", p["name"]
problems.append(f"{p['name']}: {refusal}")
return None, ("; ".join(problems)[:300] or "no provider answered"), None