diff --git a/RELEASES.json b/RELEASES.json index 84c032c50df62b3c39674769edd3a4dff338a0e6..f2bab27a950fe9faf48835fc5d624e80b34b2c80 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "39ee200", + "current": "v30 (71776ca)", "releases": [ + { + "version": "v30", + "sha": "71776ca", + "date": "2026-08-20", + "subject": "v30: W39-T29 Member-owned View first-share hotfix" + }, { "version": "v29", "sha": "55ced50", diff --git a/VERSION b/VERSION index df6e282a9892b7040437bdac3b96aa68d59e98e2..9db7eb0a25858e9592de675ebe22de39b50bf24b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -39ee200 +v30 (71776ca) diff --git a/api/ai_enrich.py b/api/ai_enrich.py index 5b1166fe9506bf6bdd2337ca50691536165dcad8..dbeb6153ad53eaab15f1ff8a38c460c0aa2164e6 100644 --- a/api/ai_enrich.py +++ b/api/ai_enrich.py @@ -314,6 +314,102 @@ def _ask(provider, model, prompt, max_tokens, timeout): return '', None, f"{provider['name']} failed: {type(exc).__name__}" +# =========================================================================== +# W37-T37 (R4 / C7) -- THE GEOCODE FIELD's pure half: turning a record into ONE query string. +# +# ⛔ WHY IT IS HERE AND NOT IN `routes_geo.py`. This module's contract, stated in its own header, +# is that everything in it is a pure function of data it is handed -- which is what lets a gate +# exercise it with no network, no store and no server. Composing an address is exactly that shape. +# `routes_geo.py` owns the part that cannot be pure: the outbound call, the throttle and the cache. +# +# ⭐ WHY IT COMPOSES AT ALL, rather than sending the one column the field names. The field config +# is `{addressField, country?}` -- a single column -- but a customers table stores an address across +# six of them. B MEASURED the fill rates on the real 3,641-row mirror (W37-T11): +# +# street 96.2% · city 96.0% · zip 94.6% · country 95.6% · state 92.0% · street2 5.2% +# street AND city together: 95.9% <- the pair this feature actually needs +# +# Sending `street` alone would ask a global geocoder to place "123 Main Street" with no town, which +# resolves confidently and often wrongly. So the named column LEADS and the conventional siblings +# follow, which is also how a person would write the address down. +# =========================================================================== + +#: The sibling columns appended after the named one, in the order an address is spoken. ⚠ A LIST +#: rather than a formatted string: a template would have to decide what to do about the ~4% of rows +#: missing any one part, and the answer is always "leave it out", never "leave a gap". +GEOCODE_PARTS = ('street2', 'city', 'state', 'zip', 'postal_code', 'country') + +#: ⛔ B's SECOND MEASURED TRAP. Odoo serves `state` as its DISPLAY name, country code included: +#: `New York (US)`, `British Columbia (CA)`. A naive join yields "SCARSDALE, New York (US)", and the +#: parenthetical is noise to a geocoder at best. Stripped here, once, rather than at each caller. +_PAREN_TAIL = re.compile(r'\s*\([^)]*\)\s*$') + + +def geocode_query(row, cfg, seen_parts=None): + """One record plus a geocode field's config -> the query string, or '' when there is nothing. + + ⛔ RETURNS '' RATHER THAN A PARTIAL GUESS when the named column is empty. B measured ~150 of + 3,641 customers with no street at all, and the honest UI for those is "no address on file", not + a request that spends a second of a shared service's budget to place a bare postcode. + + ⚠ `seen_parts=None` RESOLVES `GEOCODE_PARTS` AT CALL TIME, and that is not style. It was + `seen_parts=GEOCODE_PARTS`, which binds the tuple once at import, so the module constant stopped + being the live answer the moment anything wanted to vary it: the gate's own negative control set + `ai_enrich.GEOCODE_PARTS = ()` and the function carried on composing the full address. A default + argument that captures a module constant is a SECOND COPY of it with an earlier timestamp, and + the copy is the one that runs ([[a-constant-two-features-share]]). Caught by that control.""" + if not isinstance(row, dict) or not isinstance(cfg, dict): + return '' + if seen_parts is None: + seen_parts = GEOCODE_PARTS + key = str(cfg.get('addressField') or '').strip() + if not key: + return '' + lead = _clean_part(row.get(key)) + if not lead: + return '' + parts = [lead] + for k in seen_parts: + if k == key: + continue + v = _clean_part(row.get(k)) + # ⚠ The dedupe is not tidiness. `country` is often both the field's own value and already + # present in the street line on an imported record, and "USA, USA" measurably degrades a + # free-form geocoder's confidence. + if v and v.lower() not in {p.lower() for p in parts}: + parts.append(v) + return ', '.join(parts) + + +def _clean_part(v): + """One address component, trimmed and stripped of a trailing country parenthetical.""" + if v is None: + return '' + s = re.sub(r'\s+', ' ', str(v)).strip() + if not s or s.lower() in ('false', 'none', 'null'): + return '' + return _PAREN_TAIL.sub('', s).strip() + + +def geocode_plan(rows, cfg, coord_key=None, overwrite=False): + """Which records a run would actually ask about, and why the rest are being left out. + + ⛔ THIS IS THE R3 GUARD IN ITS ARITHMETIC FORM. It is handed the records a PERSON chose; it + never reads a table. `skipped` is returned rather than swallowed so the surface can say "142 to + look up, 9 have no address, 31 already have coordinates" instead of a bare number.""" + run, skipped = [], {} + for rid, row in (rows or {}).items(): + q = geocode_query(row, cfg) + if not q: + skipped['no_address'] = skipped.get('no_address', 0) + 1 + continue + if coord_key and not overwrite and str((row or {}).get(coord_key) or '').strip(): + skipped['already_located'] = skipped.get('already_located', 0) + 1 + continue + run.append((str(rid), q)) + return {'run': run, 'skipped': skipped} + + def on_change_fields(defn, changed_keys): """Which `ai_enrich` columns in this table want a run because one of their inputs moved. diff --git a/api/ai_review.py b/api/ai_review.py index 29f16c1b0d27c10df0e9d80a9c154d45652a9e16..03f68bd48c54f4543883f221926d84a4cd135c1f 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -17,15 +17,23 @@ the work. Nothing here can move a card somewhere the review does not already per last rather than absent — it is the quality backstop, not the default. `AIOS_AI_REVIEW_PROVIDER` pins one; `AIOS_AI_MODEL` overrides the model. -⚠ WHY RAW HTTP RATHER THAN THE `anthropic` SDK, stated because it is a deliberate deviation from -the /claude-api skill's default and not an oversight. Three of the four legs are OpenAI-chat-shaped -endpoints with no shared SDK, so a ladder built on the SDK would be one SDK leg beside three -hand-rolled ones — two implementations of the same call, the seam this repo keeps closing. And -`aios-web/api/requirements.txt` is PINNED to what the verify battery proves ([[pin-deps-space- -rebuilds]]): adding a dependency there rebuilds the container, which is a real deploy risk to take -for one leg of an optional feature. `requests` is already a dependency and `harness/analyst.py` -established per-provider raw HTTP as the house pattern. Booked as a DEBT line so the integrator -can overturn it deliberately rather than by drift. +⭐⭐ THE ANTHROPIC LEG IS ON THE OFFICIAL SDK NOW (D-346, W37-T39, 2026-08-19). ⛔ THIS PARAGRAPH +USED TO ARGUE THE OPPOSITE and is rewritten rather than deleted, because a comment left asserting +the reverse of its own code is [[two-gates-can-assert-opposite-things]] with no gate to catch it. +The retired text said raw HTTP was *"a deliberate deviation … booked as a DEBT line so the +integrator can overturn it deliberately rather than by drift"*. This is that overturn. + +Its two reasons were real and are both answered rather than waved away: + 1. *"one SDK leg beside three hand-rolled ones is two implementations of the same call."* It is + not, because the leg does not live here: `providers.anthropic_send` owns the wire, prefers + the SDK, and normalises BOTH transports to one `(status, body)` pair. This file gained no + Anthropic knowledge — it lost some. + 2. *"adding a pinned dependency rebuilds the container, a real deploy risk."* Still true, so the + import is LAZY and its absence is a FALLBACK, not a crash: with no `anthropic` package the + same raw POST runs and `LAST_ANTHROPIC_TRANSPORT` reports `'http'`. The pin is owed in BOTH + manifests ([[pin-deps-space-rebuilds]]) and is outside every worker fence this wave. +⚠ The other three legs stay OpenAI-chat-shaped over `requests` — they have no shared SDK, and that +half of the original reasoning never expired. The Messages shape below is the current one: `x-api-key` + `anthropic-version: 2023-06-01`, and `stop_reason: "refusal"` is checked BEFORE reading `content` — a refusal answers HTTP 200 with an @@ -41,9 +49,13 @@ import requests #: The ladder. Order IS the policy (R14) — cheapest capable first, Anthropic last. PROVIDERS = [ + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19). This read `llama-3.3-70b-versatile` and Groq RETIRED + # it: a real POST with the live key answers HTTP 404 `model_not_found`, and the id is absent + # from `GET /openai/v1/models`. ⚠ THE `openai/` PREFIX IS PART OF GROQ'S ID and is NOT a typo + # for the cerebras rung below, which serves the same family bare as `gpt-oss-120b`. {"name": "groq", "env": "GROQ_API_KEY", "shape": "openai", "url": "https://api.groq.com/openai/v1/chat/completions", - "model": "llama-3.3-70b-versatile"}, + "model": "openai/gpt-oss-120b"}, {"name": "cerebras", "env": "CEREBRAS_API_KEY", "shape": "openai", "url": "https://api.cerebras.ai/v1/chat/completions", "model": "gpt-oss-120b"}, @@ -56,7 +68,9 @@ PROVIDERS = [ # labels it already has in front of it. {"name": "anthropic", "env": "ANTHROPIC_API_KEY", "shape": "anthropic", "url": "https://api.anthropic.com/v1/messages", - "model": "claude-haiku-4-5"}, + # ⚠ AN ENV LEVER, NOT A NEW DEFAULT (W36-T35): haiku stays the choice for the reasons above, + # and a deployment whose side rungs are out of credit can raise the tier without a release. + "model": os.environ.get("AIOS_AI_REVIEW_ANTHROPIC_MODEL") or "claude-haiku-4-5"}, ] ANTHROPIC_VERSION = "2023-06-01" TIMEOUT_SECONDS = float(os.environ.get("AIOS_AI_REVIEW_TIMEOUT") or 20) @@ -159,16 +173,35 @@ def _call_openai(p, model, system, user, timeout): return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body +#: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any. +#: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say +#: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit. +LAST_ANTHROPIC_TRANSPORT = "" + + def _call_anthropic(p, model, system, user, timeout): - r = requests.post(p["url"], timeout=timeout, - headers={"x-api-key": os.environ[p["env"]].strip(), - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json"}, - json={"model": model, "max_tokens": 300, "system": system, - "messages": [{"role": "user", "content": user}]}) - if r.status_code >= 400: - return "", f"anthropic answered {r.status_code}", None - body = r.json() + """One classification turn on the Messages API, through `providers.anthropic_send`. + + ⭐ D-346 (W37-T39): this used to hand-roll the POST. It now goes through the shared wire, which + prefers the OFFICIAL `anthropic` SDK and falls back to the same raw POST when the package is + absent. The shape of what comes back is unchanged, deliberately: `anthropic_send` normalises + both transports to `(status, body)`, so every line below this call is untouched. + """ + global LAST_ANTHROPIC_TRANSPORT + import providers as _prov # noqa: PLC0415 + # ⚠ NO TOOLS ON THIS PATH. The builder omits `tool_choice` for exactly that case, so there is + # nothing to strip here — the rule lives in `anthropic_request`, beside the thing it constrains. + req = _prov.anthropic_request( + model=model, key=os.environ[p["env"]].strip(), system=system, + messages=[{"role": "user", "content": user}], tools=[], max_tokens=300) + status, body, transport = _prov.anthropic_send(req, timeout=timeout) + LAST_ANTHROPIC_TRANSPORT = transport + if status >= 400 or status == 0: + # ⭐ THE SENTENCE COMES FROM THE LADDER, NOT FROM HERE. `refusal_sentence` already names the + # vendor and the action for every status R4 enumerated, so a bare "anthropic answered 402" + # (which is what this line used to say, and what the owner quoted back at us) cannot recur. + return "", _prov.refusal_sentence("anthropic", status, json.dumps(body)[:400]), None + body = body or {} # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so # reading content[0] before this check turns a refusal into an IndexError inside a run. # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. @@ -277,13 +310,23 @@ MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engi MAX_PROMPT_CHARS = 2000 -def flow_providers(): - """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.""" - pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() +def flow_providers(pin=None): + """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off. + + ⭐ `pin` IS ASK D-18 (2026-08-18): the Agent chat's model toggle must configure something, and + the draft door used to read `prompt` off the body and nothing else — so the key the client sent + was accepted and dropped, and the picker was a control over nothing. + ⚠ AN UNKNOWN OR UNCONFIGURED PIN FALLS BACK TO THE LADDER rather than refusing. A model the + ladder stopped offering must not turn every later draft into an error; the caller is told which + rung actually answered, which is the honest half. + """ by_name = {p["name"]: p for p in PROVIDERS} - order = [pin] if pin else list(FLOW_PROVIDER_ORDER) - return [by_name[n] for n in order + live = [n for n in FLOW_PROVIDER_ORDER if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] + wanted = str(pin or os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() + if wanted and wanted in live: + return [by_name[wanted]] + return [by_name[n] for n in live] def flow_schema(kinds, trigger_keys, table_keys): @@ -421,7 +464,7 @@ def _flow_from_tool_call(obj): def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, - st=None, user=""): + st=None, user="", model=None): """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. Exactly one of `draft` and `refusal_sentence` is truthy — the same contract @@ -432,6 +475,9 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and D asserts it; the two keywords are what D has to pass for the line to be attributable. """ + # ⚠ DECLARED AT THE TOP OF THE FUNCTION, not beside the assignment inside the provider loop. + # It parses either way; a reader scanning for the declaration does not look inside a `for`. + global LAST_ANTHROPIC_TRANSPORT import usage_ledger # noqa: PLC0415 text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] if not text: @@ -456,7 +502,7 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou draft, refusal = _flow_from_tool_call(chat(messages, tools)) return draft, refusal, "injected" - provs = flow_providers() + provs = flow_providers(model) if not provs: # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was # never built [[flag-shipped-without-its-writer]]. @@ -464,30 +510,63 @@ def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeou "be drafted from a description yet"), None tmo = float(timeout or TIMEOUT_SECONDS) problems = [] + import providers as _prov for p in provs: - if p["shape"] != "openai": - # Anthropic's tool transport differs; it stays on the ladder for `decide()` and is - # SKIPPED here rather than half-supported. Named in `problems` so it is not invisible. - problems.append(f"{p['name']}: tool calls are not wired for this shape") - continue + # ⭐⭐ W36-T35 / R4 — THE OWNER QUOTED THIS LINE BACK AT US. It used to read + # `problems.append(f"{p['name']}: tool calls are not wired for this shape")` and `continue`, + # so the sentence *"anthropic: tool calls are not wired for this shape"* appeared under the + # Agent module verbatim. R4: *"Anthropic becomes the tool-calling path that always works."* + # It is wired now, through the ONE wire in `providers`, and it matters more than it looks: + # every side rung on this account is refusing today (cerebras 402, groq 404, openrouter + # 402), so without this branch the drafter cannot answer at all. + # ⚠ NO `effort` ON THIS DOOR — `output_config.effort` errors on Haiku 4.5, the tier this + # ladder runs. The parameter is the caller's to send, which is why the wire takes it. + # ⭐ D-346 (W37-T39): BOTH shapes now yield the SAME `(status, body)` pair, so every branch + # below reads one thing. The anthropic side gets there through `anthropic_send` (official + # SDK when installed, the identical raw POST when not); the openai side still posts here + # because there is no shared SDK across three different vendors on that wire. + global LAST_ANTHROPIC_TRANSPORT try: - r = requests.post(p["url"], timeout=tmo, - headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", - "Content-Type": "application/json"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1500}) + if p["shape"] == "anthropic": + req = _prov.anthropic_request( + model=p["model"], key=os.environ[p["env"]].strip(), system=None, + messages=messages, tools=tools, max_tokens=1500, tool_choice="required") + status, body, transport = _prov.anthropic_send(req, timeout=tmo) + LAST_ANTHROPIC_TRANSPORT = transport + else: + r = requests.post(p["url"], timeout=tmo, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1500}) + status = r.status_code + body = r.json() if (r.content and r.status_code == 200) else {"_text": r.text} except Exception as e: # noqa: BLE001 problems.append(f"{p['name']}: {type(e).__name__}") continue - if r.status_code != 200: - problems.append(f"{p['name']}: HTTP {r.status_code}") + if status != 200: + # ⛔ A SENTENCE, NOT `HTTP 402` (R4's third clause). The owner read the status codes off + # this very door. `refusal_sentence` also decides whether this was a CREDIT failure, and + # the memo makes the next turn SKIP the empty account instead of paying for it again. + # ⚠ The body is re-serialised for the substring test because the SDK path never had a + # `.text` — that is the one thing this normalisation costs, and it costs nothing else. + detail = json.dumps(body)[:600] if isinstance(body, dict) else str(body)[:600] + if _prov.is_credit_failure(status, detail): + _prov.mark_no_credit(p["name"]) + problems.append(_prov.refusal_sentence(p["name"], status, detail)) continue try: - body = r.json() - calls = (((body.get("choices") or [{}])[0].get("message") or {}) - .get("tool_calls") or []) - args = json.loads(calls[0]["function"]["arguments"]) if calls else None + body = body or {} + if p["shape"] == "anthropic": + _text, args, _refused = _prov.anthropic_read(body) + if _refused: + problems.append(f"{p['name']}: {_refused}") + continue + else: + calls = (((body.get("choices") or [{}])[0].get("message") or {}) + .get("tool_calls") or []) + args = json.loads(calls[0]["function"]["arguments"]) if calls else None except Exception as e: # noqa: BLE001 problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") continue diff --git a/api/automation_engine.py b/api/automation_engine.py index 5d60218280cb2c33520bb31163c4e64ea5f6adec..a696848fecf36f2669c5aa3d3edbe5295ae85403 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -9219,6 +9219,36 @@ ACTION_CATALOG = [ "ready": True, "detail": "Assemble this month's statements and park them for review. Nothing is sent until " "somebody opens the batch and clicks Send"}, + # ⭐⭐ WAVE 36 · W36-T39 — D-277 CLOSED THE WAY THAT ROW ITSELF RECOMMENDED: *"an + # `ACTION_CATALOG` row with `menu: false` — one line, reusing the door R18 built this same wave + # to keep the five web kinds readable but unofferable. (a) looks right; it is E's file."* + # + # ⛔ THE DEFECT WAS A KIND IN NO CATALOG ON EITHER SIDE. `routes_automation._field_agent_rows` + # emits `flow.actions[0].kind == "ai_enrich"` for every AI-enrichment column in the tenant, and + # that string appeared ZERO times here and ZERO times in `aios-web/web/src/automation/`. The + # builder resolves a stored step's caption with `catalog.find(c => c.kind === a.kind)` and falls + # back to `a.kind`, so a field agent opened in the Agents module showed a RAW TOKEN. + # + # ⚠ `menu: False`, NEVER `ready: False`. `ready: False` renders as "coming soon" — a promise — + # and `clean_actions` refuses the kind outright, which would 400 the synthetic row the moment + # anything validated it. `menu: False` is the exact shape R18 built: withheld from the picker, + # still resolvable as a caption, still valid. + # ⛔ AND IT IS NOT ADDABLE BY HAND ON PURPOSE. An enrichment belongs to a COLUMN; the automation + # canvas is not where one is created, which is why `patch_automation` already refuses a + # `field:` id with "change its prompt, model or schedule on the column itself". + # ⭐⭐ AND A THIRD INSTANCE, FOUND BY W36-T39's OWN GATE ON ITS FIRST RUN. `_odoo_sync_row` + # emits `flow.actions[0].kind == "odoo_sync"` for the connector's synthetic schedule agent, and + # that kind was in no catalog either — the same raw token on the same screen as D-277, one row + # down. Two known instances were enough to justify the check; the check then produced a third + # nobody had booked, which is the difference between a gate and a regression test. + {"kind": "odoo_sync", "label": "Sync from Odoo", "group": "Connected", + "ready": True, "menu": False, + "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not " + "here"}, + {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected", + "ready": True, "menu": False, + "detail": "Fill an AI column for the records this flow walks. Configured on the column, not " + "here"}, {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, "detail": "Needs the Slack connector"}, # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── @@ -9306,6 +9336,33 @@ def _tenant_may_use(kind, rt): return True if gate is None else bool(rt is not None and gate(rt)) +def catalog_kinds(): + """Every action kind this module knows about — the LABEL vocabulary. + + ⭐ D-277's WHOLE LESSON IN ONE SENTENCE: the client resolves a stored step's caption out of the + catalog and falls back to the raw kind token, so a kind the server can EMIT and the catalog + does not carry is a token on somebody's screen. This is the set that must cover every kind any + server path can put into a `flow.actions` entry, whether or not a person may add it. + """ + return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG) + + +def configurable_kinds(): + """The kinds a PERSON may add from the picker, and must therefore be able to configure. + + ⭐⭐ W36-T39 — THE CONTRACT BETWEEN THE TWO TREES, DERIVED AND NEVER LISTED. `ready` alone is + the wrong set: the five `web_*` kinds are ready and `menu: False` (R18), and `ai_enrich` is + ready and `menu: False` (D-277) — all six are real, runnable, captioned, and unofferable. What + a client must be able to CONFIGURE is exactly what a person can ADD. + + ⛔ DERIVED FROM THE CATALOG, so a new row joins the contract by existing. A hand-kept list + would be a third copy of the vocabulary, and the two copies this ticket exists to reconcile + were already one too many. + """ + return frozenset(str(row.get("kind") or "") for row in ACTION_CATALOG + if row.get("ready") and row.get("menu", True) is not False) + + def action_catalog(rt=None): """The catalog as the wire carries it — a copy, because a caller that mutated the module constant would change every later reader's answer. @@ -10831,6 +10888,32 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print "and nothing was sent.") counts["webBlocked"] += 1 continue + elif kind == "odoo_sync": + # Same refusal, same reason as the arm below: a catalog row with no arm is walked, + # counted, and reports success having done nothing. The connector owns this sync. + if "odoo_sync_not_run_here" not in web_notes: + web_notes.append("odoo_sync_not_run_here") + log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own " + "schedule, not from this canvas. Nothing was done") + counts["webBlocked"] += 1 + continue + elif kind == "ai_enrich": + # ⛔⛔ A REFUSAL ARM, AND IT EXISTS FOR THE REASON THE `send_statement` ARM ABOVE + # STATES: `_walk` has no terminal `else`, so a catalog row whose arm is missing is + # walked, COUNTED, reports the run `ok` and writes nothing — a step somebody + # configured that succeeds at doing nothing. Adding the row (D-277) without this + # arm would have opened exactly that window. + # ⚠ AND THE REFUSAL IS THE TRUTH, not a stub. An AI column is filled by + # `ai_enrich`, driven from the column's own editor; the synthetic agent row this + # kind appears in is DERIVED from that column and is never stored, so nothing + # reaches here through the ordinary path. If something ever does, it must say so + # rather than report success. + if "ai_enrich_not_run_here" not in web_notes: + web_notes.append("ai_enrich_not_run_here") + log("[aios-auto] ai_enrich: an AI column is filled from the column's own " + "editor, not from this canvas. Nothing was done") + counts["webBlocked"] += 1 + continue elif kind == "ai_agent": # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. # diff --git a/api/main.py b/api/main.py index 175e0173f243e89cf5f436e877e45e6b6317562b..b0bf95ce7d8a2b1bc520d88fd08fec2da6d5f0b6 100644 --- a/api/main.py +++ b/api/main.py @@ -85,6 +85,9 @@ import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack d import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) +import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router) +import routes_script_views # noqa: E402 (wave 36 R3/R10/C3 — script Views; E's router) +import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router) from core import grid_events # noqa: E402 # D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a @@ -376,6 +379,35 @@ app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one # POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does # not widen anything a tenant admin can reach — `verify_api` proves that by having one try. app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane +# ⭐⭐ WAVE 36 (R8 / C4) — THE AGENT HARNESS FILE STORE, mounted in the SAME change that created +# `routes_agent_harness.py`. Eighth consecutive wave in which this block is the artefact the +# protocol nearly loses; `verify_web_agent::section_w36_harness` asserts this path in +# `app.openapi()["paths"]` and CALLS the route, because mounted is not callable (D-107). +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or a GET answers 404 and a PUT answers 405 while every gate stays green. +# ⛔ ITS PATHS SIT UNDER `/agents/{id}/...`, WHICH `routes_slack` ALSO SERVES — and that is safe +# rather than lucky: a FastAPI path parameter never spans a `/`, so `/agents/{agent_id}` cannot +# match `/agents/x/harness`. The two routers share a prefix and no route. +app.include_router(routes_agent_harness.router) # R8 / C4 — versioned agent harness files +# ⭐⭐ WAVE 36 (R3 / R10 / C3) — THE SCRIPT VIEW, owner item 6. Mounted in the SAME change that +# created `routes_script_views.py`; `verify_script_views.py` asserts both of its paths in +# `app.openapi()["paths"]` AND calls them, and its NC comments this line out. +# ⛔ ITS RUN DOOR SPAWNS A SUBPROCESS AND IS A PLAIN `def`, so FastAPI runs it in the threadpool. +# Mounting it does not put a ten-second wait anywhere near the event loop; the router's own header +# says why that is not a style choice. +app.include_router(routes_script_views.router) # R3 / R10 / C3 — code-script database Views +# ⭐⭐ WAVE 37 (R14, amendment A11) — THE KEYLESS MAP PROVIDER SEAM, mounted on `ASK D-1` in the +# same wave that created `routes_geo.py`. NINTH consecutive wave in which this block is the +# artefact the protocol nearly loses, and the first in which the ask that carries it was itself +# lost: D raised it inside a ``` fence, so `wave_board.py` blanked it and the bus never delivered +# it (D-366). The integrator found it reading the peer mailbox by hand. +# ⚠ Placement, as for every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)` at the +# end of this file, or GET answers 404 and POST answers 405 while every one of its own gates is +# green. D is adding a `web_map` leg that reads THIS FILE and asserts this line — keep it. +# ⛔ R14 put the provider behind a seam precisely so swapping Google in later is a config change: +# tiles = OpenStreetMap · routing = OSRM · geocoding = Nominatim, none of them keyed, all of them +# rate-limited by POLITENESS rather than by a bill (~1 req/s, and bulk harvesting is forbidden). +app.include_router(routes_geo.router) # R14 / A11 — D's router, A's line (map providers) # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 9a2f2aba3fceb4b2d70b7827afec3b38d4f34928..d4c2a9d0c709c7e50b54f78709d0f29d138c24a2 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -636,8 +636,25 @@ def customer_fields(): "default": True, "pinned": True}, {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay", "default": False, "description": "The `res.partner` id. Also this row's id."}, + # ⭐⭐ W37-T11 / D-165 — THE POSTAL ADDRESS, which this table declared nowhere and served + # never. `verify_odoo_relational._prove_customer_address` recorded the blocker in its own + # docstring on 2026-08-12 — *"`street`/`street2`/`zip` are not among [the mirror's 16 + # columns] … blocked on a `sync_all()` backfill no ticket in wave 30 owns"*. That backfill + # HAS since run (`_sync_state` carries `res_partner.cols.…,street,street2,…,zip`, done + # 2026-08-15), so the half that could not be built now can be. + # ⛔ THIS IS ALSO LANE D'S DEPENDENCY. R4 puts address→coordinates in an enrichment field, + # and a geocoder pointed at a column that does not exist enriches nothing. + # ⚠ `default: False` on the postal lines, `True` on city/state — the same split + # `vendor_fields` made and for the same reason: four address columns on by default push + # the useful ones off the first screen. A locked database still allows fields, so every + # one is a click away in the picker. + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True}, {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, {"key": "country", "label": "Country", "type": "text", "source": "overlay", "default": False}, {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay", @@ -976,8 +993,13 @@ def read_customers(cur, excluded=None): # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above. rank_leg = (" OR (p.customer_rank > 0 AND p.active) " if {"customer_rank", "active"} <= have else "") + # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like + # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL + # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the + # live 500 that rule was written after). sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, " - f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col} " + f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, " + f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} " "FROM res_partner p " f"{join}" "WHERE p.id IN (" @@ -988,13 +1010,16 @@ def read_customers(cur, excluded=None): f"{rank_leg}") out = [] for r in cur.execute(sql).fetchall(): - (pid, name, city, state, country, agent, agent_id) = r + (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r out.append({ "_id": str(pid), "customer": str(name or ""), JOIN_KEY: int(pid), + "street": str(street or ""), + "street2": str(street2 or ""), "city": str(city or ""), "state": str(state or ""), + "zip": str(zipc or ""), "country": str(country or ""), "agent": str(agent or ""), AGENT_JOIN_KEY: int(agent_id) if agent_id else "", diff --git a/api/providers.py b/api/providers.py index 46db502b900cea6d9bece95beda04696cc1beb5f..d294400d33431e3b8c3bda9253357544cf451875 100644 --- a/api/providers.py +++ b/api/providers.py @@ -39,6 +39,12 @@ import time from dataclasses import dataclass, field from typing import Callable +# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in +# BOTH manifests and is already a transitive dependency of `huggingface_hub`, so it cannot be the +# line that stops a container booting. That asymmetry is the whole reason the two are imported +# differently — see `anthropic_sdk()`. +import requests + #: ⚠ APPROXIMATE, AND DELIBERATELY SO. These are list prices per RECORD in USD, used only to RANK #: providers and to estimate a run's spend for the operator. They are not billing truth — the #: vendor's own dashboard is. Wrong by 2x still ranks correctly; wrong by 100x does not, which is @@ -124,7 +130,7 @@ PROVIDERS: dict[str, Provider] = { "likes + comments + shares + saves, and play_count inline (DECLARED " "no-empties-or-zeros, UNPROVEN until one live pull)"), "tt_comments": Capability(True, _COST_BRIGHTDATA, - "separate paid dataset, opt-in — 17 fields"), + "separate paid dataset, opt-in: 17 fields"), }), "apify": Provider( key="apify", label="Apify", key_env="AIOS_APIFY_KEY", @@ -132,7 +138,7 @@ PROVIDERS: dict[str, Provider] = { # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact. "ig_post_views": Capability(True, _COST_APIFY, - "videoPlayCount — matches Instagram's displayed views"), + "videoPlayCount, matches Instagram's displayed views"), "ig_post_metrics": Capability(True, _COST_APIFY, "likes + comments (fallback)"), "ig_profile": Capability(True, _COST_APIFY, "profile fields (fallback)"), }), @@ -240,12 +246,12 @@ def run(capability, work, satisfied=None, log=None): secs = round(time.time() - started, 2) if note: attempts.append(Attempt(provider.key, False, note, 0, secs)) - log(f" {provider.label}: {note} — falling through") + log(f" {provider.label}: {note}, falling through") continue if not ok(result): attempts.append(Attempt(provider.key, False, "answered without the field asked for", 0, secs)) - log(f" {provider.label}: answered, but not with what was asked for — falling through") + log(f" {provider.label}: answered, but not with what was asked for, falling through") last = result if last is None else last continue attempts.append(Attempt(provider.key, True, "", _count(result), secs)) @@ -282,6 +288,444 @@ def wire(): } + + +# ═══════════════════ WAVE 36 · W36-T35 (ruling R4, contract C5) — THE LLM LADDER ════════════════ +# +# Owner item 4, verbatim (2026-08-18): *"I'm also getting errors everywhere when I want to use the +# assisntant: 'the assistant could not be reached just now (openrouter: HTTP 402)' and 'cerebras: +# HTTP 402; groq: HTTP 404; openrouter: HTTP 402; anthropic: tool calls are not wired for this +# shape'."* +# +# ⭐⭐ R4 IS THREE CLAUSES AND THEY LAND IN THREE DIFFERENT PLACES. (1) Anthropic becomes the +# tool-calling path that always works — a WIRE, in `routes_query`. (2) A provider with no credit is +# SKIPPED rather than tried — a memo, `mark_no_credit` below. (3) No raw `HTTP 402` ever reaches a +# screen — a SENTENCE, `refusal_sentence` below. The declaration here is what the first two read. +# +# ⚠ WHY A SECOND REGISTRY RATHER THAN ROWS IN `PROVIDERS` ABOVE. A scraping provider is +# `(key_env, caps)` and is billed per RECORD; an LLM provider is `(env, url, model, wire)` and is +# billed per TOKEN. Folding them into one dict would mean four fields that are meaningless for half +# the rows and an `estimate()` that answers $0.00 for anything LLM-shaped. What they SHARE is the +# thing worth sharing: `Capability`, so "MEASURED INCAPABLE" means exactly the same thing on both +# sides, and `llm_chain()` refuses an incapable row exactly as `chain()` does. +# +# ⛔ THE DECLARATION IS THE POINT (staged item 3). `_FAILED_GEN` in `routes_query` is a REGEX that +# recovers a tool call out of a provider's 400 — the evidence that guessing at capability failed. +# A row that says `llm_tool_calling: Capability(False, …)` is never offered for a tool-calling +# turn at all, so the guess never has to be made. + +#: How long a provider stays skipped after it tells us it is out of credit. ⚠ A MEMO, NOT A FACT: +#: the balance can be topped up at any moment, so this expires rather than latching. Fifteen +#: minutes is long enough that a chat session does not re-pay the timeout on every turn, and short +#: enough that a top-up is picked up without a restart. +CREDIT_COOLDOWN_S = float(os.environ.get("AIOS_CREDIT_COOLDOWN_S") or 900) + +#: `{provider name: unix ts when the memo expires}`. ⚠ PROCESS-LOCAL AND DELIBERATELY SO — it is a +#: latency optimisation, not a billing record. A second container learns the same thing from its +#: own first 402, and neither one can be wrong for longer than the cooldown. +_NO_CREDIT: dict[str, float] = {} + + +@dataclass(frozen=True) +class LlmProvider: + """One chat-completions endpoint, and what it is DECLARED able to do.""" + name: str + label: str + env: str + url: str + model: str + #: `openai` = the OpenAI-compatible `/chat/completions` shape. `anthropic` = the Messages API, + #: which is a different body, a different auth header and a different result shape. + wire: str + caps: dict = field(default_factory=dict) + + def configured(self) -> bool: + return bool((os.environ.get(self.env) or "").strip()) + + def can(self, capability: str) -> bool: + cap = self.caps.get(capability) + return bool(cap and cap.capable and self.configured()) + + +#: ⭐ ORDER IS THE LADDER, AND ANTHROPIC IS FIRST BECAUSE OF R4. `routes_query`'s old comment put +#: cerebras first *"because this path needs tool calling and cerebras carries this account's +#: tool-capable model"* — R4 replaces that premise: Anthropic is the tool-calling path that always +#: works, and the others are the cheap seats it falls through to. +LLM_PROVIDERS: dict[str, LlmProvider] = { + "anthropic": LlmProvider( + name="anthropic", label="Anthropic", env="ANTHROPIC_API_KEY", + url="https://api.anthropic.com/v1/messages", + # ⚠ Claude Opus 5, $5 / $25 per million tokens (2026-06 list). Override per deployment with + # `AIOS_ANTHROPIC_MODEL` — the id is read at call time, so a cheaper tier + # (`claude-sonnet-5`, $3 / $15) is an environment change, not a release. + # ⭐ W37-T39 re-verified this default against the vendor rather than the docs: a real + # Messages POST answers HTTP 200, and `claude-opus-5` is in `GET /v1/models`. + model=os.environ.get("AIOS_ANTHROPIC_MODEL") or "claude-opus-5", + wire="anthropic", + caps={ + # ⭐ MEASURED, and it is the whole of R4's first clause: the Messages API answers with a + # typed `tool_use` content block carrying parsed `input`. There is nothing to recover + # out of a 400 and no regex in the path — which is exactly what `_FAILED_GEN` exists to + # apologise for on the other wire. + "llm_tool_calling": Capability(True, 0.0, + "typed tool_use content block; no text recovery path"), + "llm_chat": Capability(True, 0.0, "Messages API"), + "llm_json_mode": Capability(True, 0.0, "output_config.format, schema-constrained"), + }), + "cerebras": LlmProvider( + name="cerebras", label="Cerebras", env="CEREBRAS_API_KEY", + url="https://api.cerebras.ai/v1/chat/completions", + # ⚠ STILL A LIVE ID, and W37-T39 checked rather than assumed: `gpt-oss-120b` is one of the + # two ids `GET https://api.cerebras.ai/v1/models` returns. What it is NOT is callable on + # this account — a real POST answers `HTTP 402 payment_required`, which `mark_no_credit` + # already knows how to survive (R4: skipped, not tried). + model="gpt-oss-120b", wire="openai", + caps={ + # ⚠ STILL "DECLARED, NOT MEASURED", DELIBERATELY. The 2026-08-19 sweep could not + # measure this rung: a 402 comes back before any tool is considered, so there is no + # observation to record. Leaving the old wording is the honest answer — upgrading it + # to MEASURED beside its two neighbours would be claiming an account balance as + # evidence about a capability. + "llm_tool_calling": Capability(True, 0.0, + "DECLARED, not measured: this account's tool-capable " + "model per the wave-32 ladder note. ⚠ 2026-08-19: the " + "account answers 402, so this stays unmeasured"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "groq": LlmProvider( + name="groq", label="Groq", env="GROQ_API_KEY", + # ⛔⛔ D-345, FIXED W37-T39 (2026-08-19): this read `llama-3.3-70b-versatile` and Groq + # RETIRED it. Measured, not inferred — a real POST with the live key answered + # `HTTP 404 {"code":"model_not_found"}`, and `GET /openai/v1/models` returns 13 ids with + # that one absent. ⚠ THE STATUS IS THE EVIDENCE AND 404 IS THE ONLY ONE THAT LICENSES A + # RENAME: cerebras answered 402 on the same sweep, which is a BILLING fact about the + # account and says nothing about its model id (`gpt-oss-120b` is in its /models list). + # "Fixing" an id behind a 401/402 is how a second bug ships behind a green probe. + # ⚠ THE PREFIX IS NOT A TYPO: Groq serves this model as `openai/gpt-oss-120b` while + # Cerebras serves the same family bare as `gpt-oss-120b`. The two rungs are DELIBERATELY + # spelled differently and a sweep that "normalises" them breaks one of them. + url="https://api.groq.com/openai/v1/chat/completions", + model="openai/gpt-oss-120b", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39, and this comment used to say the opposite. The old + # text read *"DECLARED, not measured … `routes_query._FAILED_GEN` exists because SOME + # provider on this wire answers 400 with the call in `failed_generation`; the repo + # never recorded which. Flip this to False the day it is"*. It is recorded now: on + # this id Groq answers HTTP 200 with a TYPED `tool_calls` block carrying parsed + # `arguments`, so it is not the provider `_FAILED_GEN` apologises for. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block " + "with parsed arguments; no failed_generation recovery"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), + "openrouter": LlmProvider( + name="openrouter", label="OpenRouter", env="OPENROUTER_API_KEY", + url="https://openrouter.ai/api/v1/chat/completions", + model="openai/gpt-4o-mini", wire="openai", + caps={ + # ⭐ MEASURED 2026-08-19, W37-T39 (was "DECLARED, not measured"): HTTP 200 with a typed + # `tool_calls` block. So on the OpenAI wire BOTH reachable rungs tool-call cleanly, and + # `_FAILED_GEN` is still holding a door nobody on this ladder has been seen to use. + "llm_tool_calling": Capability(True, 0.0, + "MEASURED 2026-08-19: HTTP 200, typed tool_calls block"), + "llm_chat": Capability(True, 0.0, "OpenAI-compatible"), + "llm_json_mode": Capability(True, 0.0, "response_format json_object"), + }), +} + +LLM_DEFAULT_ORDER = ("anthropic", "cerebras", "groq", "openrouter") + + +def mark_no_credit(name, seconds=None): + """Remember that `name` said it is out of credit, so the next turn SKIPS it (R4). + + ⛔ THE SECOND CLAUSE OF R4 IS "SKIPPED, NOT TRIED", and without a memo there is nowhere for + that to live: a stateless ladder re-tries the empty account on every single turn, pays its + round trip, and shows the reader a longer error each time. This is that memo. + """ + _NO_CREDIT[str(name)] = time.time() + float( + CREDIT_COOLDOWN_S if seconds is None else seconds) + return _NO_CREDIT[str(name)] + + +def no_credit(name): + """Is this provider inside its out-of-credit cooldown? Expiry is checked, never assumed.""" + until = _NO_CREDIT.get(str(name)) + if not until: + return False + if time.time() >= until: + _NO_CREDIT.pop(str(name), None) + return False + return True + + +def clear_credit_memo(name=None): + """Forget one memo, or all of them. For a gate, and for an operator after a top-up.""" + if name is None: + _NO_CREDIT.clear() + else: + _NO_CREDIT.pop(str(name), None) + + +def llm_chain(capability="llm_tool_calling"): + """The provider order for `capability` — declaration first, credit memo second. + + Three filters, in this order, and each removes a DIFFERENT kind of row: + 1. `can()` — declared capable AND configured. An incapable row is never offered, so a + turn cannot be spent discovering it (the `chain()` rule, one layer up). + 2. `no_credit()` — R4's skip. A provider that told us its balance is empty is passed over + until the memo expires. + 3. the ORDER itself, overridable with `AIOS_LLM_ORDER` (`anthropic,groq`) so a deployment + can be moved off a vendor without a release — the same clause `AIOS_PROVIDER_ORDER` + carries for the scraping side. + """ + raw = (os.environ.get("AIOS_LLM_ORDER") or "").strip() + names = [x.strip() for x in raw.split(",") if x.strip()] or list(LLM_DEFAULT_ORDER) + return [LLM_PROVIDERS[n] for n in names + if n in LLM_PROVIDERS and LLM_PROVIDERS[n].can(capability) and not no_credit(n)] + + +#: What an HTTP status MEANS, in words a person can act on. ⛔⛔ R4's THIRD CLAUSE LIVES HERE AND +#: IT IS NOT COSMETIC: `HTTP 402` on a screen tells a reader nothing they can do, and the owner +#: quoted it back at us twice. Every sentence names the VENDOR and the ACTION. +_STATUS_WORDS = { + 401: "{label} would not accept our key", + 403: "{label} would not accept our key", + 402: "{label} is out of credit", + 404: "{label} does not offer the model we asked it for", + 408: "{label} took too long", + 413: "the question was too long for {label}", + 429: "{label} is rate limiting us right now", +} + +#: Substrings that mean "no money" on a wire that does not use 402. ⚠ Anthropic answers a spent +#: balance with a 400 or 403 carrying a message, not with a status code of its own, so this is the +#: one place a body has to be read. It is a LOWERCASE substring test on the vendor's own words and +#: it only ever decides whether to SKIP a provider, never whether to trust one. +_CREDIT_WORDS = ("credit balance", "insufficient credit", "insufficient_quota", "out of credit", + "quota exceeded", "billing", "payment required", "add credits") + + +def is_credit_failure(status, body=""): + """Did this response mean "the account is empty"? Status first, then the vendor's own words.""" + if int(status or 0) == 402: + return True + if int(status or 0) not in (400, 403, 429): + return False + return any(word in str(body or "").lower() for word in _CREDIT_WORDS) + + +def refusal_sentence(name, status, body=""): + """One provider's failure, as a SENTENCE. Never a bare status code, never a vendor stack trace. + + ⚠ THE BODY IS READ AND NEVER QUOTED. A provider's error body can carry an account id, a key + prefix or an internal trace; the only thing taken out of it is the yes/no answer to "is this a + credit problem", and what reaches the caller is this module's own wording. + """ + label = (LLM_PROVIDERS.get(str(name)) or LlmProvider(name, str(name), "", "", "", "")).label + if is_credit_failure(status, body): + return f"{label} is out of credit" + code = int(status or 0) + if code in _STATUS_WORDS: + return _STATUS_WORDS[code].format(label=label) + if 500 <= code <= 599: + return f"{label} is having trouble at their end" + return f"{label} did not answer" + + +# ═══════════ THE ANTHROPIC WIRE, ONCE (W36-T35 / ASK D-18, ruling R4) ═══════════════════════════ +# +# ⛔⛔ TWO DOORS IN THIS PRODUCT CALL ANTHROPIC AND THEY MUST NOT EACH LEARN THE MESSAGES API. +# `routes_query._call_model` (the Assistant and Query) and `ai_review.draft_flow` (the automation +# drafter) both need it, and the owner quoted an error from EACH of them in one breath: +# *"the assistant could not be reached just now (openrouter: HTTP 402)"* and *"anthropic: tool +# calls are not wired for this shape"*. Two implementations of one wire is +# [[one-question-two-normalizers]] before a line is written, so the wire lives here, beside the +# ladder that declares the rung. +# +# FOUR THINGS THE OPENAI-COMPATIBLE SHAPE GETS WRONG, each a 400 on its own: +# 1. the system prompt is a TOP-LEVEL field, not a `{"role": "system"}` message +# 2. a tool is `{name, description, input_schema}` FLAT, not nested under `function` +# 3. `temperature` and friends are REMOVED on the current model family +# 4. `tool_choice` is an OBJECT (`{"type": "auto"}` / `{"type": "any"}`), not a string +# +# ⚠ AND ONE THING THAT IS NOT A SHAPE: `effort` is model-gated. `output_config.effort` errors on +# Haiku 4.5, so it is a PARAMETER here and the caller decides — the drafter runs on haiku and omits +# it, the assistant runs on the Opus tier and sends it. + +#: The Messages API version header. A DATE that pins the WIRE FORMAT, never a model. +ANTHROPIC_VERSION = "2023-06-01" + + +def anthropic_request(*, model, key, system, messages, tools, max_tokens, + tool_choice="auto", effort=None): + """`{url, headers, json}` for one Messages API call. Pure: reads no environment, sends nothing. + + `messages` is the OpenAI-shaped list this product already builds; the `system` turns are lifted + out of it, because that is where this API wants them. `tools` is the OpenAI-shaped tool list, + re-addressed rather than re-derived, so a schema change happens in one place. + """ + system_text = "\n\n".join(str(m.get("content") or "") for m in messages + if m.get("role") == "system") + if system: + system_text = (system_text + "\n\n" + str(system)).strip() if system_text else str(system) + turns = [{"role": ("assistant" if m.get("role") == "assistant" else "user"), + "content": str(m.get("content") or "")} + for m in messages if m.get("role") != "system" and str(m.get("content") or "").strip()] + wire_tools = [{"name": t["function"]["name"], + "description": t["function"]["description"], + "input_schema": t["function"]["parameters"]} for t in (tools or [])] + body = { + "model": str(model), + "max_tokens": int(max_tokens), + "system": system_text, + "messages": turns, + "tools": wire_tools, + } + # ⛔ W37-T39: `tool_choice` IS OMITTED WHEN THERE ARE NO TOOLS, and the builder decides that + # rather than each caller. The Messages API refuses `tool_choice` beside an empty `tools` list, + # so this used to be a `req["json"].pop("tool_choice", None)` at the one call site that sends no + # tools — a rule living outside the thing it constrains, which is [[limit-with-no-enforcer]]: + # the next no-tools caller inherits a 400 that reads like a model problem, not a shape problem. + # ⚠ Safe for all three callers today, checked rather than assumed: `routes_query` and + # `ai_review.draft_flow` both pass a non-empty tool list and keep the key exactly as before. + if wire_tools: + body["tool_choice"] = {"type": "any" if tool_choice in ("required", "any") else "auto"} + if effort: + body["output_config"] = {"effort": str(effort)} + return {"url": LLM_PROVIDERS["anthropic"].url, + "headers": {"x-api-key": str(key), + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json"}, + "json": body} + + +#: Cached result of `import anthropic`: the module, or `False` once it is known to be absent. +#: ⚠ `None` is NOT the "absent" sentinel — a plain falsy check would re-attempt the import on every +#: call, and a failed import is not cheap. `False` says "asked and answered". +_SDK: object = None + + +def anthropic_sdk(): + """The official `anthropic` SDK, or `None` if this deployment does not carry it. + + ⭐⭐ D-346, W37-T39. THIS FUNCTION IS THE WHOLE OF THE FIX AND ALSO THE WHOLE OF ITS RISK. + `aios-web/requirements.txt` is what the Dockerfile installs and it is OUTSIDE every lane's + fence this wave, so the pin is another session's edit. A hard `import anthropic` at module + scope would therefore turn a missing line in a manifest into a container that cannot boot at + all: the API imports this module on every request path. + + So the import is LAZY and its absence is REPORTED rather than fatal — `anthropic_send` falls + back to the raw `requests` POST that has always worked, and says which transport ran. When the + pin lands the SDK path takes over with no further change. + ⛔ The fallback is a BRIDGE, not a second implementation: both transports send the SAME body + `anthropic_request()` built and return the SAME `(status, body)` pair, so `anthropic_read`, + `is_credit_failure` and `refusal_sentence` each keep exactly one normalizer + ([[one-question-two-normalizers]]). + """ + global _SDK + if _SDK is None: + try: + import anthropic as _mod # noqa: PLC0415 - deliberately lazy; see the docstring + _SDK = _mod + except Exception: + _SDK = False + return _SDK or None + + +def anthropic_send(req, timeout=None, use_sdk=None): + """POST one `anthropic_request()` dict. Returns `(status, body, transport)`. + + ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader + who cannot tell which path ran cannot tell whether the requirements pin reached the container + [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on + `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a + check keyed to it passes on a stale value written by an earlier check in the same process. + + `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error — + the SDK raises where `requests` returns, and normalising that difference here is the only + reason this function exists rather than the caller branching on transport. + + ⚠ `use_sdk=False` forces the fallback. It exists so a gate can exercise BOTH transports + without assigning to `_SDK`: a test that mutates a module global and then throws leaves every + later check in that process running on the wrong path and passing for the wrong reason. + """ + sdk = None if use_sdk is False else anthropic_sdk() + payload = dict(req.get("json") or {}) + if sdk is None: + r = requests.post(req["url"], headers=req["headers"], json=payload, + timeout=(timeout or 60)) + try: + return r.status_code, (r.json() if r.content else {}), "http" + except Exception: + return r.status_code, {"_text": (r.text or "")[:2000]}, "http" + + key = (req.get("headers") or {}).get("x-api-key") or "" + try: + client = sdk.Anthropic(api_key=key, timeout=float(timeout or 60)) + msg = client.messages.create(**payload) + # ⚠ `.model_dump()` is what makes ONE reader serve both transports: it hands back the same + # wire-shaped dict the raw POST parses out of the response body. + return 200, msg.model_dump(), "sdk" + except Exception as exc: + # ⛔ THE STATUS IS THE PRODUCT HERE. Every sentence the reader sees is chosen by + # `refusal_sentence(status)`, so an exception that loses its code turns "Anthropic is out + # of credit" into "Anthropic did not answer" — the exact regression R4 was written to end. + status = int(getattr(exc, "status_code", 0) or 0) + body = getattr(exc, "body", None) + if not isinstance(body, dict): + body = {"_text": str(exc)[:2000]} + if not status: + # No status at all = it never reached the vendor (DNS, TLS, timeout). 408 is the one + # code `_STATUS_WORDS` already words as a reachability problem rather than a refusal. + status = 408 if isinstance(exc, getattr(sdk, "APIConnectionError", ())) else 0 + return status, body, "sdk" + + +def anthropic_read(body): + """`(text, tool_input, refusal)` out of a Messages API answer. + + ⛔ `stop_reason` IS CHECKED BEFORE `content` IS READ. A safety decline answers **HTTP 200** with + `stop_reason: "refusal"` and an empty or partial `content`, so code that indexes `content[0]` + unconditionally breaks on exactly the turn a person most needs explained. + ⭐ AND THE TOOL CALL ARRIVES PARSED. `tool_use.input` is already a dict — no `json.loads`, and + no regex recovering a call out of a 400, which is what the other wire needs. + """ + body = body if isinstance(body, dict) else {} + if str(body.get("stop_reason") or "") == "refusal": + return "", None, "the assistant declined to answer that one" + blocks = [b for b in (body.get("content") or []) if isinstance(b, dict)] + text = " ".join(str(b.get("text") or "") for b in blocks if b.get("type") == "text").strip() + calls = [b for b in blocks if b.get("type") == "tool_use"] + got = calls[0].get("input") if calls else None + return text, (got if isinstance(got, dict) else None), None + + +def llm_status(capability="llm_tool_calling"): + """Per provider: configured, declared-capable, in cooldown, and WHY — contract C5's payload. + + ⭐ ONE LIST, ONE DOOR. The Assistant's model picker and the Agent chat's toggle (W36-T34) read + THIS, so a model offered in one place cannot be missing from the other, and neither can offer a + provider the ladder would refuse to call [[permitted-is-not-answerable]]. + """ + rows = [] + for name in LLM_DEFAULT_ORDER: + p = LLM_PROVIDERS[name] + cap = p.caps.get(capability) + rows.append({ + "provider": p.name, "label": p.label, "model": p.model, "wire": p.wire, + "configured": p.configured(), + "toolCalling": bool((p.caps.get("llm_tool_calling") or Capability(False)).capable), + "jsonMode": bool((p.caps.get("llm_json_mode") or Capability(False)).capable), + "capable": bool(cap and cap.capable), + "outOfCredit": no_credit(name), + "note": (cap.note if cap else ""), + }) + return rows + + # ============================================================================================= # THE CANONICAL SCHEMA — owner ruling 2026-08-08: # *"standardize the schema between Bright Data and APIfy so we keep using the same pre-set diff --git a/api/routes_admin.py b/api/routes_admin.py index 7ee38d40363f882cd229ca9c8922db6cdbd4d6a9..2c049e137d9ea4f524a97b7b0c01c0225c5fd96e 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -46,6 +46,18 @@ router = APIRouter(prefix="/api/v1") #: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no #: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. _UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") +#: ⭐⭐ WAVE 37 · T03 (owner ruling R8) — AN EMAIL ADDRESS IS ALSO A USERNAME. Owner: the username +#: IS the email, one identity. ⛔ THIS IS NOT A NEW CAPABILITY, IT IS THE DOOR CATCHING UP WITH THE +#: DATA: measured on deployed staging before the change, FIVE of tenant #0's eight accounts already +#: carry an `@` in the `username` field (`david@fischfloralsupply.com`, `farhan@`, `florencia@`, +#: `karen@`, `naomi@`) — every one of them illegal under the slug pattern above and longer than its +#: 32-character cap, so this route could not have created any of them. +#: ⚠ DELIBERATELY PERMISSIVE, because the store key is the thing being constrained and not the +#: deliverability of the address: one `@`, a dot-bearing domain, and the same character class the +#: slug already trusts. A stricter grammar here would refuse addresses that exist. +_UNAME_EMAIL_RE = re.compile(r"^[a-z0-9][a-z0-9._%+-]*@[a-z0-9][a-z0-9.-]*\.[a-z]{2,24}$") +#: RFC 5321's cap on a whole address. The slug form keeps its own 32 via `_UNAME_RE`. +_UNAME_MAX = 254 #: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog #: enforces nothing; this does, and the two are allowed to differ because only one of them is #: reachable from the internet. @@ -168,11 +180,19 @@ def _access_summary(rec, uname="", governed=None, st=None, surface_keys=None): # ── request validation: every rule below is fail-closed ────────────────────────────────────────── def _clean_username(v): + """The ONE validator for a new account's key. R8 widened it to admit a full email address. + + ⛔ IT HAS EXACTLY ONE CALL SITE AND THAT IS LOAD-BEARING, not an oversight to tidy up. + `PATCH /admin/users/{username}`, the password route and both `/perms` routes take the username + as a PATH parameter and never re-validate it, which is precisely why the five email-keyed + accounts above are administrable today despite being unconstructable. Arming this validator on + those routes would look like hardening and would break five live accounts. + """ uname = str(v or "").strip().lower() - if not _UNAME_RE.match(uname): + if len(uname) > _UNAME_MAX or not (_UNAME_RE.match(uname) or _UNAME_EMAIL_RE.match(uname)): raise err(400, "bad_username", - "a username is 2-32 characters: lowercase letters, digits, dot, dash or " - "underscore, starting with a letter or digit") + "a username is either 2 to 32 characters of lowercase letters, digits, dot, " + "dash or underscore, or a full email address of up to 254 characters") return uname @@ -297,8 +317,35 @@ def create_user(body: dict = Body(default=None), session: Session = Depends(admi agent = str(body.get("agent") or "").strip() or None email = str(body.get("email") or "").strip() or None - if uname in _registry(): + _reg_before = _registry() + if uname in _reg_before: raise err(409, "user_exists", f"{uname} already exists — PATCH it to change it") + # ⭐⭐ WAVE 37 · T03 (R8) — THE COLLISION THE WIDENING CREATES, REFUSED AT CREATE TIME. + # `core/users.py::verify` resolves a typed string by looking up the registry KEY first and only + # scanning the `email` field when that misses (its own wave-18 R1 comment). That precedence was + # incidental while a username could not contain `@`; now it decides which of two accounts a + # person reaches, so it is stated in both files rather than left to be re-derived. + # ⛔ BOTH DIRECTIONS, because each one silently takes a login away from an account that has it: + # (a) a new USERNAME equal to somebody's EMAIL wins the key lookup and shadows their address; + # (b) a new EMAIL equal to somebody's USERNAME can never be reached, because their key wins. + # Refused with a named error rather than allowed and explained afterwards: the failure is a + # person signing in and landing in the wrong account, which nothing downstream can detect. + if "@" in uname: + _shadowed = next((k for k, r in _reg_before.items() + if isinstance(r, dict) + and str(r.get("email") or "").strip().lower() == uname), None) + if _shadowed: + raise err(409, "username_shadows_email", + f"{uname} is already the email address on the account {_shadowed}, which " + f"signs in with it today. This account would take that login over. Choose a " + f"different username, or clear the email address on {_shadowed} first.") + if email: + _e = email.strip().lower() + if _e != uname and _e in _reg_before: + raise err(409, "email_shadows_username", + f"{_e} is already a username on this deployment, so anyone signing in with " + f"it reaches that account and never this one. Choose a different email " + f"address for this account.") # Wave 18 (C1-TENANT): an admin creates accounts in their OWN company, never another's — # the tenant is stamped from the session, not taken from the body. users.create_user(uname, pw, name, role=role, bus=bus, modules=modules, @@ -708,9 +755,15 @@ def _clean_perms(v, governed_keys=None, session=None): f"saved with the bad conditions silently removed, which would store a " f"weaker wall than the one on screen.") clean_tree = {"conj": conj, "nodes": cleaned} + # ⭐⭐ W38-T19 — `metrics` IS THE FOURTH FIELD OF AN ENTRY, AND ITS DEFAULT IS GRANT. + # `raw.get("metrics", True)` rather than a required key: a PUT composed by an older + # client, or a copy of a record written before this ticket, must not read as a + # revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so + # the validator and the wall cannot disagree about what an absent key means. out[key] = {"access": bool(raw.get("access", True)), "filter": clean_tree, - "hiddenFields": sorted({str(h) for h in hidden})} + "hiddenFields": sorted({str(h) for h in hidden}), + "metrics": bool(raw.get("metrics", True))} _refuse_unshapeable_bu(out) return out @@ -1031,8 +1084,16 @@ def get_perms(username: str, session: Session = Depends(admin_gate)): _default = (perm_scope.nav_may_open(users._public(uname, rec), k) if any(m.get("surface") and m["key"] == k for m in modules) else perm_scope.may_read(users._public(uname, rec), k, st=session.runtime)) - perms_out[k] = e if isinstance(e, dict) else { - "access": bool(_default), "filter": None, "hiddenFields": []} + # ⭐⭐ W38-T19 — `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN + # BEFORE THE KEY EXISTED. The wire is TOTAL on purpose: the client's own parse already + # reads an absent key as granted, so this line changes no rendering — what it changes is + # what a HUMAN reads off the payload while debugging, and what the PUT round-trips. The + # stored dict is copied rather than mutated: `rec` is the live registry record and + # stamping a key onto it here would write a permission nobody saved. + perms_out[k] = (dict(e, metrics=bool(e.get("metrics", True))) + if isinstance(e, dict) else + {"access": bool(_default), "filter": None, "hiddenFields": [], + "metrics": True}) return {"username": uname, "perms": perms_out, # S3 ask 2 — the marker rides the GET. An absent module entry means DENY on a diff --git a/api/routes_agent_harness.py b/api/routes_agent_harness.py new file mode 100644 index 0000000000000000000000000000000000000000..cfe1ffa6958ea00035eaaaf64d5c826e246fb209 --- /dev/null +++ b/api/routes_agent_harness.py @@ -0,0 +1,455 @@ +"""routes_agent_harness.py — CONTRACT C4: the agent's HARNESS, kept as versioned files. + +Owner item 3, verbatim (2026-08-18): *"This new module under agent is supposed to host any file +pertaining to the agent skills, router etc. So we build a user can build a custom harness for each +agent through the chat interface."* + + GET /api/v1/agents/{id}/harness the file LIST (no bodies) + GET /api/v1/agents/{id}/harness?path=… one file: current body + its versions + GET /api/v1/agents/{id}/harness?path=…&version=N one older body, verbatim + PUT /api/v1/agents/{id}/harness write a NEW version of one file + DELETE /api/v1/agents/{id}/harness?path=… drop a file and its history + +⭐⭐ **R8 IS THE WHOLE SHAPE: BOTH PRINCIPALS WRITE, AND NOTHING IS EVER OVERWRITTEN.** A write +appends a version; a roll-back is a write of an older body (`restoredFrom`), never a delete. So the +history is a record of what happened rather than of what somebody last wanted it to look like. + +⛔ **R8 IS DELIBERATELY NOT R9.** An agent-authored ACTION's configuration is agent-only (item 8, +`routes_automation`); a harness file is not. Do not copy this file's posture over there or that +one's over here — the two rulings differ on purpose and they sit one screen apart in the product. + +⚠ **`author` IS THE SESSION, `authorKind` IS THE PROVENANCE, AND THEY ARE DIFFERENT FACTS.** Every +write through this router is made BY a signed-in administrator, so `author` is stamped from the +session and can never be supplied by the caller. `authorKind` says whether the BODY was drafted by +a person or by the agent in the chat — a claim the client is entitled to make, because both are +permitted (R8) and so nothing is bought by forging it. `record_version()` below is the server-side +door the automation engine uses when the agent writes with no session at all; that one stamps the +agent's own id as the author, which is the only case where `author` is not a username. + +⛔ **THE VERSION LIST IS CAPPED AND THE CAP IS REPORTED, NEVER SILENT.** `MAX_VERSIONS` versions of +one file are kept; past that the OLDEST are dropped and the count of what was dropped rides in +`trimmed` on every payload that mentions the file, so a reader can see that the history is partial +rather than infer that the file was only ever saved twice. This is the tenant document, which is +already 28.6 MB on tenant #0 and is deep-copied on every read: an unbounded per-agent history is +a store-sized leak with a UI in front of it. +""" +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's harness files: `{agent_id: {path: file_record}}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace — the same rule +#: `routes_slack.AGENTS_KEY` follows for the agent records these hang off. +HARNESS_KEY = "agent_harness" + +#: One body. Generous for a skill or a router file and far under the point where a single write +#: would move the tenant document measurably. A larger body is a REFUSAL at the door, not a +#: truncation: a silently truncated skill file is a harness that does not do what its text says. +MAX_BODY_BYTES = 128 * 1024 + +#: Files per agent. A refusal, not a trim — creating the 65th file is a different act from saving +#: the 101st version of one, and only the second can be a routine consequence of ordinary editing. +MAX_FILES = 64 + +#: Versions kept per file. Past this the oldest go and `trimmed` counts them (see the header). +MAX_VERSIONS = 100 + +MAX_PATH = 200 + +#: What a path may contain. ⛔ THIS IS NOT A FILESYSTEM PATH AND NOTHING HERE EVER TOUCHES A DISK — +#: the "files" are keys in a store bucket. The character rule exists so the key is displayable, is +#: safe to put in a URL, and cannot carry a traversal sequence that would look meaningful to a +#: future reader who assumes it IS a filesystem path. Fail-closed on the character set, not on a +#: list of forbidden sequences: an allow-list cannot be walked around by a spelling. +_PATH_OK = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-/") + +AUTHOR_KINDS = ("user", "agent") + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{agent_id: {path: record}}` for one tenant. `{}` on any failure — an unreadable bucket must + degrade to "this agent has no harness files", never to a 500 on the pane that lists them.""" + try: + found = rt.get(HARNESS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _files(rt, agent_id): + found = _all(rt).get(str(agent_id)) + return found if isinstance(found, dict) else {} + + +#: What kind of principal an id names. `None` = this tenant has no agent with that id. +#: +#: ⛔⛔ THERE ARE TWO AGENT REGISTRIES IN THIS PRODUCT AND THE FIRST DRAFT OF THIS FILE KNEW ONLY +#: ONE (ASK D-5, 2026-08-18). `routes_slack._agents` is the per-Slack-channel permission wall under +#: Manage users. The **Agents module** the owner's item 3 is about is the AUTOMATION surface — +#: `Shell.tsx` mounts ``, and +#: `AutomationDetail`'s `panelTabs` is literally the `Properties | Run history` strip R7 adds +#: "Harness" to. Keyed on the Slack bucket alone, every `GET/PUT /agents/{id}/harness` from that +#: tab would have answered **404 no_agent**: whole, gate-green and dead on arrival +#: [[reachable-is-not-the-same-as-built]]. Verified independently before acting, not taken on +#: report: `surface="Agents"` mounts `AutomationSurface`, and `ManageAgentPane` contains ZERO +#: occurrences of Canvas, Properties, Run history or panelTabs. +#: +#: ⭐ ONE STORE, BOTH PRINCIPALS — never a second bucket keyed by surface. The owner's "agent +#: skills, router" files in two places is the parallel code path item 13 exists to refuse. +AGENT_SLACK = "slack" +AGENT_AUTOMATION = "automation" + + +def agent_kind(rt, agent_id): + """Which registry holds this id — `AGENT_SLACK`, `AGENT_AUTOMATION`, or `None`. + + ⚠ A SYNTHETIC ROW IS NOT AN AGENT FOR THIS PURPOSE. `field:` and `system:` ids are DERIVED at + read time from a column definition or a connector schedule; they have no stored home, so a + harness file hung off one is an orphan the moment the column changes. `patch_automation` + already refuses those ids for the neighbouring reason, and this refuses them by simply not + finding them — `all_definitions` holds stored automations only. + """ + import routes_slack + aid = str(agent_id or "") + if isinstance(routes_slack._agents(rt).get(aid), dict): + return AGENT_SLACK + import automation_engine as engine + try: + known = engine.all_definitions(rt) or {} + except Exception: # noqa: BLE001 + return None + return AGENT_AUTOMATION if aid in known else None + + +def _known_agent(rt, agent_id): + """Does this tenant have an agent with this id, in EITHER registry?""" + return agent_kind(rt, agent_id) is not None + + +def agent_wall(session, agent_id): + """404 for an unknown id, else apply the wall THAT PRINCIPAL'S OWN SURFACE applies. + + ⛔⛔ ONE DOOR, TWO WALLS, AND THAT IS NOT A SECOND CODE PATH — it is the refusal to invent a + THIRD wall. A Slack channel agent is administered under Manage users and every `/agents/*` door + in `routes_slack` is `admin_gate`; an automation lives in the Agents module and every door in + `routes_automation` is `module_gate("automation")`. A harness file is configuration OF the + agent it hangs off, so it is reached by whoever may already configure that agent. Picking one + of the two walls for both would either lock the Agents module's own users out of a tab the + owner asked for, or hand the Slack permission wall to anyone with an automation grant. + """ + kind = agent_kind(session.runtime, agent_id) + if kind is None: + raise err(404, "no_agent", "there is no agent with that id in this workspace") + if kind == AGENT_SLACK: + import core.perms as perms + if not perms.is_admin(session.user): + raise err(403, "forbidden", "administrators only") + else: + session.require("automation") + return kind + + +def normalize_path(raw): + """THE path rule, in ONE place. Returns the cleaned path, or `None` if it is not acceptable. + + ⛔⛔ ONE RULE, TWO DOORS, AND THAT IS WHY THIS IS A FUNCTION RATHER THAN TWO IF-BLOCKS. There + are two ways into this store — the HTTP route (which must answer 400) and `record_version()` + (which must raise `ValueError`, having no response to put a status into). Written twice, the + two copies are [[one-question-two-normalizers]] waiting to happen: the first weakening of one + copy is invisible because the other still refuses, so nothing goes red and the wall is now + half there. Written once, a change to the rule is felt at both doors and by the gate. + """ + path = str(raw or "").strip().strip("/") + if not path or len(path) > MAX_PATH: + return None + if set(path) - _PATH_OK or ".." in path or "//" in path: + return None + return path + + +def _clean_path(raw): + """`normalize_path` at the HTTP door, where a refusal is a 400 with a reason.""" + path = normalize_path(raw) + if path is None: + if not str(raw or "").strip().strip("/"): + raise err(400, "no_path", "a harness file needs a path, for example skills/router.md") + if len(str(raw)) > MAX_PATH: + raise err(400, "path_too_long", f"a harness path is at most {MAX_PATH} characters") + raise err(400, "bad_path", + "a harness path may use letters, digits, dot, dash, underscore and / only") + return path + + +def _blank(path, author, author_kind): + return {"path": path, "versions": [], "trimmed": 0, + "created": _now(), "createdBy": author, "createdKind": author_kind} + + +def _append(record, body, author, author_kind, restored_from=None): + """Append ONE version to a file record, in place, and report what was trimmed. + + The version NUMBER is monotonic and survives trimming — it counts writes, not stored entries. + A version list whose numbers restart at 1 after a trim would make two different bodies share a + name, and `restoredFrom` would then point at whichever one happened to be in the window. + """ + # ⚠ A NEW LIST, NEVER `versions.append(...)` ON THE STORED ONE. `update()` hands the callback + # the live document and may run it more than once; appending in place would then stack two + # copies of the same version into the history on a retry. + prior = record.get("versions") if isinstance(record.get("versions"), list) else [] + last = max((int(v.get("version") or 0) for v in prior if isinstance(v, dict)), default=0) + entry = {"version": last + 1, "body": body, "author": author, "authorKind": author_kind, + "created": _now(), "bytes": len(body.encode("utf-8"))} + if restored_from: + entry["restoredFrom"] = int(restored_from) + versions = [*prior, entry] + dropped = max(0, len(versions) - MAX_VERSIONS) + if dropped: + versions = versions[dropped:] + record["versions"] = versions + record["trimmed"] = int(record.get("trimmed") or 0) + dropped + return entry + + +def _head(record): + """The newest version of a file record, or `None` for a record with no versions at all.""" + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return versions[-1] if versions else None + + +def _row(record): + """One file, as the LIST door reports it: everything except the bodies. + + ⚠ NO BODY, AND THAT IS THE POINT. A list door that carried every version of every file would + ship the whole harness on every pane render; the Harness tab lists first and opens one file + second, which is exactly the shape this answers. + """ + head = _head(record) or {} + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + return {"path": record.get("path") or "", + "version": int(head.get("version") or 0), + "bytes": int(head.get("bytes") or 0), + "author": head.get("author") or record.get("createdBy") or "", + "authorKind": head.get("authorKind") or record.get("createdKind") or "user", + "updated": head.get("created") or record.get("created") or "", + "created": record.get("created") or "", + "versions": len(versions), + "trimmed": int(record.get("trimmed") or 0)} + + +def _version_rows(record): + """The history of one file, newest first, WITHOUT the bodies. + + A body per version is what makes a diff possible, and it is also what makes this payload big: + 100 versions of a 128 KB file is 12 MB. The client asks for the two bodies it is diffing + (`?path=…&version=N`), which is two round trips for a diff and none for a history list. + """ + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + out = [] + for entry in reversed(versions): + if not isinstance(entry, dict): + continue + row = {"version": int(entry.get("version") or 0), + "author": entry.get("author") or "", + "authorKind": entry.get("authorKind") or "user", + "created": entry.get("created") or "", + "bytes": int(entry.get("bytes") or 0)} + if entry.get("restoredFrom"): + row["restoredFrom"] = int(entry["restoredFrom"]) + out.append(row) + return out + + +def _limits(): + """The caps, IN the payload, so a client can say "this file is full" before a write fails. + + ⚠ A limit the client cannot see is a limit the user meets as an error. `trimmed` reports the + one cap that acts without refusing; these report the three that refuse. + """ + return {"maxBodyBytes": MAX_BODY_BYTES, "maxFiles": MAX_FILES, + "maxVersions": MAX_VERSIONS, "maxPath": MAX_PATH} + + +# ── the write door the SERVER uses (no session) ──────────────────────────────────────────────── +def record_version(runtime, agent_id, path, body, author="", author_kind="agent", + restored_from=None): + """Write one version from INSIDE the server — the agent's own half of R8. + + ⭐ THIS IS THE FUNCTION, NOT THE ROUTE, THAT MAKES "editable by BOTH the user and the agent" + true. An agent acting inside an automation run holds no session and no cookie; if its only way + to write were the HTTP door it would have to borrow a person's identity, and the authorship + column would then be a record of who was logged in rather than of what wrote the file. + + Returns the appended entry. Raises nothing the caller cannot handle: an unknown agent is a + `ValueError`, because a server-side caller has no HTTP response to put a 404 into. + """ + agent_id = str(agent_id or "") + if not _known_agent(runtime, agent_id): + raise ValueError(f"no agent {agent_id!r} in this workspace") + path = normalize_path(path) + if path is None: + raise ValueError("that is not an acceptable harness path") + body = str(body or "") + if len(body.encode("utf-8")) > MAX_BODY_BYTES: + raise ValueError("harness body is over the size limit") + kind = author_kind if author_kind in AUTHOR_KINDS else "agent" + author = str(author or agent_id) + # ⛔ THE FILE-COUNT CEILING IS CHECKED HERE, NOT INSIDE `_set`. An exception raised inside a + # store `update` callback propagates out of a half-run read-modify-write, and the one thing a + # refusal must never do is leave the caller unsure whether the write happened. + existing = _files(runtime, agent_id) + if path not in existing and len(existing) >= MAX_FILES: + raise ValueError(f"this agent already has {MAX_FILES} harness files") + appended = {} + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + record = dict(files[path]) if isinstance(files.get(path), dict) else _blank(path, author, kind) + appended.clear() + appended.update(_append(record, body, author, kind, restored_from)) + files[path] = record + cur[agent_id] = files + return cur + + runtime.update(HARNESS_KEY, _set, flush="sync") + return appended + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/agents/{agent_id}/harness") +def get_harness(agent_id: str, path: str = "", version: int = 0, + session: Session = Depends(require_session)): + """The list, one file, or one older body — decided by the query string (C4). + + ⚠ ADMIN-GATED LIKE EVERY OTHER AGENT DOOR (`routes_slack`), and for a stronger reason than + consistency: a harness file is what the agent is INSTRUCTED to do, so writing one is closer to + editing a permission than to editing a document. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + files = _files(session.runtime, agent_id) + if not path: + rows = [_row(rec) for _p, rec in sorted(files.items()) if isinstance(rec, dict)] + return {"agent": agent_id, "files": rows, "limits": _limits()} + + wanted = _clean_path(path) + record = files.get(wanted) + if not isinstance(record, dict): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + versions = record.get("versions") if isinstance(record.get("versions"), list) else [] + if version: + for entry in versions: + if isinstance(entry, dict) and int(entry.get("version") or 0) == int(version): + return {"agent": agent_id, **_row(record), "body": entry.get("body") or "", + "atVersion": int(version), "history": _version_rows(record), + "limits": _limits()} + # ⛔ A TRIMMED VERSION IS A NAMED REFUSAL, NOT A 404 SHAPED LIKE A TYPO. The client asked + # for something that existed and no longer does, and telling it apart from a bad number is + # the difference between "roll back to v3" failing loudly and failing as if v3 never was. + trimmed = int(record.get("trimmed") or 0) + if trimmed and int(version) <= trimmed: + raise err(410, "version_trimmed", + f"version {int(version)} is older than the {MAX_VERSIONS} versions kept " + f"for this file, and its body is gone") + raise err(404, "no_version", f"this file has no version {int(version)}") + head = _head(record) or {} + return {"agent": agent_id, **_row(record), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(record), + "limits": _limits()} + + +@router.put("/agents/{agent_id}/harness") +def put_harness(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Write a NEW version of one harness file. `{path, body, authorKind?, restoredFrom?}` (C4). + + ⛔ THERE IS NO OVERWRITE HERE AND THERE IS NO EDIT-IN-PLACE. R8's "every version kept" is not a + UI affordance; it is this function refusing to have a code path that replaces a body. A + roll-back arrives as an ordinary write carrying `restoredFrom`, so the history records that + somebody went back rather than pretending the intervening versions never happened. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + body = body if isinstance(body, dict) else {} + path = _clean_path(body.get("path")) + text = body.get("body") + if not isinstance(text, str): + raise err(400, "no_body", "a harness file needs a body, even an empty one") + if len(text.encode("utf-8")) > MAX_BODY_BYTES: + raise err(413, "body_too_long", + f"a harness file is at most {MAX_BODY_BYTES // 1024} KB; this one is larger") + + # ⚠ THE CALLER DECLARES THE PROVENANCE AND THE SERVER STAMPS THE IDENTITY. `authorKind` is a + # claim about who WROTE the text (the person, or the agent in the chat panel); `author` is the + # session and is never read off the request. Nothing is bought by forging the first — both + # principals may write (R8) — and everything would be bought by forging the second. + kind = str(body.get("authorKind") or "user").strip().lower() + if kind not in AUTHOR_KINDS: + raise err(400, "bad_author_kind", "authorKind is either user or agent") + restored = body.get("restoredFrom") + try: + restored = int(restored) if restored else None + except (TypeError, ValueError): + raise err(400, "bad_version", "restoredFrom must be a version number") + + files = _files(session.runtime, agent_id) + if path not in files and len(files) >= MAX_FILES: + raise err(409, "too_many_files", + f"this agent already has {MAX_FILES} harness files; delete one to add another") + try: + record_version(session.runtime, agent_id, path, text, + author=session.uname, author_kind=kind, restored_from=restored) + except ValueError as exc: + raise err(400, "refused", str(exc)) + + fresh = _files(session.runtime, agent_id).get(path) + if not isinstance(fresh, dict) or not _head(fresh): + # The store took the write and did not record it. A 200 here would tell an administrator + # their skill file was saved when it was not — the shape `routes_slack` refuses too. + raise err(503, "store_unavailable", "the harness file was NOT saved") + head = _head(fresh) + return {"agent": agent_id, **_row(fresh), "body": head.get("body") or "", + "atVersion": int(head.get("version") or 0), "history": _version_rows(fresh), + "limits": _limits()} + + +@router.delete("/agents/{agent_id}/harness") +def delete_harness(agent_id: str, path: str = "", session: Session = Depends(require_session)): + """Drop one harness file AND its history. + + ⚠ THIS IS NOT THE THING R8 FORBIDS. R8 forbids a ROLL-BACK implemented as a delete — losing + versions as a side effect of an edit. Deleting a file is a person deciding the file should not + exist, which is a different act with a different button, and a store with no way to remove a + file is one where a typo'd path is permanent. + """ + agent_id = str(agent_id or "") + agent_wall(session, agent_id) + wanted = _clean_path(path) + if wanted not in _files(session.runtime, agent_id): + raise err(404, "no_file", f"this agent has no harness file at {wanted}") + + def _set(cur): + cur = dict(cur or {}) + files = dict(cur.get(agent_id) or {}) if isinstance(cur.get(agent_id), dict) else {} + files.pop(wanted, None) + # An agent with no harness files leaves NO key behind. An empty dict per agent id is how a + # bucket accumulates a row for every agent anybody ever opened the tab on. + if files: + cur[agent_id] = files + else: + cur.pop(agent_id, None) + return cur + + session.runtime.update(HARNESS_KEY, _set, flush="sync") + return {"agent": agent_id, "deleted": wanted, + "files": [_row(rec) for _p, rec in sorted(_files(session.runtime, agent_id).items()) + if isinstance(rec, dict)], + "limits": _limits()} diff --git a/api/routes_automation.py b/api/routes_automation.py index 0b8aad0a997592d356b825513ee8877b046e066b..88849c492b7823d3b84c96c4db87725749c92132 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -14,6 +14,7 @@ trigger becomes a public one — the same class of mistake as an empty-200 permi import os import re import time +from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends, Header, Request @@ -53,7 +54,7 @@ MODULE = "automation" _GATE = module_gate(MODULE) -def _wire(defn, tenant): +def _wire(defn, tenant, rt=None): """One automation, as the client reads it. `running` is PROCESS state, never store state — see the engine header on why a persisted 'running' is a permanent lock.""" live = engine.running(tenant, defn.get("id")) @@ -141,6 +142,16 @@ def _wire(defn, tenant): # back. What needs the target database's schema (an enrich binding resolved by the profile # FLAG) stays a run-time refusal — see `ACTION_REQUIRED`'s note. "unconfigured": engine.unconfigured_actions(defn), + # ⭐⭐ WAVE 36 · W36-T38 (owner item 8 / R9) — WHICH ACTIONS AN AGENT BUILT. + # `{action_id: {agent, agentName, created, by}}`, empty for an ordinary automation. + # ⛔ THE CONFIG IS NOT HIDDEN AND MUST NOT BE. R9 is a WRITE rule: the client uses this to + # draw a lock and explain who owns the step, never to withhold what the step does — an + # automation nobody can audit is worse than one nobody can edit. + # ⚠ `rt=None` YIELDS `{}` RATHER THAN OMITTING THE KEY, so a caller that forgets the + # runtime produces "nothing is agent-owned" (a visible, wrong-but-safe answer) instead of + # an absent prop the client silently reads as undefined + # [[flag-shipped-without-its-writer]]. + "agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {}, } @@ -243,7 +254,7 @@ def _field_agent_rows(session): # — the exact way `awaitingResults` shipped inert for a whole wave. Passing the # synthetic DEFINITION through the same function makes them identical by # construction; only `system` is stamped afterwards, because no stored row has it. - row = _wire(defn_syn, session.tenant) + row = _wire(defn_syn, session.tenant, rt=session.runtime) row["system"] = SYSTEM_FIELD_AGENT out.append(row) return out @@ -334,7 +345,7 @@ def _odoo_sync_row(session, detail=False): "config": {"connector": "odoo", "every": every, "frozen": frozen}}]}, } - row = _wire(defn, session.tenant) + row = _wire(defn, session.tenant, rt=session.runtime) row["system"] = SYSTEM_ODOO_SYNC # ⛔ THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence` # speaks the automation vocabulary — it builds "When , run N actions" out of @@ -745,7 +756,7 @@ def list_automations(session: Session = Depends(_GATE)): pass _STATEMENTS_SEEDED.add(session.tenant) defs = engine.all_definitions(session.runtime) - items = [_wire(d, session.tenant) for _, d in + items = [_wire(d, session.tenant, rt=session.runtime) for _, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] # ⭐⭐ WAVE 34 · CONTRACT C3 (W34-T48) — field agents join the list as SYNTHETIC rows. # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be @@ -1133,6 +1144,17 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends prompt = str((body or {}).get("prompt") or "").strip() if not prompt: raise err(400, "no_prompt", "type what you want the automation to do") + # ⭐⭐ ASK D-18 (2026-08-18) — THE KEY THE CLIENT SENDS IS NOW READ. This door took `prompt` off + # the body and nothing else, so the Agent chat's model toggle (W36-T34) was a control that + # configured nothing: the value was accepted and dropped, which is the shape three tickets in + # this wave already tripped over. D held T34 rather than mark a picker BUILT while its value + # stopped at the browser, and was right to. + # ⚠ UNKNOWN OR UNCONFIGURED FALLS BACK TO THE LADDER, never refuses (D-18's asked-for posture, + # and C5's everywhere else): a model the ladder stops offering must not turn every later draft + # into an error. The response already names the rung that ANSWERED, which is the honest half. + _model = str((body or {}).get("model") or "").strip().lower() + if _model in ("", "auto"): + _model = None # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open` # per table, so the model is shown exactly the databases this caller may already see and cannot # name one they were not granted — the permission wall re-used, never a second one built beside @@ -1170,6 +1192,7 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends # sites, this one has a real person behind it: somebody typed the sentence, so `user` is # the caller rather than the automation's owner. st=session.runtime, user=getattr(session, "uname", "") or "", + model=_model, chat=_DRAFT_CHAT[0]) if refusal or not draft: raise err(400, "draft_refused", refusal or "no automation could be drafted from that") @@ -1263,6 +1286,244 @@ def draft_automation(body: dict = Body(default=None), session: Session = Depends "provider": provider, "dropped": dropped, "notes": _notes, "saved": False} +# ══════════════════ WAVE 36 · W36-T38 (owner item 8, ruling R9) — AGENT-AUTHORED ACTIONS ══════ +# +# Owner, verbatim (2026-08-18): *"Create the ability for an automation agent to create any +# 'Action' under the Canvas, so its configuration can only be touched by the agent. Be it a tool a +# script etc. We need to really guardrail the reach of this script."* +# +# ⭐⭐ R9 IS A **WRITE** RULE, NOT A VISIBILITY RULE, and the ticket says so in as many words: the +# user MAY read an agent-authored action's configuration and MAY delete it; what they may not do +# is hand-edit it. Hiding a config from the person whose workspace it runs in is a different +# product, and a worse one — nobody can audit what they cannot see. +# +# ⛔ AND IT IS ENFORCED AT THE DOOR, NEVER IN A `disabled` ATTRIBUTE. `routes_automation` already +# carries that lesson for a different control; a client-side lock is a suggestion, and the whole +# point of R9 is that the agent's configuration stays coherent with what the agent believes it +# built. +# +# ⚠ WHY THE MARK IS A SEPARATE BUCKET RATHER THAN A KEY ON THE ACTION. `clean_actions` builds every +# action KEY BY KEY from an allowlist and drops anything it does not recognise (D-75) — so a +# marker stored inside the action would be silently erased on the next save, and the wall would +# quietly stop existing with every gate still green. This is an ANNOTATION layer keyed by +# `(automation id, action id)`; the configuration itself stays where it always was, in the +# automation, with exactly one writer. +AGENT_ACTIONS_KEY = "automation_agent_actions" + +#: Ids this door mints. ⚠ It must match `clean_actions`' `act_[a-z0-9_]{1,32}` or the engine +#: re-mints it and the annotation points at an action that no longer carries that id. +_AGENT_ACTION_PREFIX = "act_ag" + + +def _agent_marks(rt): + """`{auto_id: {action_id: mark}}` for one tenant. `{}` on any failure — an unreadable + annotation must degrade to "nothing is agent-owned", never to a 500 on the automations rail. + + ⛔ AND "DEGRADE TO NOTHING IS OWNED" IS THE SAFE DIRECTION HERE, which is worth stating because + it usually is not. The wall this feeds protects the AGENT's coherence, not the tenant's data: a + lost mark lets a person edit a config they own anyway, in their own workspace, on a step they + could always have deleted outright. A wall that failed CLOSED would instead make an automation + permanently unsavable because one annotation read timed out. + """ + try: + found = rt.get(AGENT_ACTIONS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _flat_actions(actions, out=None, depth=0): + """Every action in a flow, INCLUDING the ones nested inside a group's arms. + + ⚠ NESTED ACTIONS ARE THE ONES THIS MUST NOT MISS. `unconfigured_actions` walks them for the + same reason: a step inside an If/then branch is exactly the one a person cannot see, and a + wall that only looked at the top level would leave the agent's own nested step editable. + """ + out = [] if out is None else out + if depth > 6 or not isinstance(actions, list): + return out + for action in actions: + if not isinstance(action, dict): + continue + out.append(action) + for key in ("then", "else", "actions"): + _flat_actions(action.get(key), out, depth + 1) + return out + + +def _flow_actions(defn): + return _flat_actions(((defn or {}).get("flow") or {}).get("actions") or []) + + +def _same_action(left, right, rt): + """Do these two actions carry the SAME kind and configuration? + + ⛔ COMPARED AFTER `clean_actions`, ON BOTH SIDES, and that is the difference between a wall and + a nuisance. The stored action has already been through the cleaner; a client round-trip has + not, so it may carry a key order, a blank string or a dropped condition that means nothing. + Comparing raw shapes would refuse an edit that changes nothing, and a wall that fires on a + no-op is one an operator learns to route around [[one-question-two-normalizers]]. + """ + def _clean(action): + cleaned, error = engine.clean_actions([dict(action or {}, id="act_1")], rt=rt) + if error or not cleaned: + return None + one = dict(cleaned[0]) + one.pop("id", None) + return one + + a, b = _clean(left), _clean(right) + return a is not None and a == b + + +def _agent_owned_guard(session, auto_id, body): + """R9 AT THE DOOR: a user may not change an agent-authored action's kind or configuration. + + Three outcomes, and the middle one is the ruling: + * the action is ABSENT from the incoming flow -> a DELETE, and R9 allows it + * the action is present and CHANGED -> **409**, naming the action and the agent + * the action is present and identical -> nothing happens, so an ordinary save of a + flow that merely CONTAINS an agent action + is not refused + """ + marks = _agent_marks(session.runtime).get(str(auto_id)) or {} + if not isinstance(marks, dict) or not marks: + return + incoming = (body or {}).get("flow") + if not isinstance(incoming, dict) or "actions" not in incoming: + return + stored = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + was = {str(a.get("id") or ""): a for a in _flow_actions(stored)} + now = {str(a.get("id") or ""): a + for a in _flat_actions(incoming.get("actions") or [])} + for action_id, mark in marks.items(): + action_id = str(action_id) + if action_id not in now or action_id not in was: + continue # removed, or never stored: not an EDIT + if _same_action(was[action_id], now[action_id], session.runtime): + continue + who = str((mark or {}).get("agentName") or (mark or {}).get("agent") or "an agent") + raise err(409, "agent_owned", + f"this step was built by {who} and only {who} can change how it is set up. " + f"You can read it, and you can delete it, but it cannot be edited by hand") + + +def _prune_marks(session, auto_id): + """Drop annotations whose action is gone, and the whole entry when the automation is. + + ⚠ CALLED AFTER EVERY WRITE, because the alternative is an annotation bucket that only ever + grows: a mark on a deleted action would keep refusing an id nobody can see, and a mark on a + deleted automation would sit in the tenant document forever. + """ + auto_id = str(auto_id) + + def _set(cur): + cur = dict(cur or {}) + marks = cur.get(auto_id) + if not isinstance(marks, dict): + cur.pop(auto_id, None) + return cur + defn = (engine.all_definitions(session.runtime) or {}).get(auto_id) + if defn is None: + cur.pop(auto_id, None) + return cur + alive = {str(a.get("id") or "") for a in _flow_actions(defn)} + kept = {k: v for k, v in marks.items() if str(k) in alive} + if kept: + cur[auto_id] = kept + else: + cur.pop(auto_id, None) + return cur + + session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") + + +@router.post("/automations/{auto_id}/agent-actions") +def agent_author_action(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + """THE AGENT'S DOOR: add or replace one Action under the Canvas, marked agent-owned (R9). + + {agent: "", action: {kind, config, when?, id?}} + -> {automation, actionId, agent} + + ⛔ THE ACTION GOES THROUGH `clean_actions` LIKE EVERY OTHER ONE, and that is what "guardrail + the reach of this script" means in code: an agent cannot invent a config key, cannot name a + kind that is not in the catalog, and cannot reach a kind this tenant is not entitled to. The + agent gets a different WALL on editing, never a wider vocabulary. + + ⛔ AND IT CANNOT AUTHOR A KIND THAT IS NOT BUILT. `run_script` is `ready: False` in the + catalog, so `clean_actions` refuses it here exactly as it refuses it for a person — see this + lane's mailbox for why the script ARM is booked rather than half-built: a per-row script + contract and a client card are both missing, and a step that reports success and does nothing + is the failure this repo has already paid for. + + ⚠ THE AGENT ID IS CHECKED AGAINST THE TENANT'S OWN AGENTS. A mark naming an agent that does + not exist would refuse every future edit with a sentence naming nobody. + """ + import routes_slack # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + agent_id = str(body.get("agent") or "").strip() + agent = routes_slack._agents(session.runtime).get(agent_id) + if not isinstance(agent, dict): + raise err(404, "no_agent", "there is no agent with that id in this workspace") + action = body.get("action") + if not isinstance(action, dict) or not str(action.get("kind") or "").strip(): + raise err(400, "no_action", "an action needs a kind") + + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) + if defn is None: + raise err(404, "unknown_automation", "no automation with that id") + if str(defn.get("system") or "").strip(): + raise err(409, "system_agent", engine.system_agent_refusal(str(defn["system"]))) + + existing = list(((defn.get("flow") or {}).get("actions") or [])) + taken = {str(a.get("id") or "") for a in _flat_actions(existing)} + action_id = str(action.get("id") or "").strip() + if not action_id or action_id not in taken: + n = 1 + while f"{_AGENT_ACTION_PREFIX}{n}" in taken: + n += 1 + action_id = f"{_AGENT_ACTION_PREFIX}{n}" + fresh = dict(action, id=action_id) + + # ⚠ VALIDATED BEFORE ANYTHING IS WRITTEN, so a refusal leaves the automation exactly as it was. + checked, error = engine.clean_actions([fresh], rt=session.runtime) + if error or not checked: + raise err(400, "invalid_action", error or "that action could not be built") + + replaced = False + for i, one in enumerate(existing): + if isinstance(one, dict) and str(one.get("id") or "") == action_id: + existing[i], replaced = fresh, True + break + if not replaced: + existing.append(fresh) + + notes = [] + updated, error = engine.patch(session.runtime, str(auto_id), + {"flow": {**(defn.get("flow") or {}), "actions": existing}}, + username=session.uname, notes=notes) + if error: + raise err(400, "invalid_automation", error) + + def _set(cur): + cur = dict(cur or {}) + marks = dict(cur.get(str(auto_id)) or {}) + marks[action_id] = {"agent": agent_id, + "agentName": str(agent.get("label") or agent.get("channelName") + or agent_id), + "created": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "by": session.uname} + cur[str(auto_id)] = marks + return cur + + session.runtime.update(AGENT_ACTIONS_KEY, _set, flush="sync") + _prune_marks(session, auto_id) + return {"automation": _wire(updated, session.tenant, rt=session.runtime), + "actionId": action_id, "agent": agent_id, "notes": notes} + + @router.get("/automations/{auto_id}") def get_automation(auto_id: str, session: Session = Depends(_GATE)): """One automation, by id. @@ -1295,7 +1556,7 @@ def get_automation(auto_id: str, session: Session = Depends(_GATE)): defn = engine.all_definitions(session.runtime).get(str(auto_id)) if defn is None: raise err(404, "unknown_automation", "no automation with that id") - return {"automation": _wire(defn, session.tenant)} + return {"automation": _wire(defn, session.tenant, rt=session.runtime)} @router.post("/automations") @@ -1317,7 +1578,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend notes=_notes) if error: raise err(400, "invalid_automation", error) - return {"automation": _wire(defn, session.tenant), "notes": _notes} + return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} @router.patch("/automations/{auto_id}") @@ -1339,6 +1600,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None), return _patch_odoo_sync(session, body or {}) # D-75, same channel as `create` above — an EDIT is where this matters most, because the # person has just typed the thing that gets dropped. + # ⛔⛔ W36-T38 / R9 — THE AGENT-OWNED WALL, AT THE DOOR, BEFORE ANY WRITE. A `disabled` + # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the + # step and permits an ordinary save of a flow that merely contains one. + _agent_owned_guard(session, auto_id, body or {}) _notes = [] defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname, notes=_notes) @@ -1346,7 +1611,10 @@ def patch_automation(auto_id: str, body: dict = Body(default=None), raise err(400 if error != "no such automation" else 404, "invalid_automation" if error != "no such automation" else "unknown_automation", error) - return {"automation": _wire(defn, session.tenant), "notes": _notes} + # An action the user removed takes its annotation with it, or the mark outlives its subject + # and refuses an id nobody can see. + _prune_marks(session, auto_id) + return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} @router.delete("/automations/{auto_id}") @@ -1384,6 +1652,9 @@ def delete_automation(auto_id: str, session: Session = Depends(_GATE)): if _sys: raise err(409, "system_agent", engine.system_agent_refusal(_sys)) engine.remove(session.runtime, auto_id) + # R9's other half: the user MAY delete an agent-authored step, and deleting the whole + # automation takes its annotations with it rather than stranding them in the document. + _prune_marks(session, auto_id) return {"deleted": str(auto_id)} @@ -1439,7 +1710,7 @@ def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depend raise err(404 if error == "no such automation" else 400, "unknown_automation" if error == "no such automation" else "node_not_toggleable", error) - return {"automation": _wire(defn, session.tenant)} + return {"automation": _wire(defn, session.tenant, rt=session.runtime)} @router.post("/automations/{auto_id}/hook/{token}") diff --git a/api/routes_customers.py b/api/routes_customers.py index a8c69b255c7b87ca150dda49f227146fbe1aa732..5d1cb02b64e9415c1ede004189b052d452bf8bc6 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -141,6 +141,79 @@ def _measure_err(tag, e): pass +# ══════════════════════════ THE TENANT-WIDE STRATUM, ON THE CUSTOMER TOPIC (W38-T20 / D-425) ══ +# +# ⛔⛔ WHY THIS FILE GREW A SHARED STRATUM AT ALL. `core/shared_overlay.py` has been generic since +# W29-T62, `routes_products.py` has merged it since W30-T36 and `routes_tables.py` since W38-T16 — +# and the CUSTOMER topic had neither a write door nor a read merge. Every user-created column here +# lives in `data[username]`, so two accounts looking at "the same" column are looking at two +# columns. That is fine for a private note and fatal for a ROUTE ORDER: a visit sequence one rep +# can see and their colleague cannot is not a plan, it is a rumour. +# +# ⛔ ONE SPELLING OF THE BUCKET. `modules.customer_data.TABLE_KEY` is the per-user workspace key +# and `shared_overlay.bucket()` derives `__shared` from it. Resolving it here rather than +# writing the string means the write door and the read merge cannot disagree about where the +# values live — which is exactly the failure T16 found on the materialised `ut_*` tables, where +# `patch_shared_cell` wrote into a bucket no reader ever opened. + + +def _shared_key(): + """The store key this topic's per-user AND tenant-wide strata are both named from.""" + import modules.customer_data as cl + return cl.TABLE_KEY + + +def shared_fields(st=None): + """`{field_key: Field}` — the columns this topic shares tenant-wide. + + Unscoped on purpose, exactly as `shared_overlay.fields` is: a shared column's EXISTENCE is + tenant-wide by definition. WHO MAY SEE IT is a separate question, answered one layer up by + `perm_scope.hidden_keys` (the per-field grant wall T16 landed), and WHOSE ROWS by `cells`. + """ + from core import shared_overlay + try: + return shared_overlay.fields(_shared_key(), st=st) + except Exception: # noqa: BLE001 + # Lenient like every other display read: an unreachable store degrades to "nothing is + # shared yet", never to a 500 on a grid that would otherwise render. The WALL does not + # degrade with it — `field_grant_hidden` hides a marked column it cannot resolve. + return {} + + +def shared_cells(pids, st=None): + """`{"": {key: value}}` for the rows named by `pids`, and ONLY those. + + ⛔ `pids` IS THE ROW WALL, PASSED AND NEVER DEFAULTED. `shared_overlay.cells` refuses an + "everything" read by signature for this reason; the set handed in is the one `grid_assembly` + has already narrowed with `apply_row_scope`, so a cell belonging to the other BU has nothing + to attach itself to. + """ + from core import shared_overlay + try: + return shared_overlay.cells(_shared_key(), list(pids or ()), st=st) + except Exception: # noqa: BLE001 + return {} + + +def _merge_shared_fields(fields, defs): + """`fields` PLUS the tenant-wide columns this topic declares — `routes_tables._ut_shared_fields` + on the customer topic. + + ⚠ MERGED BEFORE THE WALL, NEVER AFTER. `hidden_keys` is a TRANSITIVE closure, so it must run + on the WHOLE contract: a formula over a shared column that reads a hidden one sits outside the + closure's reach otherwise and carries the hidden value out wearing a second name. It is also + the only order in which `field_grant_hidden` can ever see the `granted` marker at all — merge + afterwards and the per-field wall is inert while every test still passes. + ⚠ A key the canonical contract already declares WINS. A shared column is an ADDITION to this + database's contract, never a redefinition of a column it already has. + """ + if not defs: + return fields + have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} + return list(fields or ()) + [dict(f, source="overlay") + for k, f in defs.items() if k not in have] + + def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "", consume_corrections: bool = True): """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the @@ -183,29 +256,87 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # exactly this schema and 400s otherwise. `permits()` denies on anything it cannot answer. rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, aios_grid.FIELDS) pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) + # ⭐ W38-T20 — THE COLUMN DEFINITIONS ARE READ **ONCE** PER ASSEMBLY AND THREADED, because + # `core.store.get` deep-copies whatever it hands back on every call. Three consumers want this + # dict on one request (the write ctx's wall, the read merge, and the closure), and letting each + # take its own copy is the shape D-214 spent a whole ticket removing one document over. + _defs = shared_fields(st=rt) ws = grid_events.table_workspace( - _ctx_for(session, pids), allowed_pids=pids, + _ctx_for(session, pids, defs=_defs), allowed_pids=pids, consume_corrections=consume_corrections) + # ⭐⭐ W38-T20 / D-425 — THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE + # ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves + # `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same + # dict to `rows_from_pool`, so merging HERE reaches both without touching either file. + # + # ⚠ SAFE TO MUTATE, and checked rather than assumed (the same check `product_assembly` + # records): `table_workspace` reads through `store.get`, which deep-copies, so `ws` is a + # detached copy and nothing writes it back. A shared value can never leak INTO the per-user + # bucket by way of this merge. + # ⚠ SHARED WINS PER KEY. The whole point of the stratum is that every reader sees the same + # number, so a per-user leftover under the same key is stale by construction. It is also what + # makes D-423 recoverable rather than permanent: a pre-fix per-user edit is shadowed, not + # promoted. + _shared = shared_cells(pids, st=rt) + if _shared: + _ov = dict(ws.get("overlays") or {}) + for _pid, _cells in _shared.items(): + _ov[_pid] = {**(_ov.get(_pid) or {}), **_cells} + ws["overlays"] = _ov workspace, fields, views, lists = aios_grid.workspace_wire( ws, session.uname, set(pids), defs={}, scope_key=scope, storage_key=storage_key) + # ⭐⭐ W38-T20 — AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this + # position is load-bearing twice, once for the transitive closure and once because it is the + # only order in which the per-field grant marker is ever presented to `hidden_keys`. + fields = _merge_shared_fields(fields, _defs) # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so # shipping a dependent formula while withholding its input either leaks the input through # the formula's value or silently computes a wrong one; only removing both is coherent. # Applied AFTER workspace_wire because custom + measure columns are what it must cover. - hidden = perm_scope.hidden_keys(session.user, MODULE, fields) + # ⭐ W38-T20 — `st=rt` IS NOT TIDINESS. `perm_scope.field_grant_hidden` resolves a marked + # column against `object_shares`, and without a tenant handle it reads the module-default + # bucket: on any tenant but #0 that finds no grant, and no grant on a MARKED column means + # HIDDEN. So an unthreaded `st` UNDER-shares (a grantee cannot see their own column) rather + # than over-shares — visible and reportable, but still wrong, and `visible_fields`' own note + # requires the two calls to agree about it. + hidden = perm_scope.hidden_keys(session.user, MODULE, fields, st=rt) if hidden: fields = [f for f in fields if f.get("key") not in hidden] # The field LIST and the ROW payload are two different wires. Narrowing only the first # would leave the value sitting in the second, where anything can read it. rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] + # ⛔⛔ AND THE OVERLAY DICT IS A THIRD WIRE, WHICH IS NEW THIS TICKET AND WAS A HOLE + # BEFORE IT. `rows_from_pool` iterates `fields`, so a narrowed contract already keeps a + # hidden key off a ROW — but `/workspace` serves `ws["overlays"]` RAW, and that dict is + # where both the per-user cells and (as of the merge above) the tenant-wide ones sit. + # One narrowing cannot speak for a wire it never touches; `routes_tables` writes the same + # sentence over its own `shared_cells`, and `routes_odoo_tables` over its `overlays`. + # ⚠ `ws` AND NOT `workspace`: `routes_grid.workspace` copies the dict ACROSS + # (`workspace["overlays"] = g["ws"].get("overlays")`) after this returns, so narrowing the + # source is what reaches the wire. Narrowing the copy would be narrowing a key that gets + # overwritten a moment later. + _ov = ws.get("overlays") or {} + if _ov: + ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden} + for pid, cells in _ov.items()} today = time.strftime("%Y-%m-%d") stamp = _pool_stamp(rt, team_id, agent) - measures = measure_resolve.offer(team_id, on_error=_measure_err) + # ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE + # OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their + # condition answers through helpers that short-circuit on an empty `offer`, so emptying the + # offer there stops the whole feature. `core.measure_resolve` does not take an offer at all: + # `condition_sets` and `column_values` both re-derive their work from the caller's OWN saved + # views and field list. So gating only the offer here would take the Metric kind off the + # picker and refuse new creates while an EXISTING Metric column kept computing and an + # EXISTING measure condition kept resolving — a revoked capability still answering, on the + # grain with the most of them. Three calls, one predicate. + may_metrics = perm_scope.may_metrics(session.user, MODULE) + measures = measure_resolve.offer(team_id, on_error=_measure_err) if may_metrics else [] measure_sets = measure_resolve.condition_sets( [v.get("config") or {} for v in (views or [])], None, team_id, pids, today, stamp, - rt.mset_memo, on_error=_measure_err) + rt.mset_memo, on_error=_measure_err) if may_metrics else {} # The derived channel: cohort membership cells + measure column values, ONE dict — the # same read-only channel the embed host hands to rows_from_pool. derived = aios_grid.cohort_cells(lists) @@ -220,9 +351,17 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # this — a value the host knows per render and the pool has no business storing. for pid in pids: derived.setdefault(pid, {})["partner_id"] = pid - for pid, cells in measure_resolve.column_values( + # ⭐⭐ W38-T19 — SKIPPED ENTIRELY WHEN THE CAPABILITY IS REVOKED, rather than filtered after. + # `column_values` selects its own subjects (`isinstance(f.get('measure'), dict)`) off the + # field list, so there is no argument that could narrow it; not calling it is the narrowing. + # ⚠ THE COLUMN STAYS AND ITS CELLS GO BLANK, which is this channel's OWN documented degrade + # ("blank is could not compute, 0 is a real zero") and is what the product and user-table + # grains already do under an empty offer. Deleting the column instead would be a second, + # louder behaviour for the same fact on one surface out of three, and it would destroy a + # definition the admin can restore with one tick. + for pid, cells in (measure_resolve.column_values( fields, team_id, pids, today, stamp, rt.measure_memo, - on_error=_measure_err).items(): + on_error=_measure_err) if may_metrics else {}).items(): derived.setdefault(pid, {}).update(cells) return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, @@ -271,30 +410,48 @@ def _payload(session: Session): "pulled_at": time.strftime("%Y-%m-%d %H:%M")} -def _ctx_for(session: Session, pids): +def _ctx_for(session: Session, pids, defs=None): """An EventCtx for the READ path — no fallback workspace, so a store outage is a 503 rather - than a phantom in-memory workspace an API request cannot persist.""" + than a phantom in-memory workspace an API request cannot persist. + + `defs` (W38-T20) is this topic's tenant-wide column definitions when the caller already holds + them; absent, they are read. One read per assembly rather than one per question asked of it. + """ from core import grid_events return grid_events.EventCtx( uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[], - # C-PERM: the write wall's field half. Computed from the CANONICAL contract because - # `fields=[]` here — the closure only needs the schema, not this user's column list. - hidden_keys=_hidden_for(session), + # C-PERM: the write wall's field half. Computed from the CANONICAL contract PLUS the + # tenant-wide columns, because `fields=[]` here — the closure only needs the schema, not + # this user's column list, and a runtime column is part of that schema now. + hidden_keys=_hidden_for(session, defs=defs), admin=session.admin, fallback_ws=None, seen_ids={}) -def _hidden_for(session: Session): +def _hidden_for(session: Session, defs=None): """The fields this session's permissions hide — the write wall's half of C-PERM. Read paths strip these from both wires so they cannot be SEEN; this is what stops them being WRITTEN by a caller who knows the key. Evaluated against the canonical contract, the same schema `routes_admin._clean_perms` validates a hiddenFields entry against. + + ⭐⭐ W38-T20 — AND THE TENANT-WIDE COLUMNS ARE PART OF THAT CONTRACT NOW, WHICH IS A WALL AND + NOT A COMPLETENESS TIDY-UP. `patch_customer` builds its ctx with `fields=payload["fields"]`, + which carries the merged shared columns, so `grid_events.handle_one` would happily accept an + `overlay_patch` naming one. The canonical list cannot mention them (they are created at + runtime), so a wall computed from `aios_grid.FIELDS` alone answers "not hidden" for every + grant-governed column and the write door is open to a reader who was never granted it. + ⚠ Widening the field list can only ever ADD to the hidden set, never remove from it: the + closure hides what it is told to hide plus whatever depends on it. """ import aios_grid import core.perm_scope as perm_scope - return perm_scope.hidden_keys(session.user, MODULE, aios_grid.FIELDS) + if defs is None: + defs = shared_fields(st=session.runtime) + return perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), defs), + st=session.runtime) def allowed_pids(session: Session): """The pids this session may touch — the POOL's own ids, so the write wall and the read scope @@ -374,3 +531,304 @@ def patch_customer(pid: int, body: dict = Body(default=None), if refused: out["refused"] = refused return out + + +# ══════════════════════════════════ THE ROUTE-ORDER COLUMN (W38-T20 / ruling R7 / contract C1) ══ +# +# ⛔⛔ WHY THIS IS A DOOR HERE AND NOT A KIND IN THE COLUMN MENU. `ColumnMenu.onCreate` is the only +# persistence the menu has, and it lands a PER-USER `custom_` column through +# `aios_grid._field_extras`, which is a strict allowlist: a route-order bag created that way is +# silently stripped and its values are private to their author. That is the wave-19 `image` +# failure verbatim ("created, named, configured, gone", recorded in `_clean_geocode`'s own +# docstring) plus a done-when clause that cannot hold, because a colleague reading a per-user +# stratum gets a clean 200 with nothing in it. The `geocode` pseudo-kind is the precedent for the +# PICKER; its persistence half does not transfer. +# +# ⭐ SO THE COLUMN IS BORN SHARED. `shared_overlay.put_field` stores the definition verbatim (no +# allowlist), which is what lets the input fingerprint ride the DEFINITION rather than the rows — +# `shared_overlay._value` RAISES on a dict, so `{order, inputsHash}` could never be one cell, and +# one solve fingerprints the whole cohort identically anyway, so per-row would be the same string +# written N times. +# +# ⭐ AND THE NUMBER ON THE RECORD IS THE INVERSE OF THE PLANNER'S ANSWER. `mapProjection.planRoute` +# returns `order`, where `order[i]` is WHICH STOP is visited i-th; the cell holds the RANK. The +# client inverts it with `routeRanks` (gated in `map.test.ts` at the desktop shape, over a fixture +# chosen so the two differ); this door then refuses anything that is not a clean 1..N, so a +# truncated or double-posted body cannot land as a half-route. + +#: The key prefix every route-order column wears. `custom_` and `measure_` are the two existing +#: created-column namespaces and both are PER USER; this one is tenant-wide, so it takes its own +#: rather than borrowing a prefix whose readers assume a per-user home. +ROUTE_KEY_PREFIX = "route_" + +#: The marker on the stored definition that says WHAT this column is. Read by the GET below and by +#: the client; never inferred from the key, because a prefix is a naming convention and a +#: convention is not a declaration. +ROUTE_KIND = "route_order" + +#: ⛔⛔ THE TOPIC A FIELD GRANT IS NAMED UNDER, AND IT IS **NOT** THE STORE BUCKET. Two namespaces +#: meet on this column and they are spelled differently: +#: +#: the STRATUM lives at `shared_overlay.bucket(customer_data.TABLE_KEY)` +#: = `customer_table_workspace__shared` +#: the GRANT is named `shares.field_oid(, key)` +#: = `customer_data:` +#: +#: `perm_scope.hidden_keys(user, module, fields)` hands its `module` argument straight through to +#: `field_grant_hidden`, which builds the oid from it. On a `ut_*` database the module argument IS +#: the table key, so W38-T16 never had to tell them apart; on a REGISTRY topic they differ, and a +#: door that claims the grant under the bucket name writes a record the wall will never look for. +#: MEASURED, not reasoned: the first run of this ticket's gate did exactly that, and user B was +#: refused a column that had been shared with them through the real share door, with a 200 at +#: every step ([[one-question-two-normalizers]]). +SHARE_TOPIC = MODULE + + +def _route_slug(label): + """A stable store key from a human label. Lower case, non-alphanumerics collapsed to `_`.""" + import re as _re + slug = _re.sub(r"[^a-z0-9]+", "_", str(label or "").strip().lower()).strip("_") + return f"{ROUTE_KEY_PREFIX}{slug[:48]}" if slug else "" + + +def _route_defs(session: Session): + """This topic's route-order columns MINUS the ones this session was not granted. + + ⛔ THE WALL IS THE SAME ONE THE GRID USES, NOT A SECOND OPINION. `hidden_keys` is where T16 + put the per-field grant check, so filtering on it here means the listing, the grid contract + and the row payload agree by construction. A column a reader cannot see must not appear here + either: the done-when says *does not see the field at all*, and a picker that names a column + whose values are withheld has already leaked its existence. + """ + import aios_grid + import core.perm_scope as perm_scope + + all_defs = shared_fields(st=session.runtime) or {} + defs = {k: v for k, v in all_defs.items() + if isinstance(v, dict) and v.get("kind") == ROUTE_KIND} + if not defs: + return {} + hide = perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs), + st=session.runtime) + return {k: v for k, v in defs.items() if k not in hide} + + +@router.get("/customers/route-order") +def route_order_list(session: Session = Depends(module_gate(MODULE))): + """The route-order columns this session may see, with the fingerprint each was solved from. + + ⭐⭐ THE FINGERPRINT IS WHY THIS ROUTE EXISTS RATHER THAN THE CLIENT READING `fields`. + Staleness is DERIVED, never stored: what is written down is the INPUT FINGERPRINT the numbers + were produced from, so *is this order still current* is a question asked at READ time against + what is on screen NOW, and can never itself be out of date. A stored `stale: true` is a fact + about a moment that has already passed. + """ + out = [] + for key, defn in sorted(_route_defs(session).items()): + route = defn.get("route") if isinstance(defn.get("route"), dict) else {} + out.append({ + "key": key, + "label": defn.get("label") or key, + "inputsHash": str(route.get("inputsHash") or ""), + "roundTrip": bool(route.get("roundTrip")), + "startPid": route.get("startPid"), + "stops": route.get("stops"), + "solvedAt": route.get("solvedAt") or "", + "solvedBy": defn.get("createdBy") or "", + # Who may re-solve it. The same creator-or-admin wall the write door enforces, said on + # the way out so the client can grey the control instead of discovering a 403. + "mine": bool(session.admin + or str(defn.get("createdBy") or "") == session.uname), + }) + return {"fields": out} + + +@router.post("/customers/route-order") +def route_order_write(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Create (or re-solve) a tenant-wide route-order column and fill it, in ONE call. + + `{label, field?, ranks: {"": }, inputsHash, roundTrip?, startPid?, stops?}` + + ⛔ THE DEFINITION IS WRITTEN FIRST AND THE GRANT CLAIMED SECOND, which is `patch_shared_cell`'s + order and it is deliberate: the window where a column is MARKED and UNCLAIMED fails CLOSED + (governed, nobody granted, so only an admin and the creator see it, and an admin can share + it). The other order would leave a grant record pointing at nothing. + + ⛔ RE-SOLVING IS CREATOR-OR-ADMIN, WHICH CREATING IS NOT. Writing these numbers changes what + every account in the workspace reads, at once, for the whole cohort. That is the wall + `routes_tables.delete_shared_field` already applies to the destructive half of this stratum, + and a new door does not get to inherit the loose half of an asymmetry somebody has flagged. + A permitted teammate READS the numbers; they do not silently re-plan somebody's day. + """ + from core import shared_overlay + import core.perm_scope as perm_scope + from routes_grid import MAX_BULK_ROWS + + body = body if isinstance(body, dict) else {} + label = " ".join(str(body.get("label") or "").split())[:120] + key = str(body.get("field") or "").strip() or _route_slug(label) + if not key: + raise err(400, "bad_request", "a name is required for the route order column") + if not key.startswith(ROUTE_KEY_PREFIX): + raise err(400, "bad_field_key", + f"a route order column's key starts with '{ROUTE_KEY_PREFIX}', so it cannot " + f"collide with a column this database already owns") + import aios_grid + if key in {f.get("key") for f in aios_grid.FIELDS}: + raise err(400, "field_key_taken", + "this database already has a column with that key") + + defs = shared_fields(st=session.runtime) or {} + existing = defs.get(key) if isinstance(defs.get(key), dict) else None + if existing is not None: + # ⛔⛔ A REFUSAL MUST NOT DESCRIBE A COLUMN THE CALLER CANNOT SEE. The two refusals below + # name the column's KIND and its CREATOR, which is exactly the information the field wall + # exists to withhold: a stranger who guesses the key would otherwise learn that a route + # order exists on this database and who planned it. `routes_shares._can_see_object` makes + # the same choice for the same reason (a non-grantee gets the answer a non-existent id + # gets). The name is already taken either way, so the honest refusal says only that. + if key in perm_scope.hidden_keys( + session.user, MODULE, + _merge_shared_fields(list(aios_grid.FIELDS), defs), st=session.runtime): + raise err(400, "field_key_taken", + "that column name is already in use on this database") + if existing.get("kind") != ROUTE_KIND: + raise err(400, "not_a_route_column", + "that column is shared but it is not a route order column, so re-solving " + "it would overwrite values this door did not write") + owner = str(existing.get("createdBy") or "") + if not session.admin and owner != session.uname: + raise err(403, "forbidden", + f"a route order can be re-solved by the person who planned it or by an " + f"administrator. This one was planned by {owner or 'somebody else'}, and " + f"re-solving it would change the visit numbers for every account at once") + + raw = body.get("ranks") + if not isinstance(raw, dict) or not raw: + raise err(400, "bad_ranks", 'expected {ranks: {"": }}') + # ⛔ REPORTED, NEVER TRUNCATED (standing rule 1's second sentence, and `MAX_BULK_ROWS`' own + # note). The ceiling is `routes_grid`'s so there is ONE of them, not two that drift. + if len(raw) > MAX_BULK_ROWS: + raise err(400, "too_many_rows", + f"at most {MAX_BULK_ROWS} records per route; this one carried {len(raw)}") + + pool = allowed_pids(session) + ranks, not_in_pool, bad_value = {}, [], [] + for raw_pid, value in raw.items(): + try: + pid = int(raw_pid) + except (TypeError, ValueError): + not_in_pool.append(str(raw_pid)[:40]) + continue + # ⚠ THE POOL IS THE WALL, and it is the SAME predicate the read path applies + # (`apply_row_scope` inside `allowed_pids`), so a caller cannot number a record they + # could not be shown, including one in the other business unit. + if pid not in pool: + not_in_pool.append(str(raw_pid)[:40]) + continue + # ⛔ A BOOL IS AN `int` IN PYTHON and `True` would store as a visit number 1. Excluded by + # name rather than by hoping nobody sends one. + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + bad_value.append(str(raw_pid)[:40]) + continue + ranks[pid] = value + + # ⛔⛔ A PARTLY HONOURED ROUTE IS NOT A ROUTE, AND THIS REFUSES THE WHOLE CALL RATHER THAN + # WRITING THE PART IT COULD. Caught by this ticket's own gate: a body carrying one record + # outside the caller's book still produced a clean 1..N over what was left, so the door minted + # a permanent tenant-wide column and filled it with a SHORTER route than the one that was + # solved. Every number in it was plausible and the day was wrong. + # ⚠ REPORTED, WITH THE SAMPLE, which is standing rule 1's second sentence: the refusal names + # what it could not take, so the caller can fix it rather than guess. + if not_in_pool: + raise err(400, "rows_not_in_your_book", + f"{len(not_in_pool)} of those records are not in your book, so the route " + f"cannot be written as it was solved. First few: " + f"{', '.join(sorted(not_in_pool)[:5])}") + if bad_value: + raise err(400, "rows_not_a_visit_number", + f"a visit number is a whole number from 1 upwards; {len(bad_value)} records " + f"carried something else. First few: {', '.join(sorted(bad_value)[:5])}") + if not ranks: + raise err(400, "no_rows_in_your_book", + "none of those records are in your book, so there is nothing to number") + # ⛔ A ROUTE IS A SEQUENCE, SO THE NUMBERS ARE 1..N WITH NO REPEAT AND NO GAP. A body that + # arrives truncated, doubled or partly applied would otherwise land as a plausible half + # route: every row carrying a number, and the day in the wrong order. + seq = sorted(ranks.values()) + if seq != list(range(1, len(seq) + 1)): + raise err(400, "not_a_sequence", + f"a route order is the numbers 1 to {len(seq)}, each used once. This one " + f"carried {len(seq)} records numbered up to {seq[-1]} with " + f"{len(seq) - len(set(seq))} repeated") + + stamp = time.strftime("%Y-%m-%d %H:%M") + defn = { + "key": key, + "label": label or (existing or {}).get("label") or key, + # ⛔ `int`, NEVER `text`. A text column sorts 1, 10, 11, 2 — fully populated, entirely + # plausible, wrong, and named by no gate. `patch_shared_cell` defaults to text; this door + # cannot. + "type": "int", + "source": "overlay", + "shared": True, + "kind": ROUTE_KIND, + # ⭐⭐ THE PER-FIELD GRANT MARKER (T16). It is an EXPLICIT write-once declaration and it is + # what makes the wall fail CLOSED: a grant wall has the opposite absence-polarity to a + # deny wall, so keying visibility on "does a grant record exist" would publish this column + # to the whole tenant on one unreadable read of `object_shares`. Stamped, never inferred. + perm_scope.FIELD_GRANT_MARK: True, + "createdBy": (existing or {}).get("createdBy") or session.uname, + # ⭐ THE FINGERPRINT RIDES THE DEFINITION, ONCE. One `planRoute` run solves the whole + # cohort, so this string is identical for every record in it; per row it would be the same + # value written N times, and `shared_overlay._value` raises on a dict anyway. + "route": { + "inputsHash": str(body.get("inputsHash") or "")[:64], + "roundTrip": bool(body.get("roundTrip")), + "startPid": int(body["startPid"]) if isinstance(body.get("startPid"), int) + and not isinstance(body.get("startPid"), bool) else None, + "stops": len(ranks), + "solvedAt": stamp, + }, + } + shared_overlay.put_field(_shared_key(), key, defn, st=session.runtime) + if existing is None: + try: + import core.shares as shares + # ⚠ THE EMPTY ENTRY LIST IS THE POINT. `set_grants` keeps a record with an owner and + # no entries, so "shared with nobody" is STORED and is a different fact from "never + # shared". Without the owner the column is unmanageable: `may_administer` fails closed + # on an ownerless record, so nobody could ever share it. + shares.set_grants("field", shares.field_oid(SHARE_TOPIC, key), [], + owner=session.uname, st=session.runtime) + except Exception: # noqa: BLE001 + # The mark is already written, so a failed claim fails CLOSED: governed, nobody + # granted, admin-and-creator only. Recoverable. The other order is not. + pass + + # ⛔ AND THE RECORDS THAT LOST THEIR NUMBER ARE CLEARED. A re-solve over a SMALLER cohort + # would otherwise leave the previous run's ranks sitting on the records that dropped out — + # plausible integers, from a route nobody is driving. `""` is the house spelling of an empty + # overlay cell (`rows_from_pool` defaults an absent one to exactly that), so this needs no new + # vocabulary and no tombstone nobody else reads. + stale = {} + for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): + if not isinstance(cells, dict) or cells.get(key) in (None, ""): + continue + try: + gone = int(raw_pid) + except (TypeError, ValueError): + continue + if gone not in ranks: + stale[gone] = {key: ""} + written = shared_overlay.put_rows( + _shared_key(), {**{p: {key: v} for p, v in ranks.items()}, **stale}, + st=session.runtime) + + out = {"ok": True, "field": key, "label": defn["label"], "type": "int", + "stops": len(ranks), "cleared": len(stale), + "rows_written": len(written), "inputsHash": defn["route"]["inputsHash"], + "solvedAt": stamp} + return out diff --git a/api/routes_geo.py b/api/routes_geo.py new file mode 100644 index 0000000000000000000000000000000000000000..c0b7d2c67d135350f881a36d466f9b767ad34824 --- /dev/null +++ b/api/routes_geo.py @@ -0,0 +1,369 @@ +"""routes_geo.py — the MAP PROVIDER SEAM (wave 37, R3/R14, tickets T35/T36/T37). + +⭐ WHY THIS FILE EXISTS AT ALL, and why the map does not simply call a vendor. + +R14 (owner, 2026-08-19): no Google Maps API key this wave. The map work is not dropped, it is +retargeted to providers that need no key: OpenStreetMap tiles, OSRM routing, Nominatim geocoding. +The deferral only holds if swapping a paid vendor back in is a CONFIG change rather than a rewrite, +so every provider is a (base URL + attribution) pair read from the environment and served from ONE +door. The renderer receives a URL template and a credit line; it knows nothing about who is behind +them. That is the seam. + + tiles AIOS_MAP_TILE_URL default https://tile.openstreetmap.org/{z}/{x}/{y}.png + geocode AIOS_GEOCODE_URL default https://nominatim.openstreetmap.org/search + routing AIOS_ROUTE_URL default https://router.project-osrm.org/route/v1/driving/ + +⛔ R3'S ON-DEMAND RULE, AND WHY IT IS STRUCTURAL HERE RATHER THAN A COMMENT. +R3 forbids geocoding a whole table automatically. Under R14 the reason moves from money to manners +and gets sharper: Nominatim's usage policy allows roughly one request per second and explicitly +forbids bulk harvesting, so a loop over 3,636 customers does not produce a bill, it produces a +BLOCKED tenant. Three mechanisms enforce it, none of them advisory: + + 1. `/geo/geocode` takes an explicit LIST OF ADDRESSES from the caller. It cannot read a table, so + there is no code path from "a database exists" to "its rows were geocoded". Somebody had to + choose the records. + 2. `MAX_BATCH` refuses an oversized list with a named error rather than truncating it. The client + drives the loop and can therefore SHOW the wait, which is the half of the ticket a comment + cannot satisfy. + 3. `_throttle()` blocks in-process until the minimum interval has elapsed, so even a caller that + ignores everything above cannot exceed the published rate. + +⭐ AND A CACHE, because the policy asks for one: an address geocoded once is answered from memory +for the rest of the container's life. It is the cheapest way to be a good citizen and it makes a +re-run of the same records free rather than merely legal. + +⚠ NOTHING HERE PROXIES TILES. Tiles are fetched by the BROWSER, straight from the provider, which +is what every OSM client does and what the tile policy expects. Proxying them through this app +would put a free tier in the path of every pan and would breach the same policy it looks like it is +respecting. This door serves the tile URL, never the tile. +""" +import os +import re +import threading +import time +from collections import OrderedDict + +import requests +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +# --------------------------------------------------------------------- the seam +# +# Every value below is an environment override with a keyless default. Pointing the map at a paid +# vendor is three variables on the Space and no code change, which is what makes R14 a deferral. + +_TILE_URL_DEFAULT = "https://tile.openstreetmap.org/{z}/{x}/{y}.png" +_TILE_CREDIT_DEFAULT = "© OpenStreetMap contributors" +_TILE_CREDIT_HREF_DEFAULT = "https://www.openstreetmap.org/copyright" +_GEOCODE_URL_DEFAULT = "https://nominatim.openstreetmap.org/search" +_ROUTE_URL_DEFAULT = "https://router.project-osrm.org/route/v1/driving/" + +#: Both policies require a real, identifying User-Agent. A generic one is how a shared service +#: decides an application is a scraper, so this carries the product name and a contact URL. +_UA_DEFAULT = "AIOS-Loopable/1.0 (+https://runloopable.com)" + +#: Nominatim publishes one request per second. 1100 ms leaves room for clock jitter rather than +#: sitting exactly on the published edge. +_MIN_INTERVAL_MS_DEFAULT = 1100 + +#: The most addresses one call may carry. Small on purpose: the client loops and shows progress, +#: and no single request can sit on the connection for a minute waiting out the throttle. +MAX_BATCH = 25 + +#: Tile zoom the provider actually serves. OSM stops at 19. +_TILE_MAX_Z_DEFAULT = 19 + +_CACHE_MAX = 4096 + + +def _env(name, default): + return (os.environ.get(name) or "").strip() or default + + +def _env_int(name, default): + try: + n = int((os.environ.get(name) or "").strip()) + return n if n > 0 else default + except (TypeError, ValueError): + return default + + +def provider_config(): + """The whole seam as one dict. `GET /geo/providers` serves it and the gate reads it.""" + return { + "tiles": { + "url": _env("AIOS_MAP_TILE_URL", _TILE_URL_DEFAULT), + "attribution": _env("AIOS_MAP_TILE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), + "attributionUrl": _env("AIOS_MAP_TILE_ATTRIBUTION_URL", _TILE_CREDIT_HREF_DEFAULT), + "maxZoom": _env_int("AIOS_MAP_TILE_MAX_Z", _TILE_MAX_Z_DEFAULT), + }, + "geocode": { + "available": True, + "maxBatch": MAX_BATCH, + "minIntervalMs": _env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT), + # ⭐⭐ THE NUMBER THE SURFACE MUST QUOTE, AND IT IS NOT THE RATE LIMIT. + # T37 asks for the estimated wait to be shown BEFORE a run. The obvious source is + # `minIntervalMs`, and it is wrong: 1.1 s is a floor on POLITENESS, not a prediction of + # LATENCY. One real lookup was MEASURED at 3.28 s end to end on 2026-08-19. Quoting the + # limit would promise a minute and take three, and a progress estimate that runs out + # before the work does reads as a hang rather than as a wait. + "secondsPerAddress": float(_env("AIOS_GEOCODE_SECONDS_EACH", "3.3")), + "attribution": _env("AIOS_GEOCODE_ATTRIBUTION", _TILE_CREDIT_DEFAULT), + }, + "route": { + "available": True, + "attribution": _env("AIOS_ROUTE_ATTRIBUTION", "Routing by OSRM"), + }, + } + + +# ------------------------------------------------------------------ the throttle +# +# ⚠ ONE lock and ONE timestamp for the whole process, shared by geocoding and routing, because the +# rate limit belongs to the SERVICE and not to the endpoint. Two independent throttles would each +# stay honest and together double the published rate. + +_gate = threading.Lock() +_last_call = [0.0] + + +def _throttle(min_interval_ms): + """Block until the minimum interval since the last outbound call has elapsed. + + Returns the seconds actually waited, so a caller can report the wait rather than hide it.""" + wait = 0.0 + with _gate: + gap = min_interval_ms / 1000.0 + now = time.monotonic() + due = _last_call[0] + gap + if now < due: + wait = due - now + time.sleep(wait) + _last_call[0] = time.monotonic() + return wait + + +# --------------------------------------------------------------------- the cache + +_cache = OrderedDict() +_cache_lock = threading.Lock() + + +def _cache_key(q, country): + return (re.sub(r"\s+", " ", str(q or "")).strip().lower(), str(country or "").strip().lower()) + + +def _cache_get(key): + with _cache_lock: + if key not in _cache: + return None + _cache.move_to_end(key) + return _cache[key] + + +def _cache_put(key, value): + with _cache_lock: + _cache[key] = value + _cache.move_to_end(key) + while len(_cache) > _CACHE_MAX: + _cache.popitem(last=False) + + +def geocode_one(address, country=None, timeout=12): + """One address to {lat, lon, label} or None. + + ⛔ THE ONLY PLACE AN OUTBOUND GEOCODE HAPPENS. `ai_enrich.py`'s geocode field calls this rather + than reaching for `requests` itself, so the throttle and the cache cannot be walked around by a + second caller. A cache hit costs no request and no wait.""" + q = re.sub(r"\s+", " ", str(address or "")).strip() + if not q: + return None + key = _cache_key(q, country) + hit = _cache_get(key) + if hit is not None: + return dict(hit) if hit else None + + cfg = provider_config()["geocode"] + _throttle(cfg["minIntervalMs"]) + params = {"q": q, "format": "jsonv2", "limit": 1} + if country: + params["countrycodes"] = str(country).strip().lower() + try: + r = requests.get( + _env("AIOS_GEOCODE_URL", _GEOCODE_URL_DEFAULT), + params=params, + headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT), + "Accept": "application/json"}, + timeout=timeout, + ) + r.raise_for_status() + rows = r.json() + except Exception: + # ⚠ A transport failure is NOT cached. Caching it would turn one bad minute into a + # permanently empty column that no re-run could ever repair. + return None + if not isinstance(rows, list) or not rows: + # A genuine "no such place" IS cached, as a negative: asking again gets the same answer and + # spends another second of a shared service's budget to hear it. + _cache_put(key, {}) + return None + top = rows[0] or {} + try: + lat = float(top.get("lat")) + lon = float(top.get("lon")) + except (TypeError, ValueError): + _cache_put(key, {}) + return None + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + _cache_put(key, {}) + return None + out = {"lat": lat, "lon": lon, "label": str(top.get("display_name") or q)} + _cache_put(key, out) + return dict(out) + + +# ---------------------------------------------------------------------- the doors + + +@router.get("/geo/providers") +def geo_providers(session: Session = Depends(require_session)): + """What the map should draw with, and who to credit for it. + + Session gated like every other door here. The values are not secret, but an unauthenticated + endpoint on this app is a door somebody eventually hangs something else on.""" + return provider_config() + + +@router.post("/geo/geocode") +def geo_geocode(body: dict = Body(...), session: Session = Depends(require_session)): + """Turn a bounded list of addresses into coordinates. + + ⛔ Takes ADDRESSES, never a table key, a view id or a filter. There is deliberately no shape of + request that means "geocode everything", which is R3 expressed as an API rather than as a + warning.""" + items = body.get("addresses") + # ⭐ THE SECOND SHAPE, AND IT EXISTS TO KEEP ONE COMPOSER. A caller may send whole RECORDS plus + # the geocode field's config instead of finished address strings, and the server builds the + # query with `ai_enrich.geocode_query`. The alternative was a TypeScript twin of that function + # on the client, and an address composer that exists twice will disagree the first time either + # copy learns about a new column ([[one-question-two-normalizers]]) -- which is exactly the trap + # B's measured `New York (US)` suffix would spring, silently, in whichever copy forgot. + if items is None: + records = body.get("records") + cfg = body.get("config") or {} + if isinstance(records, list) and records: + import ai_enrich + items = [] + for rec in records: + if not isinstance(rec, dict): + continue + q = ai_enrich.geocode_query(rec.get("row") or {}, cfg) + # ⛔ A record with no usable address is REPORTED, not dropped: it comes back + # `found: false` with an empty address, so the caller can name it on screen instead + # of quietly returning fewer answers than it asked questions. + items.append({"key": rec.get("key"), "address": q}) + if not body.get("country") and cfg.get("country"): + body = {**body, "country": cfg.get("country")} + if not isinstance(items, list) or not items: + raise err(400, "no_addresses", "Send at least one address to look up.") + if len(items) > MAX_BATCH: + raise err( + 400, + "batch_too_large", + f"Look up at most {MAX_BATCH} addresses per request. " + f"The map sends them in batches of {MAX_BATCH} so the wait stays visible.", + ) + country = body.get("country") + out = [] + started = time.monotonic() + for raw in items: + if isinstance(raw, dict): + key, addr = raw.get("key"), raw.get("address") + else: + key, addr = None, raw + # ⛔ AN EMPTY QUERY COSTS NO REQUEST AND CARRIES ITS OWN REASON. Nominatim's policy is a + # budget shared with everyone else using it, and asking it to place "" would spend a second + # of that budget to be told nothing. `reason` is what lets the surface say "no address on + # file" rather than "not found", which are different facts and want different actions. + if not str(addr or "").strip(): + out.append({"key": key, "address": "", "found": False, "lat": None, "lon": None, + "label": None, "reason": "no_address"}) + continue + hit = geocode_one(addr, country=country) + out.append({"key": key, "address": str(addr or ""), "found": bool(hit), + "reason": None if hit else "not_found", + "lat": hit["lat"] if hit else None, + "lon": hit["lon"] if hit else None, + "label": hit["label"] if hit else None}) + elapsed = time.monotonic() - started + return {"results": out, + "found": sum(1 for r in out if r["found"]), + "asked": len(out), + "secondsElapsed": round(elapsed, 2), + "attribution": provider_config()["geocode"]["attribution"]} + + +@router.post("/geo/route") +def geo_route(body: dict = Body(...), session: Session = Depends(require_session)): + """Road distance, duration and the drawn line, for stops ALREADY put in order. + + ⭐ THE ORDERING IS NOT DONE HERE. `mapProjection.planRoute` sequences the stops on the client, + for free, and keeps working when this service does not answer. This door adds the half that + arithmetic cannot produce: what the ROADS actually cost. Splitting it that way is why the Start + picker and the round-trip toggle keep working with no network at all.""" + stops = body.get("stops") + if not isinstance(stops, list) or len(stops) < 2: + raise err(400, "too_few_stops", "A route needs at least two stops with a location.") + if len(stops) > 25: + raise err(400, "too_many_stops", + "Route at most 25 stops at once. Narrow the selection and try again.") + pairs = [] + for s in stops: + try: + lat, lon = float(s["lat"]), float(s["lon"]) + except (TypeError, ValueError, KeyError, IndexError): + raise err(400, "bad_stop", "Every stop needs a numeric latitude and longitude.") + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + raise err(400, "bad_stop", "Every stop needs a latitude and longitude on the globe.") + pairs.append(f"{lon:.6f},{lat:.6f}") + + base = _env("AIOS_ROUTE_URL", _ROUTE_URL_DEFAULT) + _throttle(_env_int("AIOS_GEOCODE_MIN_INTERVAL_MS", _MIN_INTERVAL_MS_DEFAULT)) + try: + r = requests.get( + base.rstrip("/") + "/" + ";".join(pairs), + params={"overview": "simplified", "geometries": "geojson", "steps": "false"}, + headers={"User-Agent": _env("AIOS_GEO_USER_AGENT", _UA_DEFAULT)}, + timeout=20, + ) + r.raise_for_status() + data = r.json() + except Exception: + raise err(502, "route_service_unavailable", + "The routing service did not answer. The stop order and the straight line " + "distance are still on the map.") + routes = (data or {}).get("routes") or [] + if not routes: + raise err( + 502, + "no_route", + "No road route connects these stops. They may be on different land masses, or one of " + "them may be far from any road.", + ) + top = routes[0] + geom = ((top.get("geometry") or {}).get("coordinates")) or [] + line = [] + for c in geom: + try: + line.append([float(c[0]), float(c[1])]) + except (TypeError, ValueError, IndexError): + continue + return { + "km": round(float(top.get("distance") or 0.0) / 1000.0, 1), + "minutes": int(round(float(top.get("duration") or 0.0) / 60.0)), + "line": line, + "attribution": provider_config()["route"]["attribution"], + } diff --git a/api/routes_nav.py b/api/routes_nav.py index 388473b4c8090629c965dbe8ef85b545a9dc46c7..25a055836ca0c8308102829e514ca499e2f079e1 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -494,8 +494,37 @@ def nav(session: Session = Depends(require_session)): # Manage user. An administrator unticking a shared database left it on the rail. # `nav_may_open` composes the two and cannot WIDEN past `may_open` on a `ut_*` key. import core.perm_scope as _perm_scope + # ⭐⭐ WAVE 37 · T07 (D-267, PRD amendment A8) — A RETIRED KEY CANNOT REACH THE NAV, WHATEVER + # THE TENANT DOCUMENT STILL CARRIES. + # + # W33-T44 retired `ut_odoo_customers` / `ut_odoo_products` in favour of `customer_data` / + # `product_data` — ONE SUBJECT, ONE DATABASE — and `apply_plan` pops them from a tenant's + # document on the next rebuild. The owner then reported *"Odoo products"* TWICE, and a live + # label census returned `{'Odoo products': 2}`: the retirement MECHANISM was correct and the + # PATH DID NOT RUN, for causes booked separately (D-251's `frozen(rt)` early return, and the + # rebuild being scoped to `is_royal`). + # + # ⛔ SO THIS IS A READ-SIDE GUARD, NOT THE MIGRATION, AND THE DIFFERENCE IS STATED RATHER THAN + # BLURRED: the stale ROW may still sit in a tenant's document, and removing it is + # `apply_plan`'s job in `odoo_relational.py`. What this line guarantees is that a document + # which still carries one cannot put a SECOND "Odoo products" on somebody's rail while that + # migration has not reached them. A defect whose fix depends on a migration running is a + # defect that comes back on the one tenant the migration missed + # [[a-migration-that-runs-on-the-next-write]]. + # + # ⚠ `RETIRED_KEYS` is IMPORTED, never re-typed here. A hand-copied tuple is exactly the drift + # this guard exists to prevent, and it would go stale the first time a third key is retired. + # ⚠ Inside the same `try:` as the rest of the user-table read on purpose — if the import + # fails, the block's own `except` marks the nav DEGRADED, which is the honest outcome. A + # guard that silently stops guarding is worse than one that says it could not run. + try: + from odoo_relational import RETIRED_KEYS as _retired + except Exception: + _retired = () for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, st=_lent): + if e["key"] in _retired: + continue if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): continue pages.append({"key": e["key"], "label": e["label"], @@ -838,8 +867,22 @@ def save_nav_prefs(body: dict = Body(default=None), @router.get("/nav/schema/{key}") def nav_schema(key: str, session: Session = Depends(require_session)): """The database's schema drawer payload: its field contract + the semantic measures this - session may build with. Fail-closed on the SAME predicate as the nav — a key the session - may not open answers 403, never a redacted schema.""" + session may build with. + + TWO WALLS, ANSWERING TWO QUESTIONS, AND THEY FAIL DIFFERENTLY ON PURPOSE. The DATABASE wall + is the same predicate the nav uses — a key this session may not open answers 403, never a + redacted schema. The FIELD wall runs *after* it, on a database this session may open, and + its whole job is to narrow: the drawer serves the columns this reader receives everywhere + else, and no more. + + ⭐⭐ W38-T18 — THE FIELD WALL IS NEW HERE AND THIS DOOR IS WHERE IT WAS MISSING. Every grid + door has stripped hidden columns from both wires since W36-T21; the schema drawer beside them + served the raw contract to anyone `require`/`may_open` admitted. Owner instruction 10 — + *"Metrics field lookback and value must abide by the Filter permissioning"* — is satisfied on + the picker side already (the condition kit takes `fields` as a PROP off a payload that is + walled upstream), so this route is the one place the same reader could still learn the name, + the label and the VALUE VOCABULARY of a column their grid refuses them. + """ # Wave 18 C3-UT: a user table's schema is its own definition, walled by ITS predicate # (`may_open`) rather than the module grant machinery — `session.require` would 403 every # ut key because a user table is deliberately not a module. @@ -849,6 +892,25 @@ def nav_schema(key: str, session: Session = Depends(require_session)): if not defn or not user_tables.may_open(key, session.uname, session.admin, st=session.runtime): raise err(403, "forbidden", "that database belongs to another user") + # ⭐⭐ W38-T18 — THE FIELD WALL ON A `ut_*` DATABASE, WHICH THIS DOOR HAS NEVER RUN. + # `may_open` answers *IF* you reach the database and says nothing about WHICH COLUMNS, so + # this drawer handed `defn["fields"]` whole to every account it admitted. The bypass is + # older than this ticket (it dates to the wall itself); what changed is the CATEGORY it + # leaks — since W38-T16 a hidden `ut_*` column means an administrator's `hiddenFields` + # *or* a column granted away, and a share door whose schema drawer still names the column + # tells the ungranted reader exactly what they were refused. + # ⚠ MERGED BEFORE THE WALL, NEVER AFTER — the same order `routes_tables.ut_assembly` + # runs, and for the reason `_ut_shared_fields` gives one door over: the closure must see + # the WHOLE contract, or a definition field whose formula reads a hidden tenant-wide + # column sits outside its reach and carries that value out wearing a second name. + # ⛔ ONLY THE DEFINITION'S OWN FIELDS ARE RETURNED. This drawer has never published the + # shared stratum and this ticket is not where it starts; widening the list handed to the + # closure can only ever ADD to the hidden set, never remove from it. + import routes_tables as _rt + base = list(defn.get("fields") or []) + hidden = _rt._ut_hidden(session, key, + _rt._ut_shared_fields(session, key, base), + st=session.runtime) # WAVE 19 (R8) — the drawer wears the RENAMED name. A rail that says one thing and a # schema panel opened from it that says another is the drift a rename is supposed to # remove, not create. @@ -859,7 +921,7 @@ def nav_schema(key: str, session: Session = Depends(require_session)): "fields": [{"key": f["key"], "label": f["label"], "type": f["type"], "source": f.get("source") or "overlay", "description": str(f.get("description") or "")} - for f in (defn.get("fields") or [])], + for f in base if f.get("key") not in hidden], "measures": []} session.require(key) pages = perms.nav_pages(session.user) or [] @@ -870,7 +932,26 @@ def nav_schema(key: str, session: Session = Depends(require_session)): if key in ("customer_data", "cohort", "customers"): try: import aios_grid + import routes_customers as _rc + # ⭐⭐ W38-T18 — THE FIELD WALL ON THE CUSTOMER CONTRACT, AND THE `options` LINE FOUR + # ROWS DOWN IS WHY IT IS A LEAK RATHER THAN AN UNTIDINESS. A schema drawer that names + # a hidden column has told the reader it exists; one that ships `options` has handed + # them the VALUE VOCABULARY of every choice and status column their grid refuses — + # the set of statuses the business uses, read straight off a door with no wall. + # ⛔ `_hidden_for` IS REUSED, NOT RE-DERIVED, and the difference is not cosmetic. + # `hidden_keys(user, MODULE, aios_grid.FIELDS)` is the obvious spelling and it is + # INCOMPLETE: that function's own docstring records that a wall computed from the + # canonical list alone answers "not hidden" for every runtime column, so the + # tenant-wide stratum never reaches the closure and a formula over a hidden column + # comes back. One evaluator, one caller, no second idea of what this session hides. + # ⚠ INSIDE THE `try`, DELIBERATELY. If the wall cannot be computed the route already + # answers with an empty contract rather than a full one — fail-closed is the only + # direction a schema door may fail, and the gate's ADMIN control is what stops that + # silent empty being mistaken for a wall that worked. + hidden = _rc._hidden_for(session) for f in aios_grid.FIELDS: + if f["key"] in hidden: + continue entry = {"key": f["key"], "label": f["label"], "type": f["type"], "source": f["source"], "description": str(f.get("description") or "")} @@ -882,8 +963,19 @@ def nav_schema(key: str, session: Session = Depends(require_session)): measures = [] try: from core import measure_resolve + import core.perm_scope as _perm_scope team_id = perms.scope_team_id(session.user) - for m in measure_resolve.offer(team_id) or []: + # ⭐⭐ W38-T19 — THE METRICS CAPABILITY REACHES THIS DRAWER TOO, AND IT IS THE ONE DOOR + # THE THREE GRID ASSEMBLIES DO NOT SPEAK FOR. This route calls `measure_resolve.offer` + # ITSELF rather than reading a `measures` key off an assembly, so a wall applied in + # `routes_customers` / `routes_products` / `routes_tables` narrows every grid and leaves + # the panel beside them naming the full lookback catalogue — the same one-door-short + # shape W38-T18 fixed for FIELDS in this very function, one key over. + # ⚠ THE UT_ BRANCH ABOVE RETURNS BEFORE THIS AND ALREADY SERVES `[]`, so there is + # nothing to narrow there; if it ever starts publishing an offer, it needs this line. + _offer = ((measure_resolve.offer(team_id) or []) + if _perm_scope.may_metrics(session.user, key) else []) + for m in _offer: measures.append({"key": str(m.get("key") or ""), "label": str(m.get("label") or m.get("key") or ""), "type": str(m.get("type") or "")}) diff --git a/api/routes_products.py b/api/routes_products.py index 5eb6bdf176bb4db418309fad9f635718262287b2..f7dc95faf462d51367761526e0e148992bf45df4 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -115,15 +115,207 @@ def scoped_pool(session: Session): return pids, team_id, rows_src, fields_base +#: The semantic ENTITY topic this grid IS. `model/topics/odoo_products.yml` names the same store +#: key in its `grid:` field, so the binding is stated at both ends rather than inferred. +TOPIC = "product_data" +SEM_TOPIC = "odoo_products" + + +def _measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: + pass + + +def _measure_stamp(rt, team_id): + """The cached pool's build timestamp — the DATA STAMP in every memo key, so a pool refresh + invalidates memoised measure answers exactly when the underlying rows changed. Same + discipline as `routes_customers._pool_stamp`, over THIS route's cache key.""" + entry = rt.pool_cache.get(("product_pool", team_id)) + return entry[0] if isinstance(entry, tuple) and entry else 0 + + +def product_measures(session): + """The measure OFFER for this SESSION on this database, or `[]`. + + ⭐⭐ W38-T19 — `session` IS REQUIRED, NOT DEFAULTED, AND THAT IS THE POINT OF THE EDIT. This + function was PRINCIPAL-BLIND: it answered the same catalogue to every caller, so there was + nothing for a per-user capability to be consulted by. A `session=None` default would have + left the old call shape working and silently ungated — the fail-open spelling of the same + bug, and the one `_module_fields` in `routes_admin` was rewritten to refuse. A caller with + no principal cannot ask this question and must not get an answer. + + ⛔ EMPTYING THE OFFER IS THE WHOLE WALL ON THIS GRAIN, and it is three refusals rather than + one: `_measure_cells` and `_measure_condition_sets` both return `{}` for a falsy `offer`, and + `routes_grid` passes this list into `clean_measure_field` as the ADMISSION set, which + fail-closes `measure_not_offered` on a create. The client's `offerMeasure={measures.length > + 0}` then takes the Metric kind off the picker for the same reason it hides it on a database + with no binding. + + ⭐⭐ W37-T10 / owner item 4 (R1) — this is what turns `measures: []` into a real catalogue. + Owner: *"Odoo products should have a lookbsck metrics like sales etc."* The list is DERIVED + from `model/topics/odoo_products.yml`'s `measures:` binding, never written here — a second + list would be a second definition of the same fact. + + ⚠ `[]` ON FAILURE IS THE CORRECT DEGRADE and it is not silent: `entity_measures` refuses a key + it cannot prove, the reasons are readable through `semantic.entity_measure_refusals`, and the + client's own rule (`offerMeasure={measures.length > 0}`) then hides the Metric kind rather + than offering a column that could only render blank. + """ + import core.perm_scope as perm_scope + if not perm_scope.may_metrics(session.user, MODULE): + return [] + try: + from harness import semantic as sem + return sem.entity_measures(SEM_TOPIC) + except Exception as e: # noqa: BLE001 + _measure_err("product-measure-offer", e) + return [] + + +def _measure_cells(fields, rows_src, team_id, today, stamp, memo, offer): + """`{pid: {field key: value}}` for every measure column on this grid. + + ⛔ THIS IS THE HALF THAT MAKES THE FEATURE REAL. Serving the OFFER alone flips the client's + `offerMeasure` on and lets a user mint a Metric column that would then be blank forever — + this repo's most repeated defect ([[reachable-is-not-the-same-as-built]]). Nothing else + computes product-grain measure values: `core.measure_resolve.column_values` is CUSTOMER-grain + (it resolves through `harness.measure_filter`, whose entity is the partner), which is exactly + what this route's docstring used to record as an honest absence. + + ⭐ ONE QUERY PER WINDOW, not per column. Two Metric columns over the same 90 days are one + grouped scan; the fan-out to field keys happens in memory afterwards. + + ⛔ THE JOIN IS BY SKU CODE, per contract C1's trap. `sales_lines.product_code` reproduces this + pool's identity (`default_code`, or `pid:` for the 33 codeless actives) and `pid` is + a CRC32 of it — so the map is `row['code'] -> row['pid']`, taken off the rows we already hold. + Grouping by Odoo's `product_id` instead would key cells on a value no grid row carries. + """ + import aios_grid + + mfields = aios_grid.measure_fields_of(fields) + if not mfields or not offer: + return {} + from harness import semantic as sem + from harness import windows as _wn + + by_key = {m["key"]: m for m in offer} + codes = {r["pid"]: r["code"] for r in rows_src if r.get("pid") is not None} + + # Group the columns by WINDOW: same window, same scan. + by_window = {} + for f in mfields: + spec = f.get("measure") or {} + if spec.get("key") not in by_key: + continue # not offered for this caller -> the column stays blank, honestly + w = _wn.normalize(spec.get("window")) + rng = _wn.resolve(spec.get("window"), today) if w is not None else None + if rng is None: + # ⛔ NEVER WIDEN TO ALL TIME. An unresolvable window is an unanswerable column, and + # silently answering a different question is worse than a blank one. + continue + by_window.setdefault((rng[0], rng[1]), []).append(f) + + out = {} + for (dfrom, dto), group in by_window.items(): + keys = tuple(sorted({f["measure"]["key"] for f in group})) + mkey = ("entity", SEM_TOPIC, stamp, team_id, dfrom, dto, keys) + if mkey not in memo: + try: + memo[mkey] = sem.entity_measure_values( + SEM_TOPIC, list(keys), date_from=dfrom, date_to=dto, team_id=team_id, + # ⛔ SERVICES ARE IN THIS CATALOGUE (37 active), so they must NOT be excluded + # here. The service filter exists to stop Delivery Charges polluting a SKU + # RANKING; on a per-row column the row IS the service product, and excluding + # it would print a FALSE 0 — which under C1's rule asserts "sold nothing". + exclude_services=False, offer=offer) + except Exception as e: # noqa: BLE001 + _measure_err("product-measure-column", e) + # Same memo discipline as `measure_resolve.column_values`: a PERMANENT failure + # memoises, a TRANSIENT one (store still warming after a restart) must not, or + # the cells stay blank for the rest of the session long after the data landed. + transient = False + try: + from harness import datastore as _ds + transient = not _ds.ready() + except Exception: # noqa: BLE001 + transient = False + if transient: + continue + memo[mkey] = None + vals = memo[mkey] + if vals is None: + continue + # C1's empty-window rule, applied to EVERY row — not only the ones with a group. 72% of + # this catalogue has no group in a 90-day window, so this loop IS the common case. + filled = {pid: sem.entity_zero_fill(vals.get(code), list(keys), offer) + for pid, code in codes.items()} + for f in group: + k, fkey = f["measure"]["key"], f["key"] + for pid, cell in filled.items(): + if k in cell: + out.setdefault(pid, {})[fkey] = cell[k] + if len(memo) > 100: # a lifetime cache, not a leak + memo.clear() + return out + + +def _measure_condition_sets(views, offer, rows_src, team_id, today): + """`{rule id: [pid, …]}` for every measure CONDITION across this session's saved views. + + ⚠ THE ANSWER IS KEYED BY pid, NOT BY SKU CODE. `entity_measure_sets` answers in the SOURCE + dim's own values (the code) because it cannot know how a grid hashes identity; the map back + is this route's job, exactly as it is for the column cells. + """ + if not offer: + return {} + from harness import semantic as sem + + admitted = {m["key"] for m in offer} + by_id = {} + for v in views or []: + cfg = (v.get("config") or {}) + for rule in sem.entity_measure_leaves(cfg.get("filters"), admitted): + rid = str(rule.get("id") or "") + if rid: + by_id[rid] = rule # last statement of the question wins + if not by_id: + return {} + pid_by_code = {r["code"]: r["pid"] for r in rows_src if r.get("pid") is not None} + try: + sets = sem.entity_measure_sets( + SEM_TOPIC, list(by_id.values()), today, team_id=team_id, + keys_by_id=list(pid_by_code), offer=offer, exclude_services=False) + except Exception as e: # noqa: BLE001 + _measure_err("product-measure-sets", e) + return {} + # A code the pool does not carry is dropped rather than keyed on None — the same rule the + # cell path applies, for the same reason. + return {rid: sorted(pid_by_code[c] for c in codes if c in pid_by_code) + for rid, codes in sets.items()} + + def product_assembly(session: Session, scope: str = "product", storage_key: str = "", consume_corrections: bool = True): """The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so the /workspace and events routes consume either interchangeably. - One deliberate absence, a topic fact rather than a gap: - * `measures`/`measure_sets` are EMPTY — `core.measure_resolve` is CUSTOMER-grain (the - C-TOPIC v1 descope, booked in the wave doc); the events ctx therefore refuses measure - creates on this surface, which is the correct fail-closed shape. + ⭐⭐ W37-T10 / OWNER ITEM 4 (R1) — `measures` IS NO LONGER EMPTY, and the header above it used + to record the opposite as a topic fact. It was one: `core.measure_resolve` is CUSTOMER-grain, + so nothing here could resolve a product-grain measure. What changed is that the ENTITY path + exists now (`semantic.entity_measures` / `entity_measure_values`, bound in + `model/topics/odoo_products.yml`), so this surface serves a real six-metric catalogue AND the + cells to go with it — the offer and the resolver landing together, deliberately, because an + offer whose values nobody computes is a column that renders blank forever. + + One deliberate absence remains, and it is a real one: + * `measure_sets` stays EMPTY — that is the measure CONDITION channel (a filter answered + server-side as a pid set), which still runs through the customer-grain + `measure_resolve.condition_sets`. So a Metric COLUMN works here; a Metric FILTER does + not yet. Stated rather than left to be discovered: `clean_filter_tree` drops a measure + condition whose key is not in `measure_keys`, so the refusal is already fail-closed. ⭐ WAVE 19 / R9 — `lists` IS NO LONGER EMPTY. Wave 16 passed `with_cohorts=False` because there was one customer-keyed cohort bucket and product pids are CRC32 hashes of SKU codes; @@ -172,12 +364,29 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str fields = [f for f in fields if f.get("key") not in hidden] rows_src = [perm_scope.strip_row(r, hidden) for r in rows_src] + today = time.strftime("%Y-%m-%d") + stamp = _measure_stamp(session.runtime, team_id) + measures = product_measures(session) + + # R9: the Cohorts column's cells, built from THIS topic's lists. ⭐ W37-T10 — the measure + # half of this same read-only channel is no longer empty; the customer assembly merges its + # measure columns here for exactly the same reason and through the same door. + derived = aios_grid.cohort_cells(lists) + for pid, cells in _measure_cells(fields, rows_src, team_id, today, stamp, + session.runtime.measure_memo, measures).items(): + derived.setdefault(pid, {}).update(cells) + + # ⛔ THE CONDITION CHANNEL SHIPS WITH THE OFFER, NOT AFTER IT. `routes_grid` derives + # `measure_keys` from this same list, so the moment `measures` is non-empty the client's + # filter builder offers measure conditions — and an id missing from `measureSets` renders as + # PENDING ("Calculating…") and matches nothing, permanently. Serving the offer without this + # would trade one honest absence for a spinner that never resolves. + measure_sets = _measure_condition_sets(views, measures, rows_src, team_id, today) + return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, - # R9: the Cohorts column's cells, built from THIS topic's lists. Same read-only - # `derived` channel the customer assembly uses — the measure half stays empty. - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + "derived": derived, + "measures": measures, "measure_sets": measure_sets, "today": today, "team_id": team_id} diff --git a/api/routes_query.py b/api/routes_query.py index b630526b9e3ad42eb3c0a1913addbdab7c84bf66..4074e75e5689175ade2f4c5283126e081e6a18d9 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -12,6 +12,7 @@ import io import json import os import re +import sys import uuid from fastapi import APIRouter, Body, Depends, Response @@ -44,24 +45,37 @@ MAX_HISTORY_CHARS = 6000 MAX_RATING_REASON = 200 QUERY_RATINGS = ("up", "down") MODEL_AUTO = "auto" -# ⛔⛔ WHY QUERY DECLARES THE ENV NAMES RATHER THAN READING `analyst.PROVIDERS`: MEASURED -# 2026-08-16, `import harness.analyst` costs **5 to 8 SECONDS**, warm, on this box (it pulls -# `harness.tools`, which pulls yaml and the skill recipes). `GET /query` is a LIST path that BOTH -# the Assistant and Query call on mount, so importing the ladder to answer "which models can this -# deployment actually call?" would put eight seconds on the first paint of the AI module. The map -# below is Query's own declaration, and it is deliberately the cheap half. -# ⚠ A SECOND DECLARATION IS A DIVERGENCE WAITING TO HAPPEN, so it is CHECKED rather than trusted: -# `_providers()` already holds the platform ladder in memory and reconciles there, and anything it -# finds is REPORTED into the payload instead of a provider silently never being offered. -# The ORDER is Query's own and is deliberate: cerebras first, because this path needs tool calling -# and cerebras carries this account's tool-capable model. -QUERY_PROVIDER_ENV = { - "cerebras": "CEREBRAS_API_KEY", - "groq": "GROQ_API_KEY", - "openrouter": "OPENROUTER_API_KEY", -} -QUERY_PROVIDER_ORDER = tuple(QUERY_PROVIDER_ENV) -# Filled the first time the platform ladder is loaded; empty means "not yet known", never "clean". +# ⭐⭐ WAVE 36 · W36-T35 (R4 / contract C5) — THE LADDER IS DECLARED IN ONE PLACE NOW. +# +# ⛔ THIS BLOCK USED TO BE A SECOND COPY OF THE LADDER, and it said so: *"A SECOND DECLARATION IS A +# DIVERGENCE WAITING TO HAPPEN."* It existed because reading the platform ladder meant +# `import harness.analyst`, MEASURED at **5 to 8 seconds** warm (it pulls `harness.tools`, which +# pulls yaml and the skill recipes) — eight seconds on the first paint of the AI module, to answer +# "which models can this deployment call?". +# +# `providers.LLM_PROVIDERS` is now that answer: a plain declaration in a module this file already +# imports, with no yaml and no skill recipes behind it. So the copy is DELETED rather than kept in +# step — one list, one door (C5), and the eight seconds go with it. +# ⚠ `harness.analyst.PROVIDERS` still exists and is still the ANALYST's ladder. `_ladder_drift()` +# reconciles the two WITHOUT forcing the import, so a divergence is still reported and nobody pays +# the eight seconds to hear about it. +import providers as prov # noqa: E402 + +#: The Messages API version header. ⚠ A DATE, NOT A MODEL: it pins the wire format, not the model, +#: and Anthropic requires it on every request. +ANTHROPIC_VERSION = "2023-06-01" +#: ⛔ THINKING IS ON BY DEFAULT ON THIS MODEL AND `max_tokens` CAPS THINKING PLUS TEXT TOGETHER, so +#: a budget sized around the answer alone truncates mid-thought. It is deliberately NOT disabled: +#: with thinking off, the model occasionally writes a tool call into its visible TEXT instead of +#: emitting a `tool_use` block, and the turn then reports success having called nothing — the exact +#: silent failure this ticket exists to remove. +ANTHROPIC_MAX_TOKENS = int(os.environ.get("AIOS_ANTHROPIC_MAX_TOKENS") or 8000) +#: `low` | `medium` | `high` | `xhigh` | `max`. Medium is the balance for a short, scoped +#: view-building turn; raise it per deployment if the specs come back shallow. +ANTHROPIC_EFFORT = os.environ.get("AIOS_ANTHROPIC_EFFORT") or "medium" + +#: Anything the ANALYST's ladder carries that this one does not, once something has already paid to +#: import it. Empty means "nothing known", never "clean". _LADDER_DRIFT = [] QUERY_KINDS = ("grid", "chart", "calendar", "kanban", "timeseries", "map", "list") QUERY_WORKSPACE_EVENTS = {"view_create", "view_upsert", "view_delete"} @@ -124,11 +138,37 @@ def _new_id(prefix): def model_choices(): """Choices are stable even when one is not configured, so explicit means explicit.""" - return [MODEL_AUTO, *QUERY_PROVIDER_ORDER] + return [MODEL_AUTO, *(p.name for p in prov.LLM_PROVIDERS.values())] + + +def _ladder_drift(): + """Does the ANALYST's ladder carry a provider this one does not? Answered only if it is free. + + ⛔ IT MUST NEVER FORCE `import harness.analyst`. That import is 5-8 seconds warm and this + function is reached from `GET /query`, which both AI surfaces call on mount. Reading + `sys.modules` means the reconciliation happens for whoever already paid, and costs nothing for + everybody else — a check that is free is a check that stays on [[limit-with-no-enforcer]]. + """ + analyst = sys.modules.get("harness.analyst") + if analyst is None: + return list(_LADDER_DRIFT) + rows = [row for row in getattr(analyst, "PROVIDERS", []) if isinstance(row, dict)] + known = prov.LLM_PROVIDERS + # ⛔ TWO KINDS OF DRIFT, AND THE SECOND IS THE QUIET ONE. A provider the analyst gained is + # ABSENT here, which a reader can at least notice. An env name that changed on one side leaves + # the provider listed and its availability answered from the wrong variable — present, wrong, + # and silent. Both were checked when this file held its own env map; both are still checked now + # that `providers.LLM_PROVIDERS` holds it, because the two ladders can still disagree. + _LADDER_DRIFT[:] = sorted( + {str(row.get("name")) for row in rows} - set(known) + | {str(row.get("name")) for row in rows + if str(row.get("name")) in known + and str(row.get("env") or "") != known[str(row.get("name"))].env}) + return list(_LADDER_DRIFT) def model_status(): - """Per choice: can this deployment actually CALL it, and if not, why not. + """Per choice: can this deployment actually CALL it, and if not, why not (C5). ⭐⭐ THE SAME DEFECT THE SOURCE CHIPS WERE FIXED FOR IN WAVE 33, ONE CONTROL TO THE LEFT. `permitted` is not `answerable` for a database; OFFERED is not CALLABLE for a model. The picker @@ -136,40 +176,59 @@ def model_status(): spent a click and a turn to be told "the selected model is unavailable" by the server, which knew before the click. The reason travels with the flag, exactly as `sources` does. - ⚠ It names no environment variable. "This model is not configured on this deployment" is the + ⭐ W36-T35 ADDS THREE REASONS THE FIRST VERSION COULD NOT EXPRESS, and each is a distinct fact + a picker has to be able to say out loud: + * `outOfCredit` — configured, capable, and SKIPPED because it told us its balance is empty + (R4). Before this, an empty account was indistinguishable from a working one until the + turn came back with `HTTP 402` on the screen. + * `toolCalling` — DECLARED per provider. W36-T34's `done-when` is that a model the ladder + declares incapable of tool calling is not offered for a tool-calling turn; that sentence + needs a declaration to read, and this is it. + * `jsonMode` — the same shape for the other capability staged item 3 named. + + ⚠ IT NAMES NO ENVIRONMENT VARIABLE. "This model is not configured on this deployment" is the cause a tenant user can act on (ask an admin); the variable name is our deployment's business. """ - live = [name for name in QUERY_PROVIDER_ORDER if os.environ.get(QUERY_PROVIDER_ENV[name])] - rows = [{"model": MODEL_AUTO, "available": bool(live), - "reason": "" if live else "no model is configured on this deployment"}] - for name in QUERY_PROVIDER_ORDER: - rows.append({"model": name, "available": name in live, - "reason": "" if name in live - else "this model is not configured on this deployment"}) - # Anything the platform ladder carries that Query does not offer, once a chat has taught us. - rows.extend({"model": name, "available": False, - "reason": "the platform can reach this model but Query does not offer it yet"} - for name in _LADDER_DRIFT) - return rows + rows_by_name = {row["provider"]: row for row in prov.llm_status()} + live = [name for name, row in rows_by_name.items() + if row["configured"] and row["capable"] and not row["outOfCredit"]] + out = [{"model": MODEL_AUTO, "available": bool(live), + "reason": "" if live else "no model can be reached from this deployment right now", + "toolCalling": True, "jsonMode": True, "outOfCredit": False}] + for name, row in rows_by_name.items(): + if not row["configured"]: + reason = "this model is not configured on this deployment" + elif row["outOfCredit"]: + reason = f"{row['label']} is out of credit, so it is being skipped for now" + elif not row["capable"]: + reason = f"{row['label']} cannot carry this kind of request" + else: + reason = "" + out.append({"model": name, "available": name in live, "reason": reason, + "toolCalling": row["toolCalling"], "jsonMode": row["jsonMode"], + "outOfCredit": row["outOfCredit"]}) + # Anything the ANALYST's ladder carries that Query does not offer, once a chat has taught us. + out.extend({"model": name, "available": False, "toolCalling": False, "jsonMode": False, + "outOfCredit": False, + "reason": "the platform can reach this model but Query does not offer it yet"} + for name in _ladder_drift()) + return out def _providers(model=MODEL_AUTO): - """Auto returns the permitted ladder; an explicit choice returns at most one provider.""" - import harness.analyst as analyst + """Auto returns the permitted ladder; an explicit choice returns at most one provider. + ⛔ THE FILTERING IS `providers.llm_chain`'s, NOT THIS FUNCTION'S — declared-capable AND + configured AND not inside an out-of-credit cooldown, in the ladder's own order (R4). Re-deriving + any of those three here would be a second opinion about which model to call, and the first + place it would disagree is the one nobody watches. + """ requested = str(model or MODEL_AUTO).strip().lower() - by_name = {p["name"]: p for p in analyst.PROVIDERS} - # ⚠ THE RECONCILIATION, at the ONE point the platform ladder is already in memory, so it costs - # nothing. A provider added to `analyst.PROVIDERS` would otherwise simply never appear here, - # and an env name changed there would make Query's availability answer quietly wrong. Both are - # invisible failures; this turns them into a row a reader can see [[limit-with-no-enforcer]]. - _LADDER_DRIFT[:] = sorted( - set(by_name) - set(QUERY_PROVIDER_ENV) - | {name for name, row in by_name.items() - if name in QUERY_PROVIDER_ENV and row.get("env") != QUERY_PROVIDER_ENV[name]}) - names = list(QUERY_PROVIDER_ORDER) if requested == MODEL_AUTO else [requested] - return [by_name[name] for name in names - if name in by_name and os.environ.get(by_name[name]["env"])] + chain = prov.llm_chain("llm_tool_calling") + if requested != MODEL_AUTO: + chain = [p for p in chain if p.name == requested] + return [{"name": p.name, "label": p.label, "env": p.env, "url": p.url, + "model": p.model, "wire": p.wire} for p in chain] def _history(state, thread_id): @@ -266,6 +325,100 @@ def _spec_schema(field_keys): } +# ══════════ WAVE 36 · W36-T40 — THE GROUNDING CORPUS THIS PRODUCT ALREADY OWNED ════════════════ +# +# ⭐⭐ 25 SENTENCES OF THE OWNER'S OWN SEMANTICS WERE SITTING ON DISK, UNREAD BY THE ASSISTANT. +# `platform/model/metrics/*.yml` carries an `ai_context` on 25 of its 26 registered metrics: what +# "revenue" means here (untaxed, Amazon excluded), that an AR balance takes no date window, that a +# returns figure arriving negative means a lost negation rather than no returns. They were written +# for the AIOS Analyst, which EXIT-6 deleted, and nothing has read them since. +# +# ⛔ THEY ARE VOCABULARY, NOT FIELDS, AND THE PROMPT HAS TO SAY SO. The topics they describe +# (`sales_lines`, `receivables`, `gl_lines`, ...) are NOT databases this assistant can read: its +# sources are the two built-ins plus the `ut_*` tables (`deps.assistant_source_status`). Handing a +# model 25 confident notes about `team_id` and `price_subtotal` without that fence is an invitation +# to filter on a column that is not in the snapshot. +# +# ⛔⛔ AND NO FIGURE INSIDE THEM IS EVER AN ANSWER. Five of the 25 carry a measured landmark +# (`Fisch 2025 = 518`, `$1,447,579.59 vs $1,086,381.35`, `~$192k`, `$3,187,571`, `~2% apart`), +# because a human analyst reading a metric definition wants the order of magnitude. A model that +# repeats one has put an aggregate on a screen that no `validate()` ever tied back to Odoo, which +# is the single thing standing rule 8 forbids. ⚠ THE PROMPT SAYING SO IS NOT THE CONTROL: this +# file already measured a prompt instruction failing on the very next turn (see `_no_dashes`), so +# `verify_query` asks the live model for one of those numbers and asserts it does not come back. +# +# ⚠ ONE READER, NOT TWO. `harness.semantic` is the governed registry, already parsed, validated +# and lru_cached, and `rollup_sql`/`routes_tables` reach it by this same lazy import. Measured +# after `import main`: the import costs **0.003 s** (its whole dependency set, `core.odoo`, `yaml` +# and `modules.sales` included, is already resident) and the first `metrics()` call 0.195 s, once. +# Nothing like the 5 to 8 seconds `harness.analyst` cost, which is why the ladder above stopped +# reading that one and why this does not need a second YAML parser of its own. + + +def _grounding_notes(): + """`[(topic, key, label, sentence), ...]` — this tenant's metric vocabulary, dash free. + + ⛔ FAIL SOFT, AND THE GATE IS WHAT MAKES THAT SAFE. A missing or malformed corpus returns [] + and the assistant answers without the vocabulary, because turning a YAML typo into a dead chat + door is worse than answering with less. That is precisely the silent degradation + `deploy_web.py` warns about in the comment above its own upload of these files, so + `verify_query` derives the expected count FROM the YAML and asserts the built prompt carries + exactly that many: the corpus may go missing, but it may not go missing quietly. + + ⛔ `_no_dashes` ON THE WAY IN, and it is load bearing. 11 of the 25 sentences carry an em dash, + and the prompt they are joining ends with *"Never use an em dash or an en dash"*. Shipping them + raw would demonstrate the forbidden style eleven times in the same breath as the ban. ⚠ AND + `web_prose` STRUCTURALLY CANNOT SEE IT: that gate reads Python string VALUES off the AST, and + these strings live in a data file, so prose loaded at runtime is a hole in rule 6's scanner. + """ + try: + from harness import semantic as sem + registry = sem.metrics() + except Exception: + # Named metrics are a data-layer concern; a chat turn is not the place to surface a + # corpus problem to a person who asked about their own table. + return [] + notes = [] + for key, metric in (registry or {}).items(): + if not isinstance(metric, dict): + continue + sentence = _no_dashes(str(metric.get("ai_context") or "")).strip() + if sentence: + # ⛔ THE LABEL GETS THE SAME TREATMENT, and it is not belt-and-braces: measured on + # this corpus, `sales_orders.orders_invoiced` is labelled "Orders — fully invoiced" + # and TWO em dashes reached the built prompt while every sentence was clean. One + # field of a row scrubbed and its neighbour not is [[one-question-two-normalizers]] + # at the smallest possible scale, so the rule applies to the whole row. + notes.append((str(metric.get("topic") or ""), str(key), + _no_dashes(str(metric.get("label") or key)).strip(), sentence)) + return notes + + +def _grounding_block(): + """The system-prompt section, or "" when this deployment has no corpus to state.""" + notes = _grounding_notes() + if not notes: + return "" + lines = "\n".join(f"- {topic}.{key} ({label}): {sentence}" + for topic, key, label, sentence in notes) + return f""" +BUSINESS VOCABULARY ({len(notes)} notes this tenant wrote about their own reporting): +{lines} + +Read those for what a WORD means here: which rows a number counts, which reading of an ambiguous +question is the house one, what a figure deliberately leaves out. They describe the finance topics, +not the database above, so never treat a name in that list as a field and never filter on anything +absent from FIELDS. + +Some notes quote a MEASUREMENT taken in an earlier year: a total, a customer count, a gap between +two figures. Never repeat one of those to the reader, not even labelled as old or approximate. A +number on screen is read as the answer whatever sits beside it, and nothing checked that one today. +When this database cannot produce the figure somebody asked for, say only that. Codes and formats +in the notes are different and you may use them freely: a team_id, a percentage scale, which field +carries what. Every number you state comes from build_view over the snapshot above. +""" + + def _system_prompt(snapshot): fields = snapshot["fields"] cols = json.dumps([{"key": field["key"], "label": field.get("label", field["key"]), @@ -298,7 +451,7 @@ To compare one column against ANOTHER column rather than a typed value, give the "priced below what it costs us". Both columns must be in the FIELDS list above. Set important=true when the view is one somebody should be chased about: an error, a mismatch, or money at risk. Leave it out otherwise. - +{_grounding_block()} Write with ordinary punctuation. Never use an em dash or an en dash: use a comma, a colon or a full stop instead.""" @@ -363,6 +516,78 @@ def _from_chat(answer, provider): return answer, None, provider, None +# ══════════════════ WAVE 36 · W36-T35 — TWO WIRES, ONE LOOP (ruling R4) ═════════════════════════ +# +# ⛔ THE SECOND WIRE IS NOT A SECOND CODE PATH BY CHOICE — Anthropic's Messages API genuinely is a +# different request: `x-api-key` instead of a bearer, the system prompt as a TOP-LEVEL field rather +# than a message, `input_schema` instead of `function.parameters`, and typed content BLOCKS instead +# of `choices[0].message`. What is kept single is everything around it: one ladder, one loop, one +# meter, one set of sentences. The wire is a pair of pure functions the loop dispatches on. +# +# ⚠ WHY RAW HTTP AND NOT THE `anthropic` SDK, which is the documented default. The SDK is not in +# `aios-web/api/requirements.txt`, so it is NOT in the deployed image — adding it is a dependency +# on a Space this lane cannot rebuild or test, mid-wave. `requests` is already there and is already +# how the three sibling providers in this same loop are called. Booked for the integrator as a +# `PENDING:`; the day it lands, `_anthropic_request`/`_anthropic_read` are the two functions that +# go, and nothing else moves. + + +def _openai_request(provider, messages, tools): + """The OpenAI-compatible `/chat/completions` shape — cerebras, groq, openrouter.""" + return { + "url": provider["url"], + "headers": {"Authorization": f"Bearer {os.environ[provider['env']]}"}, + "json": {"model": provider["model"], "messages": messages, "tools": tools, + # ⛔⛔ `auto`, NOT `required`, AND THAT ONE WORD IS R16. Under `required` the model + # had to emit a `build_view` call for EVERY turn: it could not answer a question, + # could not ask one back, and could not decline to build. "The AI chat must TALK + # with the user, not only build queries, and decide for itself whether to create a + # query view" is unreachable with the tool forced, whatever the prompt says. + "tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200}, + } + + +def _anthropic_request(provider, messages, tools): + """The Messages API shape — built by `providers.anthropic_request`, never re-derived here. + + ⛔ ONE WIRE, TWO DOORS. `ai_review.draft_flow` calls the same builder; the owner quoted an + error from each of these paths in one sentence, and two implementations of one API is the + defect this product keeps finding under a different name. + ⚠ THIS DOOR SENDS `effort` AND THE DRAFTER DOES NOT — `output_config.effort` errors on Haiku + 4.5, which is the tier the drafter runs on. Model-gated, so it is the caller's call. + """ + return prov.anthropic_request( + model=provider["model"], key=os.environ[provider["env"]], + system=None, messages=messages, tools=tools, + max_tokens=ANTHROPIC_MAX_TOKENS, tool_choice="auto", effort=ANTHROPIC_EFFORT) + + +def _openai_read(body): + """`(spec, said, refusal)` out of an OpenAI-compatible answer.""" + answer = ((body.get("choices") or [{}])[0] or {}).get("message") or {} + said = _said(answer.get("content")) + calls = answer.get("tool_calls") or [] + if calls: + try: + spec = json.loads(calls[0]["function"].get("arguments") or "{}") + except (TypeError, ValueError, KeyError): + return None, said, None + return (spec if isinstance(spec, dict) else None), said, None + return None, said, None + + +def _anthropic_read(body): + """`(spec, said, refusal)` — the shared reader, with this door's own dash normaliser applied.""" + text, spec, refused = prov.anthropic_read(body) + if refused: + return None, None, refused + return spec, _said(text), None + + +_WIRES = {"openai": (_openai_request, _openai_read), + "anthropic": (_anthropic_request, _anthropic_read)} + + def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None): """Return ``(spec, said, provider, reason)`` without any source-data fallback. @@ -398,9 +623,23 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s providers = _providers(requested) if not providers: - sentence = (f"the selected model ({requested}) is unavailable" if requested != MODEL_AUTO - else "the assistant is not configured on this deployment") - return None, sentence, None, "model_unavailable" if requested != MODEL_AUTO else "not_configured" + # ⛔ THREE DIFFERENT EMPTINESSES, THREE DIFFERENT SENTENCES (R4). "Not configured" and + # "every account is out of credit" are the same empty list and completely different + # problems: one needs an administrator, the other needs a card. Answering both with one + # sentence is how the owner ended up reading `HTTP 402` to find out which. + broke = [row for row in prov.llm_status() if row["outOfCredit"]] + if requested != MODEL_AUTO: + row = next((r for r in prov.llm_status() if r["provider"] == requested), None) + if row and row["outOfCredit"]: + sentence = f"{row['label']} is out of credit, so it cannot answer right now" + else: + sentence = f"the selected model ({requested}) is unavailable" + return None, sentence, None, "model_unavailable" + if broke: + names = ", ".join(row["label"] for row in broke) + return (None, f"every model this deployment can reach is out of credit ({names}). " + f"Top one up and it will answer again", None, "no_credit") + return None, "the assistant is not configured on this deployment", None, "not_configured" import requests @@ -413,9 +652,9 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s a failed one has already SPENT tokens at that vendor; booking only the winner would report a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads gets a line, including the one whose answer turned out to be empty. - ⚠ `tokens_from` returns `(None, None)` when the provider did not say, and `None` is carried - through rather than coerced to 0 — a call booked at zero is an unmeasured call presented as - a free one. `usage_ledger` counts it as UNMEASURED so the total reads as a floor. + ⚠ `tokens_from` reads `input_tokens`/`output_tokens` as well as the OpenAI spellings, so + the Messages API body is measured by the SAME reader with no branch here. That is not luck + — it is why that function takes a key LIST. ⚠ `record` never raises, by its own contract, so this cannot break the assistant. """ ins, outs = usage_ledger.tokens_from(body) @@ -424,63 +663,68 @@ def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, s total=usage_ledger.total_from(body), st=getattr(session, "runtime", None), user=getattr(session, "uname", "")) - last = None + # ⛔ EVERY FAILURE IS COLLECTED, NOT JUST THE LAST ONE. The owner's own complaint was a LIST + # ("cerebras: HTTP 402; groq: HTTP 404; openrouter: HTTP 402") and the list is the useful part + # — it is what tells an operator whether one vendor is down or the account is empty everywhere. + # What changes is that each entry is now a sentence rather than a status code. + trouble = [] for provider in providers: + build, read = _WIRES.get(provider["wire"], _WIRES["openai"]) try: - response = requests.post( - provider["url"], timeout=60, - headers={"Authorization": f"Bearer {os.environ[provider['env']]}"}, - # ⛔⛔ `auto`, NOT `required`, AND THAT ONE WORD IS R16. Under `required` the model - # had to emit a `build_view` call for EVERY turn: it could not answer a question, - # could not ask one back, and could not decline to build. "The AI chat must TALK - # with the user, not only build queries, and decide for itself whether to create a - # query view" is unreachable with the tool forced, whatever the prompt says. - # ⚠ AND IT IS A LIVE VENDOR-BEHAVIOUR CHANGE THAT NO GATE HERE CAN SEE, because - # `verify_query` injects `chat` and never builds this body. `_spec_from_400` below - # exists precisely because providers differ in how they emit a forced call; under - # `auto` a provider may also decline to build for a view-shaped question. That is - # why this ticket's evidence is a real three-turn run against a real provider and - # not a green gate. - json={"model": provider["model"], "messages": messages, "tools": tools, - "tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200}, - ) - except Exception as exc: - last = f"{provider['name']}: {type(exc).__name__}" + request = build(provider, messages, tools) + response = requests.post(request["url"], timeout=90, + headers=request["headers"], json=request["json"]) + except Exception as exc: # noqa: BLE001 + trouble.append(f"{provider['label']} could not be reached ({type(exc).__name__})") continue - if response.status_code == 400: + + if response.status_code == 400 and provider["wire"] == "openai": + # `_FAILED_GEN`'s recovery, kept for the wire that needs it and NOT extended to the one + # that does not: Anthropic answers a tool call as a typed block or not at all, so a 400 + # there is a malformed REQUEST and recovering a spec out of it would be reading tea + # leaves. spec = _spec_from_400(response.text) if spec is not None: return spec, None, provider["name"], None - last = f"{provider['name']}: 400" - continue + if response.status_code != 200: - last = f"{provider['name']}: HTTP {response.status_code}" + # ⭐⭐ R4's SECOND CLAUSE, AND THE ONLY PLACE IT CAN BE LEARNED: a provider only tells + # you the balance is empty by refusing. The memo is written here so the NEXT turn skips + # it instead of paying the round trip again. + if prov.is_credit_failure(response.status_code, response.text): + prov.mark_no_credit(provider["name"]) + trouble.append(prov.refusal_sentence(provider["name"], response.status_code, + response.text)) continue + try: body = response.json() # C7: booked from the RAW body, before anything below can raise on its shape. A # provider that answered 200 and billed for it has spent tokens whether or not this # function can read what it said. _book(provider, body) - answer = body["choices"][0]["message"] - said = _said(answer.get("content")) - calls = answer.get("tool_calls") or [] - if calls: + spec, said, refused = read(body) + if refused: + trouble.append(f"{provider['label']}: {refused}") + continue + if spec is not None: # The prose rides ALONG with the view when the model wrote both, so the reader gets # a sentence instead of `_explain`'s machine description of its own output. - spec = json.loads(calls[0]["function"].get("arguments") or "{}") return spec, said, provider["name"], None if said: return None, said, provider["name"], None # ⚠ Neither a call nor a word is a FAILED turn, not a silent one: falling through to # the next provider is right, and swallowing it as an empty answer would show the # reader a blank reply [[empty-answer-vs-unfinished-answer]]. - last = f"{provider['name']}: an empty answer" - except Exception as exc: - last = f"{provider['name']}: unreadable answer ({type(exc).__name__})" + trouble.append(f"{provider['label']} answered with nothing") + except Exception as exc: # noqa: BLE001 + trouble.append(f"{provider['label']} sent something unreadable " + f"({type(exc).__name__})") if requested != MODEL_AUTO: - return None, f"the selected model ({requested}) is unavailable", None, "model_unavailable" - return (None, "the assistant could not be reached just now (" + (last or "no provider") + ")", + return (None, f"the selected model ({requested}) could not answer: " + + (trouble[0] if trouble else "it did not respond"), None, "model_unavailable") + return (None, "the assistant could not be reached just now: " + + ("; ".join(trouble) if trouble else "no provider answered"), None, "provider_unreachable") diff --git a/api/routes_script_views.py b/api/routes_script_views.py new file mode 100644 index 0000000000000000000000000000000000000000..a4feec967331ab1c46fa8a1d6a789eda61b6f63a --- /dev/null +++ b/api/routes_script_views.py @@ -0,0 +1,439 @@ +"""routes_script_views.py — CONTRACT C3: a database View that is a PYTHON SCRIPT (R3 / R5 / R10). + +Owner item 6, verbatim (2026-08-18): *"Add code script as an interface (database View) so a user +can build whatever they want through the Agent chat interface. be able to create any dashboard +they want. User should have the ability to see the code AND the dashboard output of course… Limit +the code script View per database… Any agent can add into more AI script, so we can see different +versions or different things the AI code for us."* + + GET /api/v1/script-views?database=K the views bound to ONE database + POST /api/v1/script-views create one {database, name?, source} + GET /api/v1/script-views/{id} one view, its source and its history + PUT /api/v1/script-views/{id} a NEW VERSION of the source + DELETE /api/v1/script-views/{id} drop it + POST /api/v1/script-views/{id}/run run it -> {ok, spec | error, stdout, ms} + POST /api/v1/script-views/{id}/revert go back to an earlier version {version} + +⭐⭐ **R3 IS "SCOPED, NOT CAPPED", AND THE TWO HALVES POINT OPPOSITE WAYS.** *Scoped*: a view may +read ONLY the database it lives in, and a script that names another database is REFUSED with a +message naming both. *Not capped*: there is **no limit on how many script views a database may +carry**, because that is how an agent offers three attempts and the owner picks one. So nothing +below counts views. What IS bounded is what makes them big — one source is capped, and one view's +edit history is capped and REPORTS what it dropped. + +⛔ **THE RUN IS THE CALLER'S, NEVER THE AUTHOR'S (R5).** `script_sandbox.run_view` is handed +`session.user`, so a script written by an administrator and opened by a scoped analyst reads the +ANALYST's rows. The author decides what the code does; the reader decides what it can see. +⚠ And the reverse case is safe rather than lucky: a narrowly-scoped author cannot write a script +that exfiltrates anything, because the only thing a script can return is a render spec drawn on +the screen of the person who ran it. There is no network, no file and no second reader. + +⛔ **`run` IS `def`, NOT `async def`.** It waits on a subprocess for up to ten seconds; as a +coroutine that would block the event loop for every other request in the container. FastAPI runs a +plain `def` in the threadpool, which is what makes one slow script one slow REQUEST. +""" +import threading +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The tenant's script views: `{id: record}`. Per tenant, so it rides `runtime.store_key`. +VIEWS_KEY = "script_views" + +MAX_NAME = 80 +MAX_SOURCE_BYTES = 128 * 1024 + +#: Edit history per view. ⚠ NOT a cap on the NUMBER of views (R3 forbids that) — a cap on how far +#: back ONE view's source is kept. Past this the oldest go and `trimmed` counts them, so a reader +#: can see the history is partial instead of concluding the view was only ever saved twice. +MAX_HISTORY = 40 + +#: ⛔ HOW MANY SCRIPTS MAY BE RUNNING IN THIS CONTAINER AT ONCE, and it is a REPORTED refusal +#: rather than a queue. Each run is a real subprocess with a ten-second wall clock; without this, +#: holding down refresh forks until the box gives up, and the tenant's ONE FastAPI process is what +#: gives up. A 429 that says so is honest; an unbounded fork is not. +MAX_CONCURRENT_RUNS = 4 +_RUN_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT_RUNS) + + +def _now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _all(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this database has no script views", never to a 500 on the view rail.""" + try: + found = rt.get(VIEWS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _database_ok(session, database): + """Does this database EXIST, and may this caller read it? Answered by C1, never by a list. + + ⛔ `perm_scope.may_read` ALONE IS NOT ENOUGH and the reason is easy to miss: it answers True + for an ADMIN on any key at all, including one no database answers to. So a create validated + with `may_read` would let an administrator bind a view to a typo and leave an orphan nothing + can ever run. `scoped_fields` is the cheap half of C1 (a `ut_*` definition, no rows) and it + RAISES `UnknownTable`, which is exactly the question being asked. + """ + import core.perm_scope as perm_scope + try: + perm_scope.scoped_fields(session.user, database, st=session.runtime) + except perm_scope.UnknownTable: + raise err(404, "no_database", f"there is no database '{database}' in this workspace") + except perm_scope.Denied: + raise err(403, "forbidden", f"your account may not read '{database}'") + except perm_scope.Unresolvable as exc: + # The database is real and cannot be served under this call's constraints. Standing rule + # 1's second sentence: report the cause and the recommendation, never a bare refusal. + raise err(409, "unresolvable", str(exc)) from None + + +def _clean_source(raw, *, allow_empty=False): + """The stored source, or a 400. `allow_empty` is CREATE's alone and the asymmetry is the point. + + ⭐⭐ W37-T41 — WHY CREATE MAY BE EMPTY AND SAVE MAY NOT. + Picking "Custom View" in the mode picker mints the view immediately, before a line of code + exists, so that `mode === 'script'` always implies a real id and no reader has to carry a + "the id might be missing" branch (the create semantics handed to lane C in mailbox E-7). + A view that has been created and not yet written is therefore a REAL, legible state: the editor + shows its starter placeholder and the Run control is right there. + A PUT is a different act. It appends a VERSION to a history capped at 40, and blanking a + working script by saving nothing over it is not an edit anybody means to make. So the guard + stays exactly where it was on that door, and a person who wants the view gone deletes it. + + ⛔ THIS WAS FOUND BY THE GATE, NOT BY READING. `core.script_sandbox.check_source("")` returns + None, so "an empty script is storable" looked true and was written into a mailbox answer another + lane was about to build on. The refusal was HERE, one layer above, in a function the sandbox + knows nothing about. Two validators for one question, disagreeing + [[one-question-two-normalizers]] — and the one that would have bitten a person is the one no + unit of this feature was asserting. + """ + source = str(raw or "") + if not source.strip() and not allow_empty: + raise err(400, "no_source", "a script view needs some code") + if len(source.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + raise err(413, "source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of code") + return source + + +def _row(rec, *, source=False): + """One view as the list door reports it. ⚠ NO SOURCE unless asked: the rail lists names.""" + out = {"id": rec.get("id") or "", "database": rec.get("database") or "", + "name": rec.get("name") or "", "author": rec.get("author") or "", + "version": int(rec.get("version") or 1), + "created": rec.get("created") or "", "updated": rec.get("updated") or "", + "versions": len(rec.get("history") or []) + 1, + "trimmed": int(rec.get("trimmed") or 0), + # ⭐ W37-T46: which version this one was RESTORED from, when it was. Present on the + # row (not only the history) because the editor's header is where a person reads "what + # am I looking at", and "v5, restored from v2" is the sentence that makes a roll-back + # legible as an event rather than as a coincidence of matching code. + "restoredFrom": rec.get("restoredFrom")} + if source: + out["source"] = rec.get("source") or "" + out["history"] = [{"version": int(h.get("version") or 0), "author": h.get("author") or "", + "created": h.get("created") or "", + "bytes": len(str(h.get("source") or "").encode("utf-8", "replace"))} + for h in reversed(rec.get("history") or []) if isinstance(h, dict)] + return out + + +def _limits(): + return {"maxSourceBytes": MAX_SOURCE_BYTES, "maxName": MAX_NAME, + "maxHistory": MAX_HISTORY, "maxConcurrentRuns": MAX_CONCURRENT_RUNS, + # ⭐ SAID OUT LOUD, because R3's "not capped" half is the one a reader assumes wrong. + "maxViewsPerDatabase": None} + + +def _put(session, view_id, mutate): + """Read-modify-write ONE view, synchronously — the client re-reads the rail immediately.""" + def _set(cur): + cur = dict(cur or {}) + nxt = mutate(cur.get(view_id) if isinstance(cur.get(view_id), dict) else None) + if nxt is None: + cur.pop(view_id, None) + else: + cur[view_id] = nxt + return cur + + session.runtime.update(VIEWS_KEY, _set, flush="sync") + + +def _mine_or_admin(session, rec): + """Who may EDIT or DELETE a view: its author, or an administrator. + + ⚠ RUNNING IS A DIFFERENT QUESTION and deliberately wider — anybody who may read the database + may run any view on it, under their OWN scope. That is the whole of "so we can see different + versions or different things the AI code for us": a colleague's attempt is worth nothing if + only its author can open it. + """ + if session.admin or str(rec.get("author") or "") == session.uname: + return + raise err(403, "forbidden", "only the author or an administrator can change this script view") + + +# ── the routes ──────────────────────────────────────────────────────────────────────────────── +@router.get("/script-views") +def list_script_views(database: str = "", session: Session = Depends(require_session)): + """Every script view bound to ONE database, newest first. `database` is required.""" + key = str(database or "").strip() + if not key: + raise err(400, "no_database", "name the database whose script views you want") + _database_ok(session, key) + rows = [_row(rec) for rec in _all(session.runtime).values() + if isinstance(rec, dict) and str(rec.get("database") or "") == key] + rows.sort(key=lambda r: (r["created"], r["id"]), reverse=True) + return {"database": key, "views": rows, "limits": _limits()} + + +@router.post("/script-views") +def create_script_view(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Create one. ⛔ THE SOURCE IS CHECKED BEFORE IT IS STORED, not first at run time. + + A script view that cannot be run is a broken feature the reader discovers by pressing a button, + and the agent that wrote it is long gone by then. `check_source` is pure and costs no process, + so the refusal arrives while the author still has the code in front of them. + """ + import secrets # noqa: PLC0415 + import core.script_sandbox as sandbox # noqa: PLC0415 + + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "no_database", "a script view is bound to one database") + _database_ok(session, database) + # ⭐ `allow_empty` is CREATE's alone - see `_clean_source`. The picker mints a view the + # moment a person chooses the mode, so the empty source is the normal first state, not a slip. + source = _clean_source(body.get("source"), allow_empty=True) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + view_id = "sv_" + secrets.token_urlsafe(9) + rec = {"id": view_id, "database": database, + "name": " ".join(str(body.get("name") or "Script view").split())[:MAX_NAME], + "source": source, "author": session.uname, "version": 1, + "created": _now(), "updated": _now(), "history": [], "trimmed": 0} + _put(session, view_id, lambda _prior: rec) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell the author their + # script was saved when it was not. + raise err(503, "store_unavailable", "the script view was NOT created") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.get("/script-views/{view_id}") +def get_script_view(view_id: str, session: Session = Depends(require_session)): + """One view WITH its source and its edit history. Owner item 6's *"see the code"* half.""" + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _database_ok(session, str(rec.get("database") or "")) + return {"view": _row(rec, source=True), "limits": _limits()} + + +@router.put("/script-views/{view_id}") +def update_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """A NEW VERSION of one view's source. The prior source is KEPT, never replaced in place.""" + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + source = _clean_source(body.get("source")) + refusal = sandbox.check_source(source) + if refusal is not None: + raise err(400, refusal.code, refusal.message) + + def _mutate(prior): + prior = dict(prior or rec) + history = list(prior.get("history") or []) + history.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", "created": prior.get("updated") or ""}) + dropped = max(0, len(history) - MAX_HISTORY) + prior["history"] = history[dropped:] if dropped else history + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + if body.get("name"): + prior["name"] = " ".join(str(body["name"]).split())[:MAX_NAME] + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the new version was NOT saved") + return {"view": _row(fresh, source=True), "limits": _limits()} + + +@router.post("/script-views/{view_id}/revert") +def revert_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Put one view back to an earlier version. D-338 / W37-T46. + + ⭐⭐ A ROLL-BACK IS A NEW VERSION, NEVER A DELETE OF THE ONES AFTER IT, and that is the whole + point of the ticket rather than an implementation detail. The history is what lets a person + trust an agent with their code: if reverting destroyed the versions it stepped over, one + mistaken revert would cost exactly what the history existed to protect, and there would be no + way back from the way back. So this appends, and `restoredFrom` records where the text came + from — the same posture the agent-harness roll-back already takes, so a reader who has seen one + is not surprised by the other. + + ⛔ IT LIVES ON THE SERVER BECAUSE THE HISTORY DOES. The list door deliberately serves history + entries WITHOUT their source (`_row`: the rail lists names), so a client cannot assemble an old + version's text to re-PUT it. Handing the source out just so the client could send it straight + back would widen a payload for a round trip that does not need to exist, and would make the + revert non-atomic: two calls, and a failure between them leaves the person looking at code that + is not what is stored. + ⚠ `_clean_source` is NOT re-run. The text being restored was already validated when it was + first saved, and a source that a later, stricter rule would now reject is exactly the source a + person is most likely to want back. `check_source` still runs, because the SANDBOX's refusal is + about what the code would DO and that must never be bypassed by a route. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + body = body if isinstance(body, dict) else {} + try: + want = int(body.get("version")) + except (TypeError, ValueError): + raise err(400, "no_version", "say which version to go back to") from None + + history = [h for h in (rec.get("history") or []) if isinstance(h, dict)] + match = next((h for h in history if int(h.get("version") or 0) == want), None) + if match is None: + # ⛔ TWO REASONS A VERSION IS MISSING AND THEY ARE NOT THE SAME FACT. One was never + # written; the other was TRIMMED off the 40-deep history and the record even counts how + # many. Saying "that version is gone" for the first would be a lie a person could act on. + trimmed = int(rec.get("trimmed") or 0) + if trimmed and want <= trimmed: + raise err(410, "version_trimmed", + f"version {want} is older than this view's history keeps " + f"({trimmed} earlier versions have been dropped)") + raise err(404, "no_version", f"this view has no version {want}") + + source = str(match.get("source") or "") + refusal = sandbox.check_source(source) + if refusal is not None: + # An older version that today's sandbox refuses. The person is told which version and why, + # rather than being handed a 400 about code they did not just type. + raise err(400, refusal.code, f"version {want} cannot be restored: {refusal.message}") + + def _mutate(prior): + prior = dict(prior or rec) + history_now = list(prior.get("history") or []) + history_now.append({"version": int(prior.get("version") or 1), + "source": prior.get("source") or "", + "author": prior.get("author") or "", + "created": prior.get("updated") or ""}) + dropped = max(0, len(history_now) - MAX_HISTORY) + prior["history"] = history_now[dropped:] if dropped else history_now + prior["trimmed"] = int(prior.get("trimmed") or 0) + dropped + prior["source"] = source + prior["version"] = int(prior.get("version") or 1) + 1 + prior["updated"] = _now() + # ⚠ ON THE RECORD, so the history reads as WHAT HAPPENED rather than as a version that + # mysteriously matches an older one. Without it a reader sees v5 and v2 with identical + # code and no way to tell a revert from a coincidence. + prior["restoredFrom"] = want + return prior + + _put(session, view_id, _mutate) + fresh = _all(session.runtime).get(view_id) + if not isinstance(fresh, dict): + raise err(503, "store_unavailable", "the roll-back was NOT saved") + return {"view": _row(fresh, source=True), "restoredFrom": want, "limits": _limits()} + + +@router.delete("/script-views/{view_id}") +def delete_script_view(view_id: str, session: Session = Depends(require_session)): + view_id = str(view_id) + rec = _all(session.runtime).get(view_id) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + _mine_or_admin(session, rec) + _put(session, view_id, lambda _prior: None) + return {"deleted": view_id} + + +@router.post("/script-views/{view_id}/run") +def run_script_view(view_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """CONTRACT C3's run door: `{ok, spec | error, stdout, ms}`. + + ⛔ `spec` IS A DESCRIPTION THE CLIENT DRAWS. It is never HTML and never text the browser + executes — the sandbox refuses a spec carrying an `html`, `script`, `src`, `href` or `on*` key + before this function ever sees it, so a renderer cannot be talked into running something by a + script that was itself perfectly well behaved. + + ⛔ AND IT IS PLAIN `def`, NOT `async def` — see this module's header. A ten-second subprocess + wait on the event loop is a ten-second outage for the whole container. + """ + import core.script_sandbox as sandbox # noqa: PLC0415 + + rec = _all(session.runtime).get(str(view_id)) + if not isinstance(rec, dict): + raise err(404, "no_view", "there is no script view with that id") + database = str(rec.get("database") or "") + # A DRAFT run: the editor sends unsaved code so the author can try it before committing to it. + # It is checked exactly as a stored one is, because "unsaved" is not a permission. + draft = (body or {}).get("source") if isinstance(body, dict) else None + source = _clean_source(draft) if draft else str(rec.get("source") or "") + # ⚠ NO `_database_ok` CALL HERE. `run_view` asks C1 the identical question a line later, and + # asking twice builds a registry topic's pool twice. The three refusals are translated below + # instead, which is the same wall reached through the same door. + + if not _RUN_SLOTS.acquire(blocking=False): + raise err(429, "busy", + f"{MAX_CONCURRENT_RUNS} script views are already running on this server. " + f"Try again in a moment") + try: + out = sandbox.run_view(session.user, database, source, st=session.runtime) + finally: + _RUN_SLOTS.release() + + # ⛔ C1'S THREE REFUSALS ARE HTTP STATUSES, NOT `ok:false`. "You may not read this database" + # answered 200 would be a permission decision the client has to go looking for, and every + # other door in this app answers 403 for it. Everything BELOW this line is a well-formed + # request whose ANSWER is that the script did not produce a view — that is a 200 with + # `ok:false`, the shape `routes_web_agent.test` already uses for the same reason. + if out.get("code") == "unknown_table": + raise err(404, "no_database", out.get("message") or f"there is no database '{database}'") + if out.get("code") == "denied": + raise err(403, "forbidden", out.get("message") or "your account may not read that database") + if out.get("code") == "unresolvable": + refusal = err(409, "unresolvable", out.get("message") or "these rows cannot be served") + refusal.detail["error"]["limit"] = out.get("limit") or {} + raise refusal + + answer = {"ok": bool(out.get("ok")), "spec": out.get("spec"), + "stdout": out.get("stdout") or "", "truncated": bool(out.get("truncated")), + "ms": int(out.get("ms") or 0), "code": out.get("code") or "", + # ⭐ `caps` RIDES ON THE ANSWER (standing rule 1). On a POSIX host all three limits + # were applied; on a Windows host the memory and CPU ones were not, and a screen + # that claims an enforcement which did not happen is the failure the rule is about. + "caps": out.get("caps") or {}} + if not answer["ok"]: + answer["error"] = out.get("message") or "the script view did not produce a view" + return answer diff --git a/api/routes_shares.py b/api/routes_shares.py index 62278f5c62b12e8fbc3a8d25d03ffd9705fc8bd5..d59c96b837a5a137b3f4e8b5ff43c9570e32ecf1 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -4,9 +4,19 @@ PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set) GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]} -`kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail -already speaks, now extended to folders and databases so there is ONE vocabulary in the UI -(R10: "the same picker views use"). +`kind` ∈ view | folder | database | field. Roles are `view` | `edit` — the same two words the +view rail already speaks, now extended to folders, databases and COLUMNS so there is ONE +vocabulary in the UI (R10: "the same picker views use"). + +⭐⭐ **W38-T16 — `field` IS THE FOURTH KIND, AND ITS `oid` IS TOPIC-QUALIFIED: `":"`** +(`shares.field_oid`). A bare column key repeats across databases — `notes` exists on a dozen — +so a grant stored under one would admit the grantee to every `notes` column in the tenant at +once. ⛔ THREE functions in this file branch on kind and ALL THREE need the new one, which is not +obvious because only two of them fail loudly: `_owns_object` (without it a column's own creator +is 404'd trying to share the thing they just made) and `_object_ref` (without it `route` is None, +`_notify_new_grantees` returns early, and the grantee is **granted and never told** — owner item +18's silent half, reopened one kind over). `_can_see_object` stays deliberately CLOSED for +anything that is not a view. ⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's @@ -102,6 +112,24 @@ def _owns_object(session, kind, oid): """ if session.admin: return True + if kind == "field": + # ⭐⭐ W38-T16 — A COLUMN'S OWNER IS ITS `createdBy`, WHICH THE CREATE DOOR ALREADY STAMPS + # (`routes_tables.patch_shared_cell`) and the DELETE door already reads as its wall (R8 / + # D-172: creator-or-admin). Read from the same place by all three, so a column cannot be + # deletable by one person and shareable by another. + # ⚠ THIS BRANCH IS NOT OPTIONAL AND ITS ABSENCE FAILS SILENTLY IN THE WORST DIRECTION: + # a brand-new column has no grant record, so `put_share` falls to this predicate — and + # without it the column's own creator is answered `404 no_object` on the first attempt to + # share the thing they just made. + table_key, field_key = shares.split_field_oid(oid) + if not table_key: + return False + try: + import core.shared_overlay as shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key) + except Exception: # noqa: BLE001 + return False + return bool(defn) and str(defn.get("createdBy") or "") == str(session.uname) if kind == "database": # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table" @@ -157,6 +185,10 @@ def _can_see_object(session, kind, oid): if kind != "view": # A folder carries no per-object visibility flag of its own, and a database's `may_open` # (inside `_owns_object`) already admits grantees. Nothing wider to ask. + # ⭐ W38-T16 — AND `field` KEEPS THIS CLOSED, DELIBERATELY. A grantee never reaches here: + # `get_share` tests `role is None` first and a grant answers a role, so the only caller + # left is an account with no relationship to the column at all. Widening it would let any + # signed-in session enumerate who holds which column on a database they cannot open. return False try: import core.table_store as table_store @@ -195,7 +227,7 @@ def _entries_or_400(session, entries): raise err(400, "unknown_people", "no active account in this workspace is named " + ", ".join(sorted(set(unknown))) - + " — nothing was shared. Pick people from the list rather than typing a name.") + + ". Nothing was shared. Pick people from the list rather than typing a name.") @router.get("/share/mine") @@ -216,6 +248,12 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)): role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin, st=session.runtime) + # W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant + # registry has no owner to return from `may_administer`, even though the same caller may safely + # claim their own object through the PUT path below. Reflect that exact claim predicate here: + # a Member who owns an unshared View receives the people picker; a collaborator still does not. + if not rec["owner"] and not may_admin and _owns_object(session, kind, oid): + may_admin = True # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id: # who owns it, everyone it is granted to, and the tenant's whole username↔name directory — # to any signed-in session, about objects it cannot open. Now a caller with no role on an @@ -286,7 +324,7 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None), entries = body.get("entries") if not isinstance(entries, list): raise err(400, "bad_entries", - "entries must be a list of {user, role} — send [] to un-share, which is how " + "entries must be a list of {user, role}. Send [] to un-share, which is how " "revoking is expressed") _entries_or_400(session, entries) out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname, @@ -367,6 +405,32 @@ def _object_ref(session, kind, oid): a row that opens nowhere. """ try: + if kind == "field": + # ⭐⭐ W38-T16 — A COLUMN IS NOT ADDRESSABLE ON ITS OWN, exactly as a view is not: it + # is a column INSIDE a database, so the target that travels is the DATABASE. Without + # this branch the function falls through to the view/folder loop, finds nothing, + # answers `route=None` — and `_notify_new_grantees` returns EARLY. The grant lands and + # the receiver is never told, which is the silent half of owner item 18 reopened one + # kind over. + from routes_alerts import route_for_topic + table_key, field_key = shares.split_field_oid(oid) + if not table_key: + return ("A column", None, "") + try: + import core.shared_overlay as shared_overlay + defn = (shared_overlay.fields(table_key, st=session.runtime) or {}).get(field_key) + except Exception: # noqa: BLE001 + defn = None + label = str((defn or {}).get("label") or "").strip() or field_key + # ⚠ TWO SPELLINGS REACH THIS LINE AND ONE MAP ANSWERS BOTH. `shared_overlay` is keyed + # by whatever the calling door already held: a `ut_*` database uses its bare key, + # while a registry topic uses `_table_workspace` (`product_data.TABLE_KEY`). + # `route_for_topic` speaks the GRID SCOPE vocabulary (`customer`, not + # `customer_data`), so the suffix comes off before it is asked — rather than a second + # route table being written here, which is how the two come apart. + _WS = "_table_workspace" + scope = table_key[:-len(_WS)] if table_key.endswith(_WS) else table_key + return (label, route_for_topic(scope) or None, "") if kind == "database": import core.user_tables as ut defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {} @@ -390,4 +454,5 @@ def _object_ref(session, kind, oid): route, str(oid) if kind == "view" else "") except Exception: # noqa: BLE001 pass - return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "") + return ({"view": "A view", "folder": "A folder", + "field": "A column"}.get(kind, "An item"), None, "") diff --git a/api/routes_slack.py b/api/routes_slack.py index ce7c6c960ce3941d1a8cfdbb68ac4cc9342997b9..4cf017909fc13d069974d6d9cad3d927e8fd9fa5 100644 --- a/api/routes_slack.py +++ b/api/routes_slack.py @@ -229,7 +229,10 @@ def slack_creds(rt): def _summary(agent, modules): """One sentence per agent for the list row — computed here, because the alternative is one round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" - governed = [m for m in modules if m.get("enforced")] + # ⭐ W36-T22 / C2 — every listed database is governed now; the `enforced` flag it used to + # filter on is DELETED, and `m.get("enforced")` would have gone silently falsy here and + # summarised every agent as "" (no databases at all). + governed = list(modules) perms = agent.get("perms") or {} open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) if not governed: @@ -304,16 +307,26 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): if not isinstance(a, dict): raise err(404, "no_such_agent", "no agent with that id") modules = routes_admin._perm_modules(session) - enforced = [m["key"] for m in modules if m.get("enforced")] + # ⭐⭐ W36-T22 / C2 — EVERY listed database, `ut_*` included. ⛔ THIS IS WHY THE FLAG'S + # DELETION IS NOT A ONE-FILE CHANGE: `m.get("enforced")` on a row that no longer carries the + # key is None, so this list would have been EMPTY and the channel perms editor would have + # governed ZERO databases while looking entirely correct. A Slack channel is a principal of + # the SAME wall (`verify_perm_scope` section H) and gets the same catalogue. + governed = [m["key"] for m in modules] stored = a.get("perms") or {} principal = agent_principal(a) import core.perm_scope as perm_scope perms_out = {} - for k in enforced: + for k in governed: e = stored.get(k) + # ⛔ W36-T22 — `may_read`, the same evaluator the read door uses, for the reason spelled + # out at `routes_admin.get_perms`. ⚠ It answers DIFFERENTLY here and correctly so: a + # channel agent is not a person with a share, so a `ut_*` key it has not been granted + # defaults CLOSED — which is the fail-closed direction for a bot, and the same answer the + # table routes would give it. perms_out[k] = e if isinstance(e, dict) else { - "access": bool(perm_scope.may_access(principal, k)), "filter": None, - "hiddenFields": []} + "access": bool(perm_scope.may_read(principal, k, st=session.runtime)), + "filter": None, "hiddenFields": []} return {"id": str(agent_id), "channel": a.get("channel") or "", "channelName": a.get("channelName") or "", "label": a.get("label") or "", "active": a.get("active", True) is not False, @@ -326,7 +339,8 @@ def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): # the "Everything (admin)" state for a principal that cannot have it. "is_admin": False, "modules": modules, - "fields_by_module": {k: routes_admin._module_fields(k) for k in enforced}} + "fields_by_module": {k: routes_admin._module_fields(k, session=session) + for k in governed}} @router.put("/agents/{agent_id}/perms") @@ -341,14 +355,16 @@ def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), raise err(400, "empty_patch", "no perms to save") mods = routes_admin._perm_modules(session) # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot - # evaluate, refuses a hiddenFields key that names nothing, refuses an unenforced `ut_*` key, - # and refuses a BU condition the pushdown cannot read. Every one of those is exactly as true - # for a channel as for a person, and a second validator here is a second place for one of - # them to go missing. + # evaluate, refuses a hiddenFields key that names nothing, and refuses a BU condition the + # pushdown cannot read. Every one of those is exactly as true for a channel as for a person, + # and a second validator here is a second place for one of them to go missing. + # ⚠ CORRECTED W36-T22: this list used to end "refuses an unenforced `ut_*` key". That refusal + # is DELETED — W36-T21 armed the wall over every database, so there is no unenforced key left + # to refuse, and a comment naming a rule that no longer exists is how the next wave re-derives + # it ([[two-gates-can-assert-opposite-things]]). cleaned = routes_admin._clean_perms( body.get("perms"), - enforced_keys={m["key"] for m in mods if m.get("enforced")}, - listed_keys={m["key"] for m in mods}) or {} + governed_keys={m["key"] for m in mods}, session=session) or {} import core.perm_scope as perm_scope diff --git a/api/routes_statements.py b/api/routes_statements.py index 771ac0a8a1b1326942145be58c132991bd3e66aa..c2cbedf8619c07e441bc5a6f1f80ffb5982b0772 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -129,9 +129,14 @@ def preview(body: dict = Body(default=None), session: Session = Depends(_gate)): subject = (t.get("subject") or cs.DEFAULT_SUBJECT) try: subject = subject.format(customer=row["Customer"], company=cs.COMPANY, month=month) - except (KeyError, IndexError): + except Exception: # noqa: BLE001 # An unknown placeholder is the user's typo, not a 500. Show the template verbatim so they # can see what they typed rather than getting an opaque error. + # ⛔ W36-T42: THIS CAUGHT THREE OF THE WAYS `str.format` FAILS AND THERE ARE MORE. + # `{customer` is a ValueError, `{customer.x}` an AttributeError, `{customer:%Y}` a + # TypeError, and every one of them is the same user mistake this branch was written for. + # Naming a subset of the exception types turns a typo in the OTHER half into a 500 on the + # one screen somebody opens to avoid mailing 200 people the wrong thing. pass return { "html": cs.render_statement_html(row, t.get("intro") or cs.DEFAULT_INTRO, @@ -153,7 +158,31 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)): names = [str(n) for n in (body.get("customers") or []) if str(n).strip()] if not names: raise err(400, "bad_request", "name at least one customer") - override = (body.get("overrideTo") or "").strip() or None + # ⛔⛔ W36-T42 / D-299 — A DECLARED TEST SEND WITH NO ADDRESS REFUSES. IT DOES NOT BECOME A + # REAL ONE. This read `(body.get("overrideTo") or "").strip() or None`, so a caller who sent + # `overrideTo: ""` (the field left blank, the state not yet typed into, a trimmed-away space) + # got a REAL statement mailed to the debtor's own address, reported back as `test: false`. On + # the one route in this product that is allowed to write to Odoo, the difference between a + # rehearsal and mailing a live customer was one empty string. + # + # ⭐ THE DISCRIMINATOR IS THE CALLER'S DECLARATION, NOT THE VALUE. A body with no `overrideTo` + # key and no `test` flag is the PRODUCTION send, and refusing that would delete the feature + # D-299 exists to preserve. What can be refused is a caller who SAID this is a test: the key + # being present, or `test: true`, is that statement, and an empty address beside it is the + # mistake. So both spellings of "absent" are covered: the key present and blank, and `test` + # asserted with no key at all. + # + # ⚠ THIS IS THE SECOND OF THREE WALLS AND THE ONLY ONE A PAYLOAD CANNOT ROUTE AROUND. + # `automationApi.testSendStatement` refuses an empty address before the request is built, and + # SAFE_MODE refuses an address outside the allow-list inside `queue_statement`. The client + # wall is bypassable by construction (anything can POST); this one is not. + declared_test = ("overrideTo" in body) or bool(body.get("test")) + override = str(body.get("overrideTo") or "").strip() + if declared_test and not override: + raise err(400, "no_override_address", + "a test send needs the address to send the test to. Without one this would " + "mail the customer's own address, which is the opposite of a test") + override = override or None if override and len(names) != 1: raise err(400, "bad_request", "a test send takes exactly one customer") t = body.get("templates") or {} diff --git a/api/routes_tables.py b/api/routes_tables.py index 4777831ff4e63d6fa73933a49a95d4abb8dcef9a..79307c310d70c1d7c4d8d236bed5e69dfe917078 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -429,11 +429,36 @@ def patch_shared_cell(table_key: str, pid: int, body: dict = Body(default=None), raise err(400, "bad_request", "a field key is required") _defn_or_refuse(session, table_key) from core import shared_overlay + import core.perm_scope as perm_scope if not shared_overlay.is_shared(table_key, key, st=session.runtime): + # ⭐⭐ W38-T16 / R7 — A COLUMN IS BORN OWNED AND PRIVATE, AND THIS IS THE PRODUCT CHANGE. + # Until now this door minted a column every account in the tenant could read, with no way + # to say who. `granted` marks it as governed by `core.shares` (see + # `perm_scope.FIELD_GRANT_MARK` for why an explicit marker and not the absence of a grant + # record), and the grant record below claims it for the creator. + # + # ⚠ THE EMPTY ENTRY LIST IS THE POINT, NOT A PLACEHOLDER. `set_grants` keeps a record that + # has an owner and no entries, so "shared with nobody" is STORED and is a different fact + # from "never shared" — the same explicit-resolution shape `perms_v` uses. Without the + # owner the column would be unmanageable: `may_administer` fails closed on an ownerless + # record, so nobody could ever share or re-share it. + # ⚠ COLUMNS THAT PREDATE THIS CARRY NO MARK and stay tenant-wide, unchanged. Reading + # their absence as "granted to nobody" would blank every existing shared column at once. shared_overlay.put_field(table_key, key, { "key": key, "label": str(body.get("label") or key), "source": "overlay", "type": str(body.get("type") or "text"), "shared": True, + perm_scope.FIELD_GRANT_MARK: True, "createdBy": session.uname}, st=session.runtime) + try: + import core.shares as shares + shares.set_grants("field", shares.field_oid(table_key, key), [], + owner=session.uname, st=session.runtime) + except Exception: # noqa: BLE001 + # ⛔ THE MARK IS ALREADY WRITTEN, SO A FAILED CLAIM FAILS **CLOSED**: the column is + # governed and nobody holds a grant, i.e. only an admin sees it. That is recoverable + # (an admin can share it) and the other order is not — a marked column with a claim + # that landed first and a definition that did not would be a grant on nothing. + pass try: # ⚠ `put_cell`, not `put_cells` — this door writes exactly ONE cell, and the singular is # the API that says so. It delegates to the plural, so both stay reachable through the one @@ -477,6 +502,18 @@ def delete_shared_field(table_key: str, field_key: str, f"was added by {owner or 'somebody else'}, and dropping it would delete the " f"value for every account") dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) + # ⭐⭐ W38-T16 — THE GRANTS DIE WITH THE COLUMN, which is `shares.drop_objects`' whole reason + # for existing (wave 21, C3): a deleted object's grants would otherwise serve a ghost id into + # every receiver's "Shared with me" forever, and the ghost 404s on open. It matters twice as + # much for a field, because `drop_field` scrubs the CELLS so the key can be re-used — and a + # surviving grant record would silently re-arm on the next column that took the name. + # ⚠ R8 / D-172 IS UNCHANGED ABOVE: delete is creator-or-admin. This ticket did not widen it. + try: + import core.shares as shares + shares.drop_objects([("field", shares.field_oid(table_key, str(field_key)))], + st=session.runtime) + except Exception: # noqa: BLE001 + pass return {"ok": True, "dropped": bool(dropped), "fields": list(shared_overlay.fields(table_key, st=session.runtime))} @@ -617,7 +654,7 @@ def ut_write_ctx(session: Session, table_key: str): # `:2021`); this ctx passed `frozenset()`, so a hidden column was hidden on the READ and fully # writable on the EVENTS transport — a wall on one wire and not the other is the shape # `strip_row`'s own note warns about, with the sign flipped. - hidden = _ut_hidden(session, table_key, fields_base) + hidden = _ut_hidden(session, table_key, fields_base, st=lent) ctx = grid_events.EventCtx( uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, admin=session.admin, fallback_ws=None, seen_ids={}, st=session.runtime, @@ -626,28 +663,99 @@ def ut_write_ctx(session: Session, table_key: str): workspace, fields, views, lists = aios_grid.workspace_wire( ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key="", fields_base=fields_base) - fields, _rows, hidden = _ut_field_wall(session, table_key, fields, []) + # ⭐⭐ W38-T16 — THE SHARED COLUMNS JOIN THE **WRITE** CTX'S CONTRACT TOO, and for the reason + # W36-T21 gave one wall over: `grid_events` refuses a key by asking `ctx.hidden_keys`, so a + # column missing from this list is a column the events transport does not know it must + # refuse. The read door narrowing a name the write door has never heard of is the same + # one-wire wall with the sign flipped. + fields = _ut_shared_fields(session, table_key, fields) + fields, _rows, hidden = _ut_field_wall(session, table_key, fields, [], st=lent) return {"rows_src": [], "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, "hidden": hidden, "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + # ⭐ W37-T12 — the OFFER, because `routes_grid` reads `g["measures"]` as the ADMISSION + # list for `clean_measure_field`: without it a Metric create on this surface is + # refused `measure_not_offered` even though the read door offers the kind. + # ⛔ THE CELLS ARE DELIBERATELY ABSENT HERE. This ctx validates a write and + # materialises no rows at all (W30-T30 removed a full table build from the write + # path); computing measure values would put that work straight back. `measure_sets` + # stays empty for the same reason — a write is not a render. + # ⭐ AND THE ASYMMETRY IS SAFE, WHICH IS NOT OBVIOUS AND WAS CHECKED RATHER THAN + # ASSUMED. `routes_grid._ctx` derives TWO things from this dict and they have + # different jobs: `measure_keys` (from `measures`) is the ADMISSION list + # `clean_filter_tree` uses — non-empty here, so a saved measure CONDITION survives the + # write, which is the failure that comment records for the customer path. Empty + # `resolved_ids` (from `measure_sets`) only makes `view_upsert` answer "rerender", + # so the next `/workspace` — which DOES compute the sets, in `ut_assembly` — resolves + # it. Conservative in the right direction: one extra repaint, never a dropped filter. + # ⭐⭐ W38-T19 — THE CAPABILITY IS APPLIED HERE TOO, and this is the site that makes + # the refusal REAL rather than cosmetic: `routes_grid` reads this list as + # `measure_offer`, and `clean_measure_field` fail-closes `measure_not_offered` over + # an empty one. A picker hidden on the read door with this list still full would be + # a control that only stops the honest. + "measures": ut_measures(table_key, session), + "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), # ⭐ W31-T20 — the write door reads this to refuse a PID-BEARING event loudly rather # than letting `allowed_pids` swallow it as a no-op. See `routes_grid`'s ut_ branch. "limits": limits, "defn": defn} -def _ut_hidden(session, table_key, fields): +def _ut_hidden(session, table_key, fields, st=None): """The hidden-field closure for THIS session on THIS database — C1's field half, once. ⚠ Named rather than inlined at its four call sites for the reason `may_open`'s own note gives: four spellings of one wall is how two of them come apart. `perm_scope.hidden_keys` is the ONE evaluator; this is just the `ut_*` caller's shorthand for it. + + ⭐ W38-T16 — `st` IS THE LEND THE CALLER ALREADY HOLDS, and passing it is free rather than + merely tidy: the closure's field-grant leg resolves against `object_shares`, which is exactly + the bucket `ut_assembly`'s own lend note says the pass is already serving. """ import core.perm_scope as perm_scope - return perm_scope.hidden_keys(session.user, table_key, fields) + return perm_scope.hidden_keys(session.user, table_key, fields, st=st) + + +def _ut_shared_fields(session, table_key, fields): + """`fields` PLUS this database's TENANT-WIDE columns — the read-side merge, W38-T16. + + ⛔⛔ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into + `__shared` for any `ut_*` database since W30-T28, and only the READ-THROUGH door + (`routes_odoo_tables`) ever merged that stratum back. So a shared column written on a + MATERIALISED user table went into the store and was readable by nobody, through any door — + complete, correct, and invisible ([[reachable-is-not-the-same-as-built]]). This is the other + half of that door. + + ⚠ MERGED BEFORE THE WALL, NEVER AFTER, and `routes_odoo_tables` learned this the same way: + the hidden closure must run on the WHOLE contract, or a formula in a shared column that reads + a hidden one sits outside its reach and carries the hidden value out wearing a second name. + ⚠ A key the definition already declares WINS. A shared column is an addition to a database's + contract, never a redefinition of a column that database already has. + """ + from core import shared_overlay + defs = shared_overlay.fields(table_key, st=session.runtime) or {} + if not defs: + return fields + have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} + return list(fields or ()) + [dict(f, source="overlay") + for k, f in defs.items() if k not in have] + +def _ut_shared_cells(session, table_key, pids): + """`{"": {key: value}}` for the TENANT-WIDE stratum, over THIS session's row set. -def _ut_field_wall(session, table_key, fields, rows_src): + ⛔ `pids` IS THE ROW WALL AND IT IS PASSED, NOT DEFAULTED — `shared_overlay.cells` refuses an + "everything" read by signature for exactly this reason, and the set handed in is the one + `scoped_pool`/`scoped_pids` already narrowed. A cell for a row this session may not see + therefore has nothing to attach to. + """ + from core import shared_overlay + try: + return shared_overlay.cells(table_key, list(pids or ()), st=session.runtime) + except Exception: # noqa: BLE001 + return {} + + +def _ut_field_wall(session, table_key, fields, rows_src, st=None): """`(fields, rows_src, hidden)` with the hidden closure removed from BOTH wires. ⭐⭐ W36-T21 — the same three lines `routes_customers.grid_assembly` runs for `customer_data`, @@ -666,7 +774,7 @@ def _ut_field_wall(session, table_key, fields, rows_src): path with nothing to say ([[one-question-two-normalizers]]). """ import core.perm_scope as perm_scope - hide = perm_scope.hidden_keys(session.user, table_key, fields) + hide = perm_scope.hidden_keys(session.user, table_key, fields, st=st) if not hide: return fields, rows_src, frozenset() fields = [f for f in fields if f.get("key") not in hide] @@ -724,7 +832,7 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", ops = _ops(session, table_key, st=st) # ⭐⭐ W36-T21 — the WRITE half of the field wall. `frozenset()` here meant a column an # administrator had hidden was still writable through the events transport. - hidden = _ut_hidden(session, table_key, fields_base) + hidden = _ut_hidden(session, table_key, fields_base, st=st) ctx = grid_events.EventCtx( uname=session.uname, allowed_pids=pids, fields=[], hidden_keys=hidden, admin=session.admin, fallback_ws=None, seen_ids={}, @@ -737,20 +845,190 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", workspace, fields, views, lists = aios_grid.workspace_wire( ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key=storage_key, fields_base=fields_base) + # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM, ON A MATERIALISED GRID. Merged HERE, between + # `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two + # lines and for the same reason: the closure must be recomputed on the MERGED contract or a + # formula in a shared column reaches past it. + fields = _ut_shared_fields(session, table_key, fields) + # ⚠ THE CELLS COST A READ, SO THE ENVELOPE ARM DOES NOT PAY IT. `with_rows=False` renders no + # row at all (D-174), and `cells()` over the whole pid set would be a scoped read whose + # result nothing consumes. The COLUMNS still merge above: `/workspace` needs the contract. + shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {} # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, # so the closure covers this user's `custom_` and `measure_` columns too. - fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src) + fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) + if hidden and shared_cells: + # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the + # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak + # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict. + shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden} + for pid, cells in shared_cells.items()} + + # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database + # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real + # lookback catalogue AND the cells to go with it; every other user table still serves `[]`, + # which is the honest answer for a hand-typed one rather than a descope. + today = time.strftime("%Y-%m-%d") + measures = ut_measures(table_key, session) + derived = aios_grid.cohort_cells(lists) # R9: this table's own lists + for _pid, _cells in _ut_measure_cells(table_key, fields, pids, today, measures, + session.runtime.measure_memo).items(): + derived.setdefault(_pid, {}).update(_cells) return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, "hidden": hidden, - # R9: the Cohorts column's cells from this table's own lists. - "derived": aios_grid.cohort_cells(lists), - "measures": [], "measure_sets": {}, "today": time.strftime("%Y-%m-%d"), + "derived": derived, + # ⭐ W38-T16 — ALWAYS PRESENT, `{}` when this database shares nothing. A key a + # consumer has to test for is a key a consumer forgets to test for, and the consumer + # here (`table_rows`) is layering strata in a fixed order. + "shared_cells": shared_cells, + "measures": measures, + "measure_sets": _ut_measure_sets(table_key, views, measures, pids, today), + "today": today, # ⚠ ALWAYS PRESENT, EMPTY WHEN THERE IS NOTHING TO SAY — a key a consumer has to test # for is a key a consumer forgets to test for, and this one carries a refusal. "limits": limits, "defn": defn} +def _ut_measure_err(tag, e): + try: + import harness.telemetry as _tel + _tel.error(f"api:{tag}", e) + except Exception: # noqa: BLE001 + pass + + +def ut_measures(table_key, session): + """The lookback-measure OFFER for a user database FOR THIS SESSION, or `[]` — W37-T12 / + owner item 4 (R1), narrowed by the metrics capability in W38-T19. + + ⭐⭐ W38-T19 — `session` IS REQUIRED, NOT DEFAULTED. The function was PRINCIPAL-BLIND, so a + per-user capability had nothing to be consulted by. Defaulting it would leave both existing + call sites compiling and silently ungated, which is the fail-open spelling of the bug rather + than a fix for it — the argument `_module_fields` in `routes_admin` records for its own + `session`. ⛔ AND BOTH CALL SITES MATTER, WHICH IS EASY TO GET HALF RIGHT: `ut_assembly` is + the READ and `ut_write_ctx` is the door `routes_grid` turns into `clean_measure_field`'s + admission set. Gating only the read would take the kind off the picker and leave the CREATE + working for anyone who posts the event. + + ⭐ THE SAME ENGINE THE PRODUCT GRID USES (`semantic.entity_measures`), reached through the + topic that DESCRIBES this database. A table with no entity topic, or a topic with no + `measures:` binding, gets `[]` — which is the honest answer for a hand-typed database and is + what keeps the Metric kind hidden there (`ColumnMenu`'s `offerMeasure={measures.length > 0}`). + + ⛔ NOT A `{table_key: [keys]}` MAP IN THIS FILE. The binding is declared in the model + (`model/topics/odoo_agents.yml`'s `measures:` block) and the topic already names its own grid, + so a list here would be a second definition of one fact — the argument `_rollup_source_offer` + makes about itself one screen down. + """ + import core.perm_scope as perm_scope + if not perm_scope.may_metrics(session.user, table_key): + return [] + try: + from harness import semantic as sem + topic = sem.topic_for_grid(table_key) + return sem.entity_measures(topic) if topic else [] + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-offer", e) + return [] + + +def _ut_measure_cells(table_key, fields, pids, today, offer, memo, team_id=None): + """`{pid: {field key: value}}` for this database's Metric columns. + + ⭐ THE JOIN NEEDS NO MAP HERE, and that is worth stating because the product grid DOES need + one. `sales_lines.agent` is `rp.agent_id` — a `res.partner` id — and a `ut_odoo_agents` row's + `pid` IS its `res.partner` id (`scoped_pool` sets `pid = int(rid)` and `read_agents` keys + `_id` on the partner id). So the group key and the pid are the same integer. The product grid + keys on `default_code` instead, which is exactly why contract C1 calls the join key a trap: + the right answer differs per database and must be read off the binding, never assumed. + """ + import aios_grid + + mfields = aios_grid.measure_fields_of(fields) + if not mfields or not offer: + return {} + from harness import semantic as sem + from harness import windows as _wn + + topic = sem.topic_for_grid(table_key) + by_key = {m["key"]: m for m in offer} + by_window = {} + for f in mfields: + spec = f.get("measure") or {} + if spec.get("key") not in by_key: + continue + rng = _wn.resolve(spec.get("window"), today) if _wn.normalize(spec.get("window")) else None + if rng is None: + continue # an unresolvable window is unanswerable — never widen to all time + by_window.setdefault((rng[0], rng[1]), []).append(f) + + out = {} + for (dfrom, dto), group in by_window.items(): + keys = tuple(sorted({f["measure"]["key"] for f in group})) + mkey = ("entity", topic, table_key, today, team_id, dfrom, dto, keys) + if mkey not in memo: + try: + memo[mkey] = sem.entity_measure_values(topic, list(keys), date_from=dfrom, + date_to=dto, team_id=team_id, offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-column", e) + transient = False + try: + from harness import datastore as _ds + transient = not _ds.ready() + except Exception: # noqa: BLE001 + transient = False + if transient: + continue # a warming store must not memoise as a permanent failure + memo[mkey] = None + vals = memo[mkey] + if vals is None: + continue + # C1's empty-window rule for EVERY row, not only the grouped ones. + filled = {pid: sem.entity_zero_fill(vals.get(pid), list(keys), offer) for pid in pids} + for f in group: + k, fkey = f["measure"]["key"], f["key"] + for pid, cell in filled.items(): + if k in cell: + out.setdefault(pid, {})[fkey] = cell[k] + if len(memo) > 100: + memo.clear() + return out + + +def _ut_measure_sets(table_key, views, offer, pids, today, team_id=None): + """`{rule id: [pid, …]}` for the measure CONDITIONS in this database's saved views. + + ⛔ SHIPS WITH THE OFFER, NEVER AFTER IT. `routes_grid` derives `measure_keys` from the same + list, so a non-empty `measures` also opens the measure condition in the filter builder — and a + rule id missing from `measureSets` renders PENDING ("Calculating…") and matches nothing, + permanently. Offer and sets are one feature. + """ + if not offer: + return {} + from harness import semantic as sem + + topic = sem.topic_for_grid(table_key) + admitted = {m["key"] for m in offer} + by_id = {} + for v in views or []: + for rule in sem.entity_measure_leaves((v.get("config") or {}).get("filters"), admitted): + rid = str(rule.get("id") or "") + if rid: + by_id[rid] = rule + if not by_id: + return {} + try: + sets = sem.entity_measure_sets(topic, list(by_id.values()), today, team_id=team_id, + keys_by_id=list(pids), offer=offer) + except Exception as e: # noqa: BLE001 + _ut_measure_err("ut-measure-sets", e) + return {} + return {rid: sorted(int(k) for k in ks if str(k).lstrip("-").isdigit()) + for rid, ks in sets.items()} + + def ut_label(defn, key, meta=None): """THE name of a user table, resolved ONCE (wave 20, item 6a). @@ -857,11 +1135,38 @@ def _rollup_source_offer(): "description": m.get("description") or ""}) out = [] + indeterminate = False for tkey, t in sorted(topics.items()): dims = ((t.get("store") or {}).get("dims") or {}) measures = by_topic.get(tkey) or [] if not dims or not measures: continue + # ⛔⛔ W37-T13 / /validate-wave 2026-08-19 — THE SOURCE TABLE MUST BE LIVE, AND THIS DOOR + # DID NOT ASK. The docstring above already promises *"ONLY COMBINATIONS THAT CAN RESOLVE + # ARE OFFERED"*; adding `stock_move` to `datastore.ENTITIES` broke that promise on the one + # door nobody re-read. `semantic._one_binding_offer` — the MEASURE-CATALOG door — refuses + # the same bindings with a stated cause, so the wave shipped **two doors onto one + # vocabulary with only one of them guarded**, and the unguarded one is the picker. + # MEASURED: `_rollup_source_offer()` listed `stock_moves` with both measures and all four + # dims while `royal.duckdb` has no `stock_move` table at all, so choosing it raised + # `CatalogException: Table with name stock_move does not exist!` — five of the fifty + # (topic, measure, dim) triples in `verify_odoo_relational`. A user meets a raw catalog + # error, or a column that looks configured and is blank forever + # ([[permitted-is-not-answerable]]). + # ⚠ REUSES `_source_ready` rather than re-deriving readiness: one question, one + # normalizer. `None` (the store could not be asked) refuses TOO — fail closed, exactly as + # the measure door does. + _tbl = (t.get("store") or {}).get("table") + if _tbl: + _rdy = sem._source_ready(_tbl) + if _rdy is not True: + # ⛔ AND AN INDETERMINATE ANSWER MUST NOT BE CACHED. `_source_ready`'s own comment + # records the measured version of this: a cold container with no mirror yet + # refused every binding, the refusal was memoised, and the grid served no measures + # long after the mirror arrived. Here the same shape would freeze a truncated + # PICKER for the life of the process. + indeterminate = indeterminate or (_rdy is None) + continue out.append({ "key": tkey, "label": t.get("label") or tkey, @@ -883,7 +1188,10 @@ def _rollup_source_offer(): windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")} for k in ut.ROLLUP_SOURCE_WINDOWS] offer = {"topics": out, "windows": windows} - _ROLLUP_CACHE["offer"] = offer + # ⚠ Cache only a DEFINITE answer (see the readiness block above): if any topic was dropped + # because the store could not be asked, this offer is a snapshot of an unknown, not a fact. + if not indeterminate: + _ROLLUP_CACHE["offer"] = offer return offer @@ -1162,6 +1470,16 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): if isinstance(_ov, dict): _cells.update(_ov) merged[_pid] = _cells + # ⭐⭐ W38-T16 — THE TENANT-WIDE STRATUM GOES ON TOP, and the order is the contract rather + # than a preference. A shared column is one the whole permitted audience must read the SAME + # value in — that is the entire reason `shared_overlay` exists — so a stale per-user value + # left under the same key must not win. `routes_odoo_tables` layers the two in this exact + # order (`overlays.setdefault(pid, {}).update(cells)`) and the two doors must agree. + # ⛔ NEVER `setdefault` ON `merged`: these cells are already narrowed to this session's pids, + # but inventing a row id here would put a row in the payload that `rows_src` never admitted. + for _pid, _cells in (g.get("shared_cells") or {}).items(): + if str(_pid) in merged and isinstance(_cells, dict): + merged[str(_pid)].update(_cells) # ⭐⭐ 2026-08-10 (owner: *"wth is going on, why does it take forever to load now?"*) — THE # JSON DOCUMENTS DO NOT RIDE THE LIST. # diff --git a/api/routes_web_agent.py b/api/routes_web_agent.py index b9d119709bbbcf2501c6b372f0165fb5af1ea6c1..57453ded998810e7046aa5d14acae4d739fd235b 100644 --- a/api/routes_web_agent.py +++ b/api/routes_web_agent.py @@ -22,10 +22,11 @@ admin-gated, because it spends money and because D-51 §5's authorisation postur the tenant is authorised to use, at their instruction") is not something an ordinary member should be able to commit the tenant to. -⛔ THIS ROUTER IS NOT MOUNTED YET. `main.py` belongs to another lane this wave, so the one -`app.include_router(routes_web_agent.router)` line is a cross-fence ask — and -`verify_web_agent.py` FAILS until it lands, deliberately: three finished routers once shipped -404-dead behind entirely green gates, and a gate that tolerates it is how that happens twice. +⭐ MOUNTED, and the gate is what says so rather than this sentence: `main.py:80` imports it and +`main.py:316` includes it, and `verify_web_agent.py` asserts the route answers rather than trusting +either line. This paragraph read "NOT MOUNTED YET" for a whole wave after the mount landed, which +is the same class of stale claim as a green gate on a router nobody wired: three finished routers +once shipped 404-dead behind entirely green gates, and prose is not the control that stops it. """ from fastapi import APIRouter, Body, Depends diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 80b23e386a224df3567eb29d551b7eaa22c9f2cd..a512ac930242fe018480ae678aec50847a39c263 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -24,6 +24,7 @@ Design notes: injected-HTML path remains a read/local-write fallback when component assets are absent. """ import json +import math import re from pathlib import Path @@ -89,6 +90,22 @@ def _round(v): return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v +def _round2(v): + """D-153 (W37, asked by lane B) — MONEY AND PERCENT KEEP THEIR DECIMALS. + + `_round`'s whole-number convention is right for counts and wrong for currency: `first_cost` + 3.47 shipped as `3`, so every money cell on every Odoo grid was a whole dollar. + + ⚠ `pct` IS AFFECTED TOO, and that is the half most likely to be dropped. `cells.ts` renders a + percent as `num(v).toFixed(1) + "%"`, so a `yoy_pct` of 12.34 was `round()`ed to 12 and painted + "12.0%" — one decimal of a number that HAS one, which reads as precision rather than as loss. + ⛔ NOT written as a `_round(v, nd=0)` default argument, deliberately: `round(v, 0)` returns a + FLOAT where `round(v)` returns an INT, so every integer column's wire shape would change under + an edit that looks purely cosmetic. + """ + return round(v, 2) if isinstance(v, (int, float)) and not isinstance(v, bool) else v + + # Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors # customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step. # select — a single-select with its own `options` (the "Status" a user wants to add; distinct @@ -415,6 +432,52 @@ def _clean_formula(raw, valid_keys=None): return s +def _clean_geocode(raw): + """One `geocode` bag → the stored shape, or None (dropped). + + ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7). THE ONE validator for the geocode + pseudo-kind, called by BOTH field doors: `_field_extras` here (the Odoo grids) and + `core/user_tables._clean_field` via `_ag_geocode` (every `ut_*` user database). + + ⛔ IT WAS INLINE IN `_field_extras` UNTIL 2026-08-19 AND THAT IS WHY IT SHIPPED HALF-DONE. + `user_tables._clean_field` is a strict allowlist that returns only the bags it NAMES, so a + geocode column created on a user database was stored WITHOUT its `{addressField}` — the + *"created, named, configured, gone"* failure the `image` type shipped as in wave 19. Extracting + it rather than copying the rules is [[one-question-two-normalizers]]: a second copy is how the + two doors come to disagree about what a valid bag is. + + ⛔ WHOLE-KEY DROP (the `swipe` construction): a geocode column with no address column is not + a degraded column, it is a column no run can ever fill, and the honest state for that is the + unconfigured one the client refuses at `canCreate`. + + ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND + in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, + `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three + parity gates chain over those sets and none permits a partial landing. This is also why the + bag takes `format`'s posture and NOT `formula`'s: there is no type to pair it with, so an + absent bag is an ordinary text column rather than a refusal. + + ⚠ `addressField` is NOT checked against the table's keys: neither door is given `valid_keys`, + and the split this module already draws puts that loss on the surface that RENDERS. The + enricher must SAY the address column is gone rather than geocode an empty string. + """ + if not isinstance(raw, dict): + return None + addr = raw.get("addressField") + if not isinstance(addr, str) or not addr.strip(): + return None + out = {"addressField": addr.strip()[:80]} + # D-3 — the optional ISO 3166-1 alpha-2 hint, mapped straight onto the geocoder's + # `countrycodes` parameter. NORMALISED to upper case and stored only when it is exactly two + # letters: absent means "anywhere", and `""` would be that state's second spelling. A + # three-letter or numeric code is REFUSED rather than truncated, because a truncated code is + # a valid code for a different country. + cc = raw.get("country") + if isinstance(cc, str) and re.fullmatch(r"[A-Za-z]{2}", cc.strip() or ""): + out["country"] = cc.strip().upper() + return out + + def _field_extras(saved, ftype): """createdBy / permissions / format / scope — the validated passthrough the created strata share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it @@ -434,6 +497,27 @@ def _field_extras(saved, ftype): out["format"] = fmt if saved.get("scope") == "cohort": out["scope"] = "cohort" + # ── ⭐⭐ W37-T22 (owner item 15 / ruling R4, contract C7): THE GEOCODE COLUMN'S CONFIG ── + # + # `{addressField}` — which column this one reads. WHOLE-KEY drop (the `swipe` construction): + # a geocode column with no address column is not a degraded one, it is a column no run can + # ever fill, and the honest state for that is the unconfigured one the client refuses at + # `canCreate`. + # + # ⛔ THE FIELD'S `type` IS `text` AND THERE IS NO `geocode` FieldType. The kind is a PSEUDO-KIND + # in the create picker (the `measure` construction), so `CUSTOM_FIELD_TYPES`, `UT_FIELD_TYPES`, + # `CREATABLE_TYPES`, the shape/label tables and the cell renderer are all untouched — three + # parity gates chain over those sets and none permits a partial landing. + # ⚠ `addressField` is NOT checked against this table's keys: `_field_extras` is not given + # `valid_keys`, and the split this module already draws puts that loss on the surface that + # RENDERS ("what each mode MEANS ... is the client's business"). The enricher must SAY the + # address column is gone rather than geocode an empty string. + # ⚠ EXTRACTED to `_clean_geocode` 2026-08-19 (/validate-wave, W37-T22 clause 1). The rules + # did not change; what changed is that `core/user_tables.py` now reaches THIS function through + # `_ag_geocode` instead of carrying a second copy. See `_clean_geocode`'s own note. + geo = _clean_geocode(saved.get("geocode")) + if geo: + out["geocode"] = geo corrected_from = saved.get("labelCorrectedFrom") correction_id = saved.get("labelCorrectionId") if (isinstance(corrected_from, str) and corrected_from.strip() @@ -752,7 +836,10 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): if field.get("derived"): continue # not on the pool row — filled from `derived` below v = r.get(k) - row[k] = v if field["type"] in {"text", "status", "date"} else _round(v) + # D-153 — three roundings, not two: text/status/date pass through verbatim, currency + # and pct keep two decimals (`_round2`), everything else stays a whole number. + row[k] = v if field["type"] in {"text", "status", "date"} else ( + _round2(v) if field["type"] in {"currency", "pct"} else _round(v)) saved = overlays.get(str(pid), {}) or {} for field in overlay_fields: row[field["key"]] = saved.get(field["key"], "") @@ -789,6 +876,19 @@ COHORT_FIELD = '__cohort__' #: a set op reaching a column leaf would fall through the client engine's switch to "no #: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work. COHORT_OPS = {'anyOf', 'allOf', 'noneOf'} + +#: ⭐⭐ WAVE-37 T26 (owner item 14, contract C4) — THE VIEW-MEMBERSHIP LEAF's reserved +#: pseudo-column and its operators. Mirrors `customer-grid/types.ts` VIEW_FIELD / VIEW_OPS. +#: +#: ⛔ DISJOINT FROM `FILTER_OPS`, for the reason COHORT_OPS is: `matchFilter` answers an operator +#: it does not know by NOT NARROWING, so an op that reached a column leaf would widen the result +#: under an authoritative count. Keeping the two vocabularies apart means a column leaf carrying +#: `inView` is dropped by the same fail-closed path that drops `dropTable`. +#: ⚠ The leaf names exactly ONE view (singular where a cohort leaf names a set), so there is no +#: all-or-nothing rule to get wrong here — the id either resolves or the leaf goes. +VIEW_FIELD = '__view__' +VIEW_OPS = {'inView', 'notInView'} +MAX_VIEW_ID = 64 #: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here: #: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the #: next time it is written. Mirrors types.ts COHORT_OP_ALIASES. @@ -912,8 +1012,32 @@ def _clean_rhs(raw, valid_keys): #: written into `iconShapes.ts`, and now machine-enforced in BOTH directions by #: `verify_icons.py::mode_parity` — offering an unmounted mode is red, and mounting an unoffered #: one is red too, so the hold cannot outlive its reason the way wave 27's did). +#: ⭐⭐ WAVE-37 T20 (owner item 10, ruling R2, contract C2) — 'script', and the reason it lands +#: here is NOT the one the PRD gives. The PRD says the Custom View is invisible because this set +#: omits the name so `_clean_display` drops the mode on every write. MEASURED, that write never +#: happens: a script view is a rail PROJECTION built client-side by +#: `CustomerGrid.tsx::scriptProjectionView`, its panel is gated on `activeScriptId !== null` +#: (a row in E's per-database store), there is no `displayMode === "script"` branch at all, and the +#: autosave effect reads `views.find(...)` while the projections are appended to `railViews` ALONE. +#: Nothing was ever dropped, because nothing was ever sent. +#: ⭐ WHAT THE NAME ACTUALLY BUYS is the PRECONDITION for the create door the owner asked for: a +#: script view can only be offered through the display-mode picker once a view STORED as +#: `mode: 'script'` survives a round trip, and until this line it could not. The picker entry +#: itself is W37-T41 (lane E) plus `CREATABLE_MODES`, and `_test/icons.test.ts::HELD_MODES` keeps +#: the hold with its release condition rewritten to say so. +#: ⚠ CONTRACT C2 MAKES THIS SET AND `customer-grid/types.ts::DISPLAY_MODES` IDENTICAL NAME FOR +#: NAME, and `verify_fields_contract.py` now asserts it in BOTH directions. That retires the staged +#: hold every mode from `timeseries` to `form` was landed with — a client-first name is now RED +#: rather than merely risky. The protection the hold gave is structural instead: both halves are in +#: one lane's fence, so they ship in one change, and whether a mode may be CREATED is a separate +#: question answered by `CREATABLE_MODES` / `HELD_MODES`. DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart', - 'timeseries', 'catalog', 'swipe', 'form'} + 'timeseries', 'catalog', 'swipe', 'form', 'script'} + +#: W37-T20 (C2) — the cap on `display.script.id`. Script-view ids are minted server-side as +#: `'sv_' + secrets.token_urlsafe(9)` (`routes_script_views.py::create_script_view`), so ~15 chars; +#: 64 is a bound on a hostile payload, not an opinion about the format. +MAX_SCRIPT_ID = 64 #: ⭐⭐ WAVE-29 R7 (owner item 10, contracts C3/C4) — THE FORM INTERFACE's stored spec, at #: `views[].config.display.form`. The public door (`aios-web/api/routes_forms.py`) has read @@ -926,6 +1050,166 @@ DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'char #: so one tenant setting its token to another tenant's value would silently receive that tenant's #: submissions. The token is minted server-side and lives in a bucket no client write can reach; #: `_clean_form` drops any `token` key that arrives here, rather than validating its shape. +#: ⭐⭐ WAVE-37 T21 (owner items 1/12, rulings R5+R6, contract C3) — THE PER-VIEW CATALOG SPEC, +#: stored at `views[].config.display.catalog`. +#: +#: ⛔⛔ IT IS **`catalog`**, SINGULAR, AND IT IS NOT `catalogs`. Both keys live on `display`, they +#: are one letter apart, and that is the single most likely mistake anybody touching this file will +#: make. They are different things and BOTH are live: +#: · `catalogs` (wave-18 C6) — a LIST of up to 12 print artifacts, each with its own paper, +#: brand and pages of kind cover/intro/section/gallery. Authored CONTENT. UNTOUCHED by T21. +#: · `catalog` (wave-37 C3) — THIS view's catalog SETTINGS: which column is the picture, which +#: is the title (R5: per view, so two catalog views on one database may differ), plus the page +#: order and the free-placed items R6 asks for. +#: Neither replaces the other, and neither validator reads the other's key. +#: +#: ⛔ THE MIGRATION IS A RULE ABOUT WHAT THIS FUNCTION MUST **NOT** DO (R6, C3). An item authored +#: before free placement has no `x`/`y`, and an ABSENT coordinate must read back ABSENT — never +#: normalised to 0. Defaulting to zero would pile every pre-existing item at the origin, which is +#: "a catalog that silently loses its layout is a FAIL, not a migration" stated as code. The +#: RENDERER turns an absent coordinate into flow position N; the host's whole job is to not destroy +#: the distinction it reads. +#: ⚠ `w`/`h` carry the OPPOSITE asymmetry, deliberately: a zero WIDTH is an invisible item, so 0 +#: is refused and absent means "the renderer's default size". A zero X is the top-left corner and is +#: a real, storable position. Two keys of one shape with two rules — the `kanbanClamp` family — +#: which is why each is written out rather than looped. +MAX_CATALOG_SECTIONS = 20 +#: A bound on a hostile payload, not an opinion about the renderer's coordinate space: D owns what +#: the numbers MEAN. Stored to 2 decimals, finer than any print surface resolves. +MAX_CATALOG_COORD = 10000 +#: The page `order` bound. ABSENT => that page's ARRAY INDEX, and `order` WINS where present; ties +#: break by array index. Decided HERE rather than left to the renderer, because two readers guessing +#: differently about page order is exactly the divergence contract C3 exists to close. +MAX_CATALOG_ORDER = 9999 + + +def _clean_catalog_coord(raw, allow_zero): + """One free-placement number, or None. `allow_zero` False is the w/h leg (see the header). + + ⛔ THE ROUNDING IS `floor(v * 100 + 0.5) / 100` AND NOT `round(v, 2)`, AND THAT IS THE WHOLE + POINT OF THIS FUNCTION EXISTING SEPARATELY. Python's `round` is BANKER'S rounding and + JavaScript's `Math.round` is half-up-toward-+Infinity, so the two engines disagree on every + exact half: `round(0.125, 2)` is 0.12 here and 0.13 in the browser. One dragged item landing two + hundredths apart on the two engines is a `display` object that never compares equal — the + silent mirror drift contract C2 exists to prevent, arriving through arithmetic instead of + through a missing key. The expression below is `Math.floor(v * 100 + 0.5) / 100` written in + Python, evaluated in the same IEEE doubles, so the two agree bit for bit. + ⚠ AND AN INTEGRAL RESULT IS EMITTED AS AN `int`: `12.0` serialises as `12.0` in Python and + `12` in JSON.stringify, which is one stored value with two spellings. + """ + if isinstance(raw, bool) or not isinstance(raw, (int, float)): + return None + if raw != raw or raw in (float("inf"), float("-inf")): # NaN / +-inf + return None + val = math.floor(float(raw) * 100 + 0.5) / 100 + if val > MAX_CATALOG_COORD or val < -MAX_CATALOG_COORD: + return None + if not allow_zero and val <= 0: + return None + return int(val) if val == int(val) else val + + +def _clean_catalog_item(raw): + """One free-placed item: `{code, x?, y?, w?, h?}`. `code` REQUIRED; every coordinate optional. + + Key EMISSION ORDER is fixed and mirrored byte-for-byte by `types.ts::cleanCatalogSpec` — an + accepted save has to read back identically on both engines, and this pair is where that is + decided. + """ + if not isinstance(raw, dict): + return None + code = raw.get("code") + if not isinstance(code, str) or not code.strip(): + return None + out = {"code": code.strip()[:CATALOG_CODE_MAX]} + for key, allow_zero in (("x", True), ("y", True), ("w", False), ("h", False)): + val = _clean_catalog_coord(raw.get(key), allow_zero) + if val is not None: + out[key] = val + return out + + +def _clean_catalog_spec(raw, valid_keys): + """C3 — `display.catalog`, fail-closed. See the header block above for what it is NOT. + + Field refs follow the `dateField`/`stackField` rule this module already runs: a ref naming a + deleted column is dropped INDIVIDUALLY and the renderer falls back to its per-mode default. It + never costs the user their pages — the `charts` per-entry-drop precedent, applied to the one + payload here that is AUTHORED rather than derived from the rows. + """ + if not isinstance(raw, dict): + return None + out = {} + for ref in ("imageField", "titleField"): + if raw.get(ref) in valid_keys: + out[ref] = raw[ref] + pages_raw = raw.get("pages") + pages = [] + if isinstance(pages_raw, list): + # ONE cumulative item budget across the whole spec, spent in page order then section order + # — the `_clean_catalogs` construction, so a 40-page catalog cannot smuggle 20,000 items + # past a per-page cap. + budget = MAX_CATALOG_CODES + seen_pages = set() + for raw_page in pages_raw: + if len(pages) >= MAX_CATALOG_PAGES: + break + if not isinstance(raw_page, dict): + continue + pid = str(raw_page.get("id") or "").strip()[:CATALOG_ID_MAX] + if not pid or pid in seen_pages: + continue + seen_pages.add(pid) + page = {"id": pid} + order = raw_page.get("order") + if (isinstance(order, int) and not isinstance(order, bool) + and 0 <= order <= MAX_CATALOG_ORDER): + page["order"] = order + sections_raw = raw_page.get("sections") + sections = [] + if isinstance(sections_raw, list): + seen_sections = set() + for raw_section in sections_raw: + if len(sections) >= MAX_CATALOG_SECTIONS: + break + if not isinstance(raw_section, dict): + continue + sid = str(raw_section.get("id") or "").strip()[:CATALOG_ID_MAX] + if not sid or sid in seen_sections: + continue + seen_sections.add(sid) + section = {"id": sid} + items_raw = raw_section.get("items") + items = [] + if isinstance(items_raw, list) and budget > 0: + for raw_item in items_raw: + item = _clean_catalog_item(raw_item) + if item is None: + continue + items.append(item) + budget -= 1 + if budget <= 0: + break + # ⚠ NOT de-duplicated by `code`. The same product legitimately appears twice in + # one section at two positions — that is what free placement IS — and wave-18's + # "deduped WITHIN a page" rule was written for a LISTING, where a repeat is a + # mistake. Here it is the feature. + if items: + section["items"] = items + sections.append(section) + # An EMPTY section is KEPT: a person adds a New Section and then drags into it, and + # dropping it would delete the thing they just made, between two autosaves. + if sections: + page["sections"] = sections + pages.append(page) + if pages: + out["pages"] = pages + # A spec that reduces to nothing is DROPPED, never stored as `{}`. "No image field, no title + # field, no pages" IS the unconfigured state, and `{}` would be its second spelling — the + # no-churn law every literal-only key in `_clean_display` follows. + return out or None + + FORM_ACCESS = ('public', 'emails') MAX_FORM_FIELDS = 60 MAX_FORM_EMAILS = 200 @@ -1373,7 +1657,11 @@ def _clean_display(raw, valid_keys): # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'. mode = _LEGACY_MODES.get(mode, mode) out = {'mode': mode} - for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'): + # ⭐ W37-T28 (owner item 15) — `coordField`: ONE column holding `"lat,lon"`, so a map is no + # longer a customers-only feature. It joins this loop rather than getting a clause of its own + # because it IS a field ref with the same drop-if-deleted rule; `customer-grid/types.ts` + # gains it in the SAME change, which is contract C2's whole point. + for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField', 'coordField'): if raw.get(ref) in valid_keys: out[ref] = raw[ref] # ── C-DISP (wave 2026-08-02) ───────────────────────────────────────────────────────── @@ -1508,6 +1796,12 @@ def _clean_display(raw, valid_keys): catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys) if catalogs: out['catalogs'] = catalogs + # ── ⭐⭐ WAVE-37 C3 (T21) — `catalog`, SINGULAR, and it is NOT the line above it. See the + # header on `_clean_catalog_spec` for the two-keys-one-letter-apart warning; this call site is + # where a reader is most likely to "fix" one into the other. + catalog = _clean_catalog_spec(raw.get('catalog'), valid_keys) + if catalog: + out['catalog'] = catalog # ── ⭐ WAVE-27 C3 (item 8 / R2): the swipe binding ──────────────────────────────────── # `{fieldKey, leftOption, rightOption}` — WHOLE-KEY drop, never a partial one, and that # asymmetry against `charts`/`calendarMetrics` above is the point rather than an oversight. @@ -1536,6 +1830,26 @@ def _clean_display(raw, valid_keys): # `kanbanClamp` law) wearing a control that promises a choice. if left and right and left.casefold() != right.casefold(): out['swipe'] = {'fieldKey': f_key, 'leftOption': left, 'rightOption': right} + # ── ⭐⭐ WAVE-37 T20 (owner item 10 / R2, contract C2): WHICH SCRIPT THIS VIEW RENDERS ── + # + # `{id}` — the row in E's per-database script store (`/api/v1/script-views`) whose source this + # view draws. WHOLE-KEY drop, the `swipe` construction rather than the `charts` one: a script + # view with no id is not a degraded script view, it is a panel with nothing to run, and the + # honest state for that is the unconfigured one the client already has a surface for. + # + # ⛔ THE ID IS DELIBERATELY NOT CHECKED AGAINST `valid_keys`, AND THAT IS NOT AN OVERSIGHT. + # `valid_keys` is this table's FIELD keys; a script id names a record in a DIFFERENT store that + # this function cannot reach. So the host bounds the shape and nothing else, and the renderer + # owns the one loss this is blind to — the script row was deleted — which it must SHOW rather + # than fall back to another script (the `viewModes.tsx` house rule `_clean_swipe`'s note cites). + # ⚠ Mirrored key-for-key by `customer-grid/types.ts::cleanDisplay`. That file rebuilds the + # config key by key on every autosave, so a key accepted HERE and unknown THERE is erased on the + # next column resize — which is the half of the round trip this docstring's W33 note warns about. + script = raw.get('script') + if isinstance(script, dict): + s_id = script.get('id') + if isinstance(s_id, str) and s_id.strip(): + out['script'] = {'id': s_id.strip()[:MAX_SCRIPT_ID]} # ── ⭐⭐ WAVE-29 R7 (item 10): the FORM spec — see `_clean_form` for why the token is not here. form = _clean_form(raw.get('form'), valid_keys) if form: @@ -1743,7 +2057,59 @@ def clean_item_folders(raw, folders, valid_ids): return out -def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): +def view_refs_of(nodes): + """C4 — every view id a tree REFERS to, deduped, in order. Mirrors `types.ts::viewRefsOf`. + + ⚠ An INACTIVE leaf (no view chosen yet) is not a reference, on both engines: counting one + would refuse a save the moment somebody opened the condition and had not picked anything. + """ + out = [] + for node in nodes or []: + if not isinstance(node, dict): + continue + if 'children' in node: + for vid in view_refs_of(node.get('children')): + if vid not in out: + out.append(vid) + continue + if node.get('colId') != VIEW_FIELD: + continue + vid = str(node.get('value') or '').strip()[:MAX_VIEW_ID] + if vid and vid not in out: + out.append(vid) + return out + + +def view_filter_cycle(view_id, nodes, filters_of): + """⭐⭐ W37-T26 / C4 — THE CYCLE REFUSAL's host half. Mirrors `types.ts::viewFilterCycle`. + + View X filtered on "is in View Y" where Y is filtered on X cannot be resolved by anything: + X needs Y needs X. Returns the CYCLE PATH (ids, starting and ending at `view_id`) or None. + + ⛔ THE CLIENT REFUSES THE SAME SHAPE AND THAT IS NOT DUPLICATION. The browser's guard is what + a PERSON sees and is the only one that can explain itself; this one is what makes the rule true + for a caller that POSTs a view directly, which is every guard-in-the-browser's blind spot. + ⚠ A view referring to ITSELF is a cycle of length one and is caught by the same walk — also + the case a person reaches most easily, by duplicating a view. + """ + def walk(current, path, seen): + refs = view_refs_of(nodes if current == view_id else filters_of(current)) + for nxt in refs: + if nxt == view_id: + return path + [nxt] + if nxt in seen: + continue + seen.add(nxt) + found = walk(nxt, path + [nxt], seen) + if found: + return found + return None + + return walk(view_id, [view_id], {view_id}) + + +def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, + visible_view_ids=None): """Recursively validate an UNTRUSTED filter tree (conditions + nested groups). Returns a clean tree of leaf conditions ({colId, op, value, value2}) and @@ -1762,6 +2128,14 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): condition that can only match nothing, so `List is not [deleted]` would show an empty table forever with no way to tell why. `None` means this host has no cohorts, and then every cohort leaf is dropped: fail-closed, like every other unknown key. + + ⭐⭐ W37-T26 (C4) — `visible_view_ids` is the same parameter one leaf over: the views this + caller may name in a `__view__` membership leaf. A leaf naming anything else is DROPPED here + for the identical reason, and `None` (this host has no views to offer) drops every one. + ⚠ DEFAULTS TO `None` SO EVERY EXISTING CALLER IS BYTE-IDENTICAL. Three callers pass nothing + (`routes_admin`, `routes_query` twice) and each of them is a surface with no saved-view + vocabulary of its own; under the fail-closed rule they drop the leaf, which is correct rather + than merely safe. """ if budget is None: budget = [MAX_FILTER_NODES] @@ -1776,7 +2150,7 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): continue # too deep -> drop budget[0] -= 1 children = clean_filter_tree(node['children'], valid_keys, - depth + 1, budget, cohort_ids) + depth + 1, budget, cohort_ids, visible_view_ids) if children: out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', 'children': children}) @@ -1793,6 +2167,21 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None): out.append({'colId': COHORT_FIELD, 'op': op, 'value': ','.join(named), 'value2': ''}) continue + if node.get('colId') == VIEW_FIELD: # C4: a view-membership leaf + # ⛔ THE SAME ALL-OR-NOTHING THE COHORT ARM ABOVE APPLIES, AND FOR THE SAME REASON. + # A leaf naming a view this caller cannot see is DROPPED rather than kept, because a + # kept one can only ever match nothing: `View is not [a view you cannot see]` would + # show an empty table forever with no way to tell why. Dropping it puts the view back + # in its honest unfiltered state, which is what `clean_filter_tree`'s own docstring + # already promises for cohorts. + # ⚠ `visible_views` is what the caller passes; `None` means this host does not know, + # and then EVERY view leaf is dropped — fail-closed, like every other unknown key. + v_op = node.get('op') + v_id = str(node.get('value') or '').strip()[:MAX_VIEW_ID] + if v_op in VIEW_OPS and v_id and v_id in (visible_view_ids or ()): + budget[0] -= 1 + out.append({'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''}) + continue if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: budget[0] -= 1 # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index da186ace8dc5078cfd9e7989571f3f6f7e70d0a7..3662c13e1343ae928010f70c2484164851afa9ac 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -386,6 +386,20 @@ "source": "odoo", "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." }, + { + "key": "tier_prices", + "label": "Tier prices", + "type": "json", + "source": "odoo", + "description": "Every live pricelist that prices this SKU today, as [{pricelist, unit_price}] at qty 1. Blank when no list prices it. This is the honest SET; the three Fisch/Royal columns beside it are the DECLARED subset and cannot show a price on a list the contract does not name." + }, + { + "key": "units", + "label": "Units", + "type": "json", + "source": "odoo", + "description": "The units of measure this SKU is really sold in, as [{name, qty}] where qty is in the product's own unit. Blank means it is sold in ONE unit, not that data is missing - only 1,150 of 5,873 active SKUs (19.6%) carry a unit tier." + }, { "key": "rev_ytd", "label": "Revenue YTD", diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index b4b78bdc3ea98f45b86fbf5032c1bfd708894d4e..164f6a9c8898839a38a4a4a89d16acb1c6a8fcdc 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -801,9 +801,39 @@ def handle_one(event, ctx): # else is dropped here. Leaving it in would persist a condition the engine can only # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table # forever with nothing on screen explaining why. + # ⭐⭐ W37-T26 / CONTRACT C4 — `visible_view_ids` is the same PERMISSION list one leaf over: + # a `__view__` membership leaf naming a view this caller cannot see is dropped here, for + # the identical reason the cohort line above gives. Without it the arm is fail-closed and + # every view leaf is dropped on save, so the condition would vanish on the next read — + # D-90's shape exactly. + # ⚠ `ctx.visible_views` is already populated (`routes_grid.py` passes it, `:714` reads it), + # so this needs no new plumbing; it is the value nobody had handed to the validator. + # ⚠ Edited by lane C, not this file's owner: `platform/core/grid_events.py` is in NO fence + # and lane C was the last session open. See `mailbox/C.md` NOTE C-24. + _view_ids = {str(v.get('id')) for v in (visible_views or ()) + if isinstance(v, dict) and v.get('id')} + # ⛔⛔ THE CYCLE REFUSAL'S HOST HALF (C4), and it is the ticket's `done-when` rather than a + # guard bolted on afterwards. View X filtered on "is in View Y" where Y is filtered on X + # cannot be resolved by anything: X needs Y needs X. The browser refuses it too, with a + # sentence a person reads; this one is what makes the rule true for a caller that POSTs a + # view directly, which is every guard-in-the-browser's blind spot. + # ⚠ REFUSED, NOT REPAIRED. Dropping the offending leaf would save a view the person did not + # write and leave the panel showing a condition the config does not carry. + _cycle = _ag.view_filter_cycle( + view_id, cfg.get('filters'), + lambda vid: next((v.get('config', {}).get('filters') + for v in (visible_views or ()) + if isinstance(v, dict) and str(v.get('id')) == vid), None)) + if _cycle: + _refuse(ctx, 'view_cycle', + 'that would make a loop: ' + ' refers to '.join(_cycle) + + '. A view cannot filter on a view that filters on it, because neither can ' + 'be worked out without the other.', view_id) + return False filters = _ag.clean_filter_tree(cfg.get('filters'), valid_keys | set(measure_keys), - cohort_ids=set(cohort_ids)) + cohort_ids=set(cohort_ids), + visible_view_ids=_view_ids) sorts = [] for rule in list(cfg.get('sorts') or [])[:20]: if (isinstance(rule, dict) and rule.get('colId') in valid_keys diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index 02ce70a3e1d42292c83d998f4dce87df7c9e08ad..3b707578375d940283cecbe032f6274a9b178414 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -87,6 +87,36 @@ def may_access(user, module): return perms.may_open(user, module) # legacy record: the old grant wall +def may_metrics(user, module): + """May this account build and receive METRIC columns — lookback measures — on `module`? + + ⭐⭐ W38-T19 — A CAPABILITY, NOT A SECOND SPELLING OF ACCESS, and the distinction is the + ticket. A rollup aggregates the CHILDREN a row is linked to; a Metric answers *"this number, + over this window"* against a governed topic with no relation at all (CLAUDE.md standing rule + 9). So it reads the book behind the rows rather than the rows: an account can be exactly the + right person to see a customer list and the wrong one to mint 12-month revenue over it. Two + decisions, two controls. + + ⛔ ABSENCE GRANTS, AND THE ASYMMETRY WITH `may_access` IS DELIBERATE RATHER THAN AN + OVERSIGHT TO TIDY. `may_access` reads migrated-and-undeclared as DENY, which is right there + because every save writes an entry for every governed database — absence means an + administrator decided. No `metrics` KEY was STORABLE before this ticket, so every migrated + record in every tenant carries none, and reading that absence as DENY would revoke Metrics + for everybody on the day this shipped with nobody having decided anything. That is the exact + failure `routes_admin.get_perms` records twice already (the `ut_*` default and the surface + default), one field further down the same entry. **Only an explicit `metrics: false` + refuses.** + + ⚠ IT DOES NOT RE-ASK ACCESS. Every caller is already behind the access wall (`module_gate`, + `may_read`, `session.require`), and folding admission in here would give a denial two + possible causes with one answer — the shape that makes a permission bug take an afternoon. + """ + if perms.is_admin(user): + return True + e = entry(user, module) + return not (isinstance(e, dict) and e.get('metrics') is False) + + def nav_may_open(user, key, st=None): """May this account SEE `key` — in the nav, and at the route gate? The ADMISSION question. @@ -148,7 +178,103 @@ def assistant_entry(user, module): # ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── -def hidden_keys(user, module, fields): +#: ⭐⭐ W38-T16 — THE MARKER THAT SAYS "THIS COLUMN IS GOVERNED BY A GRANT", stamped once by the +#: door that creates a shared column (`routes_tables.patch_shared_cell`) and never flipped by any +#: door afterwards. It is the `perms_v` idiom one object down, and it is here for the reason that +#: marker exists: **absence must not be read as a decision.** +#: +#: ⛔⛔ WHY A MARKER AND NOT "DOES A GRANT RECORD EXIST". Every tenant-wide column shipped before +#: this ticket carries no `field` grant record, because none was STORABLE — `shares.KINDS` had +#: three members. Reading that absence as "granted to nobody" would blank every existing shared +#: column for every account the moment this arms, which is not a permission fix, it is an outage +#: (`may_read` leg 3 carries the same argument for `ut_*` keys, in the same words). +#: +#: ⛔⛔ AND IT IS WHAT MAKES THE WALL FAIL **CLOSED**. The polarity here is the opposite of every +#: other grant check in this repo: a field share is a GRANT, so "no grant" has to mean HIDDEN or +#: the wall does nothing — but "no grant record" also describes a legacy column and a store that +#: could not be read. The marker separates the three: `granted` on the column means an explicit +#: decision was taken, so an unreadable registry hides it; no marker means legacy, so nothing +#: changes. Without it, one unreadable read of `object_shares` would publish every governed +#: column to the whole tenant, silently ([[a-guard-for-the-dangerous-case]]). +#: +#: ⚠ IT IS NOT `shared_overlay`'s `"shared": True` AND MUST NOT BE CONFUSED WITH IT. That flag is +#: written and never read (D-414) — `is_shared` is a dict-membership test — so it is not evidence +#: of anything. This one is read HERE, on every assembly, and the only writer is the create door. +FIELD_GRANT_MARK = 'granted' + + +def granted_field_keys(fields): + """Every column in `fields` that declares itself GOVERNED by a `shares` grant. + + ⭐ THE CHEAP HALF OF THE WALL, AND IT IS WHY THE WALL COSTS NOTHING FOR ALMOST EVERYBODY. It + is a scan of dicts already in memory, so a database with no governed column reaches no store + at all and `hidden_keys` behaves exactly as it did before this ticket. The registry is only + opened once this answers non-empty. + """ + return {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key') + and f.get(FIELD_GRANT_MARK) is True} + + +def field_grant_hidden(user, table_key, fields, st=None): + """The governed columns of `table_key` this principal holds NO grant on — C1's per-FIELD wall. + + ⭐⭐ W38-T16 / R7 / R8 — THE THIRD WALL, AND IT COMPOSES ALONGSIDE THE OTHER TWO RATHER THAN + INSIDE THEM. `may_open` answers *IF* you reach a database; `may_read`'s stored `access: false` + overlay may REVOKE one; this answers *WHICH COLUMNS* of it you receive. It is deliberately not + threaded through that overlay: the overlay is **deny-only** by its own docstring (*"it may + revoke a database the wall below would admit, and it may never grant one that wall refuses"*) + and a field share is a GRANT — the widening direction. Merging them would give the codebase a + second idea of who grants what, which is the failure `may_open`'s own note is the record of. + It composes the way `may_open`'s grant leg does: additively, last, fail-closed. + + ⛔ AN ADMIN IS NOT WALLED (break-glass, as everywhere else) and neither is the column's OWNER — + `shares.role_for` answers `'owner'` for the creator, so a person cannot lose their own column + by forgetting to share it with themselves. + + ⚠ `st` IS THE TENANT HANDLE AND ITS ABSENCE IS SAFE HERE, unlike everywhere else. A caller + that cannot lend one reads the module-default bucket; on any tenant but #0 that finds no + grant, and no grant on a MARKED column means HIDDEN. So a door that has not learned to thread + `st` under-shares rather than over-shares, and the symptom is a grantee who cannot see their + column — visible, reportable, and the opposite of a leak. + """ + if perms.is_admin(user): + return set() + marked = granted_field_keys(fields) + if not marked: + return set() + uname = str((user or {}).get('username') or '').strip().lower() + try: + import core.shares as shares + except Exception: # noqa: BLE001 + return set(marked) + if not uname: + # No principal, and a marked column is an explicit decision: nobody is not somebody. + return set(marked) + # ⭐ THE CREATOR IS READ OFF THE COLUMN, NOT OUT OF THE REGISTRY, AND THAT IS NOT A SECOND + # AUTHORITY. `createdBy` is ALREADY what decides who may DELETE a shared column (R8 / D-172, + # `routes_tables.delete_shared_field`) and who may CLAIM it (`routes_shares._owns_object`); + # asking the same field here keeps one answer to "whose column is this" across all three. + # ⛔ IT IS ALSO THE ONLY THING THAT SURVIVES A CLAIM THAT NEVER LANDED. The create door writes + # the definition first and the grant record second, on purpose — so the window where a column + # is marked and unclaimed exists, and without this line its own author would be walled out of + # the column they just made, permanently and with no way to fix it but an admin. + mine = {str(f['key']) for f in (fields or ()) + if isinstance(f, dict) and f.get('key') and str(f['key']) in marked + and str(f.get('createdBy') or '').strip().lower() == uname} + hide = set() + for key in marked - mine: + try: + oid = shares.field_oid(table_key, key) + role = shares.role_for('field', oid, uname, is_admin=False, st=st) + except Exception: # noqa: BLE001 + role = None + if role is None: + hide.add(key) + return hide + + +def hidden_keys(user, module, fields, st=None): """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5). ⛔ WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER @@ -180,10 +306,26 @@ def hidden_keys(user, module, fields): and `rollup.field` is deliberately not treated as one — it names another database's column, which has its own wall. """ - e = entry(user, module) - if perms.is_admin(user) or not e: + if perms.is_admin(user): return frozenset() - hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k} + # ⭐⭐ W38-T16 — TWO SOURCES OF HIDING, ONE CLOSURE, AND THAT UNION IS THE WHOLE INTEGRATION. + # + # The administrator's `hiddenFields` and a field's own grant answer different questions and + # both end in the same place: a key this reader may not receive. Seeded together HERE, before + # the fixpoint, so the transitive argument above covers the new source unchanged — a formula + # (or a rollup) over a column this reader was not granted comes out with it, or the value + # leaks wearing the derived column's name. + # + # ⛔ AND THIS FUNCTION IS THE INSERTION POINT RATHER THAN `_scoped`, WHICH IS WHAT THE TICKET + # ASSUMED. `_scoped` is C1's evaluator and reaches C1's two doors; **the product reads through + # neither of them.** Every grid door calls THIS: `routes_customers:196`, `routes_products:345`, + # `routes_odoo_tables:920/1119`, `routes_tables._ut_hidden/_ut_field_wall`, `routes_slack:190`, + # `routes_grid:57/69/787` — and `grid_events` walls WRITES off the same set through + # `EventCtx.hidden_keys`. One evaluator, every door, both wires, read and write. + hidden = set(field_grant_hidden(user, module, fields, st=st)) + e = entry(user, module) + if e: + hidden |= {str(k) for k in (e.get('hiddenFields') or ()) if k} if not hidden: return frozenset() @@ -210,9 +352,16 @@ def hidden_keys(user, module, fields): return frozenset(hidden) -def visible_fields(fields, user, module): - """`fields` minus the hidden closure. Order preserved — the column order is the user's.""" - hide = hidden_keys(user, module, fields) +def visible_fields(fields, user, module, st=None): + """`fields` minus the hidden closure. Order preserved — the column order is the user's. + + ⚠ W38-T16 — `st` MUST MATCH WHAT ITS CALLER PASSED TO `hidden_keys`, and that is not a style + note. This recomputes the closure, and since the field-grant leg under-hides without a tenant + handle, a caller that lends one to `hidden_keys` and not to this would narrow the FIELD LIST + by MORE than it stripped from the ROWS — the value left sitting in the payload under a column + nobody can see, which is exactly the half-wall `strip_row`'s note exists to forbid. + """ + hide = hidden_keys(user, module, fields, st=st) if not hide: return list(fields or ()) return [f for f in (fields or ()) @@ -767,16 +916,19 @@ def _scoped(user, table_key, st=None, ctx=None, want_rows=True): # THE FIELD WALL — a TRANSITIVE closure, so hiding a column also hides every formula computed # FROM it. Resolved ONCE and used for both wires; see `hidden_keys` for why a set difference # is the wrong shape here. - hide = hidden_keys(user, key, fields) + # ⭐ W38-T16 — `st` RIDES INTO THE WALL, not just into the read. The field-grant leg resolves + # against `object_shares` in THIS tenant's store; without the handle it would answer from the + # module default (tenant #0) and hide a grantee's own column on every other tenant. + hide = hidden_keys(user, key, fields, st=st) if not want_rows: - return (visible_fields(fields, user, key) if hide else fields), [] + return (visible_fields(fields, user, key, st=st) if hide else fields), [] # THE ROW WALL — `permits()`, so a permanent filter this evaluator cannot answer DENIES # rather than being ignored. Evaluated against the UNSTRIPPED contract on purpose: a # permanent filter may name a column the reader is not allowed to SEE, and dropping the # predicate would widen the read rather than narrow it. rows = apply_row_scope(rows, user, key, fields, ctx) if hide: - fields = visible_fields(fields, user, key) + fields = visible_fields(fields, user, key, st=st) rows = [strip_row(r, hide) for r in rows] return fields, rows diff --git a/platform/core/script_sandbox.py b/platform/core/script_sandbox.py new file mode 100644 index 0000000000000000000000000000000000000000..babf60594d17d540e348efa836e40056260a1f96 --- /dev/null +++ b/platform/core/script_sandbox.py @@ -0,0 +1,519 @@ +"""core/script_sandbox.py — WAVE 36 (R5 / R10, contract C1): running a tenant's OWN Python. + +Owner item 6: *"Add code script as an interface (database View) so a user can build whatever they +want through the Agent chat interface."* Item 8: *"We need to really guardrail the reach of this +script. So let's really grill this down."* R10 ruled it SERVER-SIDE PYTHON after the trade was +stated, so this file is the guardrail, and one engine serves both items. + +════════════════════════════════════════════════════════════════════════════════════════════════ +⛔⛔ THE ONE PARAGRAPH TO READ BEFORE CHANGING ANYTHING HERE. + +In-process CPython cannot deliver two of this ticket's clauses. An AST allow-list plus a curated +namespace stops import, file, network and environment access — but it **cannot cap memory and +cannot interrupt a runaway loop**, because a `while True:` in the same interpreter is not a slow +request, it is the tenant's ONE FastAPI process gone. So the script runs in a **SUBPROCESS**: +`resource.setrlimit` for address space and CPU, a hard wall-clock kill from the parent, and the +allow-list inside. Neither half is sufficient; both are load-bearing. + +⭐ AND THE SUBPROCESS RECEIVES **ROWS, NEVER A STORE**. The parent calls C1's `scoped_table` under +the CALLING user's record and serialises the result; the child imports nothing from this repo and +holds no credential, no runtime and no store handle. Wiring W1 ("the sandbox has no second store +path") is then true by CONSTRUCTION rather than by discipline, and it is checkable: the child +reports its own `sys.modules`, and no `core.*` name may appear in it. + +⛔ NEVER A BLACKLIST. Every rule below is an ALLOW-LIST — a set of node types, a set of attribute +names, a dict of builtins. A blacklist of dangerous spellings is bypassable by construction, and +the bypass is usually one string method away (`"{0.__class__}".format(x)` performs its attribute +lookup inside `format`, so there is no `ast.Attribute` node to refuse). +════════════════════════════════════════════════════════════════════════════════════════════════ + +The two layers, and they refuse DIFFERENT things on purpose: + + 1. `check_source()` — a pure function over source text. Refuses a construct the language offers + and this sandbox does not: `import`, `class`, `with`, `async`, `yield`, `global`, and every + attribute name outside `ALLOWED_ATTRS`. + 2. `SANDBOX_BUILTINS` — the names that resolve at all. `__import__`, `open`, `eval`, `exec`, + `compile`, `getattr`, `globals`, `vars` and `type` are simply absent, so a source that gets + past layer 1 still finds nothing to call. + +⚠ THAT DUPLICATION IS DELIBERATE AND IT CHANGES HOW THE GATE MUST BE WRITTEN. `import os` is +refused twice, so a negative control that drops ONE layer sees the other refuse and reports +green — the shape that already cost this wave one missed control in `routes_agent_harness`. So +each layer is tested AT ITS OWN BOUNDARY: `check_source()` is called directly on source strings, +and `run()` is driven end to end. An NC drops one entry from one frozenset and the matching +boundary goes red. +""" +import ast +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +#: Wall clock, enforced by the PARENT with a kill. The one cap that works on every platform. +DEFAULT_TIMEOUT_S = 10.0 + +#: Address space for the child (`RLIMIT_AS`). POSIX only — see `run()`'s `caps` report. +DEFAULT_MEMORY_BYTES = 512 * 1024 * 1024 + +#: CPU seconds for the child (`RLIMIT_CPU`). POSIX only. Deliberately above the wall clock: the +#: wall-clock kill is the primary control and this is the backstop for a child that stops being +#: reachable. A CPU limit BELOW the timeout would make every slow script look like a CPU refusal. +DEFAULT_CPU_SECONDS = 15 + +#: What the script may print, in bytes. `print` is a curated builtin writing to a capped buffer, +#: and the child's real stdout goes to DEVNULL — so a script cannot fill a pipe, and anything +#: that escaped far enough to write to fd 1 has nowhere for it to land. +MAX_STDOUT_BYTES = 64 * 1024 + +#: The serialised ROW payload handed to the child. ⛔ A REFUSAL, NEVER A TRUNCATION (standing rule +#: 1): a short answer from a data tool is a wrong answer that looks right. Over this, `run()` +#: returns a named limit carrying its cause and a recommendation. +MAX_PAYLOAD_BYTES = 32 * 1024 * 1024 + +#: The emitted spec. A render spec is a description of a picture; one larger than this is data +#: pretending to be a description. +MAX_SPEC_BYTES = 2 * 1024 * 1024 + +MAX_SOURCE_BYTES = 128 * 1024 + + +# ══════════════════════════════════════════════════════ LAYER 1 — the AST allow-list ═══════════ +#: Every `ast` node class a script may contain. ⛔ THE ABSENCES ARE THE POLICY: `Import` / +#: `ImportFrom` (no module reaches the script), `ClassDef` (a class body is a namespace with its +#: own scoping rules and buys a data script nothing), `With` (a context manager is `__enter__` +#: by another spelling), `Global` / `Nonlocal` (rebinding the sandbox's own names), and every +#: `Async*` / `Await` / `Yield` form (this engine is synchronous; a coroutine that is never +#: awaited is a silent no-op that looks like a working script). +ALLOWED_NODES = frozenset(""" +Module Expr Assign AugAssign AnnAssign NamedExpr Return Pass Break Continue Delete Assert Raise +If For While Try TryStar ExceptHandler FunctionDef Lambda arguments arg keyword +BoolOp BinOp UnaryOp IfExp Dict Set List Tuple Starred Subscript Slice Compare Call Attribute Name +Constant JoinedStr FormattedValue ListComp SetComp DictComp GeneratorExp comprehension +Load Store Del +And Or Not Invert UAdd USub +Add Sub Mult Div FloorDiv Mod Pow LShift RShift BitOr BitXor BitAnd MatMult +Eq NotEq Lt LtE Gt GtE Is IsNot In NotIn +""".split()) + +#: Every attribute name a script may READ or CALL. ⛔⛔ THIS IS THE LOAD-BEARING SET, and it is +#: an allow-list of NAMES rather than a refusal of dunders, because the interesting escapes are +#: ordinary-looking: `f.__globals__` on any function reaches the runner's own module namespace, +#: `e.__traceback__.tb_frame.f_globals` reaches it from an exception handler, and `().__class__` +#: reaches `object.__subclasses__`. None of those names is here, and neither is any name this +#: sandbox has not been asked for. +#: ⚠ `format` IS ABSENT DELIBERATELY. `"{0.__class__}".format(x)` performs the attribute lookup +#: INSIDE `str.format`, where no `ast.Attribute` node exists for layer 1 to see. f-strings are +#: fine — `f"{x.__class__}"` compiles to a real `Attribute` node and is refused. +ALLOWED_ATTRS = frozenset(""" +append extend insert pop remove clear sort reverse copy count index +keys values items get setdefault update +add discard union intersection difference issubset issuperset +join split rsplit splitlines strip lstrip rstrip lower upper title capitalize casefold +replace startswith endswith find rfind zfill ljust rjust center partition removeprefix removesuffix +isdigit isalpha isalnum isspace isupper islower isnumeric +real imag numerator denominator +""".split()) + + +class Refused(Exception): + """A named refusal: `code` for a caller to branch on, `message` for a person to read.""" + + def __init__(self, code, message): + self.code, self.message = code, message + super().__init__(f"{code}: {message}") + + +def _attr_ok(name): + """An attribute name passes only if it is on the list AND is not private. + + ⚠ THE SECOND TEST IS NOT A BLACKLIST — it narrows an allow-list that already excludes every + private name. It is here so that adding a name to `ALLOWED_ATTRS` cannot open a dunder by + accident, which is the one edit a future reader is most likely to make in a hurry. + """ + return name in ALLOWED_ATTRS and not name.startswith("_") + + +def check_source(source): + """LAYER 1. Return a `Refused` for source this sandbox will not run, or `None`. + + ⭐ PURE, AND THAT IS WHAT MAKES IT TESTABLE AT ITS OWN BOUNDARY. It reads no file, spawns no + process and touches no store, so a gate can hand it a hundred hostile strings for free and an + NC can drop one entry from one frozenset and watch exactly this function change its answer. + """ + text = str(source or "") + if len(text.encode("utf-8", "replace")) > MAX_SOURCE_BYTES: + return Refused("source_too_long", + f"a script view is at most {MAX_SOURCE_BYTES // 1024} KB of source") + try: + tree = ast.parse(text) + except SyntaxError as exc: + return Refused("syntax", f"line {exc.lineno or 0}: {exc.msg}") + + for node in ast.walk(tree): + kind = type(node).__name__ + if kind not in ALLOWED_NODES: + return Refused("refused_construct", + f"line {getattr(node, 'lineno', 0)}: this sandbox does not run " + f"{_english(kind)}") + if isinstance(node, ast.Attribute) and not _attr_ok(node.attr): + return Refused("refused_attribute", + f"line {getattr(node, 'lineno', 0)}: the attribute " + f"'{node.attr}' is not available inside a script view") + # ⛔ A NAME may not be private either. `_` prefixed names are the runner's own, and a + # script that could bind one could shadow the machinery it runs on top of. + if isinstance(node, ast.Name) and node.id.startswith("_"): + return Refused("reserved_name", + f"line {getattr(node, 'lineno', 0)}: names starting with an " + f"underscore are reserved by the sandbox") + if isinstance(node, (ast.FunctionDef, ast.arg, ast.ExceptHandler)) and str( + getattr(node, "name", None) or getattr(node, "arg", "") or "").startswith("_"): + return Refused("reserved_name", + f"line {getattr(node, 'lineno', 0)}: names starting with an " + f"underscore are reserved by the sandbox") + if isinstance(node, ast.keyword) and str(node.arg or "").startswith("_"): + return Refused("reserved_name", + f"line {getattr(node, 'lineno', 0)}: keyword arguments starting with " + f"an underscore are reserved by the sandbox") + return None + + +_ENGLISH = { + "Import": "an import", "ImportFrom": "an import", "ClassDef": "a class definition", + "With": "a with block", "AsyncWith": "a with block", "AsyncFor": "an async loop", + "AsyncFunctionDef": "an async function", "Await": "await", "Yield": "yield", + "YieldFrom": "yield from", "Global": "a global statement", "Nonlocal": "a nonlocal statement", + "Match": "a match statement", +} + + +def _english(kind): + return _ENGLISH.get(kind, f"a {kind} expression") + + +# ══════════════════════════════════════════ LAYER 2 — the namespace, and the child program ═════ +#: The builtins a script may reach, BY NAME. Everything else is a `NameError` in the child. +#: ⛔ THE ABSENCES, again, are the policy: `__import__` `open` `eval` `exec` `compile` `input` +#: `getattr` `setattr` `delattr` `globals` `locals` `vars` `dir` `type` `super` `object` `help` +#: `exit` `breakpoint` `memoryview` `id`. Several are harmless on their own; each one is a step +#: on a published escape, and none has ever been asked for by a script that shapes rows. +#: ⚠ THE EXCEPTION CLASSES ARE HERE BECAUSE `try:` IS, and a `try` block whose `except` clause +#: cannot name what it catches is a construct that reads as supported and is not. They are safe +#: for the same reason everything else is: `Exception.__subclasses__` needs an attribute this +#: sandbox does not allow, so a class object in the namespace is a leaf, not a doorway. +SANDBOX_BUILTIN_NAMES = ( + "abs all any bool bytes callable chr dict divmod enumerate filter float frozenset hash hex " + "int isinstance issubclass iter len list map max min next oct ord pow range repr reversed " + "round set slice sorted str sum tuple zip True False None " + "Exception ValueError TypeError KeyError IndexError ZeroDivisionError ArithmeticError " + "AttributeError StopIteration OverflowError" +).split() + +#: The literal program the child runs. It is TEXT rather than a module because the child must +#: import nothing from this repo: a module would be found on `sys.path` and would drag `core` +#: with it, which is exactly the second store path W1 forbids. +#: ⚠ Every name in here is underscore-prefixed and layer 1 refuses a script from binding one, so +#: the runner's own machinery cannot be shadowed by the source it executes. +_RUNNER = r''' +import json as _json, os as _os, sys as _sys + +_pay = _json.loads(open(_sys.argv[1], "r", encoding="utf-8").read()) +_out = {"ok": False, "code": "not_run", "message": "the script did not run", + "stdout": "", "spec": None, "caps": {"wallClock": True, "memory": False, "cpu": False}} + +# ── the caps this platform can actually apply, reported either way (standing rule 1) ────────── +try: + import resource as _res + _mem = int(_pay["memoryBytes"]) + _res.setrlimit(_res.RLIMIT_AS, (_mem, _mem)) + _out["caps"]["memory"] = True + _cpu = int(_pay["cpuSeconds"]) + _res.setrlimit(_res.RLIMIT_CPU, (_cpu, _cpu)) + _out["caps"]["cpu"] = True +except Exception: + # `resource` is POSIX only. The wall-clock kill in the parent still applies, and `caps` says + # which of the three held, never a silent partial. + pass + +_printed = [] +_spent = [0] +_LIMIT = int(_pay["maxStdout"]) + + +def _print(*_a, **_k): + _text = (_k.get("sep") or " ").join(str(_x) for _x in _a) + (_k.get("end") or "\n") + _room = _LIMIT - _spent[0] + if _room > 0: + _printed.append(_text[:_room]) + _spent[0] += len(_text) + + +class _Refusal(Exception): + """The SANDBOX refusing, as distinct from the SCRIPT failing. + + Without its own class these arrive as `ValueError`, indistinguishable from a `ValueError` the + script raised itself, and the answer then says "refused" about an ordinary bug in the tenant's + own code. Two different facts, two different codes. + """ + + +_emitted = [] + + +def _emit(_spec): + if not isinstance(_spec, dict): + raise _Refusal("emit() takes a view spec, which is a dictionary") + if _emitted: + raise _Refusal("emit() was already called; a script view emits exactly one view") + _emitted.append(_spec) + + +_rows = _pay["rows"] +_fields = _pay["fields"] +_bound = _pay["table"] + + +def _scoped_table(_table=None): + if _table is not None and str(_table) != _bound: + raise _Refusal( + "this script view is bound to the database '" + _bound + "' and asked for '" + + str(_table) + "'. A script view reads its own database only") + return [dict(_r) for _r in _rows] + + +def _scoped_fields(): + return [dict(_f) for _f in _fields] + + +_ns = {"__builtins__": {_n: __builtins__[_n] if isinstance(__builtins__, dict) + else getattr(__builtins__, _n) + for _n in _pay["builtins"]}} +_ns["__builtins__"]["print"] = _print +_ns["print"] = _print +_ns["emit"] = _emit +_ns["scoped_table"] = _scoped_table +_ns["scoped_fields"] = _scoped_fields +_ns["table"] = _bound + +try: + exec(compile(_pay["source"], " diff --git a/web/src/App.tsx b/web/src/App.tsx index 77fd5fb03fc59405df22b003bfed1a8bb3b539f7..0dcffc3f4790d9b8a0842285b64f4ff08b9d5da4 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,4 +1,13 @@ -import CustomerGrid from "./customer-grid/CustomerGrid"; +import { Suspense, lazy } from "react"; +// ⭐⭐ WAVE 37 · T48 — LAZY, AND IT IS THE LAST OF THE THREE EAGER IMPORTS. `CustomerGrid` (7,715 +// lines) plus glide-data-grid sat in the ENTRY chunk because these three files imported it at the +// top level, while twelve other surfaces were already `lazy()`. Measured on the deployed build +// (`proto/P4-first-load.md` §A.2): the entry chunk was 580,102 B on the wire and took 5.6-5.8 s, +// during which `#root` was EMPTY and the screen was white — 6.0-8.0 s of the 11.6-16.0 s first open. +// ⚠ THE EMBED PAYS THIS TOO, and it is the one path where the grid IS the page, so its fallback is +// deliberately the same skeleton shape the shell uses rather than `null`: a blank iframe is +// indistinguishable from a broken one. +const CustomerGrid = lazy(() => import("./customer-grid/CustomerGrid")); import { OverlayProvider } from "./customer-grid/OverlaySurface"; import { isStreamlitComponent } from "./customer-grid/hostBridge"; import Shell from "./shell/Shell"; @@ -21,7 +30,9 @@ export default function App() { return (
- +
); diff --git a/web/src/account/SubscriptionPage.tsx b/web/src/account/SubscriptionPage.tsx index 3b815ab4b0fb47791358f302caa942923542e572..2cbc71a05dca231d01b3badbb396d5bd70452472 100644 --- a/web/src/account/SubscriptionPage.tsx +++ b/web/src/account/SubscriptionPage.tsx @@ -4,6 +4,8 @@ // Owner item 13, verbatim: *"add below settings the following button: Feedback …, // Usage credits …, and a Subscription module … Specialized for the Royal Imports // tenant, do not show it."* +// ⛔ "verbatim" IS THE POINT: W37-T02 renamed that surface to "Usage" and this quote KEEPS the +// owner's own wording. A quote edited to match a later rename is no longer evidence of anything. // // This is the third of those three, and the only one with nothing behind it yet. // So it says so, in the words this app already uses for a boarded door diff --git a/web/src/account/UsagePage.tsx b/web/src/account/UsagePage.tsx index f2d6dedc39b6b29f11d7d0b43aa8ab81a710acb6..f735d26732cc9caca5ada0eb71c190c9584ec1b4 100644 --- a/web/src/account/UsagePage.tsx +++ b/web/src/account/UsagePage.tsx @@ -142,7 +142,7 @@ export function UsageView({ }) { return (
-

Usage credits

+

Usage

{canSeeTenant ? (
diff --git a/web/src/account/account.css b/web/src/account/account.css index 135fca9b383904f11de06eda098bf69176ed435f..aa7fd6af0efc88d498fdc1045211a2043c37f39c 100644 --- a/web/src/account/account.css +++ b/web/src/account/account.css @@ -1,6 +1,6 @@ /* ───────────────────────────────────────────────────────────────────────────────────────── account/account.css — WAVE 35, owner item 13: the three surfaces that hang off the - account button (Feedback, Usage credits, Subscription). + account button (Feedback, Usage, Subscription). ⛔ ITS OWN STYLESHEET, NOT index.css. That file belongs to the INTEGRATOR for the whole wave, and the wave's own rule is that a new surface brings its own sheet. Every selector diff --git a/web/src/account/accountApi.ts b/web/src/account/accountApi.ts index 7575bafe82bc76ee5df3347953534e605a5d5c7b..b9178854bf808e39812752b527a6e28350f1898b 100644 --- a/web/src/account/accountApi.ts +++ b/web/src/account/accountApi.ts @@ -1,6 +1,6 @@ // --------------------------------------------------------------------------- // account/accountApi.ts — WAVE 35, owner item 13: the wire behind the two account -// surfaces that have one (Feedback, contract C6/R8; Usage credits, C7/R9). +// surfaces that have one (Feedback, contract C6/R8; Usage, C7/R9). // // ⛔ THE CATEGORY VOCABULARY IS THE SERVER'S AND IS FETCHED, NOT DECLARED HERE. // That is a decision, not an omission: a client constant beside a server enum is diff --git a/web/src/assistant/AssistantPage.tsx b/web/src/assistant/AssistantPage.tsx index f0a44dd2aa60a59a0268f4bb4bb875c6dc9fbb8f..e4e3020dceb1a2788bd46203b6bbaab3183e5288 100644 --- a/web/src/assistant/AssistantPage.tsx +++ b/web/src/assistant/AssistantPage.tsx @@ -105,7 +105,17 @@ const KIND_LABEL: Record = { * `modelStatus`: a model that IS offered and whose key this deployment does not hold. */ const MODEL_LABEL: Record = { - auto: "Auto", cerebras: "Cerebras", groq: "Groq", openrouter: "OpenRouter", + // ⭐ `anthropic` JOINS THE MAP (W36 · ASK E-11, ruling R4). Not a fifth option nobody wanted. + // E measured all three side rungs refusing live today (cerebras 402, groq 404, openrouter 402), + // so Anthropic is the only rung that answers at all, and C5 makes it the tool-calling path by + // ruling rather than by accident. The fall-through below still covers a provider added + // server-side; this row exists so the ONE rung that works is not the one rendered as a raw key. + // ⚠ NO `word` FOLLOWED BY A COLON IN THIS COMMENT, and that is not fussiness. `verify_query` + // reads this map with a regex over the block, comments included, so a phrase like "asked for" + // with a colon after it registered as a SIXTH key named `for` and reddened the check that + // exists to keep this map equal to the server's list [[prose-that-becomes-its-own-marker]]. + auto: "Auto", anthropic: "Anthropic", cerebras: "Cerebras", groq: "Groq", + openrouter: "OpenRouter", }; const modelLabel = (key: string) => MODEL_LABEL[key] ?? key; diff --git a/web/src/automation/AutomationBuilder.tsx b/web/src/automation/AutomationBuilder.tsx index a5881c73e1357933614c7cf91c8b14beff516004..96f1bd99ffecff61b16d127bf44a60ff6520c2a7 100644 --- a/web/src/automation/AutomationBuilder.tsx +++ b/web/src/automation/AutomationBuilder.tsx @@ -46,13 +46,19 @@ import type { FlowVocab, GraphNode, OAuthStatus, + StatementWorld, TriggerOption, UserTable, } from "./automationApi"; // ⭐ WAVE 26 · ITEM 22 / D-70 — the Run guard, imported rather than re-derived. This file and // `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides. // ITEM 9 / R11 — `createTable` + `AutomationError` for the in-place "+ New database". -import { AutomationError, createTable, runBlock } from "./automationApi"; +// ⭐ W36-T44 — the statements panel's three doors. They go to `routes_statements`, not +// `/automations`, for the reason `statementBatch`'s own note gives: that is where the send +// guardrails already are and R10's whole point is that they do not move. +import { + AutomationError, createTable, previewStatement, runBlock, statementWorld, testSendStatement, +} from "./automationApi"; import CondBuilder, { condComplete } from "./CondBuilder"; import { groupActions, groupBranches, groupByPanel, numberActions, reorderList } from "./steps"; @@ -141,7 +147,27 @@ interface Props { * ⚠ REQUIRED. An optional flag defaulting to `true` would make an unmounted toggle look like * a working panel, which is the shape this repo keeps paying for. */ + /** + * ⭐ W36-T69 / R9 — `agentActions` off the wire, keyed by action id. Optional, and absent reads + * as "nothing is agent-owned" (the wire type carries the reasoning). + */ + agentMarks?: Record; showProperties: boolean; + /** + * ⭐⭐ W36-T32 (owner item 3) — HIDE THE FLOW, KEEP THE PANEL. `true` while the Chat view holds + * the work area. + * + * ⛔ WHY THE BUILDER STAYS MOUNTED AT ALL, WHICH REVISES A WAVE-34 TRADE RATHER THAN IGNORING + * IT. `AutomationDetail`'s own note recorded that keeping this component mounted behind + * `hidden` would "preserve the selection too, at the price of two live editors of one + * automation", and chose to unmount. The owner has since asked for the opposite in as many + * words: *"even when the Chat is shown first, we still need to be able to see the + * Properties/the new module/Run history tab on the right side."* The Properties panel is a + * child of THIS component, so an unmounted builder means no Properties tab during Chat. + * ⚠ The wave-34 concern is answered rather than overruled: what it feared was two live EDITORS + * of the flow, and the flow is exactly what this prop hides. The panel is the only live half. + */ + flowHidden?: boolean; /** The Properties/Run-history switch, OWNED by `AutomationDetail` and drawn in both panel * heads so it is in the same place whichever one is showing. */ panelTabs: ReactNode; @@ -420,7 +446,9 @@ export default function AutomationBuilder({ onRunNow, renderNodeBody, scheduleFace, + agentMarks, showProperties, + flowHidden, panelTabs, onStepPicked, schedules, @@ -1091,6 +1119,30 @@ export default function AutomationBuilder({ vocabulary — it would have gone on saying the old words after the catalog changed, which is the one failure this file's header names twice. */} {row?.label || a.kind} + {/* + ⭐⭐ W36-T69 / R9 — THE LOCK, ON THE CARD, so the rule is visible BEFORE the panel + is opened. The done-when asks for a lock that NAMES its agent; the glyph carries + the name in its `title` and its `aria-label` rather than as a third line of card + text, because every card in this column has exactly two lines and a third on one + of them is the ragged edge the owner reads as unfinished. + ⚠ `aria-hidden` on the SVG and the name on the wrapper: a screen reader should + hear "set up by Odoo sync", never "image". + */} + {agentMarks?.[a.id] ? ( + + + + ) : null} {isGroup ? ( @@ -1224,7 +1276,10 @@ export default function AutomationBuilder({ return ( <> -
+ {/* ⚠ `hidden`, not unmounted (W36-T32). `.autox-flow` sets no `display`, so the UA's own + `[hidden] { display: none }` applies and no `!important` override is needed here — unlike + `.auto-panel[hidden]`, whose class rule sets `display: flex` and had to be beaten. */} +