diff --git a/RELEASES.json b/RELEASES.json index a855cc075b39979fe34ac53090074f06f2c6142c..f40634b0fc051b0bc4d6e949e9aba7b83fa58472 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,24 @@ { - "current": "v25 (bc24829)", + "current": "v29 (55ced50)", "releases": [ + { + "version": "v29", + "sha": "55ced50", + "date": "2026-08-18", + "subject": "v29: the build staging has been running since 2026-08-18 (55ced50)." + }, + { + "version": "v28", + "sha": "2793e4b", + "date": "2026-08-18", + "subject": "release v28" + }, + { + "version": "v27", + "sha": "c05a794", + "date": "2026-08-18", + "subject": "release v27" + }, { "version": "v25", "sha": "bc24829", diff --git a/VERSION b/VERSION index a91a1a22b1f8a24f211e83676adbae666a565ea0..f202b3cb765e134d529e7339649bf07176354a79 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v25 (bc24829) +v29 (55ced50) diff --git a/api/ai_enrich.py b/api/ai_enrich.py new file mode 100644 index 0000000000000000000000000000000000000000..5b1166fe9506bf6bdd2337ca50691536165dcad8 --- /dev/null +++ b/api/ai_enrich.py @@ -0,0 +1,550 @@ +"""ai_enrich.py -- the AI ENRICHMENT column (wave 34, owner ruling R13). + +R13: *"A field kind called AI enrichment: a prompt per row that populates text. Detailed +configuration in the field's own config, including a token-usage limit."* + +WHERE THE HALVES LIVE, and why the line is drawn here rather than anywhere else: + + core/user_tables.py the CONTRACT and the STORAGE -- `UT_FIELD_TYPES`, `_clean_ai_enrich`, + the per-cell provenance stratum, and the law about what an automatic + run may overwrite. It owns the store document, so it owns the marks. + api/ai_enrich.py THIS file: the PROMPT vocabulary, the run PLAN, and the token ceiling's + own report. `core` may not import upward, so anything that needs to + reach a vendor or an HTTP session lives here. + +Everything in this module is a PURE FUNCTION of data it is handed. That is deliberate: it is the +half a gate can exercise without a network, a store, or a running server, which is what makes +`api_ai_enrich` a real gate rather than a mount check. + +TOKEN ACCOUNTING, and the hole it is filling. `api/ai_review.py::decide` -- the product's only +other LLM entry -- has no token accounting of ANY kind: `max_tokens: 300` per call, one 20 s +timeout, no retry, and nothing anywhere counts what a run spent. R13 asks for a usage limit, so +there was nothing to build on and the ledger starts here. The two measured community complaints +about this feature elsewhere are both REPORTING gaps rather than capability gaps (a preview said +15 credits and the run spent 360; a finished run gave no completion signal), which is why +`ceiling_report` exists beside the ceiling itself. +""" +from __future__ import annotations + +import os +import re + +import requests + +#: A prompt names other columns the way a formula does, so a user who has written one already +#: knows this syntax. ⚠ ONE parser, exported, because `user_tables.ai_enrich_input_hash` fingerprints +#: the referenced cells and a second copy of this pattern would disagree with it the first time the +#: token syntax gained a form ([[one-question-two-normalizers]]). +_REF = re.compile(r'\{([a-zA-Z0-9_]{1,60})\}') + +#: What a cell can say about where its value came from. Mirrors +#: `user_tables.AI_ENRICH_STATES` plus the two states that are DERIVED rather than stored: +#: `stale` (the inputs moved since the value was written) and `empty` (nothing has run). +#: ⛔ Re-exported rather than re-declared: the stored vocabulary has exactly one owner. +CELL_STATES = ('empty', 'agent', 'human', 'stale', 'error') + +#: ⛔ THERE IS DELIBERATELY NO PROVIDER LIST IN THIS FILE. The product already has TWO opinions +#: about provider order (`harness/analyst.PROVIDERS` says groq-first, `routes_query. +#: QUERY_PROVIDER_ORDER` says cerebras-first) and `W34-T37` exists to reconcile them. Declaring a +#: third here would make that ticket harder and this feature wrong in a new way, so the catalogue +#: is BORROWED from `ai_review` (`PROVIDERS` + `ladder()` + the key names + the cheap-first order +#: owner ruling R14 set). ⚠ The CALL is ours because the requirements differ: `ai_review` hardcodes +#: `max_tokens: 300` and reads no `usage`, and R13 needs a per-column ceiling and a real ledger. +#: One provider catalogue, two call shapes, and the reason is written down rather than inferred. + +#: The run's per-cell wall clock. Separate from `ai_review.TIMEOUT_SECONDS` because a review is +#: one classification a person is waiting on and this is a loop over rows. +TIMEOUT_SECONDS = float(os.environ.get('AIOS_AI_ENRICH_TIMEOUT') or 30) +#: How many failing rows carry their error into the report before it is summarised. A report that +#: lists 900 identical `429`s is a report nobody reads. +MAX_REPORTED_ERRORS = 10 +#: Bound on what one row hands the model, mirroring `ai_review`'s own two caps: a table can hold a +#: 32 KB JSON cell per row and no prompt needs it. +MAX_CELL_CHARS = 400 + + +def prompt_refs(prompt): + """The column keys a prompt names, in first-appearance order, de-duplicated. + + `"Summarise {name} for {segment}"` -> `['name', 'segment']`. + """ + seen, out = set(), [] + for key in _REF.findall(str(prompt or '')): + if key not in seen: + seen.add(key) + out.append(key) + return out + + +def render_prompt(prompt, row, field_keys=()): + """`(text, unresolved)` -- the prompt with `{token}` replaced by this row's cells. + + ⛔ AN UNKNOWN TOKEN IS REPORTED, NOT SILENTLY BLANKED. A prompt naming a column that does not + exist is a prompt that will quietly ask a different question on every row, and the resulting + text looks exactly like a correct answer. `unresolved` is what the caller shows the user; + R6's standing rule is the same sentence one layer up (a limit that cannot be removed must be + reported with its cause). + + ⚠ A token naming a real column whose CELL is empty is NOT unresolved: an empty cell is data. + """ + known = set(field_keys or ()) + unresolved = [] + + def _sub(match): + key = match.group(1) + if known and key not in known: + unresolved.append(key) + return match.group(0) + return str((row or {}).get(key, '')) + + return _REF.sub(_sub, str(prompt or '')), sorted(set(unresolved)) + + +def ceiling_report(spent, ceiling, rows_done, rows_total): + """R6's second sentence as DATA, or None while the run is still inside its budget. + + `{subject, limit, spent, effect, cause, recommendation}` -- the same shape + `user_tables.limit_report` returns, so the client renders one vocabulary for every limit the + product enforces rather than a second one for this feature. + + ⛔ `effect` is `stopped`, never `truncated`. A run that quietly stopped filling cells leaves a + column that looks complete and is not, which is the failure the whole reporting rule is + arranged against. The caller STOPS and says so. + """ + if ceiling is None or spent < ceiling: + return None + left = max(0, int(rows_total or 0) - int(rows_done or 0)) + rest = f'{left:,} rows' if left != 1 else '1 row' + return { + 'subject': 'tokens', + 'limit': int(ceiling), + 'spent': int(spent), + 'effect': 'stopped', + 'cause': f'this run reached its token ceiling of {int(ceiling):,} after filling ' + f'{int(rows_done or 0):,} of {int(rows_total or 0):,} rows', + 'recommendation': f'raise the column\'s token limit, shorten its prompt, or run the ' + f'remaining {rest} again. Nothing was truncated and no cell holds a ' + f'partial answer', + } + + +def plan_rows(rows, col_id, config, marks, cost_of=None): + """Which rows an automatic run should fill, and why the others were left alone. + + Returns `{'run': [row_id, ...], 'skipped': {reason: count}, 'estimate': int}`. + + ⛔ THE SELECTION IS SEPARATE FROM THE RUN ON PURPOSE. It is pure, so a gate can prove the + never-overwrite-a-human-edit law without a vendor, and so the client can PREVIEW a run + honestly before it spends anything -- which is exactly the complaint R13 cites (a preview + that said 15 credits against a run that spent 360). + + `cost_of(row)` estimates one row's tokens; absent, the column's own ceiling is used per row, + which is the pessimistic direction and the right one for a preview. + """ + import core.user_tables as ut + + cfg = config if isinstance(config, dict) else {} + prompt = str(cfg.get('prompt') or '') + policy = str(cfg.get('overwrite') or 'blank') + refs = prompt_refs(prompt) + per_row = int(cfg.get('maxTokens') or 0) + run, skipped, estimate = [], {}, 0 + for row_id, row in (rows or {}).items(): + row = row if isinstance(row, dict) else {} + mark = (marks or {}).get(str(row_id)) or {} + fresh = ut.ai_enrich_input_hash(prompt, row, refs) + has_value = bool(str(row.get(str(col_id)) or '').strip()) + if ut.ai_enrich_may_write(mark, policy, fresh, has_value=has_value): + run.append(str(row_id)) + estimate += int(cost_of(row)) if callable(cost_of) else per_row + else: + # The same predicate again, so the count a preview shows and the cells a run leaves + # alone can never be two different sets. + reason = ('human_edited' if ut.ai_enrich_human_authored(mark, has_value) + else 'unchanged' if has_value else 'skipped') + skipped[reason] = skipped.get(reason, 0) + 1 + return {'run': run, 'skipped': skipped, 'estimate': estimate} + + +def cell_state(mark, config, row, col_id): + """What ONE cell should say about itself: one of `CELL_STATES`. + + ⚠ `stale` and `empty` are computed here and never stored -- see the stratum's own note in + `core/user_tables.py`. A stored staleness flag needs a sweep to stay true, and a sweep nobody + runs is a flag shipped without its writer. + """ + import core.user_tables as ut + + mark = mark if isinstance(mark, dict) else {} + has_value = bool(str((row or {}).get(str(col_id)) or '').strip()) + if str(mark.get('state') or '') == 'error': + return 'error' + # ⛔ THE SAME PREDICATE THE RUNNER OBEYS, called rather than restated. A cell this reports as + # the human's while `ai_enrich_may_write` would overwrite it is the worst of both answers. + if ut.ai_enrich_human_authored(mark, has_value): + return 'human' + if str(mark.get('state') or '') != 'agent': + return 'empty' + cfg = config if isinstance(config, dict) else {} + prompt = str(cfg.get('prompt') or '') + fresh = ut.ai_enrich_input_hash(prompt, row or {}, prompt_refs(prompt)) + return 'stale' if ut.ai_enrich_is_stale(mark, fresh) else 'agent' + + +# ══════════════════════════════════════════════════════════════════════ THE RUN +# ⛔⛔ THE LEDGER IS THE POINT OF THIS HALF. `ai_review.decide` (the product's only other LLM entry) +# counts nothing: it sends `max_tokens: 300` and never reads `usage` off the response, so no run in +# this product has ever been able to say what it cost. R13 asks for a token-usage limit, so the +# accounting starts here, and every leg below reads `usage` even where it is not needed, because a +# ledger with one blind provider is not a ledger. + + +def _usage_tokens(body): + """Total tokens for one call, from whichever shape the provider answered in. + + ⚠ RETURNS None WHEN THE PROVIDER DID NOT SAY, and the caller treats that as a real unknown + rather than as zero. A ledger that silently books an unmeasured call at 0 reports a cheaper + run than happened, which is exactly the cost-surprise complaint R13 cites. + + ⭐⭐ W35-T41 / C7 — THE READER MOVED AND THIS IS NOW ONE LINE OVER IT. `usage_ledger.total_from` + is the same logic, in the module that owns the meter; the alternative was a second reader in a + second file answering the same question about the same key, which is the shape this repo pays + for most often ([[one-question-two-normalizers]]). ⚠ The NAME and the None-semantics stay + exactly as wave 34 wrote them, because `run_field`'s ceiling arithmetic reads this and a gate + injects against its shape — lifting the body, not the signature. + """ + import usage_ledger + return usage_ledger.total_from(body) + + +#: R6's subject: the em dash (U+2014) and the en dash (U+2013), as a regex character class. +#: ⛔ BUILT FROM CODE POINTS, NOT TYPED, AND THIS IS NOT COSMETIC. `web_prose` reads a Python +#: string's VALUE off the AST, so a class written `'[]'` is reported as two findings in +#: the one function whose entire job is to REMOVE those characters. That is a false positive, and +#: the wrong fix would be to silence the gate or to weaken the pattern. `chr()` keeps the value +#: byte-identical while putting no dash in a literal, so the gate sees what it should see and the +#: normalizer keeps working. ⚠ Lane D hit the same wall in `routes_query` (note D-13). +_DASH = '[' + chr(0x2014) + chr(0x2013) + ']' + + +def no_dashes(text): + """R6 applied to MODEL-AUTHORED text, before it is stored. + + ⛔⛔ A PROMPT INSTRUCTION DOES NOT ENFORCE THIS, AND THE MEASUREMENT IS LANE D'S, NOT A GUESS + (mailbox note D-4, 2026-08-16): a system prompt ending *"Never use an em dash or an en dash"* + was answered on the very next live turn with *"Which view type would you like-grid, chart, + list, or another?"* carrying U+2014 (cerebras). `web_prose` scans SOURCE, so it is + structurally blind to a dash that arrives at runtime. + ⚠ AND THIS COLUMN IS THE WORST CASE IN THE WAVE, which is why D routed the note here: chat + prose is read once, but an enrichment value is WRITTEN INTO A CELL, then re-read and + re-rendered forever. Normalising after the write would never reach the rows already stored. + + ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE and gets the first rule: "10-20" means "10 to 20", and + rewriting it as "10, 20" states two numbers where the model stated a span. That is a wrong + value, not a punctuation fix. + + ⛔⛔ THIS IS THE SECOND IMPLEMENTATION IN THE PRODUCT and that is a known cost, taken + deliberately rather than by drift. The first is `routes_query._no_dashes` (lane D's, shipped + first). Importing it would point a LIBRARY module at a ROUTE module, which is backwards, and + lane D's file is not in this lane's fence to move it. So the two are held in step by a PARITY + CHECK instead of by hope: `verify_ai_enrich` section 10 imports D's function and asserts both + agree over a corpus including every shape either docstring names. One shared home is owed and + is booked as a PENDING line, not left implicit ([[one-question-two-normalizers]]). + """ + text = str(text or '') + text = re.sub(rf'(?<=\d)\s*{_DASH}\s*(?=\d)', ' to ', text) + text = re.sub(rf'\s*{_DASH}\s*(?=[,.;:!?])', '', text) # abutting punctuation: it just goes + text = re.sub(rf'(?<=[,;:])\s*{_DASH}\s*', ' ', text) # already punctuated: one space + return re.sub(rf'\s*{_DASH}\s*', ', ', text) + + +#: What the model is told about its job. It fills ONE cell, so anything conversational it adds is +#: a defect in the column rather than a nicety. `UNKNOWN` is the honest out: a row the data cannot +#: answer is recorded as an error against that row, never as an invented value. +#: ⚠ The dash sentence is here for the same reason lane D kept theirs: it costs nothing and it +#: reduces how often `no_dashes` has to act. It is NOT the enforcement; `no_dashes` is. +_SYSTEM = ('You fill in ONE cell of a spreadsheet. Answer with the value only: no preamble, no ' + 'quotes, no markdown, no explanation. Never use an em dash or an en dash; use a comma, ' + 'a colon or a full stop. If the information given does not let you answer, reply with ' + 'exactly: UNKNOWN') + + +def _ask(provider, model, prompt, max_tokens, timeout): + """One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text. + + ⚠ Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's + ceiling instead of a hardcoded 300, and they read `usage` back. + """ + key = (os.environ.get(provider['env']) or '').strip() + if not key: + return '', None, f"{provider['name']} has no key on this deployment" + try: + if provider['shape'] == 'anthropic': + r = requests.post( + provider['url'], timeout=timeout, + headers={'x-api-key': key, 'anthropic-version': '2023-06-01', + 'content-type': 'application/json'}, + json={'model': model, 'max_tokens': max_tokens, 'system': _SYSTEM, + 'messages': [{'role': 'user', 'content': prompt}]}) + if r.status_code >= 400: + return '', None, f"{provider['name']} answered {r.status_code}" + body = r.json() + # ⛔ stop_reason FIRST. A safety refusal is a 200 with an EMPTY content list, so + # reading content[0] before this turns a refusal into an IndexError inside the loop. + if body.get('stop_reason') == 'refusal': + return '', _usage_tokens(body), f"{provider['name']} declined this row" + parts = [b.get('text') or '' for b in (body.get('content') or []) + if isinstance(b, dict) and b.get('type') == 'text'] + return ''.join(parts).strip(), _usage_tokens(body), '' + r = requests.post( + provider['url'], timeout=timeout, + headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'}, + json={'model': model, 'max_tokens': max_tokens, 'temperature': 0, + 'messages': [{'role': 'system', 'content': _SYSTEM}, + {'role': 'user', 'content': prompt}]}) + if r.status_code >= 400: + return '', None, f"{provider['name']} answered {r.status_code}" + body = r.json() + choices = body.get('choices') or [] + if not choices: + return '', _usage_tokens(body), f"{provider['name']} returned no choices" + text = ((choices[0] or {}).get('message') or {}).get('content') or '' + return str(text).strip(), _usage_tokens(body), '' + except Exception as exc: # noqa: BLE001 + return '', None, f"{provider['name']} failed: {type(exc).__name__}" + + +def on_change_fields(defn, changed_keys): + """Which `ai_enrich` columns in this table want a run because one of their inputs moved. + + A column qualifies when its trigger is `on_change` AND its prompt names at least one of the + columns that just changed. ⛔ A column that names NOTHING (a prompt with no `{token}`) never + fires on change, whatever its trigger says: it would re-ask the same question of the same row + forever, once per unrelated edit, and bill for every one. + + ⚠ PURE, AND HANDED THE DEFINITION IT SHOULD USE. `automation_engine.grid_hook` calls + `all_definitions(st)` once PER EVENT, which turns a 20,000-row import into 20,000 whole + document reads (`D-134`); this function reads no store at all, so the caller can do one read + for a whole batch. Rebuilding that shape here was the one thing T54's ticket said not to do. + """ + import core.user_tables as ut + + touched = {str(k) for k in (changed_keys or ())} + if not touched: + return [] + out = [] + for field in ut.ai_enrich_fields(defn): + cfg = field.get('aiEnrich') or {} + if (cfg.get('trigger') or {}).get('mode') != 'on_change': + continue + refs = set(prompt_refs(cfg.get('prompt'))) + # ⚠ A column never fires on a change to ITSELF. Its own value moving is either the run + # that just wrote it or a person typing over it, and both would loop. + refs.discard(str(field.get('key') or '')) + if refs & touched: + out.append(field) + return out + + +def scheduled_fields(defn): + """`[(field, cron)]` for every enrichment column that runs on a cadence. + + ⛔ IT DOES NOT PARSE THE CRON AND MUST NOT. `core.user_tables` stores that string opaque + precisely so the cadence vocabulary has ONE owner, and the owner is the scheduler + (`automation_engine`), not the field layer. A second parser here would be a door that starts + refusing a cadence the other one accepts, which is exactly the divergence the whole field + contract keeps paying for ([[one-question-two-normalizers]]). + + ⚠ SO THIS IS HALF A FEATURE ON PURPOSE, and the other half is a cross-fence ask: something has + to TICK. Nothing in lane F's fence runs on a timer. + """ + import core.user_tables as ut + + out = [] + for field in ut.ai_enrich_fields(defn): + trigger = ((field.get('aiEnrich') or {}).get('trigger') or {}) + if trigger.get('mode') == 'schedule' and str(trigger.get('cron') or '').strip(): + out.append((field, str(trigger['cron']))) + return out + + +def run_field(table_key, col_id, *, st, rows=None, manual=False, policy=None, budget=None, + timeout=None, ask=None, user=''): + """Fill one `ai_enrich` column over the rows an automatic run is allowed to touch. + + Returns a REPORT and never raises for a vendor problem: + `{planned, filled, failed, skipped{reason: count}, tokens, provider, model, + limit: , errors[], problem}` + + ⛔ THE RUN BUDGET IS DERIVED FROM THE COLUMN'S OWN CEILING, and the arithmetic is the point: + `budget = maxTokens * planned`. `maxTokens` bounds the OUTPUT of one call while spend counts + INPUT tokens too, so a real run crosses this before it finishes every row. That is intended: + R13 asked for a token-usage limit and a ceiling that can never bite is not one. A caller may + pass a TIGHTER `budget`; nothing may pass a looser one. + + ⛔ TWO STORE WRITES FOR THE WHOLE RUN, not two per row: one `patch_many_cells` for the values + and one `stamp_ai_enrich` for the marks. Each is a read-modify-write of the whole tenant + document, which is why `patch_cells` in a loop was never an option (D-134's shape). + + ⚠ `ask` is injectable so a gate can exercise the ledger, the ceiling and every failure path + with no network and no vendor bill. It defaults to the real `_ask`. + + ⭐⭐ W35-T41 / C7 — ONE USAGE-LEDGER LINE PER RUN, NOT PER ROW, and the arithmetic is the reason: + this loop makes one LLM call per row, so a ledger write per call would be hundreds of store + writes for one click. The run already accumulates `report['tokens']`; `calls` and `unmeasured` + join it so the meter can say "40 calls, 12,000 tokens, 3 of them unmeasured" instead of a total + with an unknown shortfall. `user` is the ledger's attribution and is passed by this file's own + callers in `routes_tables`. + """ + import ai_review + import core.user_tables as ut + import usage_ledger + + defn = ut.get(table_key, st=st) or {} + field = next((f for f in ut.ai_enrich_fields(defn) if f.get('key') == str(col_id)), None) + if field is None: + return {'problem': f'{col_id!r} is not an AI enrichment column on this database', + 'planned': 0, 'filled': 0, 'failed': 0, 'skipped': {}, 'tokens': 0, + 'errors': [], 'limit': None, 'provider': '', 'model': ''} + cfg = field.get('aiEnrich') or {} + # ⭐ THE BULK MENU'S TWO CHOICES ARE THIS ONE OVERRIDE (`W34-T54`): "Rows never filled" is + # `blank` and "All rows" is `always`. ⛔ It overrides the column's SAVED POLICY for one run and + # NOTHING ELSE: `ai_enrich_may_write` still refuses a human-edited cell under every value, so + # "All rows" means every row the AGENT wrote, which is exactly what the menu label says. + # ⚠ An unrecognised value falls back to the column's own setting rather than to a default: a + # typo'd override must not quietly widen what a run touches. + if policy in ut.AI_ENRICH_OVERWRITE: + cfg = {**cfg, 'overwrite': policy} + all_rows = defn.get('rows') or {} + marks = ut.ai_enrich_marks(table_key, col_id, st=st) + # ⛔⛔ `rows` NARROWS; `manual` DECIDES WHETHER THE POLICY APPLIES. They are two questions and + # an early draft of this function conflated them, which would have let an ON-CHANGE fire (an + # automatic act, scoped to one row) overwrite a cell a person had typed into. That is the one + # law this whole feature is arranged around, broken by a parameter name. + # rows=None -> the automatic plan over every row + # rows=[...], manual=False -> the automatic plan, INTERSECTED with those rows (on_change) + # rows=[...], manual=True -> exactly those rows, policy skipped + # `manual` is skippable because a person clicking "run this cell" has asked, in front of the + # value being replaced; refusing them would be the product overruling an explicit act with a + # rule written for an implicit one. `ai_enrich_may_write`'s own docstring says exactly this. + # ⚠ A named row is always INTERSECTED with the table's own row set, so a stale caller cannot + # name a record that has since been deleted. + named = None if rows is None else [str(r) for r in rows if str(r) in all_rows] + if named is not None and manual: + plan = {'run': named, 'skipped': {}, + 'estimate': int(cfg.get('maxTokens') or 0) * len(named)} + else: + plan = plan_rows(all_rows, col_id, cfg, marks) + if named is not None: + keep = set(named) + dropped = [r for r in plan['run'] if r not in keep] + plan = {**plan, 'run': [r for r in plan['run'] if r in keep]} + if dropped: + plan['skipped'] = {**plan['skipped'], 'not_in_this_run': len(dropped)} + rows = all_rows + keys = [f.get('key') for f in (defn.get('fields') or []) if f.get('key')] + report = {'planned': len(plan['run']), 'filled': 0, 'failed': 0, + 'skipped': dict(plan['skipped']), 'tokens': 0, 'errors': [], 'limit': None, + 'provider': '', 'model': '', 'problem': '', + # ⭐ C7's two counters. `calls` is what the meter bills; `unmeasured` is how many of + # them the provider declined to report tokens for, so a total is never presented as + # complete when it is a floor. + 'calls': 0, 'unmeasured': 0} + if not plan['run']: + return report + + caller = ask or _ask + ceiling = int(cfg.get('maxTokens') or 0) * len(plan['run']) + if isinstance(budget, int) and not isinstance(budget, bool): + ceiling = min(ceiling, max(0, budget)) + live = ai_review.ladder() + if not live and ask is None: + # ⛔ FAIL CLOSED AND SAY SO. A run that quietly filled nothing looks identical to a run + # with nothing to do; `planned` above is the number that distinguishes them. + report['problem'] = 'no AI provider is configured on this deployment' + return report + provider = (live or [{'name': 'injected', 'model': 'injected', 'env': '', + 'shape': 'openai', 'url': ''}])[0] + model = str(cfg.get('model') or provider['model']) + report['provider'], report['model'] = provider['name'], model + tmo = float(timeout or TIMEOUT_SECONDS) + + values, stamps = {}, {} + for row_id in plan['run']: + if ceiling and report['tokens'] >= ceiling: + # ⛔ STOP AND REPORT (R6's second sentence). Cells already written STAY written and + # carry honest marks; nothing is truncated and no cell holds half an answer. + report['limit'] = ceiling_report(report['tokens'], ceiling, + report['filled'], report['planned']) + left = report['planned'] - report['filled'] - report['failed'] + if left > 0: + report['skipped']['over_budget'] = left + break + row = rows.get(row_id) or {} + text, unresolved = render_prompt(cfg.get('prompt'), row, keys) + if unresolved: + # Reported per row rather than refused up front: the prompt is valid for the column, + # and naming the missing token is what lets somebody fix it. + why = ('the prompt names ' + ', '.join(unresolved) + + ', which is not a column on this database') + report['failed'] += 1 + stamps[row_id] = {'state': 'error', 'error': why} + if len(report['errors']) < MAX_REPORTED_ERRORS: + report['errors'].append({'row': row_id, 'error': why}) + continue + answer, tokens, problem = caller(provider, model, text[:MAX_CELL_CHARS * 8], + int(cfg.get('maxTokens') or 300), tmo) + # ⚠ COUNTED HERE, ON EVERY OUTCOME — before the `problem` branch below, because a call that + # errored or answered UNKNOWN still reached the vendor and is still billed. A meter that + # counts only filled cells reports a cheap run for one that spent its whole budget failing. + report['calls'] += 1 + if isinstance(tokens, int) and not isinstance(tokens, bool): + report['tokens'] += tokens + else: + report['unmeasured'] += 1 + if problem or not answer or answer.strip().upper() == 'UNKNOWN': + # ⛔ THE CELL IS LEFT ALONE. An errored row keeps whatever it held; only the MARK + # changes, so a failed run never destroys a value it could not replace. + why = problem or ('the model had nothing to answer from' if not answer + else 'the model answered UNKNOWN for this row') + report['failed'] += 1 + stamps[row_id] = {'state': 'error', 'error': why, 'model': model} + if len(report['errors']) < MAX_REPORTED_ERRORS: + report['errors'].append({'row': row_id, 'error': why}) + continue + # ⛔ NORMALISED BEFORE THE WRITE, NOT AFTER (note D-6). This is the ONE line where model + # prose becomes stored product data; a cell fixed after the fact leaves every row already + # written carrying the dash, and `web_prose` can never see it because it scans source. + values[row_id] = no_dashes(answer)[:MAX_CELL_CHARS] + stamps[row_id] = {'state': 'agent', 'model': model, + 'hash': ut.ai_enrich_input_hash(cfg.get('prompt'), row, + prompt_refs(cfg.get('prompt'))), + 'tokens': tokens if isinstance(tokens, int) else 0} + report['filled'] += 1 + + if values: + ut.patch_many_cells(table_key, {r: {str(col_id): v} for r, v in values.items()}, st=st) + if stamps: + ut.stamp_ai_enrich(table_key, col_id, stamps, st=st) + # ⚠ The ceiling is re-asked AFTER the loop too: a run whose LAST row crossed the budget spent + # over it without re-entering the guard, and staying silent there would under-report the one + # run most likely to surprise somebody. + if report['limit'] is None and ceiling and report['tokens'] >= ceiling: + report['limit'] = ceiling_report(report['tokens'], ceiling, + report['filled'], report['planned']) + # ⭐⭐ C7's LEDGER LINE — one per run, after both store writes, and only when the run actually + # called something. `total=` rather than a split: `_ask`'s contract returns a total and a gate + # injects against that shape, so splitting it here would mean inventing two numbers from one. + # + # ⚠ TWO LINES WHEN A RUN HAS BOTH KINDS, and it is not tidiness. `record` marks a line + # `unmeasured` only when its total is None, so ONE line carrying 40 calls and a partial total + # would book all forty as measured and lose the shortfall — a total presented as complete when + # it is a floor, which is the exact failure `_usage_tokens`' None-semantics exists to prevent. + _measured = report['calls'] - report['unmeasured'] + if _measured > 0: + usage_ledger.record('field_agent', report['provider'], report['model'], + total=report['tokens'], calls=_measured, st=st, user=user) + if report['unmeasured'] > 0: + usage_ledger.record('field_agent', report['provider'], report['model'], + total=None, calls=report['unmeasured'], st=st, user=user) + return report diff --git a/api/ai_review.py b/api/ai_review.py index 5f5d12ec06e026428249349d16ac14d82c458144..29f16c1b0d27c10df0e9d80a9c154d45652a9e16 100644 --- a/api/ai_review.py +++ b/api/ai_review.py @@ -130,6 +130,17 @@ def _parse(text, options): return "", reason +# ⭐⭐ WAVE 35 · W35-T41 / C7 — THE TWO TRANSPORTS RETURN THE RESPONSE BODY'S `usage`. +# +# ⛔ THIS FILE'S OWN HEADER USED TO BE THE PRODUCT'S CONFESSION THAT NOTHING COUNTED: `ai_enrich`'s +# docstring says *"`ai_review.decide` -- the product's only other LLM entry -- has no token +# accounting of ANY kind"*. R9 asks for ONE meter for every AI surface, so the counting has to reach +# this transport rather than be bolted onto its caller — the caller never sees the body. +# +# ⚠ A THIRD RETURN VALUE, NOT A MUTATED ARGUMENT, and both call sites are in `decide` below. The +# tuple grew from `(text, err)` to `(text, err, usage)`; `usage` is the raw `usage` OBJECT (or None), +# because reading it is `usage_ledger`'s job and a second reader here would be the two-normalizers +# shape this repo keeps paying for. def _call_openai(p, model, system, user, timeout): r = requests.post(p["url"], timeout=timeout, headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", @@ -138,12 +149,14 @@ def _call_openai(p, model, system, user, timeout): "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}]}) if r.status_code >= 400: - return "", f"{p['name']} answered {r.status_code}" + # ⚠ NO BODY ON A 4xx/5xx: an error envelope carries no usage, and a provider that refused + # before running the model has nothing to bill. The call is still COUNTED by `decide`. + return "", f"{p['name']} answered {r.status_code}", None body = r.json() choices = body.get("choices") or [] if not choices: - return "", f"{p['name']} returned no choices" - return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "" + return "", f"{p['name']} returned no choices", body + return str(((choices[0] or {}).get("message") or {}).get("content") or ""), "", body def _call_anthropic(p, model, system, user, timeout): @@ -154,25 +167,34 @@ def _call_anthropic(p, model, system, user, timeout): json={"model": model, "max_tokens": 300, "system": system, "messages": [{"role": "user", "content": user}]}) if r.status_code >= 400: - return "", f"anthropic answered {r.status_code}" + return "", f"anthropic answered {r.status_code}", None body = r.json() # ⛔ stop_reason FIRST. A safety refusal is a successful 200 with an EMPTY content list, so # reading content[0] before this check turns a refusal into an IndexError inside a run. + # ⚠ A refusal IS billed and its body carries `usage`, so the body rides back on this branch too. if body.get("stop_reason") == "refusal": - return "", "anthropic declined to answer this record" + return "", "anthropic declined to answer this record", body parts = [b.get("text") or "" for b in (body.get("content") or []) if isinstance(b, dict) and b.get("type") == "text"] if not parts: - return "", "anthropic returned no text" - return "".join(parts), "" + return "", "anthropic returned no text", body + return "".join(parts), "", body -def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): +def decide(*, prompt, options, row, fields=(), label="Review", timeout=None, st=None, user=""): """Pick this record's next stage. Returns `(choice, meta)`. `choice` is "" whenever a person should decide — which is every failure mode there is. `meta` carries `provider`, `model`, `reason` on success, and `problem` on refusal to answer. + + ⭐ W35-T41 / C7 — `st` and `user` are the USAGE LEDGER's target. Both default to absent because + this function's caller is `automation_engine.ai_decide(rt, ...)`, in another lane's fence: it + HAS the runtime and does not pass it yet, so until it does, a review's tokens are counted as + unattributed and REPORTED by `GET /usage` rather than dropped. See `usage_ledger`'s header for + why they cannot be resolved implicitly (measured: a contextvar does not survive a FastAPI + dependency). """ + import usage_ledger # noqa: PLC0415 opts = [str(o) for o in (options or []) if str(o).strip()] if not opts: return "", {"problem": "the review offers no next stages"} @@ -188,11 +210,22 @@ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): problems = [] for p in live: model = override or p["model"] + body = None try: - text, err = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( + text, err, body = (_call_anthropic if p["shape"] == "anthropic" else _call_openai)( p, model, system, user, tmo) except Exception as e: # noqa: BLE001 text, err = "", f"{p['name']} failed: {type(e).__name__}" + # ⭐⭐ C7 — THE LEDGER LINE, BEFORE ANY BRANCH BELOW READS THE ANSWER. + # ⛔ IT IS WRITTEN ON EVERY OUTCOME THAT REACHED A PROVIDER, including a refusal and an + # unusable answer. A meter that books only successes reports a cheap week for a run that + # spent its budget being declined — and a declined call is billed. The only path that does + # NOT record is a transport that never reached the vendor (`body is None`), which spent + # nothing. + if body is not None: + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("ai_review", p["name"], model, ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) if err: problems.append(err) continue # ladder: a dead provider degrades to the next one @@ -205,3 +238,275 @@ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None): "problem": reason or "the model did not choose one of the stages"} return choice, {"provider": p["name"], "model": model, "reason": reason} return "", {"problem": "; ".join(problems)[:300] or "no provider answered"} + + +# ══════════════════════════════════════════════════════════════════ the FLOW WRITER +# ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION. +# +# Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the +# action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary: +# it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the +# runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object +# in that vocabulary. +# +# ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This +# is the one module in the API package that already holds an LLM ladder and its credentials, so a +# second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs +# a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23 +# shipped three finished routers 404-dead behind green gates. The DOOR is a route on +# `routes_automation.py`, which is already mounted. +# +# ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds +# present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are +# enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema +# the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters +# because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise +# and answers 200, so a draft accepted without the diff would show a person a flow that is not the +# flow the model described, with nothing anywhere saying so. +# +# ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door +# can be proven end to end with no API key and no spend. A gate that can only run where a +# credential exists is a gate that never runs. + +#: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it: +#: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you +#: have") is a model property. A cheaper rung answers an impossible request with a plausible flow, +#: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`. +FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic") +MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25 +MAX_PROMPT_CHARS = 2000 + + +def flow_providers(): + """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off.""" + pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower() + by_name = {p["name"]: p for p in PROVIDERS} + order = [pin] if pin else list(FLOW_PROVIDER_ORDER) + return [by_name[n] for n in order + if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()] + + +def flow_schema(kinds, trigger_keys, table_keys): + """The tool schema the model fills — the catalog's OWN key lists as enums. + + ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the + transport rather than by a check afterwards. A model asked for "any action name" invents + `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a + 400 from `clean_actions` carrying a sentence about a word the person never typed. + ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a + schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a + transport failure (`routes_query._spec_schema` carries the same note, for the same measured + reason). + """ + return { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": ["flow", "refused"], + "description": "refused = this cannot be built from the steps available"}, + "refusal": {"type": "string", + "description": "when kind=refused: ONE plain sentence naming what is " + "missing, in the words a non-technical person would use"}, + "name": {"type": "string", "description": "a short title for the automation"}, + "trigger": {"type": "string", "enum": sorted(trigger_keys), + "description": "what starts this automation"}, + "table": {"type": "string", "enum": sorted(table_keys), + "description": "the database whose records this flow walks, if any"}, + "actions": { + "type": "array", + "maxItems": MAX_DRAFT_ACTIONS, + "items": { + "type": "object", + "properties": { + "kind": {"type": "string", "enum": sorted(kinds)}, + "why": {"type": "string", + "description": "one short sentence: why this step is here"}, + "config": {"type": "object", "additionalProperties": True, + "description": "the step's settings, using ONLY the keys named " + "for that kind in the system message"}, + }, + "required": ["kind"], + }, + }, + }, + "required": ["kind"], + } + + +def flow_system_prompt(catalog, required, triggers, tables): + """What the model is told it may build with — DERIVED, never written down twice. + + Every list here is the server's own: the catalog rows the menu paints, the required-key table + the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases + with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to + the catalog is offered here on the same deploy and a kind removed stops being offered. + """ + lines = ["You build small automations for a business tool. You are given the EXACT set of " + "steps this tool can perform. You may use nothing else.", + "", "THE STEPS YOU MAY USE:"] + # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the + # PHRASE is the only human wording of that requirement anywhere — the one the runner's own + # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a + # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing + # what to call the setting. Handing it one and hoping is how a draft comes back configured + # against a key the validator drops. + req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()} + for row in catalog or []: + if not row.get("ready"): + continue + kind = str(row.get("kind") or "") + need = req.get(kind) or [] + lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}" + + (f" REQUIRED settings: {', '.join(need)}" if need else "")) + lines += ["", "WHAT CAN START AN AUTOMATION:"] + for t in triggers or []: + if t.get("planned") or not t.get("ready", True): + continue + lines.append(f"- {t.get('key')}: {t.get('label')}") + lines += ["", "THE DATABASES THIS PERSON HAS:"] + for t in (tables or [])[:40]: + cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25]) + lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}") + if not tables: + lines.append("- (none - do not name a database)") + lines += [ + "", + "RULES:", + "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, " + "answer kind=refused and say plainly which capability is missing.", + "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank " + "rather than inventing a web address, a CSS selector or a column name.", + "3. A web address must start with http:// or https://.", + "4. To use a value from the record the flow is walking, write it as {{Column name}}.", + "5. Name a database only from the list above, by its key.", + "6. Keep it short. Fewer steps that work beat more steps that guess.", + ] + return "\n".join(lines) + + +def _flow_from_tool_call(obj): + """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash.""" + if not isinstance(obj, dict): + return None, "the assistant's answer could not be read" + if str(obj.get("kind") or "") == "refused": + return None, (str(obj.get("refusal") or "").strip() + or "the assistant could not build this from the steps available") + acts = obj.get("actions") + if not isinstance(acts, list) or not acts: + return None, ("the assistant did not produce any steps - try describing what should " + "happen, one action at a time") + out = [] + for a in acts[:MAX_DRAFT_ACTIONS]: + if not isinstance(a, dict) or not str(a.get("kind") or "").strip(): + continue + cfg = a.get("config") + out.append({"kind": str(a["kind"]).strip(), + "config": cfg if isinstance(cfg, dict) else {}, + "why": str(a.get("why") or "").strip()[:200]}) + if not out: + return None, "the assistant's steps could not be read" + return {"name": str(obj.get("name") or "").strip()[:80] or "New automation", + "trigger": str(obj.get("trigger") or "").strip(), + "table": str(obj.get("table") or "").strip(), + # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner, + # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need + # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the + # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past + # the twelfth and said nothing, so a model that answered with a twenty-step journey had + # eight steps deleted between the answer and the screen, with no key anywhere in the + # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this + # ceiling is the draft door's own, which makes reporting it the whole obligation. + # ⚠ Found by the verifier that checked this ticket, not by a gate. + "asked": len([a for a in acts if isinstance(a, dict)]), + "actions": out}, "" + + +def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None, + st=None, user=""): + """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE. + + Exactly one of `draft` and `refusal_sentence` is truthy — the same contract + `web_agent.run_step` keeps, so a caller has no third case to get wrong. + + ⭐ W35-T41 / C7 — `st`/`user` are the usage ledger's target, exactly as on `decide` above and for + the same reason: both of this function's callers (`automation_engine._ai_agent_plan` and + `routes_automation`'s draft door) are in lane D's fence. C7 says E adds the ledger line here and + D asserts it; the two keywords are what D has to pass for the line to be attributable. + """ + import usage_ledger # noqa: PLC0415 + text = str(prompt or "").strip()[:MAX_PROMPT_CHARS] + if not text: + return None, "type what you want the automation to do", None + kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")}) + if not kinds: + return None, "this deployment offers no automation steps to build with", None + trigger_keys = sorted({str(t.get("key")) for t in (triggers or []) + if t.get("key") and not t.get("planned")}) or ["manual"] + table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""] + tools = [{"type": "function", "function": { + "name": "build_automation", + "description": "Emit the automation, or refuse.", + "parameters": flow_schema(kinds, trigger_keys, table_keys)}}] + messages = [{"role": "system", + "content": flow_system_prompt(catalog, required, triggers, tables)}, + {"role": "user", "content": text}] + + if chat is not None: + # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal + # branches as a live call; only the transport is replaced. + draft, refusal = _flow_from_tool_call(chat(messages, tools)) + return draft, refusal, "injected" + + provs = flow_providers() + if not provs: + # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was + # never built [[flag-shipped-without-its-writer]]. + return None, ("the assistant is not configured on this deployment, so an automation cannot " + "be drafted from a description yet"), None + tmo = float(timeout or TIMEOUT_SECONDS) + problems = [] + for p in provs: + if p["shape"] != "openai": + # Anthropic's tool transport differs; it stays on the ladder for `decide()` and is + # SKIPPED here rather than half-supported. Named in `problems` so it is not invisible. + problems.append(f"{p['name']}: tool calls are not wired for this shape") + continue + try: + r = requests.post(p["url"], timeout=tmo, + headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}", + "Content-Type": "application/json"}, + json={"model": p["model"], "messages": messages, "tools": tools, + "tool_choice": "required", "temperature": 0.1, + "max_tokens": 1500}) + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: {type(e).__name__}") + continue + if r.status_code != 200: + problems.append(f"{p['name']}: HTTP {r.status_code}") + continue + try: + body = r.json() + calls = (((body.get("choices") or [{}])[0].get("message") or {}) + .get("tool_calls") or []) + args = json.loads(calls[0]["function"]["arguments"]) if calls else None + except Exception as e: # noqa: BLE001 + problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})") + continue + # ⭐⭐ C7 — THE LEDGER LINE. Placed after the parse rather than before it so an UNREADABLE + # answer is not double-counted by the `continue` above... ⚠ which means an unreadable answer + # is NOT counted at all, and that is a deliberate, disclosed loss: a body this code cannot + # parse is a body whose `usage` it also cannot trust, and `body` is out of scope in that + # branch by construction. The 200-with-junk case is rare and named here rather than + # silently rounded to zero. + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record("automation_draft", p["name"], p["model"], ins, outs, + total=usage_ledger.total_from(body), st=st, user=user) + draft, refusal = _flow_from_tool_call(args) + if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused": + # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more + # expensive rung — the model has just said this cannot be built. `decide()` takes the + # same posture for the same reason. + return None, refusal, p["name"] + if draft is not None: + return draft, "", p["name"] + problems.append(f"{p['name']}: {refusal}") + return None, ("; ".join(problems)[:300] or "no provider answered"), None diff --git a/api/automation_engine.py b/api/automation_engine.py index 7b272a33112e098767ae00cc21e1a74d188df0a5..5d60218280cb2c33520bb31163c4e64ea5f6adec 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -67,9 +67,30 @@ UT_PREFIX = "ut_" MAX_AUTOMATIONS = 40 MAX_RUNS = 20 # trimmed history per automation MAX_NAME = 80 -MAX_UT_ROWS = 5000 # mirrors core.user_tables.MAX_ROWS +#: ⛔ IT DOES **NOT** MIRROR `core.user_tables.MAX_ROWS`, AND THE COMMENT THAT SAID SO WAS A 12x +#: STALE CLAIM — `MAX_ROWS` is 60,000. That sentence is most of what made D-143 hard to see: it +#: read as "the substrate's bound, kept in step", so nobody asked whether a flow was silently +#: walking 5,000 of 32,826 connected rows. It was. +#: **This is the FALLBACK ONLY.** The live answer is per TABLE and comes from +#: `core.user_tables.row_limit` via `_flow_record_cap` — `None` for a connected source (R6: no +#: cap), `MAX_ROWS` for the editable substrate, `0` for a read-through grid. This constant is +#: reached only when that import fails, and it is deliberately the CONSERVATIVE direction. +#: ⚠ Do not "fix" it by raising it to 60,000: a fallback that runs when the evaluator is missing +#: should not also be the widest one. And do not delete it — D-143's own row says the editable +#: substrate still needs a bound. +MAX_UT_ROWS = 5000 MAX_UT_TABLES = 40 +#: ⛔⛔ D-112's SENTENCE, AS A CONSTANT, BECAUSE TWO PLACES DEPEND ON IT AGREEING. +#: `enrich_selection` produces it when a step's `fromView` names a view that no longer resolves; +#: `run_flow` tests for it to turn that run `partial` instead of `ok`. Measured live on a real +#: tenant: an enrich action pointed at a deleted view walked ZERO records and reported **`ok`** — +#: indistinguishable on every surface from a run that worked, and a strong candidate for why the +#: owner's enrichment kept appearing to do nothing. +#: ⚠ THE WORDING IS FREE TO CHANGE; the AGREEMENT is not. Both sites read this name, so a better +#: sentence stays a one-line edit instead of a silent regression [[constant-two-features-share]]. +ENRICH_VIEW_UNREADABLE = "the enrich step names a view it cannot read" + #: ⚠ THE APPEND TABLES NEED A DIFFERENT CAP, and the reason is arithmetic rather than taste. #: `MAX_UT_ROWS` was sized for a scraped LIST — a page of ~500 companies that is re-read, so the #: row count is bounded by the page. The `ut_ig_*` tables are the opposite shape: every pull @@ -223,10 +244,40 @@ RETIRED_KIND_REPLACEMENT = { #: door open on purpose, and there is a gate check that says so in those words. A kind may only be #: walled off at a door once something else answers the same need. #: (`REPLACED_KINDS` was here and is deleted with the refusal it gated — see `clean_definition`.) -KIND_LABELS = {"plain": "Automation", "scrape_db": "Web page to database", +#: ⭐ WAVE 34 · R12 — `plain` READS "Agent". The owner renamed the module to Agents, so the KIND +#: that means "an ordinary one of these" is an Agent, singular (the module is the plural). +#: ⛔ THE KEY `"plain"` IS UNTOUCHED and so is every other key here: D-65's rule, restated three +#: times in this file, is that a kind is permanent — the LABELS are display and may move freely, +#: the keys are stored on every automation ever created and may not. +KIND_LABELS = {"plain": "Agent", "scrape_db": "Web page to database", "field_instagram": "Instagram profile column", "discover_instagram": "Find Instagram profiles", "discover_tiktok": "Find TikTok profiles"} +#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A SYSTEM AGENT CANNOT BE DELETED, one sentence per slug. +#: +#: ⛔ ONE PLACE, because two doors ask this question (the DELETE route refuses with it, and the +#: Canvas explains why its controls are read-only) and a refusal that says something different +#: from the surface it refuses on is worse than either sentence alone. D-65's own remedy in a new +#: family: a refusal that does not name the alternative sends somebody looking for a bug in a +#: decision made on purpose. +SYSTEM_AGENT_REASON = { + "field_agent": "this agent is an AI enrichment column. Delete the column on its database and " + "the agent goes with it", + "odoo_sync": "this is the Odoo connection's own sync schedule. Disconnect Odoo in Connectors " + "to stop it, or change how often it runs on this page", +} + + +def system_agent_refusal(slug): + """The sentence for a `system:` slug, or a general one for a slug nobody has written yet. + + ⚠ IT NEVER RETURNS EMPTY. A refusal with no sentence is a 409 a person cannot act on, and the + fallback is the one branch that will be reached by a slug added later and forgotten here. + """ + return SYSTEM_AGENT_REASON.get(str(slug or ""), "this agent is part of the workspace setup " + "and cannot be deleted here") + + EXTRACTS = ("table", "jsonld") STATES = ("idle", "running", "ok", "error", "partial") #: Which capture rung a `field_instagram` automation is allowed to reach for (R1's hybrid). @@ -453,7 +504,7 @@ def fetch(url, timeout=20.0, max_kb=2048, allowed=(), max_hops=5, headers=None): status, final = r.status_code, str(r.url) r.close() return status, final, body - raise Refused(f"more than {max_hops} redirects — refusing to follow further") + raise Refused(f"more than {max_hops} redirects. Refusing to follow further") def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): @@ -473,7 +524,7 @@ def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): data=json.dumps(body if body is not None else {}), stream=True) try: if r.status_code in (301, 302, 303, 307, 308): - raise Refused(f"the POST answered {r.status_code} — a redirected POST is refused, " + raise Refused(f"the POST answered {r.status_code}. A redirected POST is refused, " f"never re-sent to a location the server picked") return r.status_code, (r.raw.read(max_kb * 1024, decode_content=True) or b"") finally: @@ -487,7 +538,7 @@ def fetch_json(url, body, timeout=60.0, max_kb=8192, headers=None): def _soup(body): if BeautifulSoup is None: raise RuntimeError( - "beautifulsoup4 is not installed in this environment — the automation engine's " + "beautifulsoup4 is not installed in this environment. The automation engine's " "extraction half needs it (see the wave-18 mailbox: add beautifulsoup4 + lxml to " "aios-web/requirements.txt).") try: @@ -544,7 +595,7 @@ def preview(url, extract="table", table_index=0, allowed=()): status, final, body = fetch(url, allowed=allowed) if not (200 <= status < 300): return {"ok": False, "status": status, "url": final, - "note": f"the page answered {status} — it may be bot-gated or gone", + "note": f"the page answered {status}. It may be bot-gated or gone", "columns": [], "sample": [], "rowCount": 0} soup = _soup(body) rows = [] @@ -732,9 +783,38 @@ def next_fire(expr, now=None, lookahead_days=400): # THE UPSERT — pure, so the arithmetic is testable without a store or a network # --------------------------------------------------------------------------------------------- -def upsert_rows(existing, incoming, key_field, cap=None): +#: ⛔⛔ D-116 — THE SIX CELLS A CORPUS RE-FIND MUST NOT OVERWRITE ONCE AN ENRICHMENT MEASURED THEM. +#: A discovery re-find emits these on EVERY match, not only on insert, so the moment a scheduled +#: search re-matched an account somebody had PAID to enrich, six exact measurements were replaced by +#: pre-collected corpus values of unknown vintage — no error, no visible change other than the +#: number. Measured 2026-08-10 on a live tenant: 0 of 105 profiles had been re-found yet, so this +#: was a defect waiting for its first scheduled re-run rather than one anybody had seen. +#: ⚠ SCALARS ONLY, and that is the ruling. The OBSERVATION is appended either way — the corpus +#: genuinely saw that account at that follower count, and the snapshot series is where a corpus read +#: belongs. Suppressing the observation would be the same defect arriving from the other side. +CORPUS_SOFT_KEYS = ("followers", "following", "avg_engagement", "verified", "category", "bio") + + +def corpus_protect(before, _src): + """Which keys this incoming CORPUS row may not overwrite on `before`. D-116's precedence rule. + + ⛔ THE TEST IS `enriched_at`, i.e. "did an exact read ever write this row", NOT "is the cell + non-empty". The exit condition rules the second one out by name, and rightly: *"NOT by making + `upsert_rows` skip non-empty cells — that would break every re-scrape in the product."* A + re-scrape SHOULD move a number the corpus owns; what it may not do is move one an exact read + owns. + """ + return CORPUS_SOFT_KEYS if str((before or {}).get("enriched_at") or "").strip() else () + + +def upsert_rows(existing, incoming, key_field, cap=None, protect=None): """Merge scraped rows into a user table's rows BY KEY. Returns `(rows, counts)`. + `protect` is an optional `f(before, src) -> keys` naming, PER ROW, the keys this incoming row + may not overwrite. Default `None` = the old behaviour exactly, so the other nine call sites are + untouched — the precedence rule belongs to the CALLER that knows its data's provenance, not to + the merge, and a rule baked in here would apply to nine paths that never asked for one. + THE RULE THAT MATTERS: **an orphan is COUNTED, NEVER DELETED.** A row that has stopped appearing on the source page has not necessarily stopped existing — the page changed its filter, the fetch was partial, the site paginated. Deleting on absence turns any upstream @@ -792,9 +872,18 @@ def upsert_rows(existing, incoming, key_field, cap=None): counts["inserted"] += 1 continue before = rows[rid] - changed = {k: v for k, v in src.items() if str(before.get(k, "")) != str(v)} + # ⛔ D-116's PRECEDENCE RULE, applied per ROW because provenance is a property of the row. + # `held` counts the cells a lower-provenance source was refused, so the run can SAY it + # rather than quietly doing the right thing — a protection nobody is told about is + # indistinguishable from a source that happened to agree. + keep = set(protect(before, src) or ()) if protect else set() + use = {k: v for k, v in src.items() if k not in keep} if keep else src + if keep: + counts["held"] = counts.get("held", 0) + sum( + 1 for k in keep if k in src and str(before.get(k, "")) != str(src.get(k))) + changed = {k: v for k, v in use.items() if str(before.get(k, "")) != str(v)} if changed: - before.update(src) + before.update(use) counts["updated"] += 1 else: counts["unchanged"] += 1 @@ -987,7 +1076,7 @@ def bind_unbound_fields(rt): else: f["automation"]["disabled"] = True f["automation"]["statusNote"] = ( - "not bound to any flow — create an automation for this column " + "not bound to any flow. Create an automation for this column " "or delete it") return cur @@ -1438,10 +1527,10 @@ def clean_post_groups(raw, max_posts): return None, "each post group has to be a {type, limit} entry" t = str(item.get("type") or "").strip().lower() if t not in POST_TYPES: - return None, (f"'{t or 'blank'}' is not a post type — use one of " + return None, (f"'{t or 'blank'}' is not a post type. Use one of " f"{', '.join(POST_TYPES)}") if t in seen: - return None, f"the post groups name '{t}' twice — one limit per type" + return None, f"the post groups name '{t}' twice. One limit per type" seen.add(t) try: lim = int(item.get("limit")) @@ -1454,7 +1543,7 @@ def clean_post_groups(raw, max_posts): # refused HERE, where the number they typed is still on the screen in front of them. if lim > max_posts: return None, (f"this enrichment captures {max_posts} posts per profile, so a {t} " - f"group cannot keep {lim} — raise the post count first") + f"group cannot keep {lim}. Raise the post count first") out.append({"type": t, "limit": lim}) return out, None @@ -1486,7 +1575,7 @@ def clean_max_posts(raw, default=DEFAULT_POSTS_PER_PULL, submitted=True): if n < 1: return None, "keep at least one post per profile, or turn the post capture off" if n > MAX_POSTS_PER_PULL: - return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts — that is the " + return None, (f"a profile pull returns at most {MAX_POSTS_PER_PULL} posts. That is the " f"vendor's cap, not ours, so {n} would store the same " f"{MAX_POSTS_PER_PULL} and read as more") return n, None @@ -1600,7 +1689,7 @@ def clean_config(kind, raw, previous=None): _, _disc_table, _disc_label, _ = discovery_facts(kind) limit = int(raw.get("recordsLimit") or 0) if limit < 1: - return None, ("say how many profiles to fetch — an UNBOUNDED discovery query is the " + return None, ("say how many profiles to fetch. An UNBOUNDED discovery query is the " "one shape the vendor refuses outright (NOT_ENOUGH_FUNDS)") if limit > BD_MAX_RECORDS: return None, f"a single discovery run may ask for at most {BD_MAX_RECORDS} profiles" @@ -1653,7 +1742,7 @@ def clean_seed(raw): return None, f"{source or 'that seed'!r} is not one of: " + ", ".join(SEED_SOURCES) table = _s(raw.get("table"), 60).strip() if table and not table.startswith(UT_PREFIX): - return None, f"a seed reads a blank database (ut_*) — {table!r} is not one" + return None, f"a seed reads a blank database (ut_*). {table!r} is not one" basis = raw.get("basis") if isinstance(raw.get("basis"), dict) else {} derived, _err = clean_predicates(raw.get("derived") or [], "and") return {"source": source, "table": table, "id": _s(raw.get("id"), 80).strip(), @@ -1693,11 +1782,15 @@ def clean_schedule(raw, previous=None): # ⛔ `terminal` IS THE DEFAULT and every existing automation gets it, because a stored definition # that predates this field must not start moving records on its own the day the code ships. A # loop is a thing somebody turns on. -def clean_flow(raw, previous=None): - """Validate the builder's ordered action list. `(flow, error)`.""" +def clean_flow(raw, previous=None, notes=None, rt=None): + """Validate the builder's ordered action list. `(flow, error)`. + + ⚠ `rt` is the tenant wall for gated action kinds (W35-T35 / C8) and is simply forwarded. + """ raw = raw if isinstance(raw, dict) else {} prev = previous if isinstance(previous, dict) else {} - actions, err = clean_actions(raw.get("actions") if "actions" in raw else prev.get("actions")) + actions, err = clean_actions( + raw.get("actions") if "actions" in raw else prev.get("actions"), notes=notes, rt=rt) if err: return None, err return {"actions": actions}, None @@ -1972,7 +2065,7 @@ def ig_action_pinned(defn, index): return (defn or {}).get("kind") in DISCOVERY_KINDS and index == 0 -def clean_definition(raw, previous=None, username=""): +def clean_definition(raw, previous=None, username="", notes=None, rt=None): """Whole-definition validation. Returns `(defn, error)`.""" raw = raw if isinstance(raw, dict) else {} prev = previous or {} @@ -2149,7 +2242,7 @@ def clean_definition(raw, previous=None, username=""): _prev_table, _prev_enrich = discovery_seed_spec(prev.get("kind")) flow_raw = _drop_ig_seeds( flow_raw, (prev.get("config") or {}).get("targetTable") or _prev_table, _prev_enrich) - flow, ferr = clean_flow(flow_raw, prev.get("flow")) + flow, ferr = clean_flow(flow_raw, prev.get("flow"), notes=notes, rt=rt) if ferr: return None, ferr # ⭐ WAVE 30 — widened with the seeder above: TikTok now HAS a pinned step 1, so the rule that @@ -2225,6 +2318,22 @@ def clean_definition(raw, previous=None, username=""): "state": dict(prev.get("state") or {}), "created": prev.get("created") or _iso(), "createdBy": prev.get("createdBy") or username, + # ⭐⭐ WAVE 35 · T37 — `system` IS STICKY ACROSS A PATCH, AND IT IS **NEVER READ FROM `raw`**. + # + # ⛔ THE BUG THIS CLOSES WAS ALREADY WRITTEN AND WOULD HAVE SHIPPED. `routes_automation. + # delete_automation` refuses any stored row carrying `system`, which is what makes a seeded + # system agent undeletable — and this function is a WHITELIST that did not carry the key, so + # the FIRST EDIT of such an agent silently stripped its marker and made it deletable. The + # seed would then mint it again on the next list call: a delete that appears to work and + # undoes itself. Found by asserting the round trip rather than by reading the code. + # + # ⛔ FROM `prev` ONLY. Reading it from `raw` would let ANY caller POST + # `{"system": "anything"}` and mint themselves an automation nobody can delete — a + # privilege escalation through a field nobody validates. The flag is granted by the seeder + # and inherited from the stored row, never asserted by a payload. + # ⚠ Same shape and same reason as the pre-set FIELD flag, which DESIGN.md §4 already + # describes as "sticky across a PATCH". One idea, one behaviour, two objects. + **({"system": _s(prev.get("system"), 40)} if prev.get("system") else {}), }, None @@ -2475,7 +2584,7 @@ def resolve_target(rt, raw, username=""): import core.user_tables as _ut_new key = _ut_new.create(label, username or "automation", st=rt) if not key: - return None, ("the database could not be created — this workspace may be at its table " + return None, ("the database could not be created. This workspace may be at its table " "limit") return {"targetTable": key, "targetLabel": label}, None @@ -2497,7 +2606,7 @@ def _unmint(rt, key): pass -def create(rt, raw, username=""): +def create(rt, raw, username="", notes=None): existing = all_definitions(rt) if len(existing) >= MAX_AUTOMATIONS: return None, f"this workspace is at the {MAX_AUTOMATIONS}-automation limit" @@ -2526,7 +2635,9 @@ def create(rt, raw, username=""): # has, instead of letting the pure validator mint a second empty one. See # `discover_default_table`; a target the caller named is never rewritten. raw = _apply_discover_default(rt, raw) - defn, err = clean_definition(raw, None, username) + # W35-T35 (C8): `rt=` is the tenant wall for gated action kinds. Both doors pass it; a door + # that forgot would REFUSE the kind, not admit it (`TENANT_GATED_ACTIONS` is fail-closed). + defn, err = clean_definition(raw, None, username, notes=notes, rt=rt) if err: _unmint(rt, minted) return None, err @@ -2541,7 +2652,7 @@ def create(rt, raw, username=""): if defn["kind"] in RETIRED_KINDS: _unmint(rt, minted) return None, (f"{KIND_LABELS.get(defn['kind'], defn['kind'])!r} automations are no " - f"longer created — the ones already using it keep working. " + f"longer created. The ones already using it keep working. " f"{RETIRED_KIND_REPLACEMENT.get(defn['kind'], '')}") defn["id"] = _new_id(existing) @@ -2562,7 +2673,7 @@ def create(rt, raw, username=""): return defn, None -def patch(rt, auto_id, raw, username=""): +def patch(rt, auto_id, raw, username="", notes=None): existing = all_definitions(rt) prev = existing.get(str(auto_id)) if prev is None: @@ -2581,7 +2692,7 @@ def patch(rt, auto_id, raw, username=""): # config that has never carried a target through the pure validator, which mints the empty # `ut_ig_candidates`. `create` alone would have left the commonest path untouched. merged = _apply_discover_default(rt, merged) - defn, err = clean_definition(merged, prev, username) + defn, err = clean_definition(merged, prev, username, notes=notes, rt=rt) if err: return None, err defn["id"] = str(auto_id) @@ -2880,14 +2991,45 @@ def remove(rt, auto_id): # `(table, field)` pairs it found and the mutation applies exactly those — the shape # `bind_unbound_fields` in this module already uses, so a delete costs one walk instead of two # and the write touches only the fields it named. - # ⛔ HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): this still costs ONE - # full read of the tenant's `user_tables` document, and `rt.update` below takes its own strict - # read. **That second read cannot be removed here** — it belongs to the write, and answering - # "which fields did this flow tag?" without a scan needs a tagged-field INDEX, which is a - # feature and not a wave-tail edit. It is booked as D-179; what changed is the second WALK, - # not the read count. + # ⭐⭐ WAVE 33 (D-179) — THE SCAN NO LONGER READS THE ROWS, and that is where the cost was. + # + # This walk asks ONE question — "which fields did this flow tag?" — and it has never looked at + # a row to answer it. It was nevertheless taking `ut_all`, a whole deep copy of the tenant's + # `user_tables` document, of which **99.9% of the bytes are `rows` nothing here reads** + # (measured on tenant #0: 28.6 MB, 81,330 rows, 703 ms warm). A's C5 projection makes that + # ~0.1% of the bytes. + # ⛔ HANDED TO A READ AND NOTHING ELSE. `lend_defs` serves `_Projected` values, so reaching + # `defn['rows']` raises a `KeyError` NAMING the projection — by design — and `plan` carries + # only `(table_key, field_key)` STRINGS out of this scope. The mutation below takes its own + # strict document from `rt.update` and never sees a projected value, which is C5's second + # clause: a projected snapshot is never handed to a post-write read-back. + # ⚠ HONEST RESIDUAL, STATED RATHER THAN SMOOTHED (R6's second sentence): D-179's exit condition + # asks for ZERO full-document reads and this is ONE — `rt.update` below takes a strict read + # that belongs to the WRITE and cannot be removed from here. What is gone is the expensive one. + # ⚠ AND THE FALLBACK IS DELIBERATE: `all_defs` degrades to the whole read on any failure, so a + # store that cannot project still deletes correctly, only slower. + # + # ⭐⭐ WAVE 35 · T34 — THE RESIDUAL IS NOW MEASURED RATHER THAN ASSERTED, and the count is + # better than the paragraph above claims. Whole `user_tables` reads per delete, against a + # runtime that has production's `get_projection`: + # TAGGED (a discovery flow that spawned columns) → 1 whole + 1 projected + # UNTAGGED (a `plain` flow) → 0 whole + 1 projected + # The untagged case pays NONE because `if not plan: return` fires before the write. The + # tagged case's ONE is `rt.update`'s own strict read, and it is a FLOOR, not a choice: + # `core/store.py::get_projection` states that a write must always read whole, because a + # read-modify-write handed a rows-less document would upload the tenant with its rows deleted. + # ⛔ Reaching ZERO needs D-179's `{flowId: [(table_key, field_key)]}` index, written where the + # presets are spawned — a feature, not an edit to this function. Nothing here can do better. + # ⚠ THOSE NUMBERS WERE UNOBSERVABLE UNTIL W35-T34: `verify_automation`'s delete-speed double + # had no `get_projection`, so `all_defs` fell back and the gate measured 2 and 1 — the + # PRE-WAVE-33 path — for two waves. The double is producer-faithful now and asserts the counts. + try: + import core.user_tables as _ut_defs # noqa: PLC0415 + _scan = dict(_ut_defs.all_defs(rt) or {}) + except Exception: # noqa: BLE001 + _scan = ut_all(rt) plan = [(tk, str(f.get("key") or "")) - for tk, t in ut_all(rt).items() + for tk, t in _scan.items() for f in ((t or {}).get("fields") or []) if str((f.get("automation") or {}).get("flowId") or "") == aid] if not plan: @@ -2999,7 +3141,7 @@ def _commit_run(rt, auto_id, state, summary, counts, ok, affected=None, steps=No if isinstance(trg, dict): trg["paused"] = True d["statusNote"] = (f"auto-paused after {CONSECUTIVE_FAILURE_PAUSE} consecutive " - f"failures — fix the cause, then re-enable: " + f"failures. Fix the cause, then re-enable: " f"{entry['summary'][:120]}") return cur @@ -3038,7 +3180,9 @@ def _notify_outcome(rt, auto_id, state, entry): from core import alerts as _alerts _alerts.notify( owner=owner, - label=_s(defn.get("name") or "Automation", 80), + # ⭐ WAVE 34 · R12 — the fallback name a notification wears when the automation has + # none. "Agent", singular: this names ONE of them, not the module. + label=_s(defn.get("name") or "Agent", 80), topic="automation", key=str(auto_id), detail=f"{state}: {entry.get('summary') or ''}"[:300], @@ -3866,8 +4010,8 @@ BD_FIELD_LABELS = { BD_FIELD_HINTS = { "account": "The @username", "avg_engagement": "Likes and comments as a share of followers", - "id": "Instagram's internal number for the account — for matching a list you already have", - "fbid": "The linked Facebook id — for matching a list you already have", + "id": "Instagram's internal number for the account. For matching a list you already have", + "fbid": "The linked Facebook id. For matching a list you already have", "profile_name": "The name shown above the bio, when it differs from the account name", "related_accounts": "Accounts Instagram suggests alongside this one", } @@ -4019,7 +4163,7 @@ def narrowing_refusal(preds, operator="and", kind=""): # read, because the string predates the second platform. `discovery_facts` already carries the # network's name for exactly this class of string, so there is no second literal. _net = discovery_facts(kind)[0] if kind else PLATFORM_INSTAGRAM - return ("add a condition that describes the account itself — Bio contains, Handle is, a " + return ("add a condition that describes the account itself. Bio contains, Handle is, a " f"hashtag. Follower counts and yes/no conditions alone match too much of {_net} " "to search") @@ -5151,7 +5295,7 @@ def retract_foreign_presets(rt, log=print): if not (stats["columns"] or stats["flags"]): if stats["kept"]: log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they " - f"hold data — " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" + f"hold data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" for k in stats["kept"])) return stats stats = {"tables": 0, "columns": 0, "cells": 0, "flags": 0, "kept": []} @@ -5160,7 +5304,7 @@ def retract_foreign_presets(rt, log=print): f"{stats['flags']} mis-declared flag(s) across {stats['tables']} database(s)") if stats["kept"]: log(f"[aios-auto] retract: {len(stats['kept'])} foreign column(s) KEPT because they hold " - f"data — " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" + f"data. " + ", ".join(f"{k['table']}.{k['column']} ({k['rows']} rows)" for k in stats["kept"])) return stats @@ -5830,6 +5974,30 @@ def migrate_ig_tables(rt, log=print, only=""): return cur rt.update(UT_STORE_KEY, _up) + # ⭐⭐ WAVE 33 (D-187) — A SINGLE-TABLE CALL STOPS PAYING FOR A TENANT-WIDE RECONCILIATION. + # + # W31-T30 closed HALF of this: `only` naming a table outside the Instagram graph returns early + # (`if only and not targets` above). What it left is the case the row is actually about — `only` + # naming a table that IS in the graph, i.e. **every save an Instagram tenant makes**. The four + # passes below are tenant-wide by construction: two `_ig_schema_contract` + + # `_reconcile_ig_graph_fields` rounds and two backfills, each walking the whole document, none + # of them reading `only`. So one `ut_ensure` on one table paid a whole-tenant Instagram + # reconciliation, and `ut_ensure` runs several times per save. + # + # ⛔ THE ROW'S EXIT OFFERS TWO BRANCHES AND THIS TAKES THE FIRST ONE — *"the tenant-wide passes + # are reachable from a scheduled/boot caller"* — because they ARE, and were before this change: + # `routes_tables.py:882` (`_ig_forward`) calls `migrate_ig_tables(rt)` with NO `only`, once per + # tenant per process, guarded by `_IG_FORWARDED`. That is the right home for a whole-tenant + # repair: once, off the save path, rather than on every ensure. + # ⚠ THE SECOND BRANCH — "bound them by an anything-to-do test guarded on the READ" — is NOT + # available here and the comment above says why: the second contract pass exists to read what + # the first pass WROTE, so it cannot be answered out of one snapshot without silently + # un-fixing the idempotency everything else depends on. + # ⛔ NARROW AND CHECKABLE, like its W31-T30 sibling: a call that NAMES ONE TABLE does that + # table's migration. A call that names none still does the whole sweep, unchanged, and every + # tenant-wide check in `verify_automation` goes through that path. + if only: + return stats # The profile loop owns value conversions and deduplication. The schema pass owns the # declarations across ALL related databases and is safe only after those conversions, because # the stored old type is the idempotency key for unit-changing values such as engagement. @@ -5956,7 +6124,7 @@ def seed_predicates(rows): if str(r.get("related_accounts") or "").strip())}, "note": ""} if total < SEED_MIN_ROWS: - basis["note"] = ("a seed needs at least two records — one record has nothing to share " + basis["note"] = ("a seed needs at least two records. One record has nothing to share " "with anything") return [], basis ranked = [] @@ -5995,11 +6163,11 @@ def seed_predicates(rows): # asserted in the gate. If the LAW ever changes shape, this fails closed with an honest # empty rather than offering something the save door will reject. basis["note"] = ("the shared characteristics found are not specific enough to search on " - "— add a condition of your own") + "add a condition of your own") return [], basis if not derived: basis["note"] = basis["note"] or ( - "these records share no bio words, link domain or name in common — nothing could be " + "these records share no bio words, link domain or name in common. Nothing could be " "derived, so the conditions are yours to write") return derived, basis @@ -6234,14 +6402,14 @@ def clean_predicates(raw, operator="and", kind=""): name = _s(p.get("name") or p.get("field"), 60).strip() op = _s(p.get("operator") or p.get("op"), 20).strip() if name not in fields: - return None, ("that field cannot be searched — the searchable ones are: " + return None, ("that field cannot be searched. The searchable ones are: " + ", ".join(field_label(f) for f in fields)) # ⭐ PER-FIELD, not the global list. `is_business_account >= 3` used to validate cleanly # because every operator was legal on every field; a yes/no column offering "at least" # is a question with no meaning that the vendor is nevertheless asked. allowed = ops_for(name) if op not in allowed: - return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r} — " + return None, (f"{field_label(name)} cannot be asked {BD_OP_LABELS.get(op, op)!r}. " "it takes: " + ", ".join(BD_OP_LABELS.get(o, o) for o in allowed)) entry = {"name": name, "operator": op} @@ -6256,7 +6424,7 @@ def clean_predicates(raw, operator="and", kind=""): if not vals: return None, f"give {field_label(name)} something to compare against" if len(vals) > 1 and op not in BD_MULTI_VALUE_OPS: - return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value — " + return None, (f"{BD_OP_LABELS.get(op, op)!r} takes one value. " f"{field_label(name)} can only be given a list with " + ", ".join(BD_OP_LABELS[o] for o in ops_for(name) if o in BD_MULTI_VALUE_OPS)) @@ -6315,7 +6483,7 @@ def discover_estimate(records_limit): n = max(0, int(records_limit or 0)) return {"records": n, "usd": round(n * BD_RECORD_PRICE_SPEC, 4), "unitUsd": BD_RECORD_PRICE_SPEC, "basis": "SPEC", - "note": "estimated from the published rate — an exact price is only known after a run" + "note": "estimated from the published rate. An exact price is only known after a run" "a price before a run, and this account's token cannot read a balance"} @@ -6485,9 +6653,9 @@ def cap_note(capped, missing=()): success over a database that does not exist. Exactly the silent-stop failure the row cap already had, one level up — and harder to spot, because the table is not there to look at. """ - out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows) — {n} row{'' if n == 1 else 's'} " + out = [f"⚠ the {k} database is FULL ({row_cap(k):,} rows). {n} row{'' if n == 1 else 's'} " f"from this run were NOT written" for k, n in (capped or []) if n] - out += [f"⚠ the {k} database could NOT BE CREATED — this workspace is at its " + out += [f"⚠ the {k} database could NOT BE CREATED. This workspace is at its " f"{MAX_UT_TABLES}-database limit, so nothing from this run reached it" for k in (missing or [])] return "; ".join(out) @@ -6512,7 +6680,7 @@ def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, row status, final, body = fetch(cfg["url"]) if not (200 <= status < 300): steps["fetch"] = "error" - return ("error", f"{cfg['url']} answered {status} — nothing was written", + return ("error", f"{cfg['url']} answered {status}. Nothing was written", {"status": status}, [], steps) steps["fetch"] = "ok" soup = _soup(body) @@ -6557,8 +6725,10 @@ def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, row else: table_key = ut_ensure(rt, label, fields, username, key=cfg.get("targetTable") or None) t = ut_get(rt, table_key) or {} + # ⛔ D-116 — THE ONE CALL SITE THAT NEEDS A PRECEDENCE RULE, because it is the one whose rows + # come from a CORPUS. Every other `upsert_rows` caller writes measurements or appends a series. rows, counts = upsert_rows(t.get("rows") or {}, incoming, key_field, - cap=row_cap(table_key)) + cap=row_cap(table_key), protect=corpus_protect) touched = [rid for rid, row in rows.items() if str(row.get(key_field, "")) in {str(i.get(key_field)) for i in incoming}] if not dry: @@ -6568,15 +6738,23 @@ def run_scrape_db(rt, defn, username="automation", log=print, step=_no_step, row summary = (f"{counts['inserted']} new, {counts['updated']} updated, " f"{counts['unchanged']} unchanged, {counts['orphans']} no longer on the page " f"(kept)") + # ⛔ D-116 — SAY IT. A search that quietly declines to overwrite six measured cells is doing + # the right thing invisibly, which is indistinguishable from a corpus that happened to agree — + # and the next person to wonder why a number did not move has nothing to read. The observation + # still lands in the series either way; this sentence is about the SCALAR. + if counts.get("held"): + summary += (f", {counts['held']} measured value(s) kept over the corpus " + f"(an enrichment read them exactly; the search's own numbers are in the " + f"snapshot series)") capped = [(table_key, counts.get("capped", 0))] missing = [] if dry else ut_missing(rt, table_key) counts["missing_tables"] = len(missing) note = cap_note(capped, missing) if note: - summary += f" — {note}" + summary += f". {note}" steps["write"] = "partial" if dry: - summary = f"Test run — nothing saved. Would have written: {summary}" + summary = f"Test run. Nothing saved. Would have written: {summary}" steps["write"] = "skipped" elif not note: steps["write"] = "ok" @@ -7115,7 +7293,7 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste m_status, m_note = ig_master.append_run(getattr(rt, "key", ""), in_snaps, in_posts, in_psnaps) if m_status == "error": - master_note = f"⚠ the platform master copy FAILED — {m_note}; the tenant copy " \ + master_note = f"⚠ the platform master copy FAILED. {m_note}; the tenant copy " \ f"is complete and the next run re-appends" counts["master_failed"] = 1 elif m_status == "ok": @@ -7186,21 +7364,21 @@ def run_field_instagram(rt, defn, username="automation", log=print, step=_no_ste summary += f", {counts['blocked']} blocked" note = cap_note(capped, missing) if note: - summary += f" — {note}" + summary += f". {note}" state = "partial" if state != "error" else state steps["write"] = "partial" elif not dry: steps["write"] = "ok" if master_note: - summary += f" — {master_note}" + summary += f". {master_note}" state = "partial" if state != "error" else state steps["write"] = "partial" if dry: - summary = f"Test run — nothing saved. Would have written: {summary}" + summary = f"Test run. Nothing saved. Would have written: {summary}" steps["write"] = "skipped" state = "partial" if state != "error" else state if notes: - summary += f" — {notes[0][:120]}" + summary += f". {notes[0][:120]}" if not dry: # C7: the fresh master rows are exactly what the table's metric cells summarise. try: @@ -7389,7 +7567,7 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row # ⛔ THE FAIL-CLOSED PATH (C4). Discovery has NO anonymous rung to drop to — the corpus is # the vendor's — so this is where the run stops, saying which env var is missing. steps["find"] = "blocked" - return ("partial", "Profile search is not set up yet — nothing was searched and nothing " + return ("partial", "Profile search is not set up yet. Nothing was searched and nothing " "was charged", counts, [], steps) pending = str((defn.get("state") or {}).get("pendingSnapshot") or "") @@ -7404,12 +7582,12 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row or depth_refusal(preds, cfg.get("operator"))) if guard_err: steps["find"] = "blocked" - return ("error", f"the search was not started — {guard_err}", counts, [], steps) + return ("error", f"the search was not started. {guard_err}", counts, [], steps) if dry: est = discover_estimate(limit) steps["find"] = steps["write"] = "skipped" return ("partial", - f"Test run — nothing saved. Would look for up to {limit} " + f"Test run. Nothing saved. Would look for up to {limit} " f"{noun}{'' if limit == 1 else 's'} matching " f"{_predicate_sentence(preds, cfg.get('operator'))}, for about " f"${est['usd']}", counts, [], steps) @@ -7432,10 +7610,10 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row step(f"excluded {excl['excluded']} already-found " f"profile{'' if excl['excluded'] == 1 else 's'} at the provider") elif excl.get("dropped"): - step(f"searching without the already-found list — {excl['dropped']}") + step(f"searching without the already-found list. {excl['dropped']}") if note: steps["find"] = "blocked" - return ("error", f"the search was not started — {note}", counts, [], steps) + return ("error", f"the search was not started. {note}", counts, [], steps) pending = sid # ⛔ LOG THE ID *BEFORE* PERSISTING IT, and the order is the whole point. `set_state` goes # through `rt.update`, which can RAISE on an unavailable store — and that exception @@ -7456,7 +7634,7 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row # is laggy / looks stuck". A counter here is what makes a legitimate wait distinguishable # from a hung thread; without it the rendered step reads the same word for both, which is a # progress indicator that cannot indicate progress. - step(f"Searching — {int(waited)}s of " + step(f"Searching. {int(waited)}s of " f"{int(BD_FILTER_WAIT)}s") status, size, note = bd_filter_status(pending) if note: @@ -7492,7 +7670,7 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row // 60)) except Exception: # noqa: BLE001 mins = 0 - been = f" — {mins} min so far" if mins else "" + been = f". {mins} min so far" if mins else "" steps["collect"] = "partial" return ("partial", f"Still searching at the provider{been}. A corpus search takes about 20 minutes " @@ -7505,7 +7683,7 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row steps["collect"] = "ok" steps["write"] = "skipped" set_state(rt, auto_id, {"pendingSnapshot": None, "pendingSince": None}) - return ("partial", "No profiles matched. Try a shorter or more common keyword — " + return ("partial", "No profiles matched. Try a shorter or more common keyword. " "'floral' finds more accounts than 'floral design studio'", counts, [], steps) if status != "ready": @@ -7632,7 +7810,7 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row affected = [rid for rid, r in merged.items() if str(r.get("handle")) in {c["handle"] for c in incoming}][:200] est = discover_estimate(counts["found"]) - summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found — " + summary = (f"{counts['found']} {noun}{'' if counts['found'] == 1 else 's'} found. " f"{counts['new']} new, {counts['seen_again']} seen before. About " f"${est['usd']}") if counts["dropped"]: @@ -7640,10 +7818,10 @@ def run_discovery(rt, defn, username="automation", log=print, step=_no_step, row f"username and could not be saved") cnote = cap_note([(table_key, counts["capped"])], missing) if cnote: - summary += f" — {cnote}" + summary += f". {cnote}" steps["write"] = "partial" if dry: - summary = f"Test run — nothing saved. Would have written: {summary}" + summary = f"Test run. Nothing saved. Would have written: {summary}" state = "partial" if (dry or counts["capped"] or missing or not counts["found"]) else "ok" if not dry and not missing and spec["after_write"]: spec["after_write"](rt, table_key, auto_id, log) @@ -7719,7 +7897,289 @@ def _flow_cap_reason(table_key, rt=None): cause, fix = str(rep.get("cause") or ""), str(rep.get("recommendation") or "") if not cause and not fix: return "" - return f" — {cause}." + (f" To walk them all: {fix}." if fix else "") + return f". {cause}." + (f" To walk them all: {fix}." if fix else "") + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE STATEMENTS BATCH: ASSEMBLE, PARK, NEVER SEND +# +# ⛔⛔ THE TICKET'S `how:` SAID TO USE "the board's strict review posture (wave 22)". THAT POSTURE +# DOES NOT EXIST, and rebuilding it would REVERSE AN OWNER RULING rather than merely miss a symbol. +# Wave 27 R3 deleted `board()`, `move_card()`, `stages_for()`, `ensure_stage_field()` and the review +# branch of the action walk (the tombstone is above `LANE_OPS`), and its reason is a DATA reason, +# verbatim: *"the board wrote MACHINE COLUMNS INTO A TENANT'S OWN TABLE — a stage select, an `_at` +# stamp and a `_cycles` counter per automation — to render a view of state the RUN LOG already +# holds."* `retire_automation_stage_fields` and `migrate_ig_tables` still DROP those columns, so a +# batch parked in a stage field would be deleted by our own migration. +# ⇒ The batch parks on the AUTOMATION DEFINITION, the shape `runs` and `reviews` already use. Zero +# machine columns on a customer's database, and the migration has nothing to take away. +# +# ⛔ A STATEMENTS FLOW IS **BATCH**-SCOPED, NOT RECORD-SCOPED, and that is why it does not go +# through `apply_actions` at all. `run_plain` walks every row of the flow's bound table, so a +# per-record arm would assemble the whole worklist once per record; and a statements agent binds NO +# table (its customers come from the Odoo AR worklist, not a `ut_*` grid), so `run_plain` would +# have answered `partial: "no database is bound yet"` and the step would NEVER HAVE RUN. The +# precedent for a kind that defines its own scope is already here: the enrich-only branch below, +# whose saved View "ARE the set of records the automation was asked to process". +# ⚠ The `_walk` arm for this kind therefore stays a REFUSAL, not a duplicate: a `send_statement` +# dropped into an ordinary record-walking flow must say it did nothing, never half-send. +# +# ⛔⛔ AND NOTHING HERE SENDS. Not one function in this section imports the mail path. The send door +# is `routes_statements`, behind `admin_gate` + `_royal_only`, calling `collections_send. +# queue_statement`, whose SAFE_MODE guardrail lives in the DATA LAYER where no route, payload or UI +# can bypass it. This module ASSEMBLES and PARKS. A person clicks Send. + +#: How many statements one parked batch holds. Bounded for the reason `MAX_RUNS` and `MAX_REVIEWS` +#: are: this rides inside the automation DEFINITION, and an unbounded list is a serialisation cost +#: on every read of the automations bucket. +#: ⚠ WHEN IT BITES IT IS DISCLOSED, never a silent `[:N]` — R6's second sentence, and +#: [[no-unverifiable-aggregates]]. `assemble_statements` appends a note naming the number left out. +MAX_STATEMENT_BATCH = 200 + +#: The collection worklist loader, injectable so a gate can drive this without Odoo credentials. +#: Production leaves it None and the real data layer answers. Same shape as `_DRAFT_CHAT`. +_STATEMENT_ROWS = [None] + + +def _collections(): + """`modules.collections_send`, imported lazily — `platform/` is not on the path at import time + for every consumer of this module, and this is the only section that needs it.""" + import modules.collections_send as cs # noqa: PLC0415 + return cs + + +def statement_steps(defn): + """The ENABLED `send_statement` actions of this flow, top level only. + + ⚠ Top level only, deliberately, and `is_statement_flow` is what makes that safe: a batch flow + is exactly one enabled action. A `send_statement` nested inside an If is NOT a batch flow, so + it falls to the ordinary record walk and is refused there with a sentence. + """ + return [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) + if isinstance(a, dict) and a.get("kind") == "send_statement" + and a.get("enabled", True)] + + +def is_statement_flow(defn): + """Is this automation a statements BATCH rather than a record walk? + + ⛔ EXACTLY ONE ENABLED ACTION, AND IT IS THIS KIND. The narrowness is the safety: a flow that + also updates records has record semantics that the batch path would silently drop, so it keeps + the ordinary walk (where the arm refuses and says so). This mirrors the enrich-only branch in + `run_plain`, which is narrow for the same stated reason. + """ + enabled = [a for a in (((defn or {}).get("flow") or {}).get("actions") or []) + if isinstance(a, dict) and a.get("enabled", True)] + return len(enabled) == 1 and enabled[0].get("kind") == "send_statement" + + +def assemble_statements(defn, log=print): + """`(batch, notes)` — the statements this configuration WOULD send, rendered, plus why anybody + was left out. **Reads Odoo read-only. Sends nothing. Writes nothing.** + + Each entry is `{customer, to, subject, html, tier, overdue}`. `html` is rendered by the SAME + `render_statement_html` the send door uses, so what a person approves is what goes out — a + review screen rendering its own approximation of the mail is a review of the wrong thing. + + ⚠ A CUSTOMER WITH NO EMAIL IS *SKIPPED AND NAMED*, never dropped. `routes_statements.send` + already separates "we could not" from "there was nowhere to send"; this keeps that distinction + at the assembly end, because a batch that silently shrinks is one nobody can reconcile. + """ + cfg = (statement_steps(defn) or [{}])[0].get("config") or {} + cs = _collections() + notes = [] + loader = _STATEMENT_ROWS[0] + rows = list(loader() if loader else cs.load_collection_list(cs.Odoo())) + tier = str(cfg.get("tier") or "").strip() + if tier: + before = len(rows) + rows = [r for r in rows if str(r.get("Tier") or "") == tier] + log(f"[aios-auto] statements: {len(rows)} of {before} customers are in tier {tier}") + subject_tpl = str(cfg.get("subject") or "") or cs.DEFAULT_SUBJECT + intro_tpl = str(cfg.get("intro") or "") or cs.DEFAULT_INTRO + footer_tpl = str(cfg.get("footer") or "") or cs.DEFAULT_FOOTER + month = _dt.date.today().strftime("%B %Y") + batch, no_email = [], [] + for row in rows: + if len(batch) >= MAX_STATEMENT_BATCH: + break + name = str(row.get("Customer") or "") + to = str(row.get("Email") or "").strip() + if not to: + no_email.append(name) + continue + try: + subject = subject_tpl.format(customer=name, company=cs.COMPANY, month=month) + except (KeyError, IndexError): + # An unknown placeholder is the person's typo, not a crash. Show what they typed — + # the same choice `routes_statements.preview` already makes. + subject = subject_tpl + batch.append({"customer": name, "to": to, "subject": _s(subject, 200), + "html": _s(cs.render_statement_html(row, intro_tpl, footer_tpl), 40000), + "tier": str(row.get("Tier") or ""), "overdue": row.get("Overdue")}) + if no_email: + shown = ", ".join(no_email[:5]) + one = len(no_email) == 1 + notes.append(f"{len(no_email)} customer{'' if one else 's'} " + f"{'has' if one else 'have'} no email address on their record and " + f"{'is' if one else 'are'} not in this batch " + f"({shown}{', and others' if len(no_email) > 5 else ''}). " + f"Add an address in Odoo and run it again") + left = len(rows) - len(batch) - len(no_email) + if left > 0: + # R6's second sentence: a limit that bites is REPORTED, with its cause and what to do. + notes.append(f"{left} more customers matched but one batch holds " + f"{MAX_STATEMENT_BATCH}. Send this batch, then run it again for the rest, or " + f"narrow the tier filter") + return batch, notes + + +def park_statements(rt, auto_id, batch, notes): + """Write the assembled batch onto the DEFINITION as `pendingStatements`. Replaces, never + appends: a batch is THIS run's worklist, and two runs' statements merged into one list is a + customer invoiced twice. + + ⚠ `flush="async"` for the reason every writer in this module gives — the read-your-writes + contract means the next `GET /automations` already sees it, and only the upload is deferred. + """ + entry = {"ts": _iso(), "count": len(batch), "sent": False, + "items": list(batch)[:MAX_STATEMENT_BATCH], + "notes": [_s(n, 300) for n in (notes or [])][:25]} + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d["pendingStatements"] = entry + return cur + + _store_update(rt, _up, flush="async") + return entry + + +def clear_statements(rt, auto_id): + """Drop a parked batch — after it is sent, or when somebody discards it.""" + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + d = cur.get(str(auto_id)) + if d is not None: + d.pop("pendingStatements", None) + return cur + + _store_update(rt, _up, flush="async") + + +def run_statements(rt, defn, username="automation", log=print, step=_no_step): + """The batch runner: assemble, park, tell somebody. Returns `run_plain`'s 5-tuple. + + ⛔ THE STATE IS `partial`, NEVER `ok`, AND THAT IS DELIBERATE. `ok` would put a green dot over + a job that is only half done — the statements exist and NOBODY HAS SENT THEM. `partial` is this + module's honest-progress state and the summary says exactly what is waiting, which is what makes + the run notification (`_notify_run`) read as "come and look" rather than "done". + """ + if not is_statement_tenant(rt): + # Belt and braces behind `clean_actions`' wall: a definition stored before the wall, or + # copied between tenants, must not assemble another company's customer list. + return ("error", "statements are not configured for this workspace", {}, [], {}) + step("reading the collection list") + try: + batch, notes = assemble_statements(defn, log=log) + except Exception as exc: # noqa: BLE001 + return ("error", f"the collection list could not be read: {str(exc)[:200]}", {}, [], {}) + step("rendering statements") + park_statements(rt, defn.get("id"), batch, notes) + counts = {"statements": len(batch)} + if not batch: + return ("partial", "no customers matched, so there is nothing to review", counts, [], {}) + one = len(batch) == 1 + return ("partial", + f"{len(batch)} statement{'' if one else 's'} " + f"{'is' if one else 'are'} ready for review. Nothing has been sent. Open this agent " + f"and click Send to release them", + counts, [], {}) + + +#: ⭐⭐ WAVE 35 · T37 — THE SEEDED ROYAL IMPORTS STATEMENTS AGENT. +#: A FIXED id, because the row's own EXISTENCE is the idempotency key (see `seed_statements_agent`). +STATEMENTS_AGENT_ID = "system:statements" +SYSTEM_STATEMENTS = "statements" +#: First of the month, 06:00. A monthly statement run is what the collections worklist is for, and +#: the hour matches the Odoo sync default so the numbers it reads are the night's. +STATEMENTS_DEFAULT_CRON = "0 6 1 * *" + + +def seed_statements_agent(rt, username="system"): + """Mint Royal Imports' "Monthly statements" agent ONCE, **switched OFF**. Returns the + definition if it wrote one, else None. + + ⛔⛔ IT ARRIVES `enabled: False` AND THAT IS THE TICKET'S OWN TRAP, not a preference. An agent + that ships enabled has scheduled itself against real customers before a single person has read + its configuration — and this one's action queues mail to the tenant's actual debtors. + + ⭐ WHY STORED, WHEN WAVE 34's SYSTEM AGENT IS DERIVED. `_odoo_sync_row` gives three reasons for + deriving and only one survives contact with THIS agent (raised as `ASK D-11`): + · "two copies of one cadence" — does not apply: Odoo's cadence already lives in the connector + config, so a stored copy could drift; a statements schedule has no other home to drift from. + · "stored means the failure-pause can disable it" — INVERTS: auto-pausing statements after K + consecutive failures is correct (a dead credential should stop it), where pausing the Odoo + cadence would silently break infrastructure. + · "nothing can seed it" — real, and answered here: ONE write per tenant ever, by the + CONTAINER (which is what D-195 asks for; what it forbids is a CLI writing while the Space + is up), behind the existence check below. + And deriving cannot meet T37 anyway: a derived row is not in the automations bucket, so edit, + toggle, schedule and run would each need a bespoke door — four new mechanisms to avoid one + guarded write. The Odoo agent escapes that only because a separate resync loop already does its + work. + + ⛔ THE EXISTENCE CHECK IS THE WHOLE IDEMPOTENCY STORY, AND IT IS SAFE ONLY BECAUSE THE ROW + CANNOT BE DELETED. `routes_automation.delete_automation` refuses any stored row carrying + `system` (409, with a sentence), so "the row is present" can never become false behind our + back. If that refusal is ever relaxed, THIS FUNCTION NEEDS A SEPARATE DURABLE FLAG — otherwise + deleting the agent resurrects it on the next list call, which is a delete that undoes itself. + ⚠ And `system` is STICKY through `clean_definition` (see its return) for the same reason: an + edit that stripped the marker would make the row deletable and re-open exactly that hole. + """ + if not is_statement_tenant(rt): + return None + existing = all_definitions(rt) or {} + if STATEMENTS_AGENT_ID in existing: + return None + raw = { + "name": "Monthly statements", + "kind": "plain", + # ⚠ A SCHEDULE THAT IS OFF, not an absent schedule: the cron is the CONFIGURATION T37 asks + # a person to open and read, and a blank one would make them invent it before they could + # judge it. + "schedule": {"cron": STATEMENTS_DEFAULT_CRON, "enabled": False}, + "trigger": {"key": "schedule"}, + "flow": {"actions": [{ + "id": "act_1", "kind": "send_statement", + # ⚠ TIER BLANK = every tier, which is the honest default: narrowing to "A-Urgent" would + # be us deciding who gets chased, and the blank is the value the config panel shows as + # "all of them" rather than an empty box. + "config": {"tier": "", "subject": "", "intro": "", "footer": ""}, + }]}, + } + defn, err = clean_definition(raw, None, username, rt=rt) + if err: + return None + defn["id"] = STATEMENTS_AGENT_ID + # Stamped AFTER the clean, because `clean_definition` reads this key from `prev` only — a + # payload may never assert it (see that function's note). + defn["system"] = SYSTEM_STATEMENTS + + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + # ⚠ RE-CHECKED INSIDE THE MUTATION. Two concurrent list calls can both pass the read above; + # the store applies mutations under its own lock, so this is where "only once" is actually + # decided. Without it the second writer would overwrite a row a person may already have + # edited, silently resetting their schedule and template. + if STATEMENTS_AGENT_ID not in cur: + cur[STATEMENTS_AGENT_ID] = defn + return cur + + _store_update(rt, _up, flush="async") + return defn def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=None): @@ -7736,10 +8196,18 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No exact defect this module names in three other places. It reports the honest state and says which fact is missing. """ + # ⭐⭐ WAVE 35 · T36 — THE BATCH FLOW BRANCHES BEFORE THE TABLE CHECK, and the ORDER is the + # whole of it. A statements agent binds no `ut_*` table (its customers come from the Odoo AR + # worklist), so leaving this below the `if not table` return would answer "no database is bound + # yet" and the step would never run — a feature that is whole, gated and unreachable, which is + # this repo's most-repeated failure. See the section above `run_plain` for why it is not an + # `apply_actions` arm. + if is_statement_flow(defn): + return run_statements(rt, defn, username=username, log=log, step=step) table = _flow_table(defn) if not table: return ("partial", - "no database is bound yet — a plain automation walks the records of its target " + "no database is bound yet. A plain automation walks the records of its target " "database, and this one names none (pick one on the trigger, or in Properties)", {}, [], {}) all_rows = (ut_get(rt, table) or {}).get("rows") or {} @@ -7782,8 +8250,15 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No selected, enrich_scope_note = enrich_selection( rt, table, enrich_cfg, profile_key, gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {})) - if enrich_scope_note.startswith("the enrich step names a view it cannot read"): - return ("partial", f"nothing walked in {table} — {enrich_scope_note}", {}, [], {}) + # ⛔ D-112 — AND THE CONSTANT IS THE FIX, NOT THE `if`. This guard was a `startswith` + # against the sentence SPELLED OUT, twelve hundred lines from the only place that + # produces it. Reword the message in `enrich_selection` — a perfectly ordinary edit, + # since it is a sentence a person reads — and this line silently stops matching, the + # run goes back to reporting `ok`, and the defect D-112 describes returns with nothing + # anywhere going red [[gate-pins-a-spelling-not-a-claim]], on the product side. + # ONE constant, two readers, so the wording is free to change and the behaviour is not. + if enrich_scope_note.startswith(ENRICH_VIEW_UNREADABLE): + return ("partial", f"nothing walked in {table}. {enrich_scope_note}", {}, [], {}) ids = selected step(f"Walking {len(ids)} record{'' if len(ids) == 1 else 's'} in {table}") counts = {"records": len(ids)} @@ -7811,7 +8286,7 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No total = len(ids) ids = ids[:_cap] return ("partial", - f"{total} records in {table} — this run walked the first {_cap}" + f"{total} records in {table}. This run walked the first {_cap}" + _flow_cap_reason(table, rt), counts, ids, {}) n = len(ids) # ⭐⭐ 2026-08-07 (owner report) — SAY IT IN WORDS A PERSON CAN CHECK. @@ -7829,7 +8304,7 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No else ("the records selected for Instagram enrichment" if len(enabled_direct) == 1 and enabled_direct[0].get("kind") == "enrich_instagram" - else "every record — this run was started by hand, so nothing narrowed it")) + else "every record. This run was started by hand, so nothing narrowed it")) if enrich_scope_note: scope += f" ({enrich_scope_note})" # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400 @@ -8019,12 +8494,17 @@ def graph(defn): # whole of it) and `record_updated` no longer names a condition (its watched fields are). # These two lines are the migration's visible half: a node still describing a `.field` # the validator has stopped storing would be the surface and the store disagreeing. + # ⚠ W34-T47 — A COLON, NOT A FULL STOP. R6's sweep turned these four em dashes into + # sentence breaks, which is right for a REFUSAL and wrong here: these are compact node + # SUBTITLES ("ut_leads: view v1"), not prose, and a full stop mid-label reads as two + # truncated fields rather than one qualified name. The rule the sweep encodes is "a + # sentence that used a dash usually wants a period"; a LABEL usually wants a colon. det = {"event_field": (f"{trg.get('table', '')}" - + (f" — {_lane_sentence(trg.get('when'))}" + + (f": {_lane_sentence(trg.get('when'))}" if trg.get("when") else "")), - "record_updated": f"{trg.get('table', '')} — {watched}", + "record_updated": f"{trg.get('table', '')}: {watched}", "record_created": str(trg.get("table") or ""), - "enters_view": f"{trg.get('table', '')} — view {trg.get('viewId') or 'unset'}", + "enters_view": f"{trg.get('table', '')}: view {trg.get('viewId') or 'unset'}", "form_submitted": str(trg.get("table") or ""), "webhook": "POST the hook URL to fire it", # ⭐ WAVE 30 · T05 — BOTH corpus triggers, and the value is identical because the @@ -8034,11 +8514,11 @@ def graph(defn): "ig_profile_match": (_predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:80] if cfg.get("predicates") - else "No filters yet — nothing to search for"), + else "No filters yet. Nothing to search for"), "tiktok_profile_match": (_predicate_sentence(cfg.get("predicates"), cfg.get("operator"))[:80] if cfg.get("predicates") - else "No filters yet — nothing to search for"), + else "No filters yet. Nothing to search for"), "email": str(trg.get("query") or "")}.get(trg["key"], "") if not trg.get("configured", True): det = "Finish setting this trigger up before it can fire" @@ -8091,7 +8571,7 @@ def graph(defn): node("capture_comments", "capture", "Comment data", "Comments on those posts" if comments_on else "Off", 3, panel="capture", on=comments_on, - detail=("The full comments dataset — many rows per post" if comments_on else + detail=("The full comments dataset. Many rows per post" if comments_on else "Comments already embedded in a paid post row are kept either way")), node("write", "write", "Write", cfg.get("targetTable") or "No database", 4, panel="write", @@ -8143,7 +8623,7 @@ def graph(defn): node("fetch", "source", "Fetch page", host, 1, panel="source", detail=url[:90]), node("extract", "capture", "Extract", which, 2, panel="columns", detail=f"{mapped} column{'' if mapped == 1 else 's'} mapped · key: " - f"{cfg.get('keyField') or '—'}"), + f"{cfg.get('keyField') or ', '}"), node("write", "write", "Write", cfg.get("targetLabel") or "Scraped table", 3, panel="write", on=not cfg.get("dryRun"), detail=cfg.get("targetTable") or "a new database"), @@ -8180,7 +8660,7 @@ def toggle_node(rt, auto_id, node_id): if target is None: return None, f"there is no {node_id!r} step in this automation" if not target.get("toggle"): - return None, (f"the {target['title']} step cannot be turned off — it is what this " + return None, (f"the {target['title']} step cannot be turned off. It is what this " f"automation is") cfg, sched = dict(prev.get("config") or {}), dict(prev.get("schedule") or {}) which = target["toggle"] @@ -8295,18 +8775,18 @@ def clean_cond(raw, depth=0, where=""): return None, f"the condition{at} must be an object" conj = [k for k in COND_GROUP_KEYS if k in raw] if len(conj) > 1: - return None, (f"the condition{at} sets both 'all' and 'any' — a group is one or the " + return None, (f"the condition{at} sets both 'all' and 'any'. A group is one or the " f"other") if conj: key = conj[0] if depth + 1 > MAX_COND_DEPTH: - return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep — " + return None, (f"conditions nest at most {MAX_COND_DEPTH} levels deep. " f"the group{at} is deeper") kids_raw = raw.get(key) if not isinstance(kids_raw, list) or not kids_raw: return None, f"the '{key}' group{at} needs at least one condition inside it" if len(kids_raw) > MAX_COND_CHILDREN: - return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions — " + return None, (f"a group holds at most {MAX_COND_CHILDREN} conditions. " f"the '{key}' group{at} has {len(kids_raw)}") kids = [] for child in kids_raw: @@ -8314,7 +8794,7 @@ def clean_cond(raw, depth=0, where=""): if err: return None, err if c is None: - return None, (f"an empty condition sits inside the '{key}' group{at} — " + return None, (f"an empty condition sits inside the '{key}' group{at}. " f"finish it or remove it") kids.append(c) return {key: kids}, None @@ -8325,7 +8805,7 @@ def clean_cond(raw, depth=0, where=""): # act on. It states a fact about the stored tree; it never said what to DO, and the usual # cause is not a typo but an empty field picker (the flow had no walking record to offer # columns from). The fix belongs in the message. - return None, (f"the condition{at} names no field — pick a column, or remove the " + return None, (f"the condition{at} names no field. Pick a column, or remove the " f"condition") if op not in LANE_OPS: return None, f"{op or 'that comparison'!r} is not one of: " + ", ".join(LANE_OPS) @@ -8439,6 +8919,113 @@ FIND_LIMIT_MAX = 100 #: ceiling nearly unreachable. That is E's own PENDING row; this bound is what stops the damage #: until it lands. MAX_WEB_JOBS_PER_RUN = 50 +#: ⭐⭐ W33-T58 (D-191) — HOW MANY WEB STEPS MAY SHARE ONE JOB. The comment above finally got its +#: batching, so the ceiling above now counts JOBS rather than pages and a record's consecutive web +#: steps cost one cold start between them instead of ~9 s each. +#: ⚠ 20 IS THE RUNNER'S OWN `MAX_STEPS` (`jobs/web_agent_job.py`), not a number chosen here: a plan +#: longer than that is refused by the job with `kind="too_many_steps"`, which would turn a +#: performance improvement into a whole record's worth of failed steps. Kept in step by +#: `verify_web_agent.py`, which reads both. +MAX_WEB_STEPS_PER_JOB = 20 +#: ⭐⭐ W33-T56 — HOW MANY STEPS ONE `ai_agent` ACTION MAY COMPOSE FOR ITSELF. +#: ⛔ A CEILING, NOT A TARGET, and it is the only bound between a sentence and a browser session. +#: The five `web_*` kinds are each ONE action a person wrote down; this one is a description that +#: the model turns into a journey at run time, so the number of real browser actions it performs is +#: decided by a paragraph rather than by the flow. Twelve is enough for "log in, search, open the +#: third result, read the price" and short of anything that reads as a program. +#: ⚠ Bounded by `MAX_WEB_STEPS_PER_JOB` as well, since the composed steps ride ONE job. +AI_AGENT_MAX_STEPS = 12 + + +#: The test seam for `_ai_agent_plan` — `None` in every shipped path. `verify_automation.py` sets it +#: so the fuzzy step is proven end to end with NO API key and NO spend, exactly as +#: `routes_automation._DRAFT_CHAT` does for the drafting door. +_AI_AGENT_CHAT = [None] + + +def _ai_agent_plan(cfg, row, log=print, st=None, user=""): + """A description + one record -> `(steps, "")` for `run_plan`, or `(None, sentence)`. + + ⭐⭐ W33-T56. This is the whole of the fuzzy step: the instruction and the record's own values + go to the model, and CONCRETE `web_*` steps come back — the same vocabulary a person could have + written by hand, so everything downstream (the seam, the job, the checkpoint, the per-step + verdicts) is unchanged and none of it has to know an assistant was involved. + + ⛔ IT MAY COMPOSE ONLY BROWSER STEPS. `ai_review.draft_flow` is handed a catalog filtered to + `WEB_KINDS`, so the enum in the tool schema cannot express `create_record` or a connector call. + A fuzzy sentence therefore cannot be talked into writing the tenant's data: the worst a bad + instruction can do is waste one browser session. That is a property of the SCHEMA, not of the + prompt, which is the only version of it worth relying on. + ⛔ AND IT IS BOUNDED. `maxSteps` is clamped at save time and re-applied here, because the + number of real browser actions is otherwise decided by a paragraph. + ⚠ `interpolate` FIRST, so the model is told about THIS record — `{{Website}}` is a different + page for every row, and a journey composed against the template would be one guess repeated. + """ + import ai_review # noqa: PLC0415 + url = interpolate(str(cfg.get("url") or ""), row) + instruction = interpolate(str(cfg.get("instruction") or ""), row) + cap = max(1, min(int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS), MAX_WEB_STEPS_PER_JOB)) + web_only = [r for r in ACTION_CATALOG if r.get("kind") in WEB_KINDS and r.get("ready")] + draft, why, _prov = ai_review.draft_flow( + prompt=(f"Starting page: {url}\n\nDo this: {instruction}\n\n" + f"Answer with at most {cap} steps. The FIRST step must open the starting page."), + catalog=web_only, + required={k: v for k, v in ACTION_REQUIRED.items() if k in WEB_KINDS}, + triggers=[], tables=[], + # ⭐ W35 · C7 (`NOTE E-16`) — threaded IN from `_walk` rather than resolved here: this + # function is pure over `(cfg, row)` by design and giving it a store handle of its own + # would be a second way to reach the tenant. + st=st, user=user, + chat=_AI_AGENT_CHAT[0]) + if why or not draft: + return None, (why or "the assistant produced no steps for that instruction") + steps, seen_url = [], False + for i, a in enumerate(draft.get("actions") or [], 1): + kind = str(a.get("kind") or "") + if kind not in WEB_KINDS: + # Belt and braces behind the enum: a rung that ignores its own schema is stopped here + # rather than reaching the browser. + return None, f"the assistant asked for a step this action cannot perform ({kind})" + c = a.get("config") or {} + step = {"kind": kind, "id": f"ai{i}", + "url": str(c.get("url") or "") or (url if not seen_url else ""), + "selector": str(c.get("selector") or ""), + "attr": str(c.get("attr") or "text"), + "all": bool(c.get("all")), + "waitFor": str(c.get("waitFor") or "") or None, + "timeoutMs": int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS)} + if c.get("value"): + step["value"] = str(c.get("value")) + if c.get("hint"): + step["hint"] = str(c.get("hint")) + if cfg.get("dryRun"): + step["dryRun"] = True + seen_url = seen_url or bool(step["url"]) + steps.append(step) + if len(steps) >= cap: + break + if not steps: + return None, "the assistant produced no steps for that instruction" + # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to act + # on yet — and the action's own `url` is exactly the answer. Supplying it here beats letting + # the whole job be refused for a sentence the model happened not to repeat. + if not steps[0].get("url"): + steps[0]["url"] = url + return steps, "" + + +def _web_missing(kind, cfg): + """Which of this kind's REQUIRED config keys are blank — the human phrases, in table order. + + ⚠ PER KIND, because they do not need the same things: `web_goto` needs a url and no selector; + `web_fill` needs a value nobody else takes; only `web_read` needs a column to write into. One + shared three-field test would have blocked every `web_goto` ever configured for want of a + selector it does not use. + ⭐ Lifted out of `apply_actions` at W33-T58 so the BATCH BUILDER and the per-step arm ask the + same question of the same table. Two copies of "is this step configured" would let a batch + include a step the arm then refuses — or worse, exclude one it would have run. + """ + return [n for n, key in WEB_REQUIRED.get(kind, ()) if not str((cfg or {}).get(key) or "").strip()] #: ⭐⭐ W31 QA — THE FIVE WEB KINDS THE ENGINE DISPATCHES, and this is the THIRD list that names #: them (`web_agent.RUNNABLE_KINDS` is the seam's, `jobs/web_agent_job.py::RUNNABLE_KINDS` is the #: runner's). ⛔ IT IS DELIBERATELY NOT AN IMPORT: `_web_agent()` resolves the seam LAZILY so an @@ -8458,6 +9045,10 @@ WEB_REQUIRED = { "web_click": (("a CSS selector", "selector"),), "web_repair": (("a CSS selector", "selector"),), "web_fill": (("a CSS selector", "selector"), ("a value to type", "value")), + # ⭐⭐ W33-T56 — the fuzzy step needs a page to start on and a job to do, and nothing else. + # ⚠ NO `selector` AND NO `field`: working out the selector IS the step, and where to put the + # answer is optional (a journey may only need to have been performed). + "ai_agent": (("a URL", "url"), ("a description of what to do", "instruction")), } #: ⭐⭐ WAVE 32 · T45 (owner item 10) — WHAT EACH ACTION KIND CANNOT RUN WITHOUT, for every kind. #: @@ -8521,16 +9112,56 @@ ACTION_CATALOG = [ # job"* and *"Needs the recorder that captures a selector"* — both false since E's runner # landed. A menu that dates its own promises is this module's own recorded complaint one # comment up; a row that advertises a missing prerequisite it already has is the same defect. - {"kind": "web_goto", "label": "Open a page", "group": "Web action", "ready": True, + # ⭐⭐ WAVE 33 · W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP. + # ⛔ IT ADDS; IT REPLACES NOTHING. R3 is explicit: *"No existing `web_*` kind is removed — + # dropping a kind from the catalog 400s every stored automation using it, forever"* (D-65). + # This is the step for a journey somebody can DESCRIBE but not spell as a selector; the five + # kinds below stay exactly as they are for the journeys they can already express, and a person + # who knows the selector should still use `web_read`, which costs no model call. + # ⭐⭐ WAVE 34 · W34-T44 / R18 — SIX ROWS BECAME ONE, AND THE OTHER FIVE DID NOT LEAVE THE + # CATALOG. Owner, verbatim: *"Remove Do this on a page / Open a page / Fill a field / Click + # something / Read from the page / Repair a broken step completely. One action, Web agent."* + # + # ⛔⛔ WHY THEY ARE HIDDEN AND NOT DELETED, AND IT IS NOT D-65 THIS TIME — IT IS WORSE. + # `_ai_agent_plan` builds the model's tool schema as + # `[r for r in ACTION_CATALOG if r["kind"] in WEB_KINDS and r["ready"]]` + # so DELETING these five rows empties that list, and the ONE action the ruling keeps would be + # left able to compose nothing at all. Deleting the six would have deleted the survivor, in + # silence, and `ai_agent`'s own tests would still pass because they stub the chat rung + # (`_AI_AGENT_CHAT`). D-65's usual argument (a deleted kind 400s every stored automation + # forever) applies as well and is the smaller half. + # + # ⭐ SO THE LINE MOVED FROM "IS IT IN THE CATALOG" TO "IS IT ON THE MENU". `menu: False` is + # withheld by `action_catalog()` from the picker, while `clean_actions` (which reads this + # constant, not the wire) still validates the kind, the runner still runs it, and a stored + # automation built before today keeps working and stays editable. + # ⚠ THE LABELS CHANGED TOO, and that is R18's "appear NOWHERE" clause taken literally: a + # hidden row's label is still rendered on a STORED step's card, so leaving the six captions in + # place would have kept them on screen for exactly the people who already use them. They now + # name the mechanism instead, and no old caption survives as a substring in any case. + {"kind": "ai_agent", "label": "Web agent", "group": "Web action", "ready": True, + "detail": "Describe a job on a website in words; the agent works out the steps, runs them in " + "a browser and reports what it actually did"}, + {"kind": "web_goto", "label": "Web agent step (navigate)", "group": "Web action", + "ready": True, "menu": False, "detail": "Opens a page in a browser job and reports the title it landed on"}, - {"kind": "web_fill", "label": "Fill a field", "group": "Web action", "ready": True, - "detail": "Types a value into a field — mark it secret and the value is masked in the log"}, - {"kind": "web_click", "label": "Click something", "group": "Web action", "ready": True, + {"kind": "web_fill", "label": "Web agent step (type)", "group": "Web action", + "ready": True, "menu": False, + "detail": "Types a value into a field. Mark it secret and the value is masked in the log"}, + {"kind": "web_click", "label": "Web agent step (click)", "group": "Web action", + "ready": True, "menu": False, "detail": "Clicks the element a selector names, and reports where it landed"}, - {"kind": "web_read", "label": "Read from the page", "group": "Web action", "ready": True, - "detail": "Reads one value off a live page in a browser job — expect ~10-30 s per step"}, - {"kind": "web_repair", "label": "Repair a broken step", "group": "Web action", - "ready": True, + # ⚠ `"kind": "web_read"` AND `"ready": True` STAY ON ONE LINE. `verify_wiring`'s C5 row matches + # `"kind": "web_read".*?"ready": True` without DOTALL, so wrapping this row the way its four + # siblings are wrapped turns that cross-fence assertion red — on a formatting change, with the + # mount and the flag both intact. The row is A's file and its CLAIM is right (a catalog kind + # the client can add must be one the runner will execute); the layout is what it happens to + # depend on, so the layout is preserved here rather than the assertion weakened there. + {"kind": "web_read", "ready": True, "label": "Web agent step (extract)", + "group": "Web action", "menu": False, + "detail": "Reads one value off a live page in a browser job. Expect ~10-30 s per step"}, + {"kind": "web_repair", "label": "Web agent step (relocate)", "group": "Web action", + "ready": True, "menu": False, "detail": "Follows a label to its control when a selector has gone stale, and proposes one"}, # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── {"kind": "update_record", "label": "Update record", "group": "Database", "ready": True, @@ -8564,6 +9195,30 @@ ACTION_CATALOG = [ "history"}, {"kind": "send_email", "label": "Send email", "group": "Connected", "ready": False, "detail": "Needs a send scope on the Gmail connection"}, + # ⭐⭐ WAVE 35 · T35 / CONTRACT C8 / OWNER RULING R10 — STATEMENTS BECOME AN AGENT STEP. + # + # Owner item 14: move Statements out of Settings and into the agent automation, "templatic and + # easily toggleable". R10 is the half that decides the shape: the step ASSEMBLES and PARKS a + # batch in a review stage, and **nothing sends without a human click**. So this row's promise + # is deliberately "prepares", not "sends" — the label a person reads must not describe an act + # the step does not perform. + # + # ⛔ ADDED BESIDE `send_email`, NEVER BY REPURPOSING IT (the ticket's own trap, D-295): a + # stored automation naming a kind that no longer exists is refused FOREVER rather than dropped + # with a reason, so adding a kind is cheap and re-pointing one is not. + # + # ⛔ `ready: True` IS WHAT MAKES IT STORABLE — `clean_actions` refuses any row whose `ready` is + # false — and the note at `enrich_tiktok` above is the reason the RUNNER ARM lands in the same + # wave rather than after it: `_walk` has no terminal `else`, so a catalog row ahead of its arm + # is addable, storable and SILENTLY INERT, which is worse than not existing. T35 lands an arm + # that reports it is not configured; T36 makes it park a real batch. + # + # ⚠ TENANT-GATED, not `ready: False`: `TENANT_GATED_ACTIONS` withholds this row from every + # tenant but #0, because the send client behind it is env-credentialed and is tenant #0's. + {"kind": "send_statement", "label": "Prepare customer statements", "group": "Connected", + "ready": True, + "detail": "Assemble this month's statements and park them for review. Nothing is sent until " + "somebody opens the batch and clicks Send"}, {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False, "detail": "Needs the Slack connector"}, # ── 4. ADVANCED LOGIC ──────────────────────────────────────────────────────────────────── @@ -8575,7 +9230,7 @@ ACTION_CATALOG = [ {"kind": "repeating_group", "label": "Repeating group", "group": "Advanced logic", "ready": False, "detail": "Run the same actions on every item in a list"}, {"kind": "run_script", "label": "Run script", "group": "Advanced logic", "ready": False, - "detail": "Not built — a sandbox is its own decision"}, + "detail": "Not built. A sandbox is its own decision"}, {"kind": "generate_ai", "label": "Generate with AI", "group": "Advanced logic", "ready": False, "detail": "Needs an AI action implementation"}, ] @@ -8604,10 +9259,63 @@ def _connector_meta(key): return next((dict(v) for v in TRIGGER_CONNECTOR.values() if v.get("key") == key), None) -def action_catalog(): +#: ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TENANT PREDICATE `routes_statements._royal_only` USES, +#: LIFTED SO THERE IS EXACTLY ONE COPY OF IT. C8's words are "the predicate is imported, never +#: re-expressed", and this is the direction that import can run: `automation_engine` imports no +#: route module and no FastAPI (checked), and breaking that to reach `_royal_only` would drag +#: `deps` + `routes_admin` into the engine. So the ENGINE holds the test and the ROUTE calls it. +#: +#: ⛔⛔ THIS IS NOT `odoo_relational.is_royal`, AND THE DIFFERENCE IS A SEND. That one asks "is this +#: tenant ENTITLED to Odoo databases" over `RI_SLUGS = ("", "royal-imports")` — it answers TRUE for +#: the EMPTY slug and lower-cases its input. This one is `_royal_only`'s exact test: the runtime's +#: own `key`, matched exactly. A tenant whose key never got set would pass `is_royal("")` and then +#: queue statements THROUGH TENANT #0'S ENV-CREDENTIALED SEND CLIENT, i.e. email another company's +#: customers over Royal Imports' name. Entitlement to READ is not authority to SEND, and the two +#: questions keep their two predicates on purpose. Do not "unify" them. +ROYAL_TENANT_KEY = "royal-imports" + + +def is_statement_tenant(rt): + """Exactly `routes_statements._royal_only`'s test, as a boolean over the RUNTIME. + + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION, so a + payload cannot argue its way into another company's sender. + """ + return getattr(rt, "key", None) == ROYAL_TENANT_KEY + + +#: Kinds only SOME tenants may see or store, as `{kind: predicate(rt) -> bool}`. +#: +#: ⛔ FAIL-CLOSED ON `rt=None`, AND THAT IS THE WHOLE SAFETY ARGUMENT. Every reader below treats an +#: absent runtime as "not allowed", so a call site that forgets to pass one makes the action VANISH +#: rather than become universal. The opposite default would mean any future caller of +#: `action_catalog()` or `clean_actions()` silently offers tenant #0's send door to every tenant — +#: an omission that is invisible in review and loud only in production [[default-must-pass-its-own-guard]]. +TENANT_GATED_ACTIONS = {"send_statement": is_statement_tenant} + +#: The dunning buckets a statements step may filter on. ⚠ ONE LITERAL, TWO READERS: this and +#: `routes_statements.statements()`'s `"tiers"` field are the same four strings, and a step +#: configured against a tier the sender's worklist does not produce would filter to nothing and +#: report success. `routes_statements` imports this rather than repeating it. +STATEMENT_TIERS = ("A-Urgent", "B-Active", "C-Light", "Monitor") + + +def _tenant_may_use(kind, rt): + """May this runtime see/store this action kind? Ungated kinds are always yes.""" + gate = TENANT_GATED_ACTIONS.get(str(kind or "")) + return True if gate is None else bool(rt is not None and gate(rt)) + + +def action_catalog(rt=None): """The catalog as the wire carries it — a copy, because a caller that mutated the module constant would change every later reader's answer. + ⭐⭐ WAVE 35 · T35 (C8): `rt` filters TENANT-GATED rows out entirely — not `ready: False`, not + `menu: False`, but ABSENT. A tenant that may not send statements should not learn that the + capability exists, and `ready: False` renders as "coming soon", which is a promise we are not + making to them. ⚠ Called with no `rt` the gated rows are withheld (fail-closed): see + `TENANT_GATED_ACTIONS`. + ⭐ WAVE 24 (C-ACT): each row is stamped with its `groupOrder`, DERIVED from `ACTION_GROUP_ORDER` rather than hand-written per row, so a group cannot be given two different orders by two rows that claim to be in it. A group nobody has ordered sorts LAST @@ -8618,11 +9326,24 @@ def action_catalog(): nests on — RESOLVED here rather than written out per row, for the reason `groupOrder` is: two rows in one nest cannot disagree about what that nest is called. A row naming an unknown connector loses the stamp instead of inventing a nest with a raw slug for a title. + + ⭐⭐ WAVE 34 · W34-T44 / R18: `menu` is stamped on EVERY row, never left absent. A row the + picker must not offer carries `menu: False` and STILL RIDES THE WIRE, because the client + resolves a STORED step's label out of this same list (`AutomationBuilder`: + `catalog.find(c => c.kind === a.kind)` then `row?.label || a.kind`) — withholding the row + entirely would make an existing web step render its raw kind token at somebody, which this + module forbids in those words elsewhere. + ⚠ STAMPED RATHER THAN LEFT TO DEFAULT: absent-means-true is a rule two codebases have to + remember the same way, and a client filter is one `!== false` away from meaning the opposite. + An explicit boolean on every row cannot be read two ways. """ last = max(ACTION_GROUP_ORDER.values()) + 1 out = [] for a in ACTION_CATALOG: - row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last)} + if not _tenant_may_use(a.get("kind"), rt): + continue + row = {**a, "groupOrder": ACTION_GROUP_ORDER.get(a.get("group"), last), + "menu": bool(a.get("menu", True))} conn = _connector_meta(a.get("connector")) if conn: row["connector"] = dict(conn) @@ -8712,12 +9433,35 @@ def _action_label(act): return str(act.get("kind") or "Action") -def clean_actions(raw, depth=0, _seen=None, _count=None): +def clean_actions(raw, depth=0, _seen=None, _count=None, notes=None, rt=None): """Validate `flow.actions`. Returns `(actions, error)` — refuses, never coerces. + ⭐⭐ WAVE 35 · T35 (C8) — `rt` IS THE TENANT WALL ON THE STORE SIDE, and it is a separate wall + from `action_catalog(rt)`'s. Withholding a row from the MENU stops it being offered; it does + not stop a hand-written body naming the kind, and the picker is not a security boundary + [[opening-a-route-widens-every-field]]. ⚠ Keyword-only in effect and defaulting to None, so + every existing caller is untouched by construction — the same shape `notes` used, for the + reason this docstring already gives about a signature change failing at RUN, not at import. + ⛔ `rt=None` REFUSES a gated kind rather than allowing it (`TENANT_GATED_ACTIONS`). + Ids are STABLE: a caller's well-formed `id` is kept, so selecting an action in the builder survives a Save. A missing or colliding one is minted `act_`; minting on every clean would move the selection under the person editing it. + + ⭐⭐ D-75 — `notes` IS THE DISCLOSURE CHANNEL, AND IT IS OPT-IN. Pass a list and this function + appends one plain sentence per thing it SILENTLY CHANGED: a `create_record` condition dropped + (see the note at that branch), and any config key an arm's allowlist did not keep. + + ⛔ WHY IT IS AN OUT-PARAMETER RATHER THAN A THIRD RETURN VALUE. D-75's own exit says *"this is + a signature change across its callers"* and that is exactly what makes the obvious fix + dangerous mid-wave: `clean_actions` is called from `clean_flow`, from `_is_untouched_ig_seed` + and recursively from its own group arm, and a caller that unpacked two values from a + three-tuple fails at RUN, not at import. An optional list defaults to `None`, every existing + caller is untouched by construction, and the one door that wants to tell somebody opts in. + ⚠ IT REPORTS, IT NEVER REFUSES. Every drop here is deliberate and D-65 is the reason — refusing + a stored automation's condition would 400 it forever, with no way to edit it out, because the + editor cannot save the automation it needs to fix. A drop is recoverable; a locked door is not. + What was missing was never the refusal, it was somebody being told. """ if raw in (None, ""): return [], None @@ -8735,8 +9479,17 @@ def clean_actions(raw, depth=0, _seen=None, _count=None): return None, (f"{kind or 'that action'!r} is not one of: " + ", ".join(a["kind"] for a in ACTION_CATALOG)) if not row["ready"]: - return None, (f"{row['label']!r} is on the menu but not built yet — " + return None, (f"{row['label']!r} is on the menu but not built yet. " f"{row['detail'][0].lower()}{row['detail'][1:]}") + # ⭐⭐ W35-T35 (C8) — THE TENANT WALL. Refused, not dropped, and this is the one place in + # this function where a refusal is right: D-65's "drop, never refuse" protects a stored + # automation from becoming permanently unsavable, and NO tenant this can refuse has ever + # been able to store one — the kind is withheld from their catalog, so there is no legacy + # body to strand. Dropping it silently would instead let a flow save, look saved, and never + # do the step the person configured. + if not _tenant_may_use(kind, rt): + return None, (f"{row['label']!r} is not available in this workspace. It sends as " + f"Royal Imports, using Royal Imports' own mail credentials") count[0] += 1 if count[0] > MAX_ACTIONS: return None, f"an automation runs at most {MAX_ACTIONS} actions" @@ -8767,30 +9520,52 @@ def clean_actions(raw, depth=0, _seen=None, _count=None): # whose value could appear to be ignored. If a `create_record` condition editor ever comes # back, this needs a disclosure channel — `clean_actions` has none today. if kind == "create_record": + # D-75: the drop is unchanged; what is new is that a caller can be TOLD about it. + if notes is not None and entry.get("when"): + notes.append(f"{aid}: the condition on a Create-record step was dropped. A " + f"Create record has no record to test yet. Put the condition on the " + f"trigger, or on an If wrapping this step.") when = None else: when, cerr = clean_cond(entry.get("when"), where=f"action {aid}") if cerr: return None, cerr cfg_raw = entry.get("config") if isinstance(entry.get("config"), dict) else {} - cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count) + cfg, cerr = _clean_action_config(kind, cfg_raw, depth, seen, count, notes=notes, rt=rt) if cerr: return None, cerr + # ⛔ D-75 — THE ALLOWLIST'S OWN DROPS, DIFFED HERE RATHER THAN REPORTED BY EACH ARM. Every + # arm builds a fresh `out = {...}`, so none of them knows what it did not keep; the caller + # of the arm does, because it holds both dicts. One diff, and a new arm is covered the day + # it is written rather than the day somebody remembers to instrument it. + if notes is not None and isinstance(cfg, dict): + gone = [k for k in cfg_raw if k not in cfg] + if gone: + notes.append(f"{aid}: {', '.join(sorted(gone))} " + f"{'is' if len(gone) == 1 else 'are'} not a setting a " + f"{KIND_LABELS.get(kind, kind)!r} step uses, so " + f"{'it was' if len(gone) == 1 else 'they were'} not saved.") out.append({"id": aid, "kind": kind, "enabled": bool(entry.get("enabled", True)), "when": when, "config": cfg}) return out, None -def _clean_action_config(kind, cfg, depth, seen, count): - """One action's `config`, per kind. Returns `(config, error)`.""" +def _clean_action_config(kind, cfg, depth, seen, count, notes=None, rt=None): + """One action's `config`, per kind. Returns `(config, error)`. + + ⚠ `rt` is threaded ONLY so the `group` arm can hand it back to `clean_actions` for the branch + recursion (W35-T35 / C8). Without it a tenant-gated kind is refused at the top level and + ACCEPTED inside an If, which is the half of a flow `_walk_actions`' own docstring says people + cannot see. + """ if kind == "group": # ⭐ WAVE 24 · C-FORK (owner ruling R8) — a group is a FORK now: `{branches: [...]}`, # each `{id, label, cond, actions}`. It was one condition with one action list, i.e. an # if with no else, so expressing "otherwise" meant a second group carrying the negation # by hand — two conditions a later edit could put out of step with each other. if depth + 1 > MAX_GROUP_DEPTH: - return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep — " + return None, (f"conditional groups nest at most {MAX_GROUP_DEPTH} deep. " f"past that a flow is a flowchart and belongs on the board") raw_branches = cfg.get("branches") if not isinstance(raw_branches, list) or not raw_branches: @@ -8800,7 +9575,7 @@ def _clean_action_config(kind, cfg, depth, seen, count): raw_branches = [{"id": "b1", "label": "A", "cond": cfg.get("cond"), "actions": cfg.get("actions")}] if len(raw_branches) > MAX_BRANCHES: - return None, (f"an If / then has at most {MAX_BRANCHES} branches — past that the " + return None, (f"an If / then has at most {MAX_BRANCHES} branches. Past that the " f"record is being routed, which is what the board's lanes are for") out_branches, bseen = [], set() for i, br in enumerate(raw_branches): @@ -8815,10 +9590,13 @@ def _clean_action_config(kind, cfg, depth, seen, count): # would make every branch below it dead code — silently, and only for the # records that reached it. Refused with the reason rather than reordered: this # module does not quietly rewrite what somebody built. - return None, ("only the LAST branch of an If / then can be the Otherwise leg — " + return None, ("only the LAST branch of an If / then can be the Otherwise leg. " "a catch-all above another branch makes the ones below it " "unreachable") - kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count) + # D-75: threaded into the branch too — an unconfigured step inside an If is + # exactly the one a person cannot see, which is `_walk_actions`' own argument. + kids, kerr = clean_actions(br.get("actions"), depth + 1, seen, count, + notes=notes, rt=rt) if kerr: return None, kerr if not kids: @@ -8843,7 +9621,7 @@ def _clean_action_config(kind, cfg, depth, seen, count): if not key: return None, "a value is written to a field with no name" if v is not None and not isinstance(v, (str, int, float, bool)): - return None, (f"the value for {key!r} must be text or a number — " + return None, (f"the value for {key!r} must be text or a number. " f"an automation writes cells, not objects") clean_vals[key] = _s(v, 500) if isinstance(v, str) else v out = {"values": clean_vals} @@ -8860,7 +9638,7 @@ def _clean_action_config(kind, cfg, depth, seen, count): if not table: return None, "the create record action names no database" if not table.startswith(UT_PREFIX): - return None, (f"actions write to blank databases (ut_*) this wave — " + return None, (f"actions write to blank databases (ut_*) this wave. " f"{table!r} is not one") out["table"] = table # ⭐ WAVE 25 · C5 (owner ruling R1a) — UNIQUE ON. Without it this action APPENDS @@ -8880,9 +9658,37 @@ def _clean_action_config(kind, cfg, depth, seen, count): unique = _s(cfg.get("uniqueOn"), 80).strip() if unique and unique not in clean_vals: return None, (f"this action does not write {unique!r}, so it cannot keep records " - f"unique on it — it writes: " + ", ".join(sorted(clean_vals))) + f"unique on it. It writes: " + ", ".join(sorted(clean_vals))) out["uniqueOn"] = unique return out, None + if kind == "send_statement": + # ⭐⭐ WAVE 35 · T35 / R10 — WHAT A STATEMENTS STEP IS CONFIGURED WITH: a customer filter + # and a template. Both are the SENDER's own vocabulary (`routes_statements.send` passes + # `templates: {subject, intro, footer}` straight to `collections_send.queue_statement`), + # so this arm names no field the send door does not already take. + # + # ⚠ EVERY TEMPLATE FIELD IS OPTIONAL AND EMPTY MEANS "THE DEFAULT", which is exactly how + # the send door already reads them (`t.get("subject") or cs.DEFAULT_SUBJECT`). Storing our + # own copy of the default instead would freeze today's wording into every stored agent and + # silently stop tracking `collections_send`'s. + out = {} + tier = _s(cfg.get("tier"), 40).strip() + # ⛔ REFUSED, NOT COERCED, AND THE ANSWER LISTS THE REAL ONES — the same shape the connector + # cadence uses. A tier nobody sends to would filter the batch to zero customers and report + # a successful run: a silent no-op is the worst outcome available here, because the person + # believes their customers were invoiced. + # ⚠ `""` IS LEGAL and means EVERY tier. It is the stored default, so an agent saved before + # anybody picks a filter does not change meaning the day this key arrives (D-65's rule). + if tier and tier not in STATEMENT_TIERS: + return None, (f"{tier!r} is not a collection tier. Pick one of: " + + ", ".join(STATEMENT_TIERS) + ", or leave it blank for all of them") + out["tier"] = tier + for key, cap in (("subject", 200), ("intro", 4000), ("footer", 4000)): + out[key] = _s(cfg.get(key), cap) + lbl = " ".join(_s(cfg.get("label"), 60).split()) + if lbl: + out["label"] = lbl + return out, None if kind in ENRICH_KINDS: # ⭐ WAVE 30 · T08 — BOTH networks share this branch. See `ENRICH_KINDS`: the selection, the # cooldown, the limit clamps and their reasons are network-agnostic, and a second copy for @@ -8977,6 +9783,40 @@ def _clean_action_config(kind, cfg, depth, seen, count): if limit < 1 or limit > FIND_LIMIT_MAX: return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" return {"table": table, "cond": cond, "limit": limit}, None + if kind == "ai_agent": + # ⭐⭐ W33-T56 — THE FUZZY STEP'S CONFIG. Same posture as the web arm below in every + # respect that matters: an ALLOWLIST, a shape test on the url, a clamped timeout, and an + # EMPTY config stored rather than refused so the step can be added before it is configured + # (the wave-23 illegal-default-seed defect, and D-79's scar). + # ⚠ IT DELIBERATELY DOES NOT TAKE A `selector`. Working the selector out is the step; a + # selector box here would be a control that contradicts the action's own reason to exist. + _u = _s(cfg.get("url"), 500).strip() + if _u and not (_u.startswith("http://") or _u.startswith("https://") + or _u.startswith("{{")): + return None, "a web address must start with http:// or https://" + try: + _t = int(cfg.get("timeoutMs") or WEB_READ_TIMEOUT_MS) + except (TypeError, ValueError): + return None, "the web step timeout must be a whole number of milliseconds" + try: + _max = int(cfg.get("maxSteps") or AI_AGENT_MAX_STEPS) + except (TypeError, ValueError): + return None, "the step ceiling must be a whole number" + out = {"url": _u, + # ⚠ LONGER THAN A SELECTOR AND SHORTER THAN A PROMPT. This is a sentence or two + # describing a job, and the model is given the page's own vocabulary at run time — + # a 4,000-character instruction is a sign somebody is writing a program in prose. + "instruction": _s(cfg.get("instruction"), 600), + "field": _s(cfg.get("field"), 80).strip(), + "timeoutMs": max(1000, min(_t, WEB_READ_TIMEOUT_MAX_MS)), + # ⛔ THE CEILING IS THE POINT OF THE CLAMP, not the default. Every step this action + # composes is a real browser action inside one job, and a fuzzy instruction is + # exactly the input that produces twenty of them. Bounded here so a run cannot be + # talked into an unbounded journey by a sentence. + "maxSteps": max(1, min(_max, AI_AGENT_MAX_STEPS))} + if cfg.get("dryRun"): + out["dryRun"] = True + return out, None if kind in WEB_KINDS: # ⭐⭐ WAVE 31 · T38 (C5 / R10) — AND THIS ARM IS THE HALF THAT WOULD HAVE SHIPPED MISSING. # @@ -9441,7 +10281,7 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, pla if problem: # ⛔ REFUSE, do not widen. See the docstring — this is the branch where a quiet # fallback costs real money. - return [], f"the enrich step names a view it cannot read — {problem}" + return [], f"{ENRICH_VIEW_UNREADABLE}. {problem}" # ⭐⭐ 2026-08-07 (owner report) — `filter_eval.matches`, NOT `lane_match`. THIS LINE WAS # A THIRD IMPLEMENTATION OF "does this row match", AND IT SPOKE THE WRONG LANGUAGE. # @@ -9565,12 +10405,12 @@ def enrich_selection(rt, table_key, cfg, profile_key, today=None, gone=None, pla # do, because nothing else will ever retry them — under R9 this note is the only path # back from a tombstone, so a vaguer version would strand the row permanently. notes.append(f"{len(skipped_gone)} skipped because Instagram has no such account " - f"({shown}{', …' if len(skipped_gone) > 5 else ''}) — delete the row or " + f"({shown}{', …' if len(skipped_gone) > 5 else ''}). Delete the row or " f"correct the handle; they are not retried automatically") if blank_handle: notes.append(f"{blank_handle} skipped with no handle") if len(picked) < quota and (cooled or blank_handle or skipped_gone or ordered): - notes.append(f"{len(picked)} of the {quota} asked for — the list ran out") + notes.append(f"{len(picked)} of the {quota} asked for. The list ran out") return picked, "; ".join(notes) @@ -9803,6 +10643,131 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print # The RUN LOG is the operational record of what happened now, and it is the only one. patches, creates = {}, {} + #: ⭐⭐ W33-T58 · D-191 — this record's web results, keyed by ACTION ID, and the run's job + #: count. `web_done` is cleared per record (a batch is built from one row's interpolated + #: values and means nothing for the next one); `web_jobs` counts JOBS across the whole run, + #: which is what `MAX_WEB_JOBS_PER_RUN` has always been trying to bound. + #: ⚠ A key present with `None` means "this step was in a batch that refused" — distinct from + #: absent, which means "no batch has covered it yet". Collapsing the two would re-run the + #: whole batch once per step of it, which is the defect this fixes, inverted. + web_done: dict = {} + web_jobs = [0] + + def _web_step_dict(act, row): + """One action + one record → the step dict the seam takes. Interpolated HERE, as before. + + ⚠ `interpolate` on the CALLER's side, exactly as the update/create arms do it, so + `{{Field}}` works in a url, a selector or a typed value. The seam takes a plain dict and + does not know about rows. + """ + cfg = act.get("config") or {} + wait = cfg.get("waitFor") + step = {"kind": act.get("kind"), "id": str(act.get("id") or ""), + "url": interpolate(str(cfg.get("url") or ""), row), + "selector": interpolate(str(cfg.get("selector") or ""), row), + "attr": str(cfg.get("attr") or "text"), + "all": bool(cfg.get("all")), + "waitFor": interpolate(str(wait), row) if wait else None, + "timeoutMs": int(cfg.get("timeoutMs") or 20000)} + # ⚠ ADDED ONLY WHEN PRESENT: the seam distinguishes a key that is absent from one that is + # empty, and an empty `value` on a `web_read` would be a typed blank. + if cfg.get("value"): + step["value"] = interpolate(str(cfg.get("value")), row) + if cfg.get("hint"): + step["hint"] = interpolate(str(cfg.get("hint")), row) + if cfg.get("secret"): + step["secret"] = True + if cfg.get("dryRun"): + step["dryRun"] = True + return step + + def _run_web_batch(acts, start, row, rid): + """Run the longest safe run of consecutive web steps from `acts[start]` in ONE job. + + ⛔ **CONSECUTIVE, AND ONLY WHILE NOTHING IN THE BATCH DEPENDS ON THE BATCH.** `run_plan` + sends every step to one browser at once, so a step whose url/selector/value interpolates a + column an EARLIER step in the same batch writes would be interpolated against the value + that column had BEFORE the batch ran. That is a wrong answer rather than a slow one, so the + batch is cut immediately before any such step and the remainder becomes the next batch. + The single-step case is then exactly the old behaviour, which is what makes this safe to + land on a live flow. + ⚠ A step that is disabled, filtered out by its `when`, or unconfigured ENDS the batch + rather than being skipped inside it: each of those is a reason this record does not run + that step, and the loop's own arms already report them one at a time with their own + sentences. Ending here keeps exactly one place that decides what a blocked step says. + """ + batch, produced = [], set() + for act in acts[start:]: + kind = act.get("kind") + if kind not in WEB_KINDS or not act.get("enabled", True): + break + if not lane_match(act.get("when"), row): + break + cfg = act.get("config") or {} + if _web_missing(kind, cfg): + break + # The dependency cut. `interpolate` reads `{{Name}}`; a step naming a column an + # earlier step in THIS batch writes has to wait for the next job. + refs = " ".join(str(cfg.get(k) or "") for k in ("url", "selector", "value", "hint")) + if any(("{{" + f) in refs or ("{{ " + f) in refs for f in produced): + break + batch.append(act) + if str(cfg.get("field") or ""): + produced.add(str(cfg.get("field"))) + if len(batch) >= MAX_WEB_STEPS_PER_JOB: + break + if not batch: + return + steps = [_web_step_dict(a, row) for a in batch] + # ⛔ THE SEAM REFUSES A JOURNEY WHOSE FIRST STEP CARRIES NO ADDRESS — there is no page to + # act on yet — and a refusal is for the WHOLE plan. Batching a url-less first step would + # therefore take its followers down with it, where one-job-per-step only lost that step. + # A batch that cannot start is cut to one, which is exactly the old behaviour. + if not str(steps[0].get("url") or "").strip(): + batch, steps = batch[:1], steps[:1] + web_jobs[0] += 1 + + def _block(a, why_one): + web_done[str(a.get("id") or "")] = None + counts["webBlocked"] += 1 + if why_one: + log(f"[aios-auto] {a.get('kind')}: {why_one}") + + # ⛔ THE SEAM'S OWN BOUNDARY IS ON `run_step`, NOT ON `run_plan` — `run_step` wraps its + # call in `try/except` and calls that "the LAST boundary". Calling `run_plan` directly + # steps around it, and this code runs inside a record walk where an escaping exception + # ends the whole run. So the boundary moves here with the call. + try: + rows_out, why = _web_agent().run_plan( + steps, {"tenant": str(defn.get("tenant") or ""), + "automationId": str(defn.get("id") or ""), + "runId": str(defn.get("id") or ""), "log": log}) + except Exception as exc: # noqa: BLE001 — the LAST boundary + why, rows_out = (f"The web steps failed unexpectedly ({type(exc).__name__}: " + f"{str(exc).splitlines()[0][:200]}). Nothing was read."), None + if why: + log(f"[aios-auto] web: {why}") + for a in batch: + _block(a, "") + return + # ⚠ MATCHED BY ID, NEVER BY POSITION. The job returns a row for a step that FAILED and for + # every step after it that was never attempted, so the list can be shorter than, or + # misaligned with, the plan — and a positional read would hand step 3's caller step 2's + # answer, writing a wrong value into a real column, which is worse than the missing one it + # replaced. `_clean_step` mints an id for every step and the runner echoes it back, so the + # id is carried the whole way and is the only thing worth matching on. + by_id = {str(r.get("id") or ""): r for r in (rows_out or []) if isinstance(r, dict)} + for a in batch: + hit = by_id.get(str(a.get("id") or "")) + # ⛔ `ok` IS CHECKED HERE BECAUSE `run_step` USED TO CHECK IT. It turned a row with + # `ok:false` into a sentence and returned no result; reading `hit` without that test + # would take a failed step's empty `value` and write it over a real cell. + if not hit or not hit.get("ok"): + _block(a, str((hit or {}).get("error") or "") + or "The browser job returned nothing for this step.") + continue + web_done[str(a.get("id") or "")] = hit + def _walk(acts, row, rid, depth=0): """Walk the actions in order for ONE record. @@ -9811,7 +10776,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print early. With review retired nothing suspends anything, so every record walks its whole flow; the signature stays so a future gate-style action has an obvious place to say so. """ - for act in acts: + for idx, act in enumerate(acts): if not act.get("enabled", True): continue if not lane_match(act.get("when"), row): @@ -9843,6 +10808,103 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print # the table would silently apply one action's uniqueness rule to the other's rows. creates.setdefault((target, str(cfg.get("uniqueOn") or "")), []).append(vals) counts["created"] += 1 + elif kind == "send_statement": + # ⭐⭐ WAVE 35 · T35 / R10 — THE ARM LANDS WITH THE CATALOG ROW, ON PURPOSE. + # + # `_walk` has no terminal `else` (see the note on `enrich_tiktok` in the catalog): + # an unknown kind is walked, COUNTED, reports the run `ok` and writes nothing. So a + # catalog row whose arm arrives in a later ticket is addable, storable and silently + # inert — a step a person configured, that reports success and does nothing. This + # arm exists so that window never opens. + # + # ⛔ T35 DOES NOT SEND AND DOES NOT PARK. R10's review stage is W35-T36; until it + # lands this says so out loud and counts the record as blocked, which is the same + # shape `ai_agent` uses for a step it cannot perform. It must never fall through to + # "ok". + # ⛔ AND IT NEVER SENDS FROM HERE, in this wave or any later one. The send door is + # `routes_statements`, behind SAFE_MODE + `admin_gate` + the tenant gate, reached by + # a human click on the review batch. This arm's whole job is to PREPARE. + if "send_statement_pending" not in web_notes: + web_notes.append("send_statement_pending") + log("[aios-auto] send_statement: statements are assembled for review, not " + "sent. The review batch is not configured yet, so nothing was prepared " + "and nothing was sent.") + counts["webBlocked"] += 1 + continue + elif kind == "ai_agent": + # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. + # + # A description becomes concrete web steps HERE, against this record's own values, + # and then rides the ordinary seam. Composing at run time rather than at save time + # is the whole point: `{{Website}}` is a different page for every row, so a journey + # fixed at save time would be the same guess repeated. + # ⛔ IT COMPOSES ONLY `web_*` KINDS. The composer is handed the same catalog the + # AI-agent module uses, filtered to what a browser job can perform — so a fuzzy + # instruction cannot talk this action into writing a record or calling a connector. + # The blast radius of a bad sentence is one browser session, not the tenant. + # ⚠ AND IT REPORTS THE STEPS IT ACTUALLY TOOK, which is the ticket's own + # `done-when`. A step that composes a journey and reports only its final value is + # unauditable: nobody can tell a right answer from a lucky one. + _missing = _web_missing(kind, cfg) + if _missing: + _note = f"{kind}_unconfigured" + if _note not in web_notes: + web_notes.append(_note) + log(f"[aios-auto] {kind}: this step still needs " + + ", ".join(_missing) + ". Nothing was done") + counts["webBlocked"] += 1 + continue + if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: + if "web_cap" not in web_notes: + web_notes.append("web_cap") + log(f"[aios-auto] web: this run stopped after {MAX_WEB_JOBS_PER_RUN} " + f"browser jobs of about 10-30 seconds each.") + counts["webBlocked"] += 1 + continue + # W35 · C7: `st` + `user` so the model spend is attributed (`NOTE E-16`). + _plan, _why = _ai_agent_plan(cfg, row, log, st=rt, + user=str(defn.get("createdBy") or "")) + if _why: + # ⛔ NAMED, NEVER OPAQUE — the second half of the `done-when`. "The assistant + # could not work out how to do that" with the reason attached is actionable; + # a blank cell is not. + log(f"[aios-auto] ai_agent: {_why}") + counts["webBlocked"] += 1 + continue + web_jobs[0] += 1 + try: + _rows_out, _why2 = _web_agent().run_plan( + _plan, {"tenant": str(defn.get("tenant") or ""), + "automationId": str(defn.get("id") or ""), + "runId": str(defn.get("id") or ""), "log": log}) + except Exception as _exc: # noqa: BLE001 — the LAST boundary + _rows_out, _why2 = None, ( + f"the browser job failed unexpectedly ({type(_exc).__name__}: " + f"{str(_exc).splitlines()[0][:200]}). Nothing was done.") + if _why2: + log(f"[aios-auto] ai_agent: {_why2}") + counts["webBlocked"] += 1 + continue + # THE ACCOUNT OF WHAT IT DID — one line per step, in order, with each step's own + # verdict. This is what makes a fuzzy step auditable at all. + _done = [r for r in (_rows_out or []) if isinstance(r, dict)] + for _i, _r in enumerate(_done, 1): + log(f"[aios-auto] ai_agent step {_i}/{len(_plan)}: {_r.get('kind')} " + f"{'ok' if _r.get('ok') else 'FAILED'}" + + (f". {str(_r.get('error'))[:160]}" if not _r.get("ok") else "")) + _last = _done[-1] if _done else {} + if not _done or not _last.get("ok"): + log("[aios-auto] ai_agent: the journey did not finish. " + + str((_last or {}).get("error") + or "the browser job returned nothing for the last step")) + counts["webBlocked"] += 1 + continue # ⛔ NOTHING IS WRITTEN on an unfinished journey. + _target = str(cfg.get("field") or "") + if _target: + vals = {_target: _last.get("value")} + row.update(vals) + _act_row_patch(patches, table, rid, vals) + counts["webRead"] += len(_done) elif kind in WEB_KINDS: # ⭐⭐ WAVE 31 · T38 (C5) — THE WEB-BROWSING AGENT'S LIVE ARM, ALL FIVE KINDS. # @@ -9875,8 +10937,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print # no selector; `web_fill` needs a value nobody else takes; only `web_read` needs a # column to write into. One shared three-field test would have blocked every # `web_goto` ever configured for want of a selector it does not use. - _missing = [n for n, key in WEB_REQUIRED.get(kind, ()) - if not str(cfg.get(key) or "").strip()] + _missing = _web_missing(kind, cfg) if _missing: # ⚠ THE NOTE KEY CARRIES THE KIND. It used to be the literal # `"web_read_unconfigured"`, so a flow with an unconfigured `web_goto` AND an @@ -9886,57 +10947,37 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if _note not in web_notes: web_notes.append(_note) log(f"[aios-auto] {kind}: this step still needs " - + ", ".join(_missing) + " — nothing was done") + + ", ".join(_missing) + ". Nothing was done") counts["webBlocked"] += 1 continue - # ⛔⛔ THE PER-RUN JOB CEILING (E-4). One browser job per RECORD, ~9-32 s each, - # against HF's 6-concurrent cap — a flow over a few thousand rows would submit a few - # thousand jobs and run for days. Reported ONCE with its cause AND the fix, which is - # R6's second sentence: a limit that cannot be removed today must say why and what - # would remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning. - if counts["webRead"] >= MAX_WEB_JOBS_PER_RUN: + # ⛔⛔ THE PER-RUN JOB CEILING (E-4). A browser job is ~9-32 s against HF's + # 6-concurrent cap — a flow over a few thousand rows would submit a few thousand + # jobs and run for days. Reported ONCE with its cause AND the fix, which is R6's + # second sentence: a limit that cannot be removed today must say why and what would + # remove it. `MAX_WEB_JOBS_PER_RUN` carries the reasoning. + # ⭐⭐ W33-T58 (D-191) — THE CEILING NOW COUNTS **JOBS**, NOT PAGES, because the + # two stopped being the same thing on the line below. A record whose three web + # steps batch into one job spends ONE of these, not three. + if web_jobs[0] >= MAX_WEB_JOBS_PER_RUN: if "web_cap" not in web_notes: web_notes.append("web_cap") log(f"[aios-auto] web_read: this run stopped after " - f"{MAX_WEB_JOBS_PER_RUN} pages — each one is a separate browser job " - f"of about 10-30 seconds, and they do not run in parallel. To read " - f"more, narrow the flow's records, or batch the flow's web steps into " - f"one job (web_agent.run_plan).") + f"{MAX_WEB_JOBS_PER_RUN} browser jobs of about 10-30 seconds each, " + f"which do not run in parallel. A record's consecutive web steps " + f"already share ONE job; to read more, narrow the flow's records.") counts["webBlocked"] += 1 continue - _url = interpolate(str(cfg.get("url") or ""), row) - _sel = interpolate(str(cfg.get("selector") or ""), row) - _wait = cfg.get("waitFor") - _step = {"kind": kind, "id": str(act.get("id") or ""), - "url": _url, "selector": _sel, - "attr": str(cfg.get("attr") or "text"), - "all": bool(cfg.get("all")), - "waitFor": interpolate(str(_wait), row) if _wait else None, - "timeoutMs": int(cfg.get("timeoutMs") or 20000)} - # ⚠ `value` IS INTERPOLATED LIKE THE URL AND THE SELECTOR — a `web_fill` typing - # `{{Email}}` is the whole point of running it over a record walk. ⛔ AND IT IS - # ADDED ONLY WHEN PRESENT: the seam distinguishes a key that is absent from one - # that is empty, and an empty `value` on a `web_read` would be a typed blank. - if cfg.get("value"): - _step["value"] = interpolate(str(cfg.get("value")), row) - if cfg.get("hint"): - _step["hint"] = interpolate(str(cfg.get("hint")), row) - if cfg.get("secret"): - _step["secret"] = True - if cfg.get("dryRun"): - _step["dryRun"] = True - result, why = _web_agent().run_step( - _step, - {"tenant": str(defn.get("tenant") or ""), - "automationId": str(defn.get("id") or ""), - "runId": str(defn.get("id") or ""), "log": log}) - # ⛔ EXACTLY ONE OF THE TWO IS TRUTHY — E's contract, and `if why:` is the COMPLETE - # client-side branch. There is no exception path to catch: `run_step` turns - # everything into a sentence, which is R6's second sentence built into the seam. - if why: - log(f"[aios-auto] web_read: {why}") - counts["webBlocked"] += 1 - continue # ⛔ NOTHING IS WRITTEN on a refusal. + # ⭐⭐ W33-T58 · D-191 — ONE JOB FOR A RECORD'S CONSECUTIVE WEB STEPS. + # `run_plan(steps, ctx)` has always taken a list and nothing ever called it with + # more than one: the arm called `run_step`, which wraps `[step]`, so a flow with + # three web reads paid THREE ~9 s cold starts to do what one job does. The batch is + # built at WALK time rather than from the stored flow, because whether a step runs + # at all depends on this record (`enabled`, `when`, and whether it is configured). + if str(act.get("id") or "") not in web_done: + _run_web_batch(acts, idx, row, rid) + result = web_done.get(str(act.get("id") or "")) + if result is None: + continue # its batch refused; the reason was logged once, there _target = str(cfg.get("field") or "") if _target: vals = {_target: (result or {}).get("value")} @@ -9954,7 +10995,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if "unbound" not in enrich["notes"]: enrich["notes"].append("unbound") log("[aios-auto] enrich_instagram: no profile column on " - f"{table} — name one on the action, or mark a text column as an " + f"{table}. Name one on the action, or mark a text column as an " "Instagram profile") continue # ⭐ 2026-08-07 (owner ruling) — IS THIS RECORD IN THIS RUN'S BUDGET? @@ -10099,7 +11140,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if "unbound" not in tt["notes"]: tt["notes"].append("unbound") log("[aios-auto] enrich_tiktok: no profile column on " - f"{table} — name one on the action, or mark a text column as a " + f"{table}. Name one on the action, or mark a text column as a " "TikTok profile") continue aid = str(act.get("id") or "") @@ -10240,6 +11281,10 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print row = dict(rows.get(str(rid)) or {}) if not row: continue + # ⭐ D-191 — the web batch is built from THIS row's interpolated values, so it means + # nothing for the next one. Cleared here rather than inside `_walk`, which recurses + # into branches and would wipe a batch its own caller is still consuming. + web_done.clear() _walk(actions, row, str(rid)) # ⭐⭐ WAVE 30 · T08 — TWO FLUSHES, ONE `counts`, AND THE NOTES ARE CONCATENATED RATHER THAN # OVERWRITTEN. `RUN_NOTES_KEY` is the one key both flushes legitimately produce, so a plain @@ -10425,7 +11470,7 @@ def _enrich_flush(rt, defn, username, acc, log): acc["posts"], acc["psnaps"]) if m_status == "error": out["enrichMasterFailed"] = 1 - log(f"[aios-auto] enrich_instagram: the platform master copy FAILED — {m_note}; " + log(f"[aios-auto] enrich_instagram: the platform master copy FAILED. {m_note}; " f"the tenant copy is complete and the next run re-appends") elif m_status == "ok": out["enrichMaster"] = len(acc["snaps"]) + len(acc["psnaps"]) @@ -10469,7 +11514,7 @@ def _unbound_hint(rt, table_key): return "" shown = ", ".join(names[:UNBOUND_HINT_MAX]) more = len(names) - UNBOUND_HINT_MAX - return (f" — this database's text columns are {shown}" + return (f". This database's text columns are {shown}" + (f" (+{more} more)" if more > 0 else "")) @@ -10931,7 +11976,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print if stale_h is not None and stale_h >= PENDING_PROFILE_MAX_HOURS: closed += 1 note = (f"@{task['influencer']}: the source never finished the profile read " - f"queued {int(stale_h)}h ago ({task['snapshotId']}) — it was dropped; " + f"queued {int(stale_h)}h ago ({task['snapshotId']}). It was dropped; " f"the next run will ask again") notes.append(note) acc["notes"].append(note) @@ -10949,7 +11994,7 @@ def collect_pending_profile_snapshots(rt, defn, username="automation", log=print acc["profiles"] += 1 acc["blocked"] += 1 note = (f"@{task['influencer']}: the source delivered {len(rows)} profile" - f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle — " + f"{'' if len(rows) == 1 else 's'} for that batch, none of them this handle. " f"it was dropped from the batch and the next run will ask again") notes.append(note) acc["notes"].append(note) @@ -11210,9 +12255,16 @@ def ai_decide(rt, defn, act, row, row_id="", log=print): (ut_get(rt, ((defn.get("config") or {}).get("targetTable") or (defn.get("trigger") or {}).get("table") or "")) or {} ).get("fields") or []] + # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — ATTRIBUTE THE SPEND. `st=` is what lets the ledger write + # to the right tenant's store and `user=` is who it is billed to; without them the call is + # counted as UNATTRIBUTED, which is a meter that reports a total nobody can act on. + # ⚠ `createdBy` is the honest actor here: an AI review decision is made ON BEHALF of the + # automation, by a scheduler, with no person at the keyboard. Naming whoever last edited it + # would attribute a nightly run to an editor who was asleep. choice, meta = ai_review.decide(prompt=cfg.get("prompt") or "", options=options, row=row, fields=[f for f in fields if f], - label=cfg.get("label") or "Review") + label=cfg.get("label") or "Review", + st=rt, user=str(defn.get("createdBy") or "")) if not choice: if meta.get("problem"): log(f"[aios-auto] ai review declined to answer: {meta['problem']}") @@ -11280,7 +12332,7 @@ def _commit_action_writes(rt, table, patches, creates, username, log): t = ut_get(rt, tkey) if t is None: n = sum(len(r) for _u, r in groups) - log(f"[aios-auto] create_record: {tkey} no longer exists — {n} skipped") + log(f"[aios-auto] create_record: {tkey} no longer exists. {n} skipped") out["createSkipped"] += n continue # The PATCHED rows when this table was also stamped this run, so the creates land on top @@ -11300,7 +12352,7 @@ def _commit_action_writes(rt, table, patches, creates, username, log): # because the symptom is otherwise a run that says it created less than it walked. out["createSkipped"] += c["skipped"] if c["capped"]: - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — " + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " f"{c['capped']} row(s) not written") continue nxt = max([int(r) for r in cur if str(r).isdigit()] or [0]) + 1 @@ -11310,7 +12362,7 @@ def _commit_action_writes(rt, table, patches, creates, username, log): # named, here and in the run's counts. This used to `break` with a log line # that said the cap was hit and never said how much was lost. out["createCapped"] += len(new_rows) - i - log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap — " + log(f"[aios-auto] create_record: {tkey} at its {cap}-row cap. " f"{len(new_rows) - i} row(s) not written") break cur[str(nxt)] = dict(vals) @@ -12270,9 +13322,19 @@ TRIGGER_LABELS = { # * THE GATE fails HARD — `verify_automation` asserts that NO shipped trigger lands in `other`, # so the fallback is provably dead code in production and the classification is still # mandatory. The fallback catches the accident; the gate stops it shipping. +# ⭐ WAVE 34 · R19 — CONNECTOR SITS ABOVE DATABASE, DIRECTLY UNDER TIME. Owner, verbatim: +# *"In the Trigger picker, the Connector section sits directly above the Database section, just +# under the Time trigger types."* So the two orders below are SWAPPED against wave 24's, and +# nothing else moved: same keys, same labels, same fallback. +# ⛔ THIS IS THE WHOLE OF R19 AND IT IS DELIBERATELY NOT A CLIENT CHANGE. `steps.ts::groupTriggers` +# sorts by each option's `groupOrder` and by nothing else, so re-ordering an array on the client +# would look right in a fixture and be wrong in production the moment the server re-sorted. +# ⚠ `verify_steps.py` CANNOT WITNESS THIS EDIT: its C2 leg supplies its OWN `groupOrder` in a +# fixture and asserts the client honours it, which stays true whatever these numbers say. The +# check that binds R19 to this table lives in `verify_automation` beside the vocab section. TRIGGER_GROUPS = {"time": {"label": "Time", "order": 1}, - "database": {"label": "Database", "order": 2}, - "connector": {"label": "Connector", "order": 3}, + "connector": {"label": "Connector", "order": 2}, + "database": {"label": "Database", "order": 3}, "other": {"label": "Other", "order": 99}} #: Where an unclassified trigger goes. ⚠ Reaching this in production is a BUG the gate exists to #: prevent — it is the soft landing, not a category anybody should be adding triggers to. @@ -12398,7 +13460,7 @@ def clean_trigger(raw, previous=None): if key in TRIGGER_PLANNED: # Declared on the wire, refused at the door — see the TRIGGER_PLANNED note. The sentence # says WHY rather than "unknown trigger", because the picker legitimately showed it. - return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet — " + return None, (f"{TRIGGER_LABELS[key]!r} is on the list but not built yet. " f"it renders so you can see it is coming, and it cannot be saved") if key not in TRIGGER_KEYS: return None, (f"{key or 'that trigger'!r} is not one of: " + ", ".join(TRIGGER_KEYS)) @@ -12439,7 +13501,7 @@ def clean_trigger(raw, previous=None): if key in TRIGGER_TABLE_KEYS: table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() if table and not table.startswith(UT_PREFIX): - return None, ("event triggers watch blank databases (ut_*) this wave — " + return None, ("event triggers watch blank databases (ut_*) this wave. " f"{table!r} is not one") out["table"] = table if key == "event_field": @@ -12466,7 +13528,7 @@ def clean_trigger(raw, previous=None): watch = [_s(f, 80).strip() for f in watch_raw if _s(f, 80).strip()] if len(watch) > MAX_WATCH_FIELDS: return None, (f"a record-updated trigger watches at most {MAX_WATCH_FIELDS} " - f"fields — leave the list empty to watch every field") + f"fields. Leave the list empty to watch every field") out["fields"] = watch # ⭐ WAVE 24 · C-TRIG LAW 5 (owner item 7) — THE CONDITION IS GONE, and this REMOVES A # SHIPPED CAPABILITY. Watched `fields` is now the whole of this trigger's configuration: @@ -12496,7 +13558,7 @@ def clean_trigger(raw, previous=None): # every existing webhook trigger would go unconfigured and stop firing. table = _s(raw.get("table") if "table" in raw else prev.get("table"), 60).strip() if table and not table.startswith(UT_PREFIX): - return None, ("a webhook writes into a blank database (ut_*) — " + return None, ("a webhook writes into a blank database (ut_*). " f"{table!r} is not one") out["table"] = table fmap, ferr = clean_body_map(raw.get("fieldMap") if "fieldMap" in raw @@ -12537,7 +13599,7 @@ def clean_body_map(raw): if not p: return None, "a webhook mapping has an empty body path" if len(p.split(".")) > MAX_BODY_DEPTH: - return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload — " + return None, (f"{p!r} reaches more than {MAX_BODY_DEPTH} levels into the payload. " f"map a shallower value") fk = re.sub(r"[^a-z0-9_]+", "_", _s(field, 60).strip().lower()).strip("_") if not fk: @@ -12587,7 +13649,7 @@ def webhook_row(rt, defn, body): if v is not None: values[fkey] = _s(v, 500) if isinstance(v, str) else v if not values: - return "", 0, ("none of the mapped paths resolved to a value in this payload — " + return "", 0, ("none of the mapped paths resolved to a value in this payload. " "check the paths against what you are sending") rows = dict((t.get("rows") or {})) if len(rows) >= row_cap(table): @@ -12672,7 +13734,7 @@ def trigger_fire(rt, tenant, auto_id, log=print, rows=None): start (breaker, or already running — both are answers, not errors).""" if _breaker_trips(tenant, auto_id): note = (f"auto-paused: more than {FIRE_LIMIT} trigger fires in " - f"{FIRE_WINDOW_SECONDS // 60} minutes — something is writing this trigger's " + f"{FIRE_WINDOW_SECONDS // 60} minutes. Something is writing this trigger's " f"subject in a loop") _pause_trigger(rt, auto_id, note) log(f"[aios-auto] breaker: {auto_id} {note}") @@ -12816,7 +13878,7 @@ def _settle_eval(rt, tenant, auto_id, log=print): fired = list(touched) # "the field changed" — the burst is the edge if len(fired) > FLOOD_LIMIT: _commit_run(rt, auto_id, "partial", - f"the trigger matched {len(fired)} records in one evaluation — more than " + f"the trigger matched {len(fired)} records in one evaluation. More than " f"the {FLOOD_LIMIT}-record flood hold, so nothing ran. Press Run now to " f"process them deliberately", {"held": len(fired)}, True) return @@ -13009,7 +14071,7 @@ def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): reader = _read or (lambda mid: oauth_connect.gmail_message(token, mid)) ids, lerr = lister(trg.get("query") or "", EMAIL_MAX_PER_POLL + FLOOD_LIMIT) if lerr: - return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer — {lerr}", + return _commit_run(rt, auto_id, "partial", f"the Gmail poll did not answer. {lerr}", {}, True) seen = set((defn.get("state") or {}).get("emailSeen") or []) fresh = [m for m in ids if m not in seen] @@ -13017,7 +14079,7 @@ def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): return None # nothing new is not a run — no history spam if len(fresh) > FLOOD_LIMIT: return _commit_run(rt, auto_id, "partial", - f"{len(fresh)} new emails matched in one poll — more than the " + f"{len(fresh)} new emails matched in one poll. More than the " f"{FLOOD_LIMIT}-record flood hold, so nothing was written. Narrow " f"the query, or press Run now after adjusting it", {"held": len(fresh)}, True) @@ -13032,7 +14094,7 @@ def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): table_key = (defn.get("config") or {}).get("targetTable") or "" if not table_key: return _commit_run(rt, auto_id, "error", - "the email trigger has nowhere to write — the automation names no " + "the email trigger has nowhere to write. The automation names no " "target database", {}, False) ut_ensure(rt, (defn.get("config") or {}).get("targetLabel") or defn.get("name") or "Inbox", EMAIL_FIELDS, username=str(defn.get("createdBy") or "automation"), key=table_key) @@ -13041,10 +14103,10 @@ def email_poll(rt, tenant, auto_id, defn, log=print, _list=None, _read=None): ut_write_rows(rt, table_key, merged) new_seen = (list(seen) + fresh)[-EMAIL_SEEN_CAP:] set_state(rt, auto_id, {"emailSeen": new_seen}) - summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched — " + summary = (f"{len(fresh)} new email{'' if len(fresh) == 1 else 's'} matched. " f"{counts['inserted']} row{'' if counts['inserted'] == 1 else 's'} written") if notes: - summary += f" — {notes[0][:100]}" + summary += f". {notes[0][:100]}" entry = _commit_run(rt, auto_id, "ok" if not notes else "partial", summary, counts, True, affected=list(merged)[:200]) trigger_fire(rt, tenant, auto_id, log=log) @@ -13106,7 +14168,7 @@ def compose_sentence(defn): n = sum(1 for _ in walk_actions((defn.get("flow") or {}).get("actions"))) tbl = cfg.get("targetTable") or trg.get("table") or "" body = ((f"run {n} action{'' if n == 1 else 's'}" + (f" on {tbl}" if tbl else "")) - if n else "do nothing yet — this automation has no actions") + if n else "do nothing yet. This automation has no actions") lanes = cfg.get("lanes") or [] tail = f", then route each record across {len(lanes)} lanes" if lanes else "" return f"{head}, {body}{tail}." @@ -13216,7 +14278,7 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): if a_counts.get("enrichMetricBatchesPending"): batches = int(a_counts["enrichMetricBatchesPending"]) state = "partial" if state != "error" else state - summary += (f" — {batches} post-engagement batch" + summary += (f". {batches} post-engagement batch" f"{'' if batches == 1 else 'es'} still building; Views and other " "metrics will be collected automatically without another paid read") if a_counts.get("enrichUnbound"): @@ -13225,7 +14287,7 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): # the sentence names the ONE thing that has to change, in the two places a person # can change it. The old behaviour was `ok` with an empty table. state = "partial" if state != "error" else state - summary += (" — the Instagram step did not run: this database has no profile " + summary += (". The Instagram step did not run: this database has no profile " "column. Name one on the step, or mark a text column as the " "Instagram profile" + _unbound_hint(rt, _flow_table(defn))) @@ -13236,21 +14298,21 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): # so a single sentence covering "the enrich step" would send them to the wrong # place half the time. Both may appear on one run, which is correct. state = "partial" if state != "error" else state - summary += (" — the TikTok step did not run: this database has no TikTok profile " + summary += (". The TikTok step did not run: this database has no TikTok profile " "column. Name one on the step, or mark a text column as a TikTok " "profile" + _unbound_hint(rt, _flow_table(defn))) if a_counts.get("ttEnrichBlocked"): state = "partial" if state != "error" else state tt_note = next((n for n in run_notes if "(TikTok): " in n), "") - summary += (f" — {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " + summary += (f". {int(a_counts['ttEnrichBlocked'])} TikTok profile read(s) were " "blocked" + (f". {_s(tt_note, 220)}" if tt_note else "")) if a_counts.get("enrichProfileBatchesPending"): # ⭐ The paid profile the vendor is still building. Said out loud so a run that # looks like a failure is read as the handoff it is — the tick finishes it. n = int(a_counts["enrichProfileBatchesPending"]) state = "partial" if state != "error" else state - summary += (f" — {n} profile read{'' if n == 1 else 's'} took longer than the " + summary += (f". {n} profile read{'' if n == 1 else 's'} took longer than the " "wait allows and will be collected automatically, at no extra cost") if a_counts.get("enrichBlocked"): state = "partial" if state != "error" else state @@ -13262,7 +14324,7 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): # reason. Two sentences quoting each other's vendor reason is worse than one. blocked_note = next((n for n in run_notes if ": " in n and "(TikTok): " not in n), "") - summary += (f" — {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + summary += (f". {int(a_counts['enrichBlocked'])} profile read(s) were blocked" + (f". {_s(blocked_note, 220)}" if blocked_note else "")) # ⭐ WAVE 25 · C5 — A FULL TARGET IS A `partial` RUN THAT SAYS SO. D-11 made this the # law for the runners' OWN writes (`cap_note`), and `create_record` never joined: it @@ -13270,12 +14332,12 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None): # looked exactly like one that had nothing to write. Same rule, same sentence shape. if a_counts.get("createCapped"): state = "partial" if state != "error" else state - summary = (f"{summary} — {a_counts['createCapped']} row(s) NOT created: a target " + summary = (f"{summary}. {a_counts['createCapped']} row(s) NOT created: a target " f"database is at its row cap") except Exception as e: # noqa: BLE001 log(f"[aios-auto] actions on {auto_id} failed: {type(e).__name__}: {e}") state = "partial" if state != "error" else state - summary = f"{summary} — the actions did not finish ({type(e).__name__})" + summary = f"{summary}. The actions did not finish ({type(e).__name__})" return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected, steps, notes=run_notes) finally: diff --git a/api/deps.py b/api/deps.py index 3339bd471c143589a0cec3ba60832f76ebb4cf8f..04900582fc1b0e4c242390e9834a786a6cb639a9 100644 --- a/api/deps.py +++ b/api/deps.py @@ -10,6 +10,7 @@ epoch-revoked, or naming an unknown tenant → **401**. Authenticated but not gr **403**. Never an empty 200: an empty list is a legitimate answer meaning "no rows", and using it to mean "you are not allowed" is how a permission bug becomes invisible. """ +import copy import os import sys import time @@ -68,6 +69,418 @@ class Session: f"this workspace does not include {module_key}") +# ── Assistant: app-stored, permissioned snapshot boundary (W33-T71 / C8) ─────────────────────── +# +# This is deliberately an ordinary function rather than a route. E owns the Assistant's tool +# routes and model/provider choices; every one of those tools must come through this one boundary +# before it can see a database. Keeping it here gives the reader the already-verified Session and +# prevents Query's schema-only `_target` helper from becoming an accidental data authority. +# +# The important absence is as much part of the contract as the code below: this module never calls +# the Customer/Product pool builders, connector routes, provider routes, Odoo, or an environment +# lookup from this boundary. A cache miss is a named refusal, never permission to refresh from the +# source of truth. That is what "data the app already stores or serves" means in executable form. +_ASSISTANT_BUILT_INS = { + "customer_data": {"label": "Odoo customers", "cache_prefix": "pool"}, + "product_data": {"label": "Odoo products", "cache_prefix": "product_pool"}, +} + + +def _assistant_refuse(status, code, message): + """Raise the public error shape without exposing a source implementation detail.""" + raise err(status, code, message) + + +def _assistant_tenant_allows(session: Session, database: str) -> bool: + """Read the served tenant catalogue without reusing the legacy account gate.""" + tenant = getattr(getattr(session, "runtime", None), "tenant", None) + config = getattr(tenant, "config", None) or {} + modules = config.get("modules", "all") if isinstance(config, dict) else "all" + return modules == "all" or str(database) in {str(key) for key in (modules or ())} + + +def _assistant_preflight_request(session: Session, database: str, canonical_fields, + requested_fields, filters, *, require_explicit_grant: bool): + """Refuse an unresolved grant, field or filter before a stored record is touched.""" + import core.perm_scope as perm_scope + + grant = perm_scope.assistant_entry(session.user, database) + if require_explicit_grant and grant is None: + _assistant_refuse(403, "assistant_forbidden", + "your current permissions do not allow this database") + visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, database) + if grant is not None else list(canonical_fields or ())) + try: + perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields) + _assistant_requested_fields(requested_fields, visible) + perm_scope.validate_assistant_filter(filters, visible) + except ValueError as exc: + _assistant_refuse(400, "assistant_bad_filter", str(exc)) + return grant, visible + + +def _assistant_cached_builtin(session: Session, database: str, requested_fields=None, filters=None): + """The exact in-process snapshot for one built-in database, or a loud cache-miss refusal. + + This intentionally reads `runtime.pool_cache` directly. Calling either route's `_pool_for` + would make a first Assistant request call Odoo; calling a cache helper would make a stale entry + refresh. Neither is data already stored/served by this process. + """ + import core.perm_scope as perm_scope + + # Unlike an interactive grid, the Assistant has no legacy-grant compatibility path: the + # caller needs a declared, migrated database grant. `may_access` retains the old wall for + # the grid rollout, so it is necessary but not sufficient at this boundary. + declared_access = perm_scope.assistant_entry(session.user, database) is not None + if not _assistant_tenant_allows(session, database) or not declared_access: + _assistant_refuse(403, "assistant_forbidden", + "your current permissions do not allow this database") + + # These are static field contracts, not the route assemblies: route assemblies can add + # per-user workspace state and can call a source builder. The Assistant begins with the + # already-stored base rows only; resolve their grant/field/filter contract *before* the cache + # read so a denied field or malformed operand cannot touch a stored record. + import aios_grid + fields = aios_grid.FIELDS if database == "customer_data" else aios_grid.product_fields() + _assistant_preflight_request(session, database, fields, requested_fields, filters, + require_explicit_grant=True) + + team_id, agent = perm_scope.derive_pool_scope(session.user, database) + cache_key = (("pool", team_id, agent) if database == "customer_data" + else ("product_pool", team_id)) + cache = getattr(session.runtime, "pool_cache", None) + entry = cache.get(cache_key) if isinstance(cache, dict) else None + if (not isinstance(entry, tuple) or len(entry) < 2 + or not isinstance(entry[1], (list, tuple)) + or any(not isinstance(row, dict) for row in entry[1])): + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database has no app-stored snapshot for your current scope") + return list(entry[1]), [dict(field) for field in fields if isinstance(field, dict)], entry[0] + + +def _assistant_user_table(session: Session, database: str, requested_fields=None, filters=None): + """Read a materialised user-table document after a rows-free database grant check.""" + from core import user_tables + + # A guessed `ut_*` key must never make `user_tables.get()` copy the tenant's rows before the + # table resolver refuses it. `all_defs` can fall back to a whole-document read for older + # stores, so use the runtime's projection primitive directly and fail closed if it is absent. + # The existing `may_open` remains THE resolver; it is lent this rows-dropped document. + try: + definitions = session.runtime.get_projection('user_tables', drop=('rows',)) + except Exception: + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database has no readable definition snapshot") + if not isinstance(definitions, dict): + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database has no readable definition snapshot") + definition = definitions.get(database) + if not isinstance(definition, dict): + _assistant_refuse(404, "assistant_unknown_database", "that database is not available") + definition_runtime = user_tables.lend(session.runtime, user_tables=definitions) + if not user_tables.may_open(database, session.uname, session.admin, st=definition_runtime): + # Deliberately no label/owner in this response: knowing a key must not become an oracle + # for somebody else's database. + _assistant_refuse(403, "assistant_forbidden", + "your current permissions do not allow this database") + + fields = [dict(field) for field in (definition.get("fields") or []) + if isinstance(field, dict) and field.get("key")] + _assistant_preflight_request(session, database, fields, requested_fields, filters, + require_explicit_grant=False) + + # Only an admitted table may now cross the app-stored record seam. This call still has no + # upstream fallback: it is the tenant's materialised document already held by the runtime. + table = user_tables.get(database, st=session.runtime) + if not isinstance(table, dict): + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database changed before its stored snapshot could be read") + if not user_tables.materialises(database, st=session.runtime, defn=table): + # Read-through rows belong to the app's DuckDB mirror, but this fence does not own its + # tenant-checked query binding. Returning an empty table would be a false answer; E must + # consume the mirror through its own `runtime.mirror_cursor`-guarded seam. + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database is mirror-served and needs its materialised reader") + + # A table's field contract was authorized from the rows-free definition. If it changed while + # the materialised document was fetched, fail rather than projecting a schema that was never + # checked. + actual_fields = [dict(field) for field in (table.get("fields") or []) + if isinstance(field, dict) and field.get("key")] + if actual_fields != fields: + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database changed before its stored snapshot could be read") + raw_rows = table.get("rows") or {} + if not isinstance(raw_rows, dict): + _assistant_refuse(409, "assistant_snapshot_unavailable", + "this database does not have a readable stored snapshot") + keys = {field["key"] for field in fields} + rows = [] + for record_id, raw in raw_rows.items(): + if not str(record_id).isdigit() or not isinstance(raw, dict): + continue + # Matches `routes_tables.scoped_pool`: user-table permission is table-level, and only + # declared fields cross this boundary. `pid` is the opaque record identity required to + # bind a returned record to a single-source Query artefact; it is not a selectable field. + rows.append({"pid": int(record_id), + **{key: raw.get(key) for key in keys if key in raw}}) + rows.sort(key=lambda row: row["pid"]) + return rows, fields, (table.get("updated") or table.get("created")), table.get("label") or database + + +def _assistant_requested_fields(fields, visible_fields): + """Resolve an explicit field selection without turning an unknown key into an omission.""" + by_key = {str(field.get("key")): field for field in visible_fields + if isinstance(field, dict) and field.get("key")} + if fields is None: + return list(by_key) + if not isinstance(fields, (list, tuple)): + _assistant_refuse(400, "assistant_bad_fields", "fields must be a list of field keys") + out = [] + for raw in fields: + key = raw.strip() if isinstance(raw, str) else "" + if not key or key not in by_key or key in out: + _assistant_refuse(400, "assistant_unknown_field", + "every requested field must be visible in this database") + out.append(key) + return out + + +def assistant_read_scope(session: Session, database, fields=None, filters=None): + """Return a detached, permissioned snapshot for one Assistant tool call. + + Signature (the C8 contract for E): + assistant_read_scope(session, database, fields, filters) -> { + database, label, source_kind, source_version, retrieved_at, + permission_scope_applied, fields, filters, records + } + + `database` is exactly one source. `fields` is an explicit list of visible field keys (or + ``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by + `core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and + unsupported operands before a row is read. The returned dictionaries are deep copies, never + a live cache/store object, cursor, connector, route, provider, or credential. + """ + import core.perm_scope as perm_scope + from harness import filter_eval + + key = str(database or "").strip() + if key in _ASSISTANT_BUILT_INS: + rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters) + label = _ASSISTANT_BUILT_INS[key]["label"] + source_kind = "runtime_cache" + elif key.startswith("ut_"): + rows, canonical_fields, source_version, label = _assistant_user_table(session, key, fields, filters) + source_kind = "materialised_user_table" + else: + _assistant_refuse(404, "assistant_unknown_database", "that database is not available") + + # Permanent permissions run against the full source contract. A permission filter may name + # a field that is intentionally hidden from the returned schema; resolving it after the field + # projection would make that filter unanswerable and (depending on the evaluator) could widen. + # Validate even that grant before evaluating it: `permits` would deny an unknown operand row + # by row, but the Assistant contract requires an explicit refusal for unresolved grants. + grant = perm_scope.assistant_entry(session.user, key) + if source_kind != "materialised_user_table" and grant is None: + # `_assistant_cached_builtin` already made this decision before touching its cache. Keep + # this guard adjacent to the field/row projection so a future source cannot forget it. + _assistant_refuse(403, "assistant_forbidden", + "your current permissions do not allow this database") + try: + perm_scope.validate_assistant_filter((grant or {}).get("filter"), canonical_fields) + except ValueError as exc: + _assistant_refuse(400, "assistant_bad_filter", str(exc)) + permitted_rows = (perm_scope.assistant_apply_row_scope(rows, session.user, key, canonical_fields) + if grant is not None else list(rows or ())) + visible = (perm_scope.assistant_visible_fields(canonical_fields, session.user, key) + if grant is not None else list(canonical_fields)) + selected_keys = _assistant_requested_fields(fields, visible) + try: + requested_filter = perm_scope.validate_assistant_filter(filters, visible) + except ValueError as exc: + _assistant_refuse(400, "assistant_bad_filter", str(exc)) + if requested_filter is not None: + permitted_rows = [row for row in permitted_rows + if filter_eval.permits(requested_filter, row, visible)] + + records = [] + for row in permitted_rows: + if not isinstance(row, dict) or row.get("pid") is None: + continue + records.append({"pid": copy.deepcopy(row["pid"]), + **{field: copy.deepcopy(row.get(field)) for field in selected_keys}}) + + return { + "database": key, + "label": str(label), + "source_kind": source_kind, + "source_version": copy.deepcopy(source_version), + "retrieved_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "permission_scope_applied": True, + "fields": [copy.deepcopy(field) for field in visible + if field.get("key") in selected_keys], + "filters": copy.deepcopy(requested_filter), + "records": records, + } + + +def assistant_permitted_sources(session: Session, databases): + """Which of `databases` the caller may still read — WITHOUT reading one record. + + ⛔ THIS EXISTS BECAUSE `GET /query` LISTS ARTEFACTS, AND AN ARTEFACT IS NOT A READ. + `routes_query._public_state` has to drop every saved view whose source the caller has since + lost. The obvious way to ask that question is `assistant_read_scope(..., fields=[])` — and it + is the wrong one: for a `ut_*` source that call runs `user_tables.get()`, i.e. a WHOLE-DOCUMENT + read of the tenant store (measured at **28.6 MB / 703 ms warm on tenant #0**, and 99.9% of those + bytes are `rows` this question does not look at), then projects every row down to its `pid` and + throws the result away. Once per saved view, up to `MAX_ARTIFACTS = 200` of them, on a route + that BOTH the Assistant and Query call on mount. That is `/nav`'s old 1+N whole-document read + (D-175, an 11.0 s median) rebuilt inside a new route, and every fixture with two artefacts + passes it [[measure-the-real-call]]. + + So this asks the permission question and ONLY the permission question: + * the rows-free `user_tables` projection is read at most ONCE for the whole batch, and lent + to `may_open` exactly as `_assistant_user_table` lends it; + * a built-in resolves from the tenant catalogue and the declared grant, with no `pool_cache` + read — a cache MISS is a data-availability fact, not a permission one, and refusing to list + an artefact because its snapshot is cold would silently delete the user's history from the + rail every time the process restarted [[empty-answer-vs-unfinished-answer]]. + + ⚠ It is deliberately NOT a second permission wall. Every predicate here is the same call + `assistant_read_scope` makes for the same source; nothing may be admitted here that the read + boundary would refuse, which is what `verify_query.py` asserts in both directions. + """ + return {key for key, row in assistant_source_status(session, databases).items() + if row["permitted"]} + + +def assistant_source_status(session: Session, databases=None): + """`{key: {permitted, answerable, reason}}` — WITHOUT reading one record. + + ⭐⭐ `answerable` EXISTS BECAUSE `permitted` IS NOT THE QUESTION THE READER IS ASKING. + Measured on tenant #0, signed in, 2026-08-15: the Assistant offered **twelve** source chips + and could answer for **two**. The other ten are the `ut_odoo_*` grids, and + `_assistant_user_table` refuses every one of them — *"this database is mirror-served and needs + its materialised reader"* — because their rows live UNCAPPED in the DuckDB mirror and are read + through it (W30/R6 is exactly why). The refusal is correct, named and fail-closed. The defect + is that it arrives one spent prompt at a time: a picker that offers ten doors which cannot + open is the same shape as the rail that painted two rows differently from five. + ⚠ It is R6's SECOND SENTENCE, in the family that rule was written for — a limit that cannot be + removed must be REPORTED with its cause, never silently enforced. + + ⛔ AND IT COSTS NOTHING, WHICH IS WHY IT IS HERE RATHER THAN IN THE REGISTER. + `user_tables.materialises()` reads the `readThrough` flag off the DEFINITION, and the + rows-free projection this function already reads carries it. So the batch that answers + "may they?" answers "can it?" in the same pass, with no extra store read. + + ⚠ `databases=None` enumerates every source this tenant HAS (the two built-ins plus every + `ut_*` in the projection) rather than taking a caller-supplied list. A client list would make + the answer depend on what the caller thought to ask about, and the nav has already told them + what they hold. + """ + import core.perm_scope as perm_scope + from core import user_tables + + definitions = None + read_definitions = False + + def _definitions(): + nonlocal definitions, read_definitions + if not read_definitions: + read_definitions = True + try: + candidate = session.runtime.get_projection('user_tables', drop=('rows',)) + except Exception: + candidate = None + definitions = candidate if isinstance(candidate, dict) else None + return definitions + + if databases is None: + wanted = list(_ASSISTANT_BUILT_INS) + [ + key for key in sorted((_definitions() or {})) + if isinstance(key, str) and key.startswith("ut_") + ] + else: + wanted = [str(key or "").strip() for key in (databases or ())] + wanted = list(dict.fromkeys(key for key in wanted if key)) + + out = {} + for key in wanted: + # `visible` = "the caller can open this database elsewhere in the product". D-276 uses it + # to decide whether an unusable source is REPORTED or stays absent; default False, so a + # branch that never sets it keeps today's silence rather than inheriting a disclosure. + row = {"permitted": False, "answerable": False, "reason": "", "visible": False} + out[key] = row + if key in _ASSISTANT_BUILT_INS: + # ⭐⭐ D-276 (2026-08-15) — AN OMISSION MUST STATE ITS CAUSE, and this branch used to + # `continue` in silence. MEASURED on tenant #0, both `admin` and `leadership`: + # `GET /query` listed nine sources and `product_data` was not among them, while + # `POST /query/chat {database:"product_data"}` answered 403 `assistant_forbidden` — + # for a database the SAME account opens from the nav, holds saved views on, and can + # see in the Database flyout. The two mirror-served grids are also un-askable and they + # ARE listed, with their cause in the chip; this one said nothing at all. + # ⛔ THE ASYMMETRY IS THE DEFECT, NOT THE REFUSAL. R6's second sentence: a limit that + # cannot be removed must be REPORTED with its cause, never silently enforced. + # ⚠ `visible` IS WHAT KEEPS THIS FROM BEING A DISCLOSURE. A source the caller cannot + # open ANYWHERE stays absent — naming it would tell them a database exists that they + # were never allowed to know about. `visible` is true only when the ordinary product + # wall already lets them open it, i.e. when the silence was incoherent rather than + # protective. `permitted` is untouched: it gates which stored artefacts survive, and + # widening it here would resurrect views the read boundary refuses. + if not _assistant_tenant_allows(session, key): + row["visible"] = bool(perm_scope.may_access(session.user, key)) + row["reason"] = ("this workspace's catalogue does not list it as an assistant " + "source — an administrator can add it") + continue + if perm_scope.assistant_entry(session.user, key) is None: + row["visible"] = bool(perm_scope.may_access(session.user, key)) + # ⛔⛔ THIS SENTENCE DELIBERATELY PROMISES NO REMEDY, AND THAT IS A CORRECTION. + # Its first version said *"an administrator can re-save them"*. MEASURED on the + # deployed build the same hour: re-saving `leadership`'s permissions through + # `PUT /admin/users/{u}/perms` returned 200 with the block byte-identical and + # `perms_v` unchanged at 1 — and the source stayed refused. So the advice was + # wrong, and a refusal that hands the reader a fix which does not work is worse + # than one that admits it does not know: they spend the trip and land back here. + # ⚠ The cause is still open (D-276). `assistant_entry` reads only `entry`, + # `is_migrated` and `may_access`, and `product_data` carries the SAME grant as + # `customer_data` under one record — so the divergence is not visible in the perms + # document and guessing at it once already produced a wrong answer. + row["reason"] = ("the assistant cannot resolve a data grant for it yet — this is a " + "known gap (D-276), not something you have set wrongly") + continue + row["permitted"] = True + # ⚠ A CACHE MISS IS A DATA-AVAILABILITY FACT, NOT A PERMISSION ONE, and it must not + # drop the source from the list — an artefact already built on it stays listed. It is + # reported as not-yet-answerable, with the action that fixes it. MEASURED: the pool is + # filled by a grid REQUEST, not by `AIOS_PREWARM` — a fresh process refuses until + # somebody opens the database once. + team_id, agent = perm_scope.derive_pool_scope(session.user, key) + cache_key = (("pool", team_id, agent) if key == "customer_data" + else ("product_pool", team_id)) + cache = getattr(session.runtime, "pool_cache", None) + entry = cache.get(cache_key) if isinstance(cache, dict) else None + if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)): + row["answerable"] = True + else: + row["reason"] = "no snapshot loaded yet — open this database once, then ask" + continue + if not key.startswith("ut_"): + continue + defs = _definitions() + definition = (defs or {}).get(key) + if not isinstance(definition, dict): + continue + lent = user_tables.lend(session.runtime, user_tables=defs) + if not user_tables.may_open(key, session.uname, session.admin, st=lent): + continue + row["permitted"] = True + if user_tables.materialises(key, st=lent, defn=definition): + row["answerable"] = True + else: + row["reason"] = "served live from its connector mirror — the assistant cannot read it yet" + return out + + def _user_for(claims): """The account named by a verified cookie, or None — with the epoch check that makes a stateless cookie revocable. @@ -110,7 +523,14 @@ def _user_for(claims): # (login validated the slug's existence, never the membership) — with # AIOS_ENABLE_QA_TENANT=1 that was a working cross-tenant session. A signature proves # the cookie is ours; THIS proves the account belongs where the cookie says it does. - if claim_tenant != pub.get("tenant", "royal-imports"): + # ⚠ WAVE 33 (W33-T37) — the default was DELETED, not changed. It read + # `pub.get("tenant", "royal-imports")` and could never fire: `users._public` normalises a + # missing tenant to `royal-imports` before this line ever sees it. A dead default that + # names a real tenant reads like a live fallback, and the cross-tenant audit spent a row + # proving it was not one. Bare `.get` is also the safer shape if that ever stops being + # true: `None != claim_tenant` denies, where the old default would have ADMITTED a + # tenant-less record into tenant #0's session. + if claim_tenant != pub.get("tenant"): return None return pub if (uname == "admin" and os.environ.get("APP_PASSWORD", "") diff --git a/api/main.py b/api/main.py index fce0849c6a534b5c559406db113a7e57924dd6cf..175e0173f243e89cf5f436e877e45e6b6317562b 100644 --- a/api/main.py +++ b/api/main.py @@ -79,7 +79,18 @@ import routes_odoo_tables # noqa: E402 (wave 27 item 17 — Odoo relational; S import routes_connected_tables # noqa: E402 (wave 31 T49/C4 — the source-neutral grid door; D's) import routes_web_agent # noqa: E402 (wave 31 R10/C5 — the web-browsing agent; E's router, A's line) import routes_query # noqa: E402 (wave 32 R1/C5 — the Query module; E's router, A's line) +import routes_publish # noqa: E402 (wave 33 R5/C2 — the publish door; C's router, A's line) +import routes_brand # noqa: E402 (wave 33 R4/C2/C6 — connector brand marks; G's router, A's line) +import routes_slack # noqa: E402 (wave 33 R4/C2 — Manage agent + the Slack door; D's router, A's line) +import routes_starred # noqa: E402 (wave 35 R4/C2/C3/C5 — the star; E's router, E's line) +import routes_usage # noqa: E402 (wave 35 R9/C7 — the AI usage meter; E's router, E's line) +import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operator plane; E's router) from core import grid_events # noqa: E402 +# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below. +# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a +# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not. +from core import data_binding # noqa: E402 +from core import store # noqa: E402 from deps import Session, module_gate # noqa: E402 _WEB_DIST = _HERE.parents[1] / "web" / "dist" @@ -164,7 +175,49 @@ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable): """ return JSONResponse( {"error": {"code": "store_unavailable", - "message": "the tenant store is unavailable — no change was saved"}}, + "message": "the tenant store is unavailable. No change was saved"}}, + status_code=503) + + +@app.exception_handler(data_binding.StoreWriteRefused) +def _store_write_refused(request: Request, exc: data_binding.StoreWriteRefused): + """D-315 — this deployment may not write that store. 503, with the REASON. + + ⛔ IT ECHOES THE EXCEPTION'S OWN MESSAGE, WHERE ITS NEIGHBOUR ABOVE USES A FIXED SENTENCE, and + the difference is the point. "The store is unavailable" is the whole truth about an outage; a + refusal has a cause the operator cannot otherwise discover, because per D-160 the Space + environment cannot be read from outside. A refusal that said only "unavailable" would send + somebody hunting for a network fault that does not exist ([[report-the-cause-before-you-fix-it]]). + The message names the deployment, the store and the fix, and carries no customer data. + + ⛔ AND IT IS ITS OWN HANDLER RATHER THAN A SUBCLASS OF `StoreUnavailable`. Several routes carry + `except StoreUnavailable:` blocks that DEGRADE to a session-scoped fallback workspace — correct + for an outage, catastrophic here, because the user would be told the change was saved. A + distinct type falls through every one of them to this handler. + """ + return JSONResponse( + {"error": {"code": "store_write_refused", + "message": f"this deployment may not write the tenant store. No change was " + f"saved. {exc}"}}, + status_code=503) + + +@app.exception_handler(store.StoreConflict) +def _store_conflict(request: Request, exc: store.StoreConflict): + """D-305 — the document moved under this write more times than it could be rebased. 503. + + ⚠ 503 RATHER THAN 409, DELIBERATELY. A 409 invites the client to resolve a conflict, and there + is nothing here for it to resolve: the store already re-read and re-applied this change up to + `_MAX_REBASE` times before giving up. What the caller needs to know is that the write did not + land and nothing was damaged, which is the same contract as the outage above and the same + retry-later shape. ⛔ The one answer this must never be is 200: reaching this handler means the + only way to have "succeeded" was to overwrite somebody else's work. + """ + return JSONResponse( + {"error": {"code": "store_conflict", + "message": f"somebody else changed this database while your change was being " + f"saved, and it could not be merged. Nothing was saved or " + f"overwritten. Try again. {exc}"}}, status_code=503) @@ -269,6 +322,60 @@ app.include_router(routes_web_agent.router) # R10 / C5 — E's router, A's lin # ⚠ `verify_api` asserts `/api/v1/query` in `app.openapi()["paths"]` — NEVER `{r.path for r in # app.routes}`, which finds nothing in this app because FastAPI wraps included routers (W31). app.include_router(routes_query.router) # R1 / C5 — E's router, A's line +# ⭐⭐ WAVE 33 (C2) — THE SIXTH CONSECUTIVE WAVE IN WHICH THIS BLOCK IS THE THING THE PROTOCOL LOSES. +# Contract C2 is written for exactly that: a lane creating a `routes_*.py` posts an ASK, and the +# INTEGRATOR adds the line in the SAME wave. Both of these arrived that way (C's `ASK C-1`, G's +# `ASK G-1`), and `verify_api::section_w23_mounts` asserts each path in `app.openapi()["paths"]` +# with an NC that comments a mount out and goes RED. +# ⚠ Same placement rule as the line above — ABOVE the `app.mount("/", _AppStatic(...), html=True)` +# at the end of the file, never after it. +# ⭐⭐ THE PUBLISH DOOR IS MOUNTED AGAIN (2026-08-15). It was withheld for the wave-33 deploy +# because QA reproduced three HIGH defects that are armed ONLY by mounting it. All three are fixed +# in `routes_publish.py`, each at its cause rather than at its symptom: +# 1. W33-T68 — `form` LEFT `PUBLISHABLE_MODES`. A form view's rows ARE the submissions people +# sent it, so publishing one served other people's answers to anyone holding the link. A form +# still has its own public door (`#/form/`) which serves the BLANK form and never rows. +# 2. W33-T69 — `_visible_keys` returns `[]` when a view STORED a `visible` list and none of its +# keys survive, instead of falling back to the table default. The fallback still applies to a +# view that never stored one, which is what it was written for. +# 3. W33-T70 — three separate leaks closed: the rate limit no longer keys on the caller-supplied +# `x-forwarded-for` (it keys on the socket peer and counts FAILURES only, so a shared proxy +# peer cannot become one global bucket); every failing path now spends the same PBKDF2 the +# success path spends, closing the 206x timing gap; and an unknown token on the sibling GET +# answers the LOCKED shape rather than a 403, so that route stops sorting real tokens from +# fake ones for free. +# ⚠ `verify_api::section_w23_mounts` asserts this path in `app.openapi()["paths"]`, so its four +# EXPECTED reds should now go GREEN. A red here after this line means the mount broke, not the gate. +app.include_router(routes_publish.router) # R5 / C2 — C's router, A's line (the publish door) +# ⛔ NOT BEHIND `module_gate("product_data")`, WHICH IS THE WHOLE REASON IT IS A SECOND ASSET DOOR. +# `routes_assets.py` gates EVERY one of its routes on that module, so a connector logo served from +# there would 403 for any account without the product-data grant — i.e. the Connectors directory +# would lose its logos for exactly the accounts most likely to be setting a connector up (C6). +app.include_router(routes_brand.router) # R4 / C2 / C6 — G's router, A's line (brand marks) +# ⛔ TWO OF ITS SIX PATHS ARE UNAUTHENTICATED BY DESIGN (`/slack/events`, `/slack/interact`) and +# that is the DOOR, not an omission — Slack posts to them with no session and could not carry one. +# They are built on `routes_forms`' proven public shape: signing-secret verification, a sliding-window +# rate limit, a body cap and ONE non-oracle 403, so a bad signature cannot distinguish "no such +# workspace" from "not yours". ⚠ There is no auth middleware and no exempt-path allow-list in this +# file to register them in: "public" here IS the absence of `Depends(require_session)`, which is why +# `verify_api` asserts the absence rather than an entry in a list that does not exist. +app.include_router(routes_slack.router) # R4 / C2 — D's router, A's line (Manage agent + Slack) +# ⭐⭐ WAVE 35 (R4/R5/R7, contracts C2/C3/C5) — THE STAR. Mounted in the SAME change that created +# `routes_starred.py`, which is the seventh consecutive wave in which this block is the artefact the +# protocol nearly loses — and the first in which the router's own lane also owns `main.py`, so there +# is no ask/mount pair to drop. `W35-T46` asserts every one of this wave's paths in +# `app.openapi()["paths"]` and CALLS one route from each. +# ⚠ Same placement rule as every line above: ABOVE `app.mount("/", _AppStatic(...), html=True)`, or +# the router answers 404 on GET and 405 on POST while every one of its own tests passes. +app.include_router(routes_starred.router) # R4 / C2 — the star, counts, and record stars +# ⛔ R9 SAYS THIS ROUTE REPORTS AND NEVER ENFORCES, so mounting it cannot cut anybody off — there is +# no ceiling anywhere behind it (`usage_ledger` has no refusal in it). The one AI limit the product +# enforces is per COLUMN and is unchanged (`ai_enrich.ceiling_report`). +app.include_router(routes_usage.router) # R9 / C7 — GET /usage, the one AI meter +# ⛔ ITS TWO DOORS CARRY DIFFERENT WALLS ON PURPOSE (R8, and D-221 is the booked precedent): the +# POST is any authenticated session's own act, the GET is `is_platform_admin` only. Mounting it does +# not widen anything a tenant admin can reach — `verify_api` proves that by having one try. +app.include_router(routes_feedback.router) # R8 / C6 — feedback to the operator plane # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working) @@ -302,18 +409,18 @@ def _assert_contract_floor(): if [f["key"] for f in base] != [f["key"] for f in FIELDS]: raise RuntimeError( "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) " - "disagree on the base field set — embed and standalone would serve different schemas. " + "disagree on the base field set. Embed and standalone would serve different schemas. " "Run aios-web/verify_fields_contract.py.") def _startup_notes(): """Say the two things an operator must know, once, at import — never in a response body.""" if aios_session.EPHEMERAL_SECRET: - print("[aios-api] AIOS_SESSION_SECRET is not set — signing with a random per-process " + print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process " "key. Sessions will not survive a restart and will not work across workers. " "Set it in production.") if os.environ.get("AIOS_INSECURE_SSL") == "1": - print("[aios-api] AIOS_INSECURE_SSL=1 — TLS verification is DISABLED for outbound " + print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound " "requests. Local development only; never in a deployed environment.") @@ -431,6 +538,12 @@ def _seed_and_sync_store(): # is D-107's own hypothesis 3: a partial population trips `MAX_SHRINK` and the rebuild # refuses — correctly, but for a reason that reads like a data loss scare. # ⚠ Same thread on purpose: it is already a daemon and nothing serves requests behind it. + # ⭐⭐ W35-T45 / R11 — TENANT #0'S ODOO CREDENTIAL MOVES ONTO ITS KEYCHAIN, IN THE CONTAINER. + # ⚠ BEFORE the relational rebuild, deliberately: the rebuild resolves its connector through + # `rt.odoo_source()`, so running the migration first means the very next read already goes + # through the keychain branch and a broken migration is visible in THIS boot's log rather + # than in tomorrow's. It is idempotent, so every later boot is one `list_entries` read. + _migrate_env_odoo("boot") _pull_meta("boot") _rebuild_odoo_relational("boot") # ⭐⭐ W32-T07 — owner items 13 and 15, DELIVERED. See `_sweep_automation_schemas`. @@ -442,6 +555,35 @@ def _seed_and_sync_store(): print(f"[aios-api] datastore seed/sync skipped: {e}") +def _migrate_env_odoo(why): + """⭐⭐ W35-T45 / R11 — move tenant #0's environment Odoo credential onto its keychain. + + ⛔⛔ IN THE CONTAINER, WHICH IS THE WHOLE REASON THIS IS A BOOT LINE AND NOT A SCRIPT. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and the write REPORTS SUCCESS + every time, then a fresh read confirms it, and it is gone by the next poll. A CLI migration would + be a dry run that lies, and what it would lie about here is a credential. + + ⚠ IDEMPOTENT AND SCOPED TO TENANT #0 by `routes_keychain.env_odoo_available`, so on every other + tenant and on every later boot this is one `list_entries` read and a line. + ⚠ FAIL-QUIET: this must never take a boot down. But it is never SILENT — a skip prints its reason, + because "already migrated", "no keychain key on this deployment" and "the env is incomplete" are + three different operator actions and a blank Keychain page cannot tell them apart. + """ + try: + import routes_keychain as _kc_routes + from harness import runtime as _runtime + rep = _kc_routes.migrate_env_odoo(_runtime.get_runtime("royal-imports")) + if rep.get("done"): + print(f"[aios-api] odoo credential migrated onto the keychain ({why}): " + f"entry={rep['entry']} carried_pause={rep['carried_pause']}" + + (f" PROBLEM: {rep['why']}" if rep.get("why") else "")) + else: + print(f"[aios-api] odoo keychain migration skipped ({why}): {rep.get('why')}") + except Exception as e: # noqa: BLE001 + print(f"[aios-api] odoo keychain migration FAILED ({why}): {type(e).__name__}: {e}") + + def _pull_meta(why): """Pull Meta Ads into THIS container's mirror, before the relational rebuild reads it. @@ -546,18 +688,18 @@ def _sweep_automation_schemas(why): import automation_engine as _eng from harness import runtime as _runtime except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED — import failed: {e}") + print(f"[aios-api] schema sweep ({why}) SKIPPED. Import failed: {e}") return try: schemas = _eng.platform_schemas() except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED — no declarations: {e}") + print(f"[aios-api] schema sweep ({why}) SKIPPED. No declarations: {e}") return tenants = [] try: tenants = _runtime.known_tenants() except Exception as e: # noqa: BLE001 - print(f"[aios-api] schema sweep ({why}) SKIPPED — tenant list unreadable: {e}") + print(f"[aios-api] schema sweep ({why}) SKIPPED. Tenant list unreadable: {e}") return for slug in tenants: try: @@ -599,7 +741,7 @@ def _sweep_automation_schemas(why): # which is how the previous behaviour survived review in the first place. print(f"[aios-api] schema sweep ({why}) {slug}: {len(skipped)} child table(s) not " f"present in this tenant, so nothing was created for them " - f"({', '.join(sorted(skipped))}) — they are spawned by an automation that " + f"({', '.join(sorted(skipped))}). They are spawned by an automation that " f"needs them, never by this sweep") # ⭐ THE RETRACTION (D's `retract_foreign_presets`, D-152), AFTER the children loop. # ⛔ THE SWEEP ABOVE MAKES COLUMNS **ARRIVE** AND CANNOT MAKE STALE ONES **LEAVE**, and @@ -714,11 +856,31 @@ def _store_resync_loop(): # next cycle rather than at the next container restart, which is what makes it a setting. # ⚠ FAIL-SAFE TO 1800 — an unreadable config must not become a tight loop. The floor is # enforced in `sync_seconds` as well as at the write door, for the same reason. + # ⛔⛔ WAVE 34 · W34-T46 / D-246 — THE PARAGRAPH ABOVE DESCRIBED A BRANCH THAT DID NOT + # EXIST, AND THAT IS WHY NOBODY LOOKED. It said `manual` "parks at the default cadence + # instead and simply DOES NO WORK". The first half was true; the second half was not + # implemented anywhere: `_secs is None` set the sleep to 1800 and then fell into an + # UNCONDITIONAL `sync_all()`. So a tenant who chose "manual" was synced every thirty + # minutes exactly like everyone else, with the setting stored, displayed, clamped and + # obeyed by nothing — the identical failure this block's own W32/R11 note is about, one + # layer further in. A comment asserting a fix is why the second reader stops reading. + # ⚠ FIXED, AND THE FIX IS NARROW ON PURPOSE. `manual` is an ODOO connector setting, so it + # skips the Odoo mirror sync and its reconcile and NOTHING ELSE: the Meta pull and the + # relational rebuild below still run, because a tenant asking not to have Odoo polled has + # not asked for the other connector to stop. + # ⚠ AND IT STILL SLEEPS AND STILL LOOPS. The config is re-read every pass, so switching + # back off `manual` takes effect on the next cycle rather than at the next restart. _every = 1800 + _manual = False try: import odoo_relational as _rel_cad from harness import runtime as _rt_cad + # ⚠ D-245, KNOWN AND NOT FIXED HERE: the slug is hardcoded, so this reads TENANT #0's + # cadence and applies it to a loop that syncs the shared mirror. Correct while one + # tenant has Odoo; wrong the moment a second one does. Out of this ticket's scope and + # named rather than silently inherited. _secs = _rel_cad.sync_seconds(_rt_cad.get_runtime("royal-imports")) + _manual = _secs is None _every = 1800 if _secs is None else max(int(_secs), 60) except Exception: # noqa: BLE001 pass @@ -731,12 +893,19 @@ def _store_resync_loop(): _odoo._tlocal.client = _odoo.OdooClient() except Exception: pass - _ds.sync_all(log=lambda *a, **k: None) - # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot - # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; - # without it deleted Odoo lines inflate every sum on the mirror FOREVER. - if passes % 4 == 0: - _ds.reconcile_deletes(log=lambda *a, **k: None) + # ⚠ A PLAIN BRANCH, NOT AN EARLY EXIT AND NOT AN EXCEPTION. A `raise` here would be + # caught by this block's own `except` and printed as "store resync failed", turning a + # setting a person chose into a recurring error in the log; a `continue` would skip + # the Meta pull and the relational rebuild below, which `manual` says nothing about. + if not _manual: + _ds.sync_all(log=lambda *a, **k: None) + # Wave 21 — every 4th pass (~2h): purge hard-deleted rows the cursor sync cannot + # see (the boot pass's comment has the measured case). Cheap id-sweep per entity; + # without it deleted Odoo lines inflate every sum on the mirror FOREVER. + # ⚠ INSIDE the branch: a reconcile is a pass over the mirror this loop was just + # told not to advance. + if passes % 4 == 0: + _ds.reconcile_deletes(log=lambda *a, **k: None) except Exception as e: # noqa: BLE001 print(f"[aios-api] store resync failed: {e}") # ⭐ WAVE 27 item 17 (E's ASK ->A, resolved): the Odoo RELATIONAL tables are rebuilt diff --git a/api/odoo_relational.py b/api/odoo_relational.py index f28a4fc365e17ecc722663b748df5af48148fb5e..9a2f2aba3fceb4b2d70b7827afec3b38d4f34928 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -117,6 +117,13 @@ def _ut(): return user_tables +def _registry(): + """`core.registry`, imported the same lazy way `_ut` is — this module is imported by the + route layer before `platform/` is necessarily on the path.""" + import core.registry as registry + return registry + + def _iso_today(): return _dt.date.today().strftime("%Y-%m-%d") @@ -180,9 +187,14 @@ def agent_fields(): "source": "overlay", "default": True}, # ⭐ THE INVERSE HALF: the customers whose `agent_id` names this agent. MEASURED: 2,093 # customers carry one and ALL 2,093 resolve to a row in this table (zero dangling). - {"key": "customers", "label": "Customers", "type": "link", "source": "overlay", - "default": True, - "link": {"table": CUSTOMERS_KEY, "on": AGENT_JOIN_KEY, "from": AGENT_JOIN_KEY}}, + # ⛔ W33-T43 / AMENDMENT A2 — the `customers` REVERSE link DELETED. See `invoice_fields` + # for the ruling. ⚠ This one costs more than the other two and the difference is worth + # recording: those were a click-through to a record whose id stays on the row, while this + # was an agent's BOOK — the list of customers assigned to them. The id side survives + # (`agent_id` here, and `agent_id` on `customer_data`), so the relationship is intact in + # the data and only the rendered list is gone; the same question is answerable on the + # customer grid by filtering `agent_id`, and at analytical grain via the `agent` dim on + # `sales_lines` / `sales_orders`. _refreshed_field(), )] @@ -237,8 +249,34 @@ def vendor_fields(): "default": False, "description": "The `res.partner` id. Also this row's id."}, {"key": VENDOR_JOIN_KEY, "label": "Odoo vendor id", "type": "int", "source": "overlay", "default": False}, + # ⭐⭐ W33-T48 / owner item 13 — the columns the census proved are there and were not shown. + # ⚠ `default` is deliberately split: the ones a vendor list is READ for (contact + tax id) + # arrive visible; the postal lines arrive HIDDEN, because four address columns turned on by + # default would push the useful ones off the first screen, and DESIGN.md's "never + # over-explain" applies to columns as much as to prose. Every one is one click away in the + # field picker, and a locked database still allows fields (item 4's vocabulary). {"key": "country", "label": "Country", "type": "text", "source": "overlay", "default": True}, + {"key": "email", "label": "Email", "type": "text", "source": "overlay", "default": True}, + {"key": "phone", "label": "Phone", "type": "text", "source": "overlay", "default": True}, + {"key": "mobile", "label": "Mobile", "type": "text", "source": "overlay", + "default": False}, + {"key": "vat", "label": "Tax ID", "type": "text", "source": "overlay", "default": True, + "description": "Odoo `vat` — the vendor's tax/VAT registration number."}, + # ⚠ `vendor_ref`, not `ref`: `ref` is Odoo's own name for it, and the bills grid already + # uses `ref` for the VENDOR'S INVOICE NUMBER on a document. Two different facts, and a + # shared spelling across two linked grids is how a rollup ends up summing the wrong column. + {"key": "vendor_ref", "label": "Vendor reference", "type": "text", "source": "overlay", + "default": False, + "description": "Odoo `res.partner.ref` — our internal reference for this vendor."}, + {"key": "website", "label": "Website", "type": "url", "source": "overlay", + "default": False}, + {"key": "street", "label": "Street", "type": "text", "source": "overlay", + "default": False}, + {"key": "street2", "label": "Street 2", "type": "text", "source": "overlay", + "default": False}, + {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": False}, + {"key": "zip", "label": "ZIP", "type": "text", "source": "overlay", "default": False}, {"key": "bills", "label": "Bills", "type": "link", "source": "overlay", "default": True, "link": {"table": BILLS_KEY, "on": VENDOR_JOIN_KEY, "from": VENDOR_JOIN_KEY}}, _refreshed_field(), @@ -316,9 +354,21 @@ def invoice_fields(): _scope_field(), # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like # its twin, so the engine owns the cell and no human can edit a relation Odoo decided. - {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # ⛔⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED, and the loss is stated here rather + # than left to be inferred from a green gate. + # + # It pointed at `ut_odoo_customers`, which W33-T44 retires (R2: one identity per subject). + # A2 ruled OPTION 2 — retire both twins, DROP the three link columns, keep every data + # column — so this grid loses the CLICK-THROUGH to a customer record and NOTHING else: + # `customer` (the name) and `partner_id` (the Odoo id) are plain columns on this same row, + # and `customer_data` now carries `partner_id` too, so both ends of the join still exist. + # + # ⛔ THE ALTERNATIVE WAS REFUSED IN WRITING, and the reason belongs beside the deletion: + # re-pointing this bag at `customer_data` needs `core/user_tables.py::_clean_link`'s `ut_` + # prefix test relaxed — which makes EVERY GATE GREEN while + # `automation_engine::compute_relation_cells` still resolves out of the `user_tables` + # document alone and returns `{}`. Empty cells, blank rollups, nothing red. A2 forbids + # touching that line this wave for exactly that reason. # ⭐ DEBT D-88, closed 2026-08-09. `invoice_origin` carries the ORDER NAME an invoice was # raised from, and the mirror did not sync it until this wave — so order->invoice was a # two-hop join through 963,783 `account_move_line` rows and wave 27 shipped no link at all. @@ -358,9 +408,9 @@ def order_fields(): {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay", "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]}, _scope_field(), - {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay", - "default": False, - "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}}, + # ⛔ W33-T43 / AMENDMENT A2 — `customer_link` DELETED here too; see the note on the same + # column in `invoice_fields` above for the ruling and the refused alternative. This row + # keeps `customer` and `partner_id`, so only the click-through is gone. # The reciprocal of the invoice's `order_link` (D-88): the invoices raised from THIS # order, matched on the order's own name. {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay", @@ -1071,18 +1121,44 @@ def read_vendors(cur): have = columns(cur, "res_partner") if not have or not columns(cur, "account_move"): return [] - sql = (f"SELECT p.id, p.name, {_col(have, 'p.country_name', chr(39) + chr(39))} " + # ⭐⭐ W33-T48 (owner item 13). The grid served FIVE columns off a partner the census measured + # at SEVENTY-SIX populated fields. The nine added here are the ones a buyer actually asks a + # vendor record for — how to reach them, who they are for tax, and where they are. + # ⛔ EVERY ONE GOES THROUGH `_col`, which substitutes a literal when the mirror lacks the + # column. That is not defensive habit: this projection has to keep working against a mirror + # that has not been re-synced since the widening, and the alternative is a reader that raises + # on the exact box the widening was meant to help. The columns arrive when the backfill runs; + # until then these read blank rather than failing. + blank = chr(39) + chr(39) + sql = ("SELECT p.id, p.name, " + f"{_col(have, 'p.country_name', blank)}, {_col(have, 'p.email', blank)}, " + f"{_col(have, 'p.phone', blank)}, {_col(have, 'p.mobile', blank)}, " + f"{_col(have, 'p.website', blank)}, {_col(have, 'p.vat', blank)}, " + f"{_col(have, 'p.ref', blank)}, {_col(have, 'p.street', blank)}, " + f"{_col(have, 'p.street2', blank)}, {_col(have, 'p.city', blank)}, " + f"{_col(have, 'p.zip', blank)} " "FROM res_partner p WHERE p.id IN " f" (SELECT partner_id FROM account_move WHERE {_VENDOR_DOCS} " " AND partner_id IS NOT NULL)") out = [] - for (pid, name, country) in cur.execute(sql).fetchall(): + for (pid, name, country, email, phone, mobile, website, vat, ref, + street, street2, city, zipc) in cur.execute(sql).fetchall(): out.append({ "_id": str(pid), "vendor": str(name or ""), "odoo_id": int(pid), VENDOR_JOIN_KEY: int(pid), "country": str(country or ""), + "email": str(email or ""), + "phone": str(phone or ""), + "mobile": str(mobile or ""), + "website": str(website or ""), + "vat": str(vat or ""), + "vendor_ref": str(ref or ""), + "street": str(street or ""), + "street2": str(street2 or ""), + "city": str(city or ""), + "zip": str(zipc or ""), }) return out @@ -1118,8 +1194,14 @@ class Refused(Exception): #: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code. #: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits. TABLES = ( - ("customers", CUSTOMERS_KEY, "Odoo customers", customer_fields), - ("products", PRODUCTS_KEY, "Odoo products", product_fields), + # ⛔⛔ `customers` AND `products` ARE GONE FROM THIS TUPLE — W33-T44 / owner item 12 / + # AMENDMENT A2. They presented the same SUBJECTS as the compiled registry modules + # `customer_data` and `product_data` ("why do we have 'Odoo products' already with the current + # Products database? The Unique ID is redundant"), and R2 ruled the LEGACY key survives: it + # keeps its store bucket, so no saved view, grant, cohort or formula moves, and it now carries + # the twins' join keys (`partner_id`; `product_id` pending the ask in `mailbox/E.md`). + # Dropping the row here is what stops them being planned, built or re-created; + # `RETIRED_KEYS` below is what removes the rows a tenant already has. ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields), ("orders", ORDERS_KEY, "Odoo orders", order_fields), # ⭐ WAVE 28 / R1. Measured populations: 19 / 192 / 6,538 / 393 — every one of them two orders @@ -1139,6 +1221,66 @@ TABLES = ( ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), ) +#: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` +#: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. +#: +#: ⛔ A SEPARATE MAP RATHER THAN A FIFTH ELEMENT ON EACH `TABLES` ROW, and that is not tidiness: +#: six sites in this file and two in the gate unpack `for bucket, key, _l, _f in TABLES`, so +#: widening the tuple is eight edits that all fail loudly at once and one — in a gate — that +#: would fail QUIETLY, having already been rewritten to match. The subject is a fact ABOUT the +#: key, and this is the shape that says so. +#: +#: ⚠ THE THREE res.partner SUBSETS ARE THREE SUBJECTS, NOT ONE. Customers, agents and vendors all +#: read `res_partner`, and they are different POPULATIONS of it — the claim is over who is in the +#: database, never over which Odoo model was queried. Same for `account_move`, which is the +#: customer-invoice book under one WHERE and the vendor-bill book under another. +TABLE_SUBJECTS = { + # ⛔ `CUSTOMERS_KEY` and `PRODUCTS_KEY` are absent — W33-T44 retired them, and their subjects + # (`odoo:res.partner`, `odoo:product.product`) are claimed by `core.registry`'s `customer_data` + # and `product_data` rows, which is now the ONLY claim on each. That is item 12 satisfied: one + # subject, one database, and `subject_conflict` would REFUSE either of these keys if a future + # spec row tried to bring it back. + INVOICES_KEY: "odoo:account.move.customer", + ORDERS_KEY: "odoo:sale.order", + AGENTS_KEY: "odoo:res.partner.agent", + ACCOUNTS_KEY: "odoo:account.account", + VENDORS_KEY: "odoo:res.partner.vendor", + BILLS_KEY: "odoo:account.move.vendor", + ORDER_LINES_KEY: "odoo:sale.order.line", + GL_LINES_KEY: "odoo:account.move.line", +} + +#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS. +#: +#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the +#: definitions and rows already sitting in a tenant's `user_tables` document, which is what the +#: owner actually sees in the nav. This set is what removes them, and it is applied inside +#: `apply_plan`'s single atomic updater — i.e. BY THE CONTAINER, on the boot rebuild and every +#: resync. +#: +#: ⛔ IT MUST BE THE CONTAINER AND NOT A CLI, AND THIS IS MEASURED, NOT CAUTIOUS (D-195): a +#: developer's script CAN write the tenant store, the write returns clean, a fresh read confirms +#: it — and the running Space reverts it within a minute, because the container holds the document +#: and re-uploads its own copy (download-modify-upload, last write wins). A removal shipped as a +#: script is a dry run that reports success. +#: +#: ⛔ AND A SWEEP THAT DELIVERS MUST NOT CREATE. Wave 32's `ut_ensure` was handed a merge-only job +#: and minted 8 empty databases in every tenant, because the door it used creates when absent. This +#: is a `pop`, it runs only over keys already present, and the gate asserts BOTH halves — removed, +#: AND not re-created on the next pass. Gating only the removal would pass on a tree that deletes +#: and re-adds the table every 30 minutes. +RETIRED_KEYS = (CUSTOMERS_KEY, PRODUCTS_KEY) + +#: ⛔ THE GRANDFATHER LIST IS DELETED — W33-T44 did what it was written to force. +#: +#: It existed for exactly one wave, to keep the product runnable between W33-T41 (the uniqueness +#: check) and W33-T44 (the retirement): the check is CORRECT and the collision it forbids was +#: LIVE, so without a named exemption the spawn refused on any fresh document and tenant #0 wrote +#: nothing at all. The exemption was ratcheted BOTH ways — a collision outside it was a new +#: duplicate, a member that stopped colliding was a stale exemption — so retiring the twins turned +#: the gate RED until this constant went with them. It did, and that is the ratchet working. +#: `verify_odoo_relational::_prove_subject_uniqueness` now asserts the collision set is EMPTY. + #: The bucket -> reader map. ⛔ ITS ABSENCES ARE LOAD-BEARING: a bucket with no reader has no #: python row builder ANYWHERE, which is what makes "never materialised" structural rather than a #: policy `plan()` could forget. The two line grains are absent for that reason and no other. @@ -1412,7 +1554,7 @@ def plan(cur, rt=None): return built -def apply_plan(rt, built, username="automation", today=None): +def apply_plan(rt, built, username="automation", today=None, report=None): """Create-or-merge every table in `built` and its rows. Idempotent by construction. Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the @@ -1425,6 +1567,9 @@ def apply_plan(rt, built, username="automation", today=None): raise Refused("; ".join(built["problems"])) stamp = today or _iso_today() written = {} + # ⚠ An OPTIONAL out-parameter, not a return-shape change: `written` has one value shape and + # keeps it. A caller that wants to SAY what was retired passes a dict; `refresh` does. + retired = [] plans = [(key, label, fields(), built[bucket]) for bucket, key, label, fields in TABLES if bucket in built] @@ -1442,11 +1587,29 @@ def apply_plan(rt, built, username="automation", today=None): # create. Inside one updater, a raise aborts before anything is persisted. def _apply_all(cur): cur = cur if isinstance(cur, dict) else {} + # ⛔⛔ W33-T44 — THE RETIREMENT, INSIDE THE SAME ATOMIC WRITE. This updater is the ONE + # sync write the whole spawn makes, and it runs in the CONTAINER (boot rebuild + every + # resync), which is the only place a change to the tenant document survives (D-195: the + # same edit from a CLI reports success and is reverted within a minute). + # ⚠ It POPS and never creates: `RETIRED_KEYS` are absent from `TABLES`, so `plans` cannot + # contain them, and there is no door here that could re-add one. `removed` is reported so + # a caller can SAY what happened rather than infer it from a table going missing. + # ⛔ THE REMOVED KEYS DO **NOT** GO INTO `written`, and the first cut of this put them + # there. `written` maps table key -> a COUNTS dict, and every consumer iterates its + # `.values()` expecting `c["added"]`; a list under `"_retired"` made the very next leg die + # with `TypeError: list indices must be integers`. One dict, two value shapes, is a + # sentinel in a result set [[sentinel-in-a-sort-key]] — so the retirement reports through + # its OWN channel and the return type stays exactly what it was. + for key in RETIRED_KEYS: + if cur.pop(key, None) is not None: + retired.append(key) for key, label, fields, rows in plans: written[key] = _ensure_table_inplace(cur, key, label, fields, rows, username, stamp) return cur rt.update(_ut().STORE_KEY, _apply_all, flush="sync") + if report is not None and retired: + report["retired"] = sorted(retired) return written @@ -1478,12 +1641,37 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): for row in wanted.values(): row["refreshed"] = stamp counts = {"added": 0, "updated": 0, "removed": 0, "rows": len(wanted)} + subject = TABLE_SUBJECTS.get(key) table = cur.get(key) if table is None: if len(cur) >= ut.MAX_TABLES: # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a # successful refresh over a table that does not exist. raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created") + # ⭐⭐ W33-T41 / item 12a — THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE. + # + # ⛔ IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This + # function is the boot rebuild and the 1800 s resync; it adopts an existing table by key + # on every pass. A claim test outside this `if` refuses the table it created last boot, + # `_apply_all` raises inside the updater, and tenant #0 spawns NOTHING — the check would + # take the product down to prevent a duplicate that already exists. Refusing the SECOND + # birth is what "make sure this never happens" asks for; the FIRST one is T44's job to + # remove, and it is removed by dropping its `TABLES` row, not by a guard here. + # + # ⚠ Claims are gathered from THIS TENANT'S document (`cur`) plus the compiled registry. + # Never a module-level cache: one Space process serves every tenant, and two tenants both + # holding an "Odoo customers" is correct (that is D-169's shape, and it is not repeated + # here). A stored definition with no `subject` claims nothing. + held = {t["subject"]: k for k, t in cur.items() + if isinstance(t, dict) and t.get("subject")} + # ⭐ NO EXEMPTION ANY MORE. The grandfather list is deleted with the two tables it covered + # (W33-T44), so this is now the plain rule the owner asked for: one subject, one database. + other = _registry().subject_conflict(subject, key, claimed=held) + if other: + raise Refused( + f"{key}: refusing to create a second database for {subject!r} — {other!r} " + f"already presents it. One subject, one database (item 12a); if this table is " + f"meant to replace {other!r}, retire {other!r} first rather than shipping both") table = cur[key] = { "key": key, "label": label, "source": ut.AUTOMATION_SOURCE, "createdBy": username, "created": stamp, "fields": [], "rows": {}, @@ -1499,8 +1687,33 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp): # an empty grid. if key in READ_THROUGH_KEYS: table["readThrough"] = True + # W33-T41 — the claim, made DURABLE on the definition. Written on every pass for the same + # reason `readThrough` is: it is derived from this module's own structure, so the code is what + # it has to agree with, and a definition that lost the stamp should regain it rather than go + # on being invisible to the next table's claim test. ⚠ It is also the ONLY way the check sees + # a `ut_*` table at all — those are per-tenant DATA, never compiled registry rows, so a cold + # process reading a fresh document has nothing else to read the claim off. + if subject: + table["subject"] = subject have = {str(f.get("key")): f for f in (table.get("fields") or [])} for field in fields: + # ⛔⛔ THE ONE DOOR THAT BYPASSES THE FIELD VALIDATOR, HARDENED WHERE IT BYPASSES IT. + # + # `user_tables._clean_field` stamps `source: 'overlay'` unconditionally on create AND + # patch, so every field that goes through the normal door has one. This function does NOT + # go through that door — it writes definitions straight into the document — and + # `aios_grid.rows_from_pool` reads `field["source"]` as a HARD KEY, so a contract that + # ever omitted it would 500 the entire rows route rather than degrade one column. Measured + # today across 9 automation-owned tables and 173 fields: zero are missing it, so this is + # LATENT, not live (A's PENDING row, raised as D-9 and corrected by D-23 — the crash that + # prompted it came from a hand-written test fixture, not from production). + # + # ⚠ Fixed HERE rather than by softening `rows_from_pool` to `.get`, deliberately: a + # missing `source` means the definition does not say which stratum owns the column, and + # rendering it as blank would bury that. The default matches what the validator would + # have stamped, so the bypass stops being a hole without inventing a second rule. + if isinstance(field, dict) and not field.get("source"): + field = {**field, "source": "overlay"} fkey = str(field.get("key")) if fkey not in have: table.setdefault("fields", []).append(dict(field)) @@ -1599,8 +1812,13 @@ def refresh(rt, tenant, username="automation", cur=None, today=None): built.pop(bucket, None) else: skipped = [] - written = apply_plan(rt, built, username=username, today=today) + # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing. + # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no + # longer built) SAYS SO, with the keys. + plan_report = {} + written = apply_plan(rt, built, username=username, today=today, report=plan_report) return {"tables": written, + **({"retired": plan_report["retired"]} if plan_report.get("retired") else {}), # R6's second sentence: a set that was deliberately not built SAYS SO, with the keys. **({"skipped": skipped} if skipped else {}), **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES if bucket in built}} diff --git a/api/routes_admin.py b/api/routes_admin.py index b588db85769adb835b18d15c99734329b084a1f4..06206f5ac37f2853b507004197e0a80b57df6bcb 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -356,7 +356,78 @@ def set_password(username: str, body: dict = Body(default=None), #: so a migrated account would be locked out of Product with no control anywhere to grant it: #: fail-closed, but unadministrable. The editor grows its section with zero client edits #: (S3 built it off this route's `modules` + `fields_by_module`). -_PERM_MODULES = ("customer_data", "product_data") +#: +#: ⭐⭐ WAVE 33 (owner item 11, W33-T33) — **THIS IS A CATALOGUE, NOT AN ANSWER.** It used to be +#: both, and that was the defect the owner flagged four times: `get_perms` served this tuple +#: verbatim and never touched `session.runtime`, so EVERY tenant was handed tenant #0's two grid +#: modules. What this names now is a fact about THIS FILE — the registry topics `_module_fields` +#: below has a field-schema arm for — and the answer a tenant receives is `_perm_modules(session)`, +#: which filters this by the tenant's own provisioning and merges the tenant's own `ut_*` +#: databases. ⛔ Serving this list directly again re-opens item 11; `verify_perm_scope`'s +#: `section_tenant_derived` NC does exactly that and reds. +#: ⭐ CONTRACT C3, CONSUMED (lane E's `W33-T46`, answered as `E-3`). The topic list is DERIVED from +#: `registry.governable_modules()` — `{topic: {'key','label','subject'}}` over every non-archived, +#: non-group_only registry row that declares a `topic`. What stays hand-written is +#: `_FIELD_PROVIDER_KEYS` below, and that is CODE, not policy: each name has its own `aios_grid` +#: contract behind it in `_module_fields`, and a topic with no arm cannot be governed by this +#: editor at all. The intersection is the honest answer to "what can this module build pickers for". +#: +#: ⚠ KEPT AS A SYMBOL RATHER THAN DELETED, deliberately. `verify_api`'s W30a reads +#: `routes_admin._PERM_MODULES` to assert `perm_scope.BU_FILTERABLE_MODULES` names exactly the +#: governed topics whose contract carries `dba`. Deleting the name would make that gate raise +#: `AttributeError` — a CRASH, which scores as "the gate is broken" rather than as a red +#: [[gate-must-go-red-not-crash]] — and `verify_api.py` is lane A's fence. Derived-and-kept gives +#: that assertion a truer subject than the literal ever did, and costs nobody a cross-fence edit. +_FIELD_PROVIDER_KEYS = frozenset({"customer_data", "product_data"}) + + +def _governable_catalogue(): + """The REGISTRY topics this module can govern: `[{key, label}]`, in registry order. + + ⛔ No tenant filtering here — that is `perms.tenant_governable_modules`' job, and E's map is + tenant-blind by design. Two filters in two files answering one question is how the perms route + and `routes_nav` came apart in the first place. + """ + import core.registry as registry + return [{"key": v["key"], "label": v["label"]} + for v in registry.governable_modules().values() + if v.get("key") in _FIELD_PROVIDER_KEYS] + + +_PERM_MODULES = tuple(m["key"] for m in _governable_catalogue()) + + +def _perm_modules(session): + """The databases THIS tenant's permission editor governs: `[{key, label, enforced}]`. + + Three inputs, none of them a literal: the topic catalogue above, the tenant's provisioned + module set (`tenant.config['modules']`, exactly the rule `routes_nav.py::nav` applies), and + the tenant's own `ut_*` databases read through the DEFINITIONS projection. + + ⭐ `user_tables.nav_entries` rather than a fresh listing, deliberately: it is already the ONE + resolver for "which databases may this account see", it is `may_open`-filtered, and it reads + the projected document (`all_defs`) so this costs definitions, never the 28.6 MB of rows. + The caller is admin-gated, so the filter admits the whole tenant — but asking the resolver + instead of assuming that is what keeps this from becoming the SECOND idea of who may see a + table (`user_tables.may_open`'s own docstring is the record of the first time that happened). + """ + import core.perms as _perms + import core.user_tables as user_tables + try: + ut = user_tables.nav_entries(viewer=session.uname, is_admin=True, + st=session.runtime) + except Exception: + # A store that cannot list the tenant's databases must not take the whole editor down — + # the topic half is still administrable, and an empty `ut_*` half is visibly empty. + ut = () + return _perms.tenant_governable_modules(session.runtime, _governable_catalogue(), + ut_entries=ut) + + +def _enforced_keys(session): + """The module keys a permission block may actually be STORED against for this tenant.""" + import core.perms as _perms + return _perms.enforced_module_keys(_perm_modules(session)) def _module_fields(key): @@ -374,23 +445,98 @@ def _module_fields(key): a filter naming a field the product rows lack would DENY every row via `permits()`. The product contract is served CONSOLIDATED (every column) on purpose: a BU-scoped reader is served fewer columns, but the ADMIN is choosing what may ever be hidden. + + ⛔⛔ WAVE 33 — THE `else` USED TO BE A FALL-THROUGH, AND IT WAS SILENT-WRONG. The body read + `aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS`, so ANY key that + was not `product_data` got the CUSTOMER contract: the moment the governed list stopped being + a two-element literal, a `ut_*` database or a third topic would have been handed customer + columns, `_clean_perms` would have validated an admin's `hiddenFields` against them, and the + stored wall would name columns that module does not have — which `permits()` resolves by + DENYING every row. No error at any point. So the map is explicit and an unknown key RAISES: + a topic without a schema arm is a missing arm, never a customer grid in disguise. """ import aios_grid - src = aios_grid.product_fields() if key == "product_data" else aios_grid.FIELDS + providers = {"customer_data": lambda: aios_grid.FIELDS, + "product_data": aios_grid.product_fields} + provider = providers.get(str(key or "")) + if provider is None: + # Reached only by a caller that skipped `_perm_modules`. Loud, because the alternative + # (`[]`) is an empty option list read as an ANSWER — wave 26 item 24, `if ([])` is truthy. + raise err(400, "ungoverned_module", + f"{key!r} has no permission-editor field schema — the editor governs " + f"{sorted(providers)}") + src = provider() + known = _server_side_vocabularies() out = [] for f in src: if not isinstance(f, dict) or not f.get("key"): continue + opts = f.get("options") + if not isinstance(opts, list) or not opts: + # ⭐ D-76 / W33-T35 — SUPPLY WHAT THE SERVER ALREADY KNOWS. See + # `_server_side_vocabularies`. + opts = known.get(f["key"]) out.append({"key": f["key"], "label": f.get("label") or f["key"], "type": f.get("type") or "text", - **({"options": f["options"]} if isinstance(f.get("options"), list) else {}), + **({"options": list(opts)} if isinstance(opts, list) and opts else {}), **({"pinned": True} if f.get("pinned") else {})}) return out -def _clean_perms(v): +def _server_side_vocabularies(): + """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, + but which the grid contract does not declare. + + ⛔ D-76 — WHY THE PERMISSION EDITOR'S DROPDOWNS WERE EMPTY, AND WHY THE FIX IS HERE. The grid + resolves a choice column through `types.ts::choiceVocabulary`: the DECLARED list when the + field has one, otherwise the values seen in the loaded ROWS. This editor loads no rows, so + every column that leans on the second branch renders `Select…` with nothing in it — the + identical shape wave 26 item 24 closed on the Product grid, one surface over. ⚠ And the fix + belongs at the CALLER, never in the panel: `FilterBuilderPanel::choicesFor` treats a supplied + list as authoritative *including when it is empty* ("this column has no values" is an answer), + which is correct and must not be weakened. So the caller stops handing it nothing. + + ⭐ IMPORTED, NEVER RESTATED. `stock_bucket`'s labels are `modules/inventory.COVERAGE_LABELS`, + the same constant `_bucket()` mints from — a copy here would be a second vocabulary that + drifts the first time a band is renamed [[constant-two-features-share]]. The import is lazy + for the same reason `aios_grid`'s is: this route should not pull the analytics stack at + module load. + + ⚠ TWO COLUMNS ARE DELIBERATELY ABSENT, and R6's second sentence says to name them rather than + let them look handled: + · `category` (product) — an OPEN vocabulary read off Odoo's product categories. There is no + closed list to declare, and discovering it would mean a pool read on an admin route. + · `odoo_status` (customer) — WAS bare for the same reason and is NOT any more: lane E + answered `ASK D-15` by declaring `options: ['Active','Archived']` on the CONTRACT, which + is the right home (the grid picks a declared list up for free) and is why this function + never needed an arm for it. Recorded because the ask, not a guess, is what settled it. + """ + try: + import modules.inventory as inventory + labels = list(inventory.COVERAGE_LABELS or ()) + except Exception: + # A missing analytics import must not take the permission editor down; the dropdown + # degrades to today's empty one rather than the page to a 500. + return {} + return {"stock_bucket": labels} if labels else {} + + +def _clean_perms(v, enforced_keys=None, listed_keys=None): """Validate a whole `perms` block. FAIL-CLOSED, and LOUD rather than lenient. + ⭐ WAVE 33 (W33-T33) — `enforced_keys` IS PER TENANT AND IT IS NOT OPTIONAL IN PRACTICE. It + used to be `set(_PERM_MODULES)`, a literal, which is the same tenant-blindness owner item 11 + flagged: an admin of a tenant provisioned for neither grid module could still store a wall + for both. It now comes from `_enforced_keys(session)`, i.e. from the tenant's own catalogue. + The default is the catalogue itself so a test or a future caller cannot silently widen it. + + ⛔ AND "LISTED" IS NOT "ENFORCED". `listed_keys` is every database the editor SHOWS, which + includes this tenant's `ut_*` ones. `perm_scope` is never consulted on those (visibility + there is `user_tables.may_open` — creator, admin, or a `core.shares` grant), so a wall stored + against one would be INERT: the editor would say DENY and the table routes would keep + serving. It is refused with a message that says WHICH of the two it is, because + "unknown module key" for a database the admin can see on screen is the wrong sentence. + ⛔ WHY THIS REFUSES WHERE `clean_filter_tree` DROPS. The view sanitiser is deliberately drop-per-node: one bad rule must never cost a user their whole saved view. A PERMISSION filter is the opposite situation — an admin types "agent is Tara", a leaf is silently @@ -409,11 +555,21 @@ def _clean_perms(v): return None if not isinstance(v, dict): raise err(400, "bad_perms", "perms must be an object keyed by module") - unknown = sorted(set(v) - set(_PERM_MODULES)) + governed = set(enforced_keys) if enforced_keys is not None else set(_PERM_MODULES) + shown = set(listed_keys) if listed_keys is not None else governed + inert = sorted(set(v) & (shown - governed)) + if inert: + raise err(400, "unenforced_module", + f"{inert}: this tenant can SEE these databases, but a permission wall stored " + f"against them would never be applied — `routes_tables` gates a `ut_*` " + f"database on `user_tables.may_open` (creator, admin, or a share), not on " + f"`perm_scope`. Saved silently, it would read as a restriction that is not " + f"there. Share the database instead, or arm `perm_scope` over `ut_*` first.") + unknown = sorted(set(v) - shown) if unknown: raise err(400, "bad_perms", - f"unknown module keys: {unknown} — the permission editor governs " - f"{list(_PERM_MODULES)}") + f"unknown module keys: {unknown} — this tenant's permission editor governs " + f"{sorted(governed)}") out = {} for key, raw in v.items(): if not isinstance(raw, dict): @@ -730,8 +886,16 @@ def get_perms(username: str, session: Session = Depends(admin_gate)): # omitted from `perms` would become an explicit DENY the moment anyone pressed Save. The # default sent is the EFFECTIVE one (`may_access`), so an un-migrated record shows the # access its legacy grant currently gives rather than a guess in either direction. + # + # ⭐⭐ WAVE 33 (W33-T33, owner item 11) — THE LIST IS THIS TENANT'S, NOT A LITERAL. Everything + # below was built from `_PERM_MODULES` and never touched `session.runtime`, which is why a + # nurilab or gtmlab admin opened Manage user and saw Royal Imports' Customer and Product + # databases. `_perm_modules` applies the tenant's own provisioning (the rule `routes_nav` + # has applied since wave 18) and merges the tenant's own `ut_*` databases. + modules = _perm_modules(session) + enforced = [m["key"] for m in modules if m.get("enforced")] perms_out = {} - for k in _PERM_MODULES: + for k in enforced: e = stored.get(k) perms_out[k] = e if isinstance(e, dict) else { "access": bool(perm_scope.may_access(rec, k)), "filter": None, "hiddenFields": []} @@ -744,11 +908,20 @@ def get_perms(username: str, session: Session = Depends(admin_gate)): # `role == 'admin'` bypasses perms entirely (amendment 4) — sent so the editor can # say "Everything (admin)" instead of rendering stored rules that do not apply. "is_admin": perms.is_admin(rec), - "modules": [{"key": k, "label": _MODULE_LABELS.get(k, k)} for k in _PERM_MODULES], + # ⭐ `enforced` rides every row. A `ut_*` database is LISTED because the owner asked to + # see every database, and declared UNENFORCED because `perm_scope` is not consulted on + # that path — so the editor can say which it is instead of painting a dead panel + # (wave 26 item 24: an empty answer that reads as an answer). The LABEL comes from the + # registry / the table's own definition; `_MODULE_LABELS` was a hardcoded copy of two + # registry labels and is deleted with this change. + "modules": modules, # S3 ask 1 — CANONICAL SHAPE: a BARE ARRAY of fields per module key. Not the nav # schema envelope; the editor needs the field list and nothing else, and one shape # beats two readings of a sentence. - "fields_by_module": {k: _module_fields(k) for k in _PERM_MODULES}} + # ENFORCED MODULES ONLY, and that is the honest shape: a picker for a database whose + # wall is never applied is a control that lies. An unenforced row carries no entry + # here at all, so a client cannot mistake `[]` for "this database has no fields". + "fields_by_module": {k: _module_fields(k) for k in enforced}} @router.put("/admin/users/{username}/perms") @@ -781,7 +954,10 @@ def put_perms(username: str, body: dict = Body(default=None), raise err(400, "self_perms", "you cannot set permissions on your own account — ask another admin") - cleaned = _clean_perms(body.get("perms")) or {} + _mods = _perm_modules(session) + cleaned = _clean_perms(body.get("perms"), + enforced_keys={m["key"] for m in _mods if m.get("enforced")}, + listed_keys={m["key"] for m in _mods}) or {} cleaned = _fold_legacy_scope(reg[uname], cleaned) _guard_bu_widening(reg[uname], cleaned, bool(body.get("confirm_widen"))) users.set_access(uname, perms=cleaned) @@ -796,7 +972,29 @@ def put_perms(username: str, body: dict = Body(default=None), "perms_v": int(rec.get("perms_v") or 0)} -_MODULE_LABELS = {"customer_data": "Customer", "product_data": "Product"} +def _store_binding(session): + """`data_binding.describe()` for the store THIS SESSION's tenant actually writes. + + ⭐ THE TENANT'S STORE, NOT THE PROCESS DEFAULT, AND THAT DISTINCTION IS THE POINT. Tenant #0 + rides `core.store`'s module default, but nurilab/gtmlab/loopable are bound to their OWN dataset + repos by `harness/runtime.py:470`, straight out of the control-plane record — which is the path + that carried three tenants' PRODUCTION stores onto staging while `--data-repo` isolated only + tenant #0. Reporting the process default here would show "staging store, all fine" to exactly + the tenants that were not fine. + + ⚠ Degrades to a stated `unknown` rather than raising: a settings page that 500s because it + could not describe the store is worse than one that says it does not know. + """ + try: + import core.data_binding as data_binding # noqa: PLC0415 + import core.store as store # noqa: PLC0415 + bound = getattr(session.runtime, "store_handle", None) + repo = getattr(bound, "repo", None) or store.REPO + return data_binding.describe(repo) + except Exception as e: # noqa: BLE001 + return {"deployment": "unknown", "production": False, "repo": "", + "writable": False, "refusal": f"could not resolve the store binding: {e}", + "localOverride": False, "allowlist": [], "observed": {}} @router.get("/settings") @@ -827,6 +1025,20 @@ def settings(session: Session = Depends(require_session)): }, "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name", session.tenant)}, + # ⭐⭐ D-315 — WHICH STORE THIS CONTAINER IS BOUND TO, AND WHETHER IT MAY WRITE IT. + # + # ⛔ THIS IS NOT DIAGNOSTIC GARNISH; IT IS THE ONLY WAY THE QUESTION CAN BE ANSWERED. + # D-160 established that a Space's environment cannot be read from outside, so "is staging + # pointed at production data?" is a question only the container can answer — and it went + # unanswered for the whole window in which two builds wrote tenant #0's real store. + # `verify_live` asserts this field after logging in, which is a check that survives a role + # swap in a way that reading a deploy log never did. + # + # ⚠ Session-gated, beside `version`, for `main.py::health`'s reason: a Space id and a + # dataset repo id are public names, but a health endpoint is the first URL a scanner finds + # and it may not describe the deployment. No customer data is here — only which store, and + # yes or no. + "store": _store_binding(session), # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client # does not paint a "Manage users" button that 403s. "admin": perms.is_admin(session.user), diff --git a/api/routes_alerts.py b/api/routes_alerts.py index 5db8ccd3fe295c43e401817adff02344918291af..c651d88cf37431431a9588f38c540b71c89597df 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -172,8 +172,36 @@ def notification_view(item): # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it # is D-101's dead one; leaving it through would give the client two vocabularies for one # question, which is the defect this wave's item 6 is about in a different file. + # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST. + # + # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was + # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" — + # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in + # the model or in the markup. Mail has a from. This is it. + # + # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point. + # An alert firing and an automation landing rows have no person behind them; their honest + # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a + # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and + # this reads it back. ⛔ It is NOT parsed out of the body prose (" shared this with + # you") — a sender recovered by regexing a sentence breaks the first time the sentence is + # reworded, and it would break silently, in the header. + # + # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an + # empty from column reads as a broken inbox rather than as an old notification. + actor = str(item.get("actor") or "").strip() + if kind == NOTIF_KIND_SHARE: + sender = actor or "A teammate" + elif kind == NOTIF_KIND_AUTOMATION: + # ⛔ "Agents", not "Automation" (W34-T40, corrected at QA 2026-08-17). The client's + # `senderOf` already falls back to `AGENTS_MODULE_LABEL` — but `if (sent) return sent` + # runs FIRST, so this server literal won and every actor-less automation notification + # showed the retired module name in the inbox's From column. + sender = actor or "Agents" + else: + sender = actor or "Alerts" out = {**item, "read": bool(item.get("read")), "kind": kind, - "subject": subject or "Notification"} + "subject": subject or "Notification", "sender": sender} if target is not None: out["target"] = target return out @@ -191,7 +219,61 @@ def inbox_view(box): items = box.get("items") if not isinstance(items, list): return box - return {**box, "items": [notification_view(n) for n in items]} + # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client + # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`. + # + # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and + # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH + # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?" + # against a browser clock would re-introduce the drift the offset exists to remove — a reader a + # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands + # now come from the same machine. + # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree — + # the note two lines up records what happened last time only one of them was enriched. + return {**box, "now": alerts._now_iso(), + "items": [notification_view(n) for n in items]} + + +def _view_by_id(g, view_id): + """One saved view out of an assembly, by id. `None` when there is no such view. + + ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS + FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND + `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes + it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on + it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the + `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.** + + ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In + `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare + FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our + side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above + the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical + line is swallowed by `/notifications`' `except Exception: continue`, so **every stored + view-alert was silently dropped from the Inbox** and nobody had anything to report at all. + One expression, one loud symptom and one silent one. + + ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site + be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]]. + + ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was + keyed `{id: view}` — which is precisely why the gate was green while production raised on + every call. The fixture is moving to the production shape in this same change, and tolerating + both here means a caller that legitimately holds one cannot resurrect the bug. + """ + want = str(view_id or "") + if not want: + return None + views = (g or {}).get("views") + if isinstance(views, dict): + found = views.get(want) + return found if isinstance(found, dict) else None + if not isinstance(views, list): + return None + for v in views: + if isinstance(v, dict) and str(v.get("id") or "") == want: + return v + return None def _topic_or_400(raw): @@ -283,7 +365,7 @@ def _evaluate(session: Session, rec: dict, assemblies=None): if isinstance(assemblies, dict): assemblies[memo_key] = g - view = (g.get("views") or {}).get(str(rec.get("viewId"))) + view = _view_by_id(g, rec.get("viewId")) if not isinstance(view, dict): # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than # deleting the alert: an alert that silently vanishes is indistinguishable from one that @@ -293,7 +375,40 @@ def _evaluate(session: Session, rec: dict, assemblies=None): # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived # and overlay values on a row. Evaluating a filter against raw pool dicts would silently # never match any condition on a user-created or measure column. - rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"), + # + # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND + # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths. + # + # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows + # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner + # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it: + # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell + # evaluated as BLANK. Measured on one assembly, same view, same rows: + # rows_src state='unpaid' / 'paid' + # _evaluate saw state='' / '' ⇐ every base cell blank + # the GRID saw state='unpaid' / 'paid' + # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty` + # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it + # inverted.** End to end: a row whose value arrived by import, automation, paste or the create + # door never fired; only a value typed as a hand EDIT did. + # + # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit; + # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`); + # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns + # `skipped: window_required`). + # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it + # would let a stale definition value shadow an edit the user has just made — the same defect + # `table_rows`' own note records, arriving from the other side. + _ov = (g.get("ws") or {}).get("overlays") or {} + _merged = {} + for _r in g["rows_src"]: + _pid = str(_r.get("pid")) + _cells = {k: v for k, v in _r.items() if k != "pid"} + _o = _ov.get(_pid) + if isinstance(_o, dict): + _cells.update(_o) + _merged[_pid] = _cells + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged, derived=g.get("derived")) config = view.get("config") or view ctx = filter_eval.EvalCtx( @@ -391,12 +506,28 @@ def _require_filtered_view(session: Session, topic: str, view_id: str): # the one path that reaches this branch is also the one path with nothing to debug from. raise err(503, "unavailable", f"the table could not be read ({type(e).__name__}) — try again in a moment") - view = (g.get("views") or {}).get(str(view_id)) + view = _view_by_id(g, view_id) if not isinstance(view, dict): raise err(404, "no_view", "that view does not exist on this table") nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or []) - # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT, AND THAT IS OWNER ITEM 17.** + # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT + # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS + # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have + # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines + # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the + # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix. + # + # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the + # fix was right, and the three reasons it hid are the most transferable thing in this file. + # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each + # naming themselves as the origin of the same screenshot are mutually exclusive, and the next + # reader believes whichever they meet first — which is why this correction sits above rather + # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead + # of the working tree. [[grep-output-is-not-source]] + # + # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly, + # that this was owner item 17 — see the correction directly above). # # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF diff --git a/api/routes_automation.py b/api/routes_automation.py index 214128a74849f05cb9b698bbea3522ea568948e2..0b8aad0a997592d356b825513ee8877b046e066b 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -12,10 +12,12 @@ rather than admitting everyone. A "no token configured means no check" default i trigger becomes a public one — the same class of mistake as an empty-200 permission answer. """ import os +import re import time from fastapi import APIRouter, Body, Depends, Header, Request +import ai_review import automation_engine as engine import oauth_connect import routes_oauth @@ -27,11 +29,22 @@ router = APIRouter(prefix="/api/v1") # C5 (wave 22): the OAuth connector surface rides INSIDE this router — main.py belongs to no # session this wave, and this router is already mounted there. `/api/v1` + `/oauth/...`. router.include_router(routes_oauth.router) -# ⭐ WAVE 23 (C11): the connectors directory rides this router for the same reason the OAuth -# routes do — it is mounted already, so E's page needs no main.py change to reach it. (C still -# owns main.py; this avoids an unnecessary ask across a fence.) -import routes_connectors # noqa: E402 -router.include_router(routes_connectors.router) +# ⛔⛔ D-203 — THE NESTED INCLUDE OF `routes_connectors` IS REMOVED, and the reason it existed is +# worth keeping because it was a GOOD reason that expired. +# +# Wave 23 (C11) mounted the connectors directory here rather than in `main.py`, because this router +# was already mounted and `main.py` belonged to another session — a sensible way to avoid a +# cross-fence ask. `main.py:227` includes `routes_connectors` DIRECTLY now, so this line stopped +# being the only door and became a SECOND one. +# ⚠ AND A ROUTER MOUNTED TWICE DOES NOT SERVE THE SAME PATHS TWICE — it serves the prefix twice. +# `routes_connectors` carries its own `/api/v1`, so nesting it inside this router's `/api/v1` +# produced **`/api/v1/api/v1/connectors/directory`**: a live, session-gated, entirely dead path +# that nothing links to and every route audit has to explain. MEASURED before removal: 123 served +# paths, exactly 1 of them doubled. +# ⚠ `routes_oauth` above is NOT the same case and stays: `main.py` does not mount it, so this +# router is genuinely its only door. Deleting it because its neighbour was wrong is how a real +# route dies for a tidy-up. +import routes_connectors # noqa: E402,F401 #: The registry key this surface carries (C-AUTONAV — A adds the row; the gate is live now, so #: the day the row lands the wall is already the one that was tested). @@ -78,6 +91,27 @@ def _wire(defn, tenant): # C3 (wave 22): the trigger config rides whole — the webhook token included, because # the person configuring the external caller has to be shown the URL somewhere, and # this payload is session-gated behind the same wall as everything else here. + # ⭐⭐ WAVE 35 · T36 / R10 — THE PARKED BATCH, SUMMARISED. Without this line the whole + # ticket would be unreachable: `_wire` is a WHITELIST, so a key the engine parks on the + # definition simply never arrives, and the detail pane could not know a batch was waiting + # ([[reachable-is-not-the-same-as-built]] — the exact shape `flow` was caught in above). + # ⚠ COUNT AND NOTES ONLY, NEVER `items`. A 200-statement batch carries 200 rendered HTML + # mails; this payload rides on the automations LIST, which every rail paint reads. + # `GET /admin/statements/agent/{id}` serves the detail when somebody opens the batch. + "pendingStatements": ({ + "count": int((defn.get("pendingStatements") or {}).get("count") or 0), + "ts": (defn.get("pendingStatements") or {}).get("ts") or "", + "notes": list((defn.get("pendingStatements") or {}).get("notes") or [])[:25], + } if isinstance(defn.get("pendingStatements"), dict) else None), + # ⭐⭐ WAVE 35 · T37 — `system` FOR A **STORED** ROW. Until now only SYNTHETIC rows carried + # it, stamped by their builders AFTER this function (`_odoo_sync_row`, `_field_agent_rows`), + # so a stored row with the marker had it enforced on the server (`delete_automation` 409s) + # and INVISIBLE to the client — which paints a live-looking Delete that answers with a + # refusal. `AutomationDetail` already reads `automation.system` to disable that button and + # explain why; this is the line that lets it. + # ⚠ Absent stays absent rather than becoming `""`: every ordinary automation is not-a-system + # -agent, and an empty string is a value a client could accidentally treat as one. + **({"system": str(defn.get("system") or "")} if defn.get("system") else {}), "trigger": defn.get("trigger") or None, "statusNote": defn.get("statusNote") or "", # The one-sentence summary (airtable-brief rec 7), composed from the definition so it @@ -110,6 +144,302 @@ def _wire(defn, tenant): } +#: ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — WHY A ROW CANNOT BE DELETED, as a slug. +#: +#: ⚠ ALWAYS A STRING, NEVER A BOOLEAN, and the two contracts disagreed about that (C3 says +#: `system: true`, C4 says `system: ""`). Raised as ask `E-13`; built as a slug because one +#: key with two types is [[one-question-two-normalizers]] before a line is written, and because a +#: refusal has to be able to NAME the reason. Truthiness is unchanged for any reader that only +#: asks "is this a system agent". +SYSTEM_FIELD_AGENT = "field_agent" +SYSTEM_ODOO_SYNC = "odoo_sync" + + +def _field_agent_rows(session): + """⭐⭐ CONTRACT C3 — one synthetic Agent row per `ai_enrich` column in the tenant. + + R13: *"Every field agent must be VIEWABLE under the Agent module as a simple Trigger whose + first step is the field enrichment, with the Field and Database shown under Config."* + + ⛔⛔ THESE ARE DERIVED, NOT STORED, AND NOTHING HERE MAY WRITE. A field agent's only source of + truth is the column definition F's `_clean_field` stamps; a copy in the automations bucket + would be a second one, and the first PATCH through the enrichment editor would silently + diverge from it. `delete_automation` refuses these ids for the same reason: accepting a delete + that cannot delete anything is the `view_upsert` failure mode (200 OK, nothing written) in a + new place. + + ⛔⛔ AND IT USES THE ROWS-FREE PROJECTION, WHICH IS THE WHOLE ENGINEERING PROBLEM OF THIS + TICKET. `GET /automations` is the route W29-T01 and W30-T12 spent two waves taking the + whole-document read OFF: it reads `user_tables` exactly ONCE PER TENANT PER PROCESS now, and + this file's own `unconfigured` note says *"a label is not worth putting it back"*. Walking + every table's fields for `ai_enrich` columns is exactly that read. So this uses + `user_tables.all_defs` — the same projection `/nav` uses, MEASURED there at 1,750 ms -> 1.4 ms, + because tenant #0's document is 99.89% rows and a field definition is not a row. + ⚠ `all_defs` RAISES on `rows` rather than answering `{}`. Nothing here wants a row; if that + ever changes, the error names the projection instead of painting an empty grid. + + ⚠ THE PERMISSION WALL IS CALLED, NEVER RE-IMPLEMENTED. `may_open` decides which databases this + caller may see, lent the projected document exactly as `/nav` lends it. This route's own + `automation_tables` carries the scar that makes that non-negotiable: it once re-implemented + the wall, got it WIDER, and showed people databases they were refused the moment they clicked. + """ + import core.user_tables as user_tables + + try: + defs = user_tables.all_defs(st=session.runtime) or {} + lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs}) + except Exception: # noqa: BLE001 + # A store blip must not take the whole rail down with it. The stored automations are the + # payload's real subject; field agents are an addition to it. + return [] + + out = [] + for key, defn in sorted(defs.items(), key=lambda kv: (kv[1].get("label") or "").lower()): + if not user_tables.may_open(key, session.uname, session.admin, st=lent): + continue + table_label = defn.get("label") or key + for f in user_tables.ai_enrich_fields(defn): + bag = f.get("automation") if isinstance(f.get("automation"), dict) else {} + if bag.get("kind") != SYSTEM_FIELD_AGENT: + continue + col = str(f.get("key") or "") + if not col: + continue + # ⚠ THE ID CARRIES THE TABLE AND THE COLUMN, and `table` is NOT read off the bag: + # F's `_clean_field` is handed ONE field dict with no table context, so it cannot + # stamp one (mailbox `F-3`). E holds that key because E is the side that iterates + # tables. Two sources for one fact was the alternative, and it drifts. + trig = bag.get("trigger") if isinstance(bag.get("trigger"), dict) else {} + mode = str(trig.get("mode") or "manual") + cron = str(trig.get("cron") or "").strip() + defn_syn = { + "id": f"field:{key}:{col}", + "name": f.get("label") or col, + "kind": SYSTEM_FIELD_AGENT, + # ⛔⛔ TWO TRIGGER VOCABULARIES MEET HERE AND THEY ARE NOT THE SAME ONE. A field + # agent's trigger is the ENRICHMENT vocabulary (`{mode, cron}`, values + # manual/on_change/schedule); an automation's is the ENGINE's (`{key}` plus a + # separate `schedule {cron, enabled}`). Handing the engine's shape a `mode` it + # cannot read is not a type error — it is SILENT: `compose_sentence` fell back to + # "When you press Run now" for a column scheduled at 06:00, and `graph()` drew a + # trigger node with no trigger. Measured before this mapping existed, which is the + # only reason it does. + # ⚠ THE MAP IS EXACT WHERE AN EQUIVALENT EXISTS AND HONEST WHERE IT DOES NOT: + # `manual` and `schedule` are the engine's own keys, and `on_change` becomes + # `event_field` ("When a record matches conditions"), the nearest thing the engine + # has to "a cell this column reads has moved". `mode` is kept verbatim beside it so + # nothing is lost in translation and the enrichment editor stays the source. + "trigger": {"key": {"manual": "manual", "schedule": "schedule", + "on_change": "event_field"}.get(mode, "manual"), + "mode": mode, **({"cron": cron} if cron else {})}, + **({"schedule": {"cron": cron, "enabled": True}} + if mode == "schedule" and cron else {}), + "flow": {"actions": [{"id": "act_1", "kind": "ai_enrich", + "config": {"table": key, "field": col, + "tableLabel": table_label}}]}, + } + # ⭐ THROUGH `_wire`, NOT BESIDE IT. A synthetic row hand-built to "look like" a + # stored one is a second shape that agrees until somebody adds a key to the real one + # — the exact way `awaitingResults` shipped inert for a whole wave. Passing the + # synthetic DEFINITION through the same function makes them identical by + # construction; only `system` is stamped afterwards, because no stored row has it. + row = _wire(defn_syn, session.tenant) + row["system"] = SYSTEM_FIELD_AGENT + out.append(row) + return out + + +#: ⭐⭐ WAVE 34 · W34-T47 (mailbox D-5) — R6 APPLIED TO WORDS A MODEL WROTE. +#: +#: ⛔ `web_prose` READS FILES, so it is structurally blind to a dash that arrives at RUN TIME. +#: `POST /automations/draft` returns three strings a language model authored (the flow's name, +#: each step's `why`, and the dropped/notes sentences) and `W34-T42` is what put them on a screen. +#: Lane D MEASURED that a prompt instruction does not hold: their system prompt ends "Never use an +#: em dash" and the very next live cerebras turn came back with one. So this is code at the +#: boundary, not a better prompt. +#: ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE and gets the first rule: "10-20" means "10 to 20", and +#: turning it into "10, 20" states two numbers where the model stated a span. +_DRAFT_DASH = "[" + chr(0x2014) + chr(0x2013) + "]" + + +def _draft_no_dashes(text): + """Model prose with no em or en dash, and no meaning changed on the way.""" + text = str(text or "") + text = re.sub(rf"(?<=\d)\s*{_DRAFT_DASH}\s*(?=\d)", " to ", text) + text = re.sub(rf"\s*{_DRAFT_DASH}\s*(?=[,.;:!?])", "", text) + text = re.sub(rf"(?<=[,;:])\s*{_DRAFT_DASH}\s*", " ", text) + return re.sub(rf"\s*{_DRAFT_DASH}\s*", ", ", text) + + +ODOO_SYNC_ID = "system:odoo_sync" +#: The cadence presets in the words a person reads. ⚠ Keyed on `odoo_relational.SYNC_PRESETS`, and +#: a key with no phrase here falls back to "Every " rather than raising — a missing caption +#: must degrade to something readable, not take the agents list down. +ODOO_CADENCE_PHRASE = {"30m": "Every 30 minutes", "1h": "Every hour", + "4h": "Every 4 hours", "daily": "Every day"} + + +def _odoo_sync_row(session, detail=False): + """⭐⭐ CONTRACT C4 (ruling R22) — the Odoo sync, as an agent you can SEE. + + R22: *"Implicit time-triggered work (Odoo syncing) becomes EXPLICIT: a pre-set agent that + cannot be deleted, whose Canvas shows Trigger = time and Action = data syncing from Odoo, with + the keychain/API configuration shown."* + + ⛔⛔ DERIVED, NOT STORED, AND C4 SAYS "a stored automation may carry `system:`". THE MECHANISM + C4 ASKS FOR IS BUILT (`delete_automation` refuses any stored row carrying `system`); this + particular agent does not use it, for three measured reasons, and the deviation is raised as + ask `E-15` rather than taken quietly: + 1. NOTHING CAN SEED IT. There is no system-automation seeding path anywhere, and `D-195` + measured three times that a CLI write to the tenant store while the Space is live REPORTS + SUCCESS and is reverted within a minute. A stored row would have to be minted by the + container, i.e. a write-on-read on the busiest list route in the product. + 2. STORED MEANS RUNNABLE BY MACHINERY BUILT FOR USER AUTOMATIONS. If its runs went through + `_commit_run`, `CONSECUTIVE_FAILURE_PAUSE` would flip `schedule.enabled` off after K bad + passes — silently disabling the tenant's REAL Odoo cadence, not a cosmetic card. + 3. TWO COPIES OF ONE CADENCE. The loop reads `odoo_relational.sync_seconds`; a stored + automation would carry its own `schedule.cron`, and C4's own requirement is that the card + "cannot drift from the running loop". Deriving makes drift impossible instead of + forbidden. + + ⚠ `detail=False` IS THE LIST PATH AND IT STAYS CHEAP. The keychain label and the last-sync + stamp each cost their own store read, and `GET /automations` is the route two waves were spent + taking reads OFF. They are resolved only on the DETAIL fetch, which is one automation and one + click. Returns `None` when this tenant has no Odoo at all — an agent for a connector nobody + connected is a card that lies. + """ + try: + import odoo_relational as rel + except Exception: # noqa: BLE001 + return None + try: + cfg = rel.read_config(session.runtime) + secs = rel.sync_seconds(session.runtime) + frozen = bool(rel.frozen(session.runtime)) + except Exception: # noqa: BLE001 + return None + + every = str(cfg.get("syncEvery") or rel.DEFAULT_SYNC) + manual = secs is None + defn = { + "id": ODOO_SYNC_ID, + "name": "Odoo sync", + "kind": SYSTEM_ODOO_SYNC, + # ⚠ THE CADENCE IS READ, NEVER STORED HERE. `everySeconds` is what the loop will actually + # sleep for, so the card cannot claim a schedule the loop is not keeping. + "trigger": {"key": "manual" if manual else "schedule", "preset": every, + "everySeconds": secs, "presets": sorted(rel.SYNC_PRESETS)}, + **({} if manual else {"schedule": {"cron": "", "enabled": True}}), + "flow": {"actions": [{"id": "act_1", "kind": "odoo_sync", + "config": {"connector": "odoo", "every": every, + "frozen": frozen}}]}, + } + row = _wire(defn, session.tenant) + row["system"] = SYSTEM_ODOO_SYNC + # ⛔ THE SENTENCE IS OVERRIDDEN, AND IT IS A FIX RATHER THAN A PREFERENCE. `compose_sentence` + # speaks the automation vocabulary — it builds "When , run N actions" out of + # `schedule.cron` — and this agent has no cron: its cadence is a PRESET in seconds that the + # resync loop sleeps on. MEASURED before this line existed, the card read exactly + # `", run 1 action."`: a leading comma where the trigger phrase should have been. Two + # vocabularies again, and this one fails in punctuation rather than in behaviour, which is + # why only reading the output catches it. + # ⚠ AND THE CADENCE IS SPELLED OUT RATHER THAN INTERPOLATED. `f"Every {every}"` produced + # "Every daily, sync data from Odoo." — the preset KEYS are storage tokens ("30m", "daily"), + # not English, and a sentence built by pasting one in is only accidentally readable for the + # three that happen to be durations. Caught by reading the output, not by any assertion. + row["sentence"] = ("Odoo is disconnected, so nothing syncs." if frozen else + "Manually, sync data from Odoo." if manual else + f"{ODOO_CADENCE_PHRASE.get(every, f'Every {every}')}, sync data from Odoo.") + # ⛔ R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that cannot be removed + # must be REPORTED with its cause. Two are reported here rather than left for somebody to + # discover by watching a mirror not move. + notes = [] + if frozen: + notes.append("Odoo is disconnected, so this agent is not syncing. Reconnect it in " + "Connectors.") + if manual: + notes.append("The cadence is set to manual, so nothing syncs on a timer.") + row["statusNote"] = " ".join(notes) + if not detail: + return row + + # ── the DETAIL half: two store reads, on a route that fetches one automation ────────────── + try: + import routes_keychain as kc_routes # noqa: PLC0415 + row["flow"]["actions"][0]["config"]["keychain"] = ( + kc_routes.odoo_config(session).get("label") or "Odoo") + except Exception: # noqa: BLE001 + # A label is a nicety; failing to read one must not make the agent unopenable. + row["flow"]["actions"][0]["config"]["keychain"] = "Odoo" + try: + from harness import datastore as ds # noqa: PLC0415 + # ⭐ THE LAST RUN IS THE MIRROR'S OWN `_sync_state`, not a run history we would have to + # write. That table already carries a real per-entity `updated` stamp, so the card reports + # what actually happened rather than what this module remembers happening. + st = ds.status() or {} + stamps = sorted(str((v or {}).get("updated") or "") for v in st.values() if v) + last = [s for s in stamps if s] + if last: + row["status"] = {**(row.get("status") or {}), "state": "ok", "lastRunAt": last[-1], + "lastSummary": f"{len(last)} datasets synced"} + except Exception: # noqa: BLE001 + pass + return row + + +def _patch_odoo_sync(session, body): + """⭐⭐ CONTRACT C4 — THE ONE THING ON THIS CARD THAT IS EDITABLE: how often it runs. + + C4: *"Its Canvas is READ-ONLY except `trigger.config.everySeconds`, which writes through to the + same value `odoo_relational.sync_seconds` reads, so the card cannot drift from the running + loop."* + + ⛔ IT WRITES THE CONNECTOR'S OWN CONFIG KEY, NOT AN AUTOMATION. `rel.CONFIG_KEY` / + `rel.SYNC_PRESETS` are the same key and the same vocabulary `sync_seconds` reads and + `_store_resync_loop` sleeps on, so there is one value and the card reads back exactly what the + loop will use. Storing a cron on a synthetic automation would have been a second copy with a + guaranteed drift date. + + ⚠ EVERY OTHER FIELD IS REFUSED RATHER THAN IGNORED. A PATCH that silently kept only the part + it liked would let somebody rename this agent, watch the 200, and find the name gone on reload + — the failure shape this module has already paid for twice. + + ⚠ AND THE LATENCY IS REPORTED, because it is real and a person would otherwise call it a bug: + the loop re-reads the cadence at the TOP of its cycle and then sleeps, so a change takes effect + on the NEXT pass — up to one full OLD interval away (24 hours if it was on `daily`). + """ + import odoo_relational as rel # noqa: PLC0415 + + known = {"trigger", "every", "syncEvery"} + extra = sorted(k for k in body if k not in known) + if extra: + raise err(409, "system_agent", + "this agent is the Odoo connection's own sync. Only how often it runs can be " + f"changed here; {', '.join(extra)} belongs to the connector in Connectors") + every = str(body.get("syncEvery") or body.get("every") + or ((body.get("trigger") or {}) if isinstance(body.get("trigger"), dict) else {}) + .get("preset") or "").strip().lower() + if every not in rel.SYNC_PRESETS: + raise err(400, "bad_cadence", + f"pick one of: {', '.join(sorted(rel.SYNC_PRESETS))}") + + def _up(cur): + cur = dict(cur) if isinstance(cur, dict) else {} + cur["syncEvery"] = every + return cur + + session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") + row = _odoo_sync_row(session, detail=True) + if row is None: + raise err(404, "unknown_automation", "no automation with that id") + secs = row["trigger"].get("everySeconds") + note = ("It syncs on a timer again from the next cycle." if secs + else "Nothing will sync on a timer now.") + return {"automation": row, + "notes": [f"The Odoo sync is set to {every}. {note} A change takes effect on the " + f"loop's next pass, so it can be up to one of the OLD intervals away."]} + + def _triggers_vocab(session): """C3's server-owned trigger list: `[{key, label, ready, needs, planned}]`, keys EXACTLY `engine.TRIGGER_KEYS` + `engine.TRIGGER_PLANNED`. `ready:false` + `needs` renders as a @@ -241,6 +571,13 @@ def _tick_state(): #: or delete — a new database cannot contain the legacy stage cells this retires. _BOARD_RETIRED = set() +#: ⭐ WAVE 35 · T37 — tenants whose statements agent has been considered THIS PROCESS. Same shape as +#: `_BOARD_RETIRED` and, like it, purely a cost saver: the DURABLE idempotency is the row's own +#: existence, which `engine.seed_statements_agent` re-checks inside its own store mutation. +#: ⚠ SO A RESTART RE-CHECKING IS HARMLESS BY CONSTRUCTION, which is the property to preserve. If +#: this set were ever the only guard, a process restart would mint a second agent. +_STATEMENTS_SEEDED = set() + #: ⭐⭐ WAVE 30 · T12 — THE PICKER DEFAULT, MEMOISED. `{tenant: (stamp, table_key)}`, the shape #: `scope_cache` stores. #: @@ -393,9 +730,31 @@ def list_automations(session: Session = Depends(_GATE)): # Idempotent Board retirement removes only engine-marked stage fields and legacy Board # state. User-created Status/Stage columns remain intact. engine.retire_automation_board_state(session.runtime, tables=tables) + # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY + # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds + # it present and returns immediately. See `engine.seed_statements_agent` for why this one is + # STORED where wave 34's Odoo agent is derived, and why the row's existence is a sufficient + # idempotency key (it cannot be deleted). + # ⚠ GUARDED SO IT CAN NEVER TAKE THE RAIL DOWN. A tenant with no Odoo, a store mid-outage or a + # validator change must degrade to "no statements agent", never to a 500 on the one route the + # whole Agents surface polls. The same posture `_odoo_sync_row` takes for the same reason. + if session.tenant not in _STATEMENTS_SEEDED: + try: + engine.seed_statements_agent(session.runtime, username=session.uname or "system") + except Exception: # noqa: BLE001 + pass + _STATEMENTS_SEEDED.add(session.tenant) defs = engine.all_definitions(session.runtime) items = [_wire(d, session.tenant) for _, d in sorted(defs.items(), key=lambda kv: (kv[1].get("name") or "").lower())] + # ⭐⭐ WAVE 34 · CONTRACT C3 (W34-T48) — field agents join the list as SYNTHETIC rows. + # ⚠ MERGED AND RE-SORTED, not appended in a block at the end. R13 asks for a field agent to be + # "VIEWABLE under the Agent module", i.e. one list of agents, not a list with a second list + # stapled to it — and a rail that sorts by name everywhere except its last few rows reads as a + # rendering bug. `_field_agent_rows` costs no whole-document read; see its docstring. + _sys_rows = [r for r in (_odoo_sync_row(session),) if r] + items = sorted(items + _field_agent_rows(session) + _sys_rows, + key=lambda r: str(r.get("name") or "").lower()) return {"automations": items, # ⭐ WAVE 24 — DERIVED from the engine's own `KINDS`, not a hand-written trio. It was # three literals that happened to match, which is a second copy of a server @@ -484,7 +843,11 @@ def list_automations(session: Session = Depends(_GATE)): # own reason instead of omitting them — the owner asked for Airtable's full menu, and # a shorter list would imply those actions do not exist. `clean_actions` REFUSES an # unready kind, so the faded state is a wall rather than a styling choice. - "actionsCatalog": engine.action_catalog(), + # ⭐ W35-T35 (C8): `session.runtime` is what withholds a TENANT-GATED row. Passing it + # here is the whole of "the menu does not offer Send statements to another tenant"; + # the STORE-side wall is `clean_actions(rt=)`, deliberately separate, because a picker + # is not a security boundary [[opening-a-route-widens-every-field]]. + "actionsCatalog": engine.action_catalog(session.runtime), # The builder's own vocabulary: how deep a condition tree may nest, how deep groups # may nest, and the ceilings. B reads these instead of hard-coding the same numbers # into its "+ Add condition" affordance. @@ -505,7 +868,25 @@ def list_automations(session: Session = Depends(_GATE)): # The ceiling a group's limit is judged against. `clean_post_groups` bounds a # group by the action's OWN `maxPosts`, not by a constant, so the control can # only warn honestly if it reads the same number the validator uses. - "maxPostsPerPull": engine.MAX_POSTS_PER_PULL}, + "maxPostsPerPull": engine.MAX_POSTS_PER_PULL, + # ⭐⭐ W33-T52 (owner item 16) — WHICH CONFIG KEYS A KIND REQUIRES, on the + # wire, for exactly the reason `postTypes` above is: the panel paints a red + # `*` beside a required control, and a client-side table of which keys those + # are would be a SECOND copy of `ACTION_REQUIRED` living in another file. The + # two would agree on the day they were written and diverge the first time a + # kind gained a key — the panel would then mark a control optional that the + # runner blocks on, and the person would read "this is fine" from the one + # surface that is meant to tell them it is not. + # ⛔ BOTH HALVES RIDE, phrase AND key, because the phrase is the ONLY human + # wording of that requirement anywhere ("a column to write into"), and the + # runner's own refusal sentence is built from it. A client that re-worded it + # would give the same requirement two names. + # ⚠ It is `ACTION_REQUIRED`, not `WEB_REQUIRED`: the table is the one the + # run refusal and `unconfigured_actions` already read, so a kind added to it + # later paints its `*` with no client change at all. + "actionRequired": {kind: [{"phrase": phrase, "key": key} + for phrase, key in reqs] + for kind, reqs in engine.ACTION_REQUIRED.items()}}, # Wave 21 C6-A1: {"enabled": bool, "source": "in-process"|"external"|""} — see # `_tick_state` for why one boolean off AIOS_AUTOMATIONS alone would lie. "tick": _tick_state()} @@ -714,8 +1095,203 @@ def automation_seed_derive(body: dict = Body(default=None), session: Session = D "derived": derived, "basis": basis} +#: The injected chat transport, for gates only — `None` in every shipped path, so the route takes +#: the real ladder. Set by `verify_automation.py` to prove this door end to end with NO API key and +#: NO spend (`routes_query._call_model`'s `chat` argument is the same idea and the same reason). +_DRAFT_CHAT = [None] + + +@router.post("/automations/draft") +def draft_automation(body: dict = Body(default=None), session: Session = Depends(_GATE)): + """⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A DESCRIPTION BECOMES A DRAFT FLOW. + + `{prompt}` → `{draft: {name, trigger, table, actions:[{id,kind,config,why}]}, provider, + dropped: [...], saved: false}` — or a 400 carrying ONE plain sentence. + + ⛔⛔ NOTHING IS SAVED HERE, AND THE `saved: false` ON THE WIRE IS NOT DECORATION. T54's + `done-when` is *"gets back a DRAFT flow they can see BEFORE anything is saved"*; a door that + wrote first and showed second would satisfy every check about the flow's contents and none about + the promise. Accepting a draft is the ORDINARY `POST /automations` — R3's *"indistinguishable + from a hand-built one"* is achieved by there being no second write path, not by making the + second one look similar. + + ⛔⛔ CONTRACT C7, AND THIS ROUTE IS WHERE IT IS ENFORCED: *"the output must pass `clean_actions` + unchanged."* So it is RUN, here, before the person ever sees the draft — and the result is + DIFFED against what the model wrote. `clean_actions` has no disclosure channel (D-75): it drops + a config key it does not recognise and answers 200. Without this diff a person would accept a + flow, open it, and find a step configured differently from the one they were shown, with nothing + anywhere having said so. `dropped` is that channel, built here because this is the first caller + that needed one. + ⚠ A draft whose actions `clean_actions` REFUSES outright is a 400, never a partial flow: half a + flow presented as a whole one is the one outcome worse than a refusal. + + ⚠ DECLARED ABOVE `/automations/{auto_id}`, and that is load-bearing rather than tidy — FastAPI + matches in DECLARATION order, so a literal path registered after a sibling path-parameter route + is never reached, and `get_automation` would answer *"no automation with that id"* for the id + `draft`. The same reason `/automations/tables` and `/automations/presets` sit up here. + """ + prompt = str((body or {}).get("prompt") or "").strip() + if not prompt: + raise err(400, "no_prompt", "type what you want the automation to do") + # ⚠ THE TENANT'S OWN TABLES, THROUGH THE EXISTING WALL. `automation_tables` applies `may_open` + # per table, so the model is shown exactly the databases this caller may already see and cannot + # name one they were not granted — the permission wall re-used, never a second one built beside + # it. (`QueryPage`'s `granted` prop carries the same clause on the client side.) + tables = (automation_tables(session) or {}).get("tables") or [] + # ⭐⭐ WAVE 34 · W34-T42 / R18 — THE DRAFTER IS OFFERED WHAT THE PICKER OFFERS, nothing more. + # + # R18 collapsed six web actions into one on the menu. The other five stay in the catalog + # because the surviving "Web agent" composes its own steps out of them server-side — but a + # DRAFT is read and accepted by a PERSON, who then edits it on the Canvas. Handing the model + # the unfiltered catalog would let it draft a step kind that is not in the action picker: the + # reader could not add a second one, could not reason about where it came from, and the flow + # they accepted would not be one they could have built by hand. R3's *"indistinguishable from + # a hand-built one"* is a claim about what a person can REACH, not only about the write path. + # ⚠ This is the ONE place the filter is applied on this route, and it is applied to the + # drafter's vocabulary only. `clean_actions` below still validates against the FULL catalog, + # so a flow that already holds a hidden kind is unaffected and nothing 400s (D-65). + # ⚠ AND `required` IS NARROWED WITH IT, which is the half that is easy to miss. `draft_flow` + # builds the ENUM from `catalog` and the "settings you must write" prose from `required`, so + # filtering one and not the other describes settings for kinds the model cannot choose. That is + # not merely untidy: it spends prompt on unreachable options and invites the model to reach for + # one. `_ai_agent_plan` already narrows both in exactly this way; this follows it. + # W35-T35 (C8): tenant-gated rows are withheld here too, or the DRAFTER would offer a kind + # the same tenant's own save door then refuses — a flow the assistant proposes and the product + # will not accept. + menu_catalog = [r for r in engine.action_catalog(session.runtime) if r.get("menu") is not False] + _offered = {r["kind"] for r in menu_catalog} + draft, refusal, provider = ai_review.draft_flow( + prompt=prompt, + catalog=menu_catalog, + required={k: v for k, v in engine.ACTION_REQUIRED.items() if k in _offered}, + triggers=_triggers_vocab(session), + tables=tables, + # ⭐ W35 · CONTRACT C7 (`NOTE E-16`) — the spend is ATTRIBUTED. Unlike the two engine call + # sites, this one has a real person behind it: somebody typed the sentence, so `user` is + # the caller rather than the automation's owner. + st=session.runtime, user=getattr(session, "uname", "") or "", + chat=_DRAFT_CHAT[0]) + if refusal or not draft: + raise err(400, "draft_refused", refusal or "no automation could be drafted from that") + + # ── C7: run the real save-door validator and DIFF it ────────────────────────────────── + wrote = draft.get("actions") or [] + # W35-T35 (C8): the SAME `rt` the menu was built with. Without it this validator would refuse + # a tenant-gated kind it had just offered the model — the draft door disagreeing with the save + # door about one tenant, which reads as "the assistant produced an invalid flow". + cleaned, why = engine.clean_actions([{"kind": a.get("kind"), "config": a.get("config") or {}} + for a in wrote], rt=session.runtime) + if why or cleaned is None: + raise err(400, "draft_invalid", + f"the assistant produced a flow this deployment will not accept: {why}") + dropped = [] + for before, after in zip(wrote, cleaned): + b_cfg = before.get("config") or {} + a_cfg = after.get("config") or {} + gone = sorted(k for k in b_cfg + if k not in a_cfg or str(a_cfg.get(k)) != str(b_cfg.get(k))) + if gone: + dropped.append({"kind": str(after.get("kind") or ""), "keys": gone}) + # ⛔⛔ WHAT IS STILL MISSING FROM EACH STEP, PER STEP — and this is the fix for the sharpest + # thing the verifier found. The system prompt TELLS the model *"leave one blank rather than + # inventing a web address, a CSS selector or a column name"* (which is right), and the panel + # skips empty values (which is also right, on its own). Together they meant a `web_read` with + # no selector and no target column painted as a FINISHED step — url, attr, timeout, nothing + # else — and the ordinary create door then accepted it. A person approved a step that reads + # nothing into nowhere, having been shown no blank at all. + # ⚠ `engine.action_needs` IS THE PREDICATE, not a second list: the same function the builder's + # Configured/Unconfigured label and the run refusal read. Three readers, one answer. + _tables_by_key = {str(t.get("key")): t for t in tables} + _target_cols = {str(f.get("key")) for f + in (_tables_by_key.get(str(draft.get("table") or "")) or {}).get("fields") or []} + shown = [] + for i, after in enumerate(cleaned): + row = dict(after, why=str((wrote[i] or {}).get("why") or ""), + needs=engine.action_needs(after)) + # ⛔ AND A COLUMN NAME THE DATABASE DOES NOT HAVE. `field` is a free string that nobody + # validated at draft, at accept or at store — so a model writing the column's LABEL + # ("Price") instead of its key ("price"), which is exactly what a person would say and an + # obedient model would echo, stored a write into a column that does not exist. No schema + # was violated; the flow was well-formed and did something other than what was asked. + _f = str((after.get("config") or {}).get("field") or "") + if _f and _target_cols and _f not in _target_cols: + row["unknownField"] = _f + shown.append(row) + # ⚠ A TRIGGER THE CREATE DOOR WOULD REFUSE IS CORRECTED HERE, NOT SHOWN AND THEN 400'd. The + # enum makes this need a model that ignores its own schema, but the failure mode is ugly: the + # draft paints, the person clicks Accept, and the save refuses with a sentence about a word + # they never chose. Falling back to `manual` and SAYING SO keeps the draft usable. + _trig = str(draft.get("trigger") or "").strip() + _notes = [] + if _trig and _trig not in tuple(engine.TRIGGER_KEYS): + _notes.append(f"the assistant asked for a trigger this deployment does not have " + f"({_trig}). Set to manual instead") + _trig = "" + _asked = int(draft.get("asked") or len(wrote)) + if _asked > len(shown): + _notes.append(f"the assistant wrote {_asked} steps and a draft carries at most " + f"{ai_review.MAX_DRAFT_ACTIONS}; the last {_asked - len(shown)} were not " + f"kept. Describe the job in two automations, or shorten it.") + # ⭐⭐ WAVE 34 · W34-T47 (mailbox D-5) — THE MODEL'S OWN WORDS ARE NORMALISED, BECAUSE R6 + # CANNOT BE ENFORCED BY A SOURCE SCAN. + # + # ⛔ `web_prose` READS FILES. Every string below is written by a language model at run time, + # so the gate is structurally blind to it: the sweep every lane did this wave is undone by our + # own drafter the first time it answers with an em dash. Lane D MEASURED that a prompt + # instruction does not hold (their system prompt says "Never use an em dash" and the very next + # cerebras turn came back with one), so this is code, at the boundary, not a nicer prompt. + # ⚠ THREE STRINGS REACH A SCREEN FROM HERE and all three are covered: the flow's NAME, each + # step's `why`, and the `dropped`/`notes` sentences. `W34-T42` is what made them visible. + # ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE: "10-20" means "10 to 20", and rewriting it as + # "10, 20" states two numbers where the model stated a span. It gets its own rule, first. + # Same shape as `routes_query._no_dashes`, deliberately: one behaviour, two doors. + # ⚠ MODULE-LEVEL, not a closure. A normaliser defined inside this handler is unreachable to + # anything that is not an HTTP request, so the only way to test it would be to drive the whole + # route and read the answer — which is how a rule ends up asserted by a grep instead of a call. + # ⛔ `dropped` IS NOT NORMALISED AND THAT IS DELIBERATE: it is `[{kind, keys}]`, an action kind + # from OUR catalog and config keys from OUR allowlist, never a model's sentence. The first + # version of this block mapped the normaliser over it and turned each dict into the string + # `"{'kind': ..., 'keys': [...]}"` — `verify_automation`'s own W33 section caught it with a + # `TypeError` on `d["kind"]`. Applying a text rule to a structure is how a channel stops + # carrying what its reader expects, and the reader here is a person's list of what changed. + shown = [dict(r, why=_draft_no_dashes(r.get("why"))) for r in shown] + _notes = [_draft_no_dashes(n) for n in _notes] + return {"draft": {"name": _draft_no_dashes(draft.get("name")) or "New automation", + "trigger": _trig or "manual", + "table": draft.get("table") or "", + "actions": shown}, + "provider": provider, "dropped": dropped, "notes": _notes, "saved": False} + + @router.get("/automations/{auto_id}") def get_automation(auto_id: str, session: Session = Depends(_GATE)): + """One automation, by id. + + ⛔⛔ WAVE 34 · CONTRACT C3 — A SYNTHETIC ROW MUST BE FETCHABLE BY ID, NOT ONLY LISTABLE, AND + THIS IS THE HALF THAT IS EASY TO SHIP BROKEN. `list_automations` and this route are SEPARATE + lookups: a row appended only to the list appears in the rail, looks entirely real, and 404s + the instant somebody clicks it. `W34-T48`'s `done-when` is *"opening it shows a Trigger and + one step"* — i.e. this route, not the list — so the field agents are resolved here too, from + the same builder, and a person cannot reach a row the detail route does not know. + ⚠ THE SAME BUILDER, NOT A SECOND ONE. `_field_agent_rows` is called and the id looked up in + its output rather than re-deriving one row from the field: two derivations of one row is how + the list and the editor start disagreeing about a name. + """ + if str(auto_id) == ODOO_SYNC_ID: + # ⚠ `detail=True` — the keychain label and the last-sync stamp cost a store read each and + # are resolved HERE, on a route that fetches one automation, never on the list. + row = _odoo_sync_row(session, detail=True) + if row is None: + raise err(404, "unknown_automation", "no automation with that id") + return {"automation": row} + if str(auto_id).startswith("field:"): + row = next((r for r in _field_agent_rows(session) if r.get("id") == str(auto_id)), None) + if row is None: + # ⚠ The SAME 404 as any other unknown id. A field agent whose column has been deleted + # is genuinely gone, and inventing a different error for it would make a normal + # outcome look like a fault. + raise err(404, "unknown_automation", "no automation with that id") + return {"automation": row} defn = engine.all_definitions(session.runtime).get(str(auto_id)) if defn is None: raise err(404, "unknown_automation", "no automation with that id") @@ -729,33 +1305,84 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend answered before the kind. `new` mints the blank database in the same call, so the automation is never saved pointing at a table that does not exist yet.""" if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - defn, error = engine.create(session.runtime, body or {}, username=session.uname) + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") + # ⭐⭐ D-75 — THE DISCLOSURE CHANNEL, CONSUMED. `clean_actions` silently drops a + # `create_record` condition (D-65's law: dropping is recoverable, refusing locks the door) and + # any config key an arm's allowlist does not keep. Both are deliberate; what was missing was + # anybody being TOLD, so a person saved a flow and got a different one with nothing said. + # ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the + # person is looking — a note in a log they never open is the same silence with extra steps. + _notes = [] + defn, error = engine.create(session.runtime, body or {}, username=session.uname, + notes=_notes) if error: raise err(400, "invalid_automation", error) - return {"automation": _wire(defn, session.tenant)} + return {"automation": _wire(defn, session.tenant), "notes": _notes} @router.patch("/automations/{auto_id}") def patch_automation(auto_id: str, body: dict = Body(default=None), session: Session = Depends(_GATE)): if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname) + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") + # ⛔⛔ WAVE 34 · CONTRACT C3 — A SYNTHETIC ROW IS NOT PATCHABLE, AND IT MUST SAY SO. A field + # agent is DERIVED from a column definition; there is nothing in the automations bucket to + # write. `engine.patch` would answer "no such automation" (a 404 that reads as "your agent + # vanished") or, worse for a future id shape, find nothing and report success. The refusal + # names where the setting actually lives, because a wall that does not say what to do instead + # sends somebody looking for a bug in a decision made on purpose (D-65's own lesson). + if str(auto_id).startswith("field:"): + raise err(409, "system_agent", + "this agent is an AI enrichment column. Change its prompt, model or schedule on " + "the column itself and this page follows") + if str(auto_id) == ODOO_SYNC_ID: + return _patch_odoo_sync(session, body or {}) + # D-75, same channel as `create` above — an EDIT is where this matters most, because the + # person has just typed the thing that gets dropped. + _notes = [] + defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname, + notes=_notes) if error: raise err(400 if error != "no such automation" else 404, "invalid_automation" if error != "no such automation" else "unknown_automation", error) - return {"automation": _wire(defn, session.tenant)} + return {"automation": _wire(defn, session.tenant), "notes": _notes} @router.delete("/automations/{auto_id}") def delete_automation(auto_id: str, session: Session = Depends(_GATE)): """Delete the DEFINITION. ⚠ The database it filled is NOT touched — an automation is the thing that writes rows, not the thing that owns them, and deleting a job must never be a way - to lose data (the same rule the orphan count encodes).""" + to lose data (the same rule the orphan count encodes). + + ⭐⭐ WAVE 34 · CONTRACTS C3 + C4 — A SYSTEM AGENT REFUSES, LOUDLY, AND THE REFUSAL SAYS WHERE + TO GO INSTEAD. Two families are undeletable and they are undeletable for different reasons, so + the sentence names the reason rather than saying "no": a FIELD AGENT is a column (delete the + column), and the ODOO SYNC is the connector's own schedule (disconnect the connector). + + ⛔ REFUSED, NOT IGNORED, AND THAT IS THE POINT OF THE FIRST BRANCH. A synthetic id is not in + the automations bucket, so `engine.remove` would find nothing, do nothing, and this route + would answer `200 {"deleted": ...}` — a delete that reports success and changes nothing, which + is the `view_upsert` failure mode (200 OK, zero writes) arriving in a new place. The row would + then reappear on the next poll and read as a bug in the rail. + """ + # ⛔⛔ EVERY SYNTHETIC ID, NOT JUST THE FIELD ONES. The first version of this guard listed + # `field:` alone, and `verify_automation`'s own C4 leg caught what that left open: deleting + # `system:odoo_sync` fell straight through to `engine.remove`, which popped a key that was + # never in the bucket and answered `200 {"deleted": ...}`. A delete that reports success and + # changes nothing, on the one row the ruling says must be undeletable — the exact defect the + # guard exists for, one id shape away. Both prefixes are refused by the same branch now. + if str(auto_id).startswith("field:"): + raise err(409, "system_agent", engine.system_agent_refusal(SYSTEM_FIELD_AGENT)) + if str(auto_id) == ODOO_SYNC_ID: + raise err(409, "system_agent", engine.system_agent_refusal(SYSTEM_ODOO_SYNC)) if engine.running(session.tenant, auto_id): - raise err(409, "automation_running", "it is running — wait for it to finish") + raise err(409, "automation_running", "it is running. Wait for it to finish") + # ⚠ READ BEFORE THE REMOVE, not after: `engine.remove` is the thing being refused. + _defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + _sys = str(_defn.get("system") or "").strip() + if _sys: + raise err(409, "system_agent", engine.system_agent_refusal(_sys)) engine.remove(session.runtime, auto_id) return {"deleted": str(auto_id)} @@ -773,7 +1400,7 @@ def preview_source(body: dict = Body(default=None), session: Session = Depends(_ except engine.Refused as e: raise err(400, "refused_url", str(e)) except Exception as e: # noqa: BLE001 - raise err(502, "fetch_failed", f"could not read that page — {type(e).__name__}: " + raise err(502, "fetch_failed", f"could not read that page. {type(e).__name__}: " f"{str(e)[:160]}") @@ -804,9 +1431,9 @@ def toggle_automation_node(auto_id: str, node_id: str, session: Session = Depend nothing: a control that appears to work and does not is worse than one that refuses. """ if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") if engine.running(session.tenant, auto_id): - raise err(409, "automation_running", "it is running — wait for it to finish") + raise err(409, "automation_running", "it is running. Wait for it to finish") defn, error = engine.toggle_node(session.runtime, auto_id, node_id) if error: raise err(404 if error == "no such automation" else 400, @@ -873,7 +1500,7 @@ def tick(request: Request, x_aios_tick_token: str = Header(default="")): want = os.environ.get("AIOS_AUTOMATION_TICK_TOKEN") or "" if not want: raise err(403, "tick_disabled", - "AIOS_AUTOMATION_TICK_TOKEN is not configured — the tick endpoint is closed") + "AIOS_AUTOMATION_TICK_TOKEN is not configured. The tick endpoint is closed") got = x_aios_tick_token or request.headers.get("X-AIOS-TICK-TOKEN") or "" if got != want: raise err(403, "bad_tick_token", "that token is not valid for this deployment") @@ -891,7 +1518,7 @@ def purge_ig_subject(body: dict = Body(default=None), session: Session = Depends if not session.admin: raise err(403, "admin_only", "erasing a subject is an admin action") if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was " + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was " "purged") handle = str((body or {}).get("handle") or "").strip() if not handle: @@ -930,7 +1557,7 @@ def metric_drill(table_key: str, field_key: str, row_id: str, "measure": bag.get("measure"), "window": bag.get("window"), "value": value, "rows": rows[:200], "note": "" if value is not None else - "no master data answers this window — the cell is honestly blank"} + "no master data answers this window. The cell is honestly blank"} @router.get("/automations/{auto_id}/rows") diff --git a/api/routes_brand.py b/api/routes_brand.py new file mode 100644 index 0000000000000000000000000000000000000000..9c85792eaa3cf9703c223f358ea6e985e7af3213 --- /dev/null +++ b/api/routes_brand.py @@ -0,0 +1,140 @@ +"""routes_brand.py — THE CONNECTOR BRAND-ASSET DOOR (wave 33, owner item 6 / ruling R4 / contract C6). + +⭐ WHAT THE OWNER ASKED FOR, verbatim: *"Do not generate your own logo for Connectors, they look +really uncanny /browse them the PNG, cut appropriately for our needs, verify visually."* So the marks +are the VENDORS' OWN artwork, on disk, with their provenance recorded — `web/public/brand/` and its +`SOURCES.md`. This module is the door R4 asks for and nothing else: it maps a brand KEY to a file, +serves it, and refuses anything else. + +⛔ WHY A NEW DOOR AND NOT `routes_assets.py`. That module gates EVERY one of its routes behind +`module_gate("product_data")` (`routes_assets.py:45`) because it serves product imagery and the grant +that owns product imagery is the right wall for it. A connector logo is not product data: an account +with no product grant still opens Connectors, still opens the automation pickers, and would have got +a 403 where a picture should be. Contract C6 says session-gated, NOT product-gated, and that is the +whole reason this file exists rather than one more path over there. + +⚠ SESSION-GATED, AND THAT IS AN HONEST WALL RATHER THAN A SECRET. Public brand artwork is public by +definition, and the same bytes are ALSO reachable through the static bundle — `web/public/**` is +copied into `web/dist/**` by `vite build`, which `main.py` mounts unauthenticated at `/`. So the +session gate here is consistency with every other `/api/v1` route, not a claim that these files are +confidential. What the door actually buys, and what a raw static path could not: + + · ONE KEY VOCABULARY. The client asks for `instagram`, the same string `BRAND_OF_CONNECTOR` maps + to — never a filename. The extension is ours to change (a vendor publishing SVG tomorrow moves + one line here) and no caller learns about it. + · A CLEAN 404 for a key we have no art for, which is what lets the client keep its two-letter tile + fallback honest instead of painting a broken-image glyph. + · IMMUTABLE CACHING on a response the browser then never re-fetches. + +⛔ THE FALLBACK IS NOT DEAD CODE AND MUST NOT BE "TIDIED". `ConnectorsPage.tsx:204 tileText` draws two +letters for any connector with no entry in `BRAND_OF_CONNECTOR` — which is most of the planned list, +by design, because the directory deliberately lists things we have not built (R8). A 404 from here is +the *correct* answer for those rows, not a failure. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +from fastapi import APIRouter, Depends +from fastapi.responses import FileResponse + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: ⛔ CLOSED CHARSET, CHECKED BEFORE THE KEY TOUCHES A PATH — same discipline as +#: `routes_assets._CODE_RE`. No dot, no slash, no backslash, no colon, so `..` and every absolute +#: path are refused at the door (400) rather than resolved and then defended against. +_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,31}$") + +#: A year, immutable. The artwork is versioned by its own key: a NEW mark for a vendor is a new file +#: with the same name, so a rebrand ships with a deploy and a hard reload, not with a cache bust. +#: ⚠ Deliberately the same string as `routes_assets._IMMUTABLE` — two asset doors telling a browser +#: two different things about the same class of file would be a defect nobody could see. +_IMMUTABLE = "public, max-age=31536000, immutable" + +_MEDIA = {".png": "image/png", ".svg": "image/svg+xml"} + +#: ⭐ THE ROOTS, IN ORDER, AND THE ORDER IS THE POINT. +#: +#: `web/dist/brand` FIRST because that is what a container has: `deploy_web.py::_web_source_files` +#: SKIPS `dist`, so the Space uploads `web/public/**` as source and its own `vite build` produces +#: `dist/`. `web/public/brand` SECOND because that is what a DEV BOX has before anything is built — +#: and a door that only works after `npm run build` is a door that fails for the person writing it. +#: `AIOS_BRAND_DIR` FIRST OF ALL, so an operator can point at either without a redeploy. +#: +#: ⚠ RESOLVED PER REQUEST, NOT AT IMPORT. A module-level constant computed at import is a fact about +#: the process's startup, and this one would be wrong for the whole life of any container that built +#: its bundle after the API module was first imported. +#: +#: ⚠ THE ONE TRAP THIS ORDER CREATES, AND IT ONLY BITES A DEV BOX. `dist` winning means a mark +#: replaced in `public/brand/` is NOT served until the next `npm run build` — locally the two can +#: disagree, and the door will confidently serve yesterday's logo. In the image they cannot: the +#: Space uploads `public/` (`deploy_web.py::_web_source_files` SKIPS `dist`) and builds `dist/` from +#: it in the same step. If a local mark looks stale, rebuild before believing the door is wrong. +_HERE = Path(__file__).resolve().parent + + +def _roots(): + env = (os.environ.get("AIOS_BRAND_DIR") or "").strip() + out = [Path(env)] if env else [] + web = _HERE.parent / "web" # _HERE is `aios-web/api`; its parent is `aios-web` + out.append(web / "dist" / "brand") + out.append(web / "public" / "brand") + return out + + +def brand_path(key): + """The file for a brand key, or None. Pure and importable, so the gate answers this question + without a request — the same shape `routes_connectors.directory` uses for the same reason.""" + k = str(key or "") + if not _KEY_RE.match(k): + return None + for root in _roots(): + for ext in (".png", ".svg"): + p = root / f"{k}{ext}" + # ⚠ `is_file()` and not `exists()`: a DIRECTORY named `instagram.png` would otherwise + # resolve and then fail inside FileResponse as a 500. + if p.is_file(): + return p + return None + + +def available(): + """Every brand key that has artwork on disk, sorted. The directory listing this module can + answer cheaply — used by `verify_meta` to assert parity against `BRAND_OF_CONNECTOR` rather than + against a list somebody typed twice.""" + keys = set() + for root in _roots(): + if not root.is_dir(): + continue + for p in root.iterdir(): + if p.is_file() and p.suffix in _MEDIA and _KEY_RE.match(p.stem): + keys.add(p.stem) + return sorted(keys) + + +@router.get("/brand/{key}") +def brand_asset(key: str, session: Session = Depends(require_session)): + """R4/C6: one connector's official mark, to any authenticated session. + + ⛔ THREE ANSWERS, AND THE THIRD IS THE ONE THE TICKET ASKS ABOUT: 400 for a key that is not a + key (it never reaches the filesystem), **404 for a key we simply have no art for**, and the file + otherwise. A stack trace is not one of the answers — an unknown key is an ordinary, expected + request from a client whose connector list is longer than our logo set. + ⚠ `session` is required and then unused, which is intentional: the dependency IS the wall, and + naming it in the signature is what puts it there. Deleting the "unused" parameter would open the + route to the internet with no other symptom. + """ + if not _KEY_RE.match(str(key or "")): + raise err(400, "bad_brand_key", "that is not a brand key") + p = brand_path(key) + if p is None: + raise err(404, "brand_not_found", f"no brand mark for '{key}'") + return FileResponse( + str(p), + media_type=_MEDIA.get(p.suffix, "application/octet-stream"), + headers={"Cache-Control": _IMMUTABLE}, + ) diff --git a/api/routes_changes.py b/api/routes_changes.py index 82de7d68cf1de8fcbd681de089bff07b4de73f0c..e3e439226e8697d03142aca0e830c510e1888c73 100644 --- a/api/routes_changes.py +++ b/api/routes_changes.py @@ -55,7 +55,7 @@ So: a valid session, a scope sanitised to a bucket-safe shape, and — for a `ut cached verdict that this session may open it. """ import time -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request import core.shared_overlay as shared_overlay import core.store as store @@ -224,3 +224,87 @@ def changes(scope: str = "", session: Session = Depends(require_session)): # a token frozen at 0 would promise liveness the backend is not delivering. tokens[label] = rev.get("token") if isinstance(rev, dict) else None return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()} + + +# ── ⭐⭐ D-144 (W33-T32) — A CAUGHT RENDER ERROR TELLS SOMEBODY ──────────────────────────────── +# +# ⛔ THE DEBT, in its own words: *"A caught render error tells nobody."* W30-T39 gave the app its +# first error boundary — measured at the time, `componentDidCatch|getDerivedStateFromError` was +# **0 hits** across all of `web/src`, so any throw anywhere blanked the whole product. The +# boundary CONTAINS the failure; it does not REPORT it, so the next user complaint still arrives +# as a sentence rather than as a stack. That is why owner item 2's crash could not be diagnosed +# and had to be fixed at both candidate causes. +# +# ⚠ IT LIVES HERE, IN THE CHANGE-TOKEN ROUTER, AND THE REASON IS WORTH ONE LINE. It is not about +# change tokens. It is here because this router is already mounted, already in one lane's fence, +# and a NEW `routes_*.py` costs a cross-fence mount (contract C2) that this wave has already spent +# three times — and a fourth finished-but-unmounted router is precisely the failure wave 23 shipped +# three of. Subject purity would have been bought with the exact risk the contract exists to +# prevent. ⚠ It performs no `store.get()`, so this file's zero-deep-copy claim (and +# `verify_live_workspace`'s count of it) is untouched: `telemetry.log` appends one line to a +# container-local JSONL. +# +# ⛔⛔ AND IT CARRIES NO ROW DATA, WHICH IS THE DEBT'S OWN HARD CONDITION. The body is rebuilt KEY +# BY KEY from exactly two strings; there is no `**body` anywhere below. A crash reporter that +# forwarded props or state would quietly ship a customer's cell values into a log — the one way +# this feature could do more harm than the silence it replaces. +_CLIENT_ERR_MAX = 4 * 1024 +_CLIENT_ERR_WINDOW_S = 60 +_CLIENT_ERR_PER_WINDOW = 12 +_CLIENT_ERR_HITS: dict = {} + + +@router.post("/client-error") +async def client_error(request: Request, session: Session = Depends(require_session)): + """One caught render error, forwarded to telemetry. `{message, componentStack}`. + + ⚠ SESSION-GATED, deliberately. A public crash sink is an unauthenticated append to a log file, + which is a disk-filling primitive with a URL. Only signed-in accounts can crash this app in a + way we need to hear about. + ⚠ RATE-LIMITED PER ACCOUNT, because the failure mode is a render LOOP: a boundary that catches, + resets and re-throws would otherwise post several times a second, for ever, from every open + tab. Beyond the window the report is DROPPED and the request still answers 200 — the client + must never learn to retry a report. + """ + now = time.time() + who = session.uname or "?" + seen = [t for t in _CLIENT_ERR_HITS.get(who, ()) if now - t < _CLIENT_ERR_WINDOW_S] + seen.append(now) + _CLIENT_ERR_HITS[who] = seen + if len(_CLIENT_ERR_HITS) > 2048: + for k in [k for k, v in _CLIENT_ERR_HITS.items() + if not v or now - v[-1] > _CLIENT_ERR_WINDOW_S]: + _CLIENT_ERR_HITS.pop(k, None) + if len(seen) > _CLIENT_ERR_PER_WINDOW: + return {"ok": True, "recorded": False} + + # The body, BOUNDED BY THE STREAM. `Body(...)` would already have been read and parsed before + # this line ran, and `content-length` is caller-supplied — the same argument `routes_forms` + # and `routes_publish` both carry. + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > _CLIENT_ERR_MAX: + raise err(413, "body_too_large", "that report is too large") + chunks.append(chunk) + import json as _json + try: + body = _json.loads(b"".join(chunks) or b"{}") + except ValueError: + body = {} + if not isinstance(body, dict): + body = {} + + # ⛔ TWO KEYS, BOTH TRUNCATED, NOTHING ELSE. See the block note above. + message = str(body.get("message") or "")[:500] + stack = str(body.get("componentStack") or "")[:4000] + if not message and not stack: + return {"ok": True, "recorded": False} + try: + import harness.telemetry as _tel + _tel.log("client_error", user=session.uname, tenant=session.tenant, + message=message, componentStack=stack) + except Exception: # noqa: BLE001 + # Telemetry may never break a request — `deps.py` carries the same rule at its own site. + return {"ok": True, "recorded": False} + return {"ok": True, "recorded": True} diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 309c29c1507de67daf16e5804144cb405d832fc4..25bafcd1158e54d207444bff1a56c86d08e7870b 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -30,7 +30,7 @@ from deps import Session, require_session #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a #: second correct one here would leave the same two-copies shape that produced the defect. -from routes_keychain import env_odoo_available +from routes_keychain import TENANT_WIDE_TYPES, env_odoo_available # ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit). # This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory` @@ -396,15 +396,31 @@ def directory(rt, uname, is_admin=False, incomplete=None): # reader. A token connector is the case R4 is really about: a member's own Stripe # key is theirs, and the card must say which kind of connection it is looking at. "scope": scopes.get(t["entry"]) or "business", - # ⚠ `manage` STAYS ADMIN-ONLY HERE, deliberately, and it is a smaller door than - # R4 allows. A member may now hold a PERSONAL connection — the room for it is - # Settings → Keychains, which W32-T11 opens to them — but this card's action word - # has been "the tenant-wide credential door" since wave 23 and a sibling gate - # asserts a member is offered none. Widening it is a one-word change once that - # assertion is retargeted; booked rather than done, because reddening another - # lane's gate to add a second door to a room that is already reachable is a bad - # trade. - "manage": "keychain" if is_admin else "", + # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A + # WIDENED ONE. This route's own docstring has promised since wave 23 that a + # non-admin may *"connect their OWN per-user accounts"*, and the next sentence + # then handed them an empty `manage` — a contradiction inside one paragraph, and + # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from + # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing + # still missing was a way to reach it from the map. + # + # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid. + # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"` + # row, and its own comment says why in the right words: *"the line is TENANT-WIDE + # vs PER-USER, not admin vs everyone … what they must never be offered is the + # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion + # is CORRECT and stays green. A member's personal connection is a different door + # into the same room, so it gets its own word. + # + # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE. + # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so + # offering a personal door for one of those types would be offering a button that + # 400s — the exact failure R6 removed from the Google card. Today that means + # Stripe and Shopify open for a member and Meta Ads does not + # ([[one-question-two-normalizers]]). + "manage": ("keychain" if is_admin + else "" if t["entry"] in TENANT_WIDE_TYPES + else "keychain_personal"), # Said once, on the row, rather than in a paragraph somewhere (R13): storing the # key is real and useful, and it is not the same as the data being on screen. "note": "Storing the key connects the account; reading its data lands in a later " @@ -566,9 +582,22 @@ def connectors_directory(session: Session = Depends(require_session)): """C11: every connector this platform knows about, with its state for THIS reader. Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the - workspace is connected to (and connect their OWN per-user accounts, which is the whole point - of R7's per-user slots). `manage` is what carries admin-ness — it is empty for a non-admin, - so the client renders the card without an action rather than offering a door that 403s. + workspace is connected to, and connect their OWN per-user accounts — which is the whole point + of R7's per-user slots. + + ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and + then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction + inside a single paragraph, which is how the promise survived three waves without a door behind + it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row: + + · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay + that way: it edits the key the whole workspace reads through. + · `keychain_personal` — the same room, for a connection that is this reader's own. Offered + to a non-admin **only** where a personal credential is actually + allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so + Odoo and Meta Ads never get it). + · `""` — nothing this reader can do here; the client renders no control + rather than a door that would 403. """ # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as # careful and is silently False for every admin on earth — the manage actions would simply diff --git a/api/routes_customers.py b/api/routes_customers.py index f7cee1791ee0de4552c2bb0c634bbec16ce7d2b4..a8c69b255c7b87ca150dda49f227146fbe1aa732 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -209,6 +209,17 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # The derived channel: cohort membership cells + measure column values, ONE dict — the # same read-only channel the embed host hands to rows_from_pool. derived = aios_grid.cohort_cells(lists) + # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried + # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id` + # as its join key — so the merge would have lost the one value every Odoo document joins on. + # + # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS + # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is + # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for + # one fact, which is the class of defect item 12 is about. This channel exists for exactly + # this — a value the host knows per render and the pool has no business storing. + for pid in pids: + derived.setdefault(pid, {})["partner_id"] = pid for pid, cells in measure_resolve.column_values( fields, team_id, pids, today, stamp, rt.measure_memo, on_error=_measure_err).items(): diff --git a/api/routes_feedback.py b/api/routes_feedback.py new file mode 100644 index 0000000000000000000000000000000000000000..6e4dd9bf8160d337571a10e4c5045d9016dd3ef5 --- /dev/null +++ b/api/routes_feedback.py @@ -0,0 +1,180 @@ +"""routes_feedback.py — WAVE 35 (ruling R8, contract C6): FEEDBACK GOES TO THE OPERATOR PLANE. + +R8: *"Feedback goes to the platform-wide `loopable` operator plane. No tenant sees another's, and +the submitting tenant cannot edit or delete what it sent."* + +⛔⛔ TWO DOORS, TWO DIFFERENT WALLS, AND THAT ASYMMETRY IS THE WHOLE TICKET. D-221 is already on the +register for a cross-tenant WRITE gated on `admin_gate` alone, so getting this backwards is a repeat +rather than a novelty: + + POST /feedback ANY authenticated session, any tenant. It is the TENANT'S OWN ACT. + GET /feedback `core.platform_admin.is_platform_admin` ONLY. It is the OPERATOR'S read. + +A single wall cannot express that. Gating the write on the operator predicate would mean nobody can +send feedback; gating the read on `admin_gate` would hand every tenant admin every other tenant's +words. So the walls are declared per route, and `verify_api` drives BOTH from a tenant admin, a +plain user and a platform admin rather than asserting the source. + +⛔ THE PROVENANCE IS STAMPED SERVER-SIDE AND NEVER READ FROM THE BODY. `tenant`, `user` and `ts` +come from the verified session and the clock. A body-supplied tenant would let any account file +words against another company's name, in a store only the operator reads and therefore only the +operator could ever be misled by. + +⛔ NO EDIT AND NO DELETE EXIST AT ALL — not for the tenant (R8 says so) and not for the operator +either. R8 grants the operator a READ; a delete door is a capability nobody asked for on a store +whose whole value is that it is append-only. There is nothing here to forget to wall. + +⚠ ONE STORE, AND WHERE IT PHYSICALLY LIVES IS STATED RATHER THAN IMPLIED. This uses `core.store` +with a raw key, which is the same thing `routes_platform_admin` does for `runtime.TENANTS_KEY` and +for the same reason: a platform fact belongs in the default store, not inside any tenant's +namespace. ⚠ That default is currently tenant #0's dataset repo (`OS_DATA_REPO`), so "the platform +store" and "tenant #0's repo" are the same bucket today. That is pre-existing (the control plane's +tenant records are already there) and it is a hosting fact, not a permission one — no tenant route +can read this key, because none of them names it. +""" +import time + +from fastapi import APIRouter, Body, Depends + +import core.platform_admin as platform_admin +import core.store as store +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The platform-wide bucket. `{"rows": [ {id, tenant, user, category, text, ts} ]}` — newest LAST +#: in the store, newest FIRST on the wire. +STORE_KEY = "platform_feedback" + +#: ⭐ THE CATEGORY VOCABULARY IS SERVER-OWNED AND SERVED (B's `ASK B-1 (3)`, agreed). A client +#: constant beside a server enum drifts, and the drift shows up as a dropdown offering a value the +#: door refuses. B renders exactly this list; the door accepts exactly these keys. +CATEGORIES = ( + {"key": "bug", "label": "Something is broken"}, + {"key": "idea", "label": "An idea or a request"}, + {"key": "data", "label": "The numbers look wrong"}, + {"key": "speed", "label": "Something is slow"}, + {"key": "other", "label": "Something else"}, +) + +MAX_CHARS = 4000 + +#: A ceiling on the whole store. ⛔ REPORTED AT BOTH ENDS, never a silent drop: the submitter is +#: refused with the cause, and the operator's own payload says the store is full so they can act. +#: Dropping the OLDEST rows instead would delete the operator's earliest feedback to make room for +#: the newest, which is the one direction this store must never move in. +MAX_ROWS = 5000 + + +def _rows(): + try: + raw = store.get(STORE_KEY) or {} + except Exception: + return [] + rows = raw.get("rows") if isinstance(raw, dict) else None + return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else [] + + +@router.get("/feedback/form") +def feedback_form(session: Session = Depends(require_session)): + """What the composer renders. Any session — every account may send feedback. + + ⚠ It deliberately returns NO submissions. R8 gives the tenant no read of what it sent, so a + "your feedback" list would be a second door to a thing the ruling closed. The 201 is the receipt. + """ + return {"categories": [dict(c) for c in CATEGORIES], "maxChars": MAX_CHARS} + + +@router.post("/feedback", status_code=201) +def submit_feedback(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Send one piece of feedback to the operator plane. `{"category": key, "text": str}`. + + ⚠ ANY authenticated session, deliberately — see the header. The provenance is stamped from the + VERIFIED session, never from the body. + """ + body = body if isinstance(body, dict) else {} + category = str(body.get("category") or "").strip() + text = str(body.get("text") or "").strip() + if category not in {c["key"] for c in CATEGORIES}: + # Named, and it names the fix: a client whose dropdown has drifted from this list is a + # client whose user is about to lose what they typed. + raise err(400, "unknown_category", + "that is not one of the feedback categories. Reload the page and try again") + if not text: + raise err(400, "bad_request", "type what you want to tell us") + if len(text) > MAX_CHARS: + raise err(400, "too_long", + f"feedback is limited to {MAX_CHARS:,} characters. Yours is {len(text):,}") + if not store.available(): + raise err(503, "store_unavailable", + "feedback could not be sent right now. Nothing was saved.") + if len(_rows()) >= MAX_ROWS: + raise err(503, "feedback_full", + "the feedback store is full and we have been told. Nothing was saved, so please " + "send this again later") + row = { + # ⛔ EVERY ONE OF THESE FOUR IS SERVER-SIDE. A body-supplied tenant or user would let any + # account file words against somebody else's name in a store only the operator reads. + "tenant": str(session.tenant), + "user": str(session.uname), + "ts": int(time.time()), + "category": category, + "text": text, + } + row["id"] = f"fb_{row['ts']}_{abs(hash((row['tenant'], row['user'], text))) % 10**8:08d}" + + def _up(data): + data = data if isinstance(data, dict) else {} + rows = data.get("rows") + data["rows"] = (rows if isinstance(rows, list) else []) + [row] + return data + + try: + # `flush='sync'`: a person pressed Send and is being told it landed. The coalescing mode is + # for autosaves nobody is waiting on. + store.update(STORE_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "feedback could not be sent right now. Nothing was saved.") + return {"ok": True, "id": row["id"]} + + +@router.get("/feedback") +def list_feedback(tenant: str = "", session: Session = Depends(require_session)): + """R8's operator read: every tenant's feedback, newest first. PLATFORM ADMIN ONLY. + + ⛔ THE WALL IS `core.platform_admin.is_platform_admin`, IMPORTED AND CALLED — the double lock + (the record flag AND the `loopable` tenant), never re-expressed here. A tenant admin is not + admitted: `role: 'admin'` is one company's authority over its own workspace, and this is every + company's words in one list. + ⚠ `?tenant=` NARROWS an already-admitted operator's view. It is a convenience, not a wall, and + it is applied after the gate so it can never be the thing that admits anybody. + """ + if not platform_admin.is_platform_admin(session.user): + raise err(403, "forbidden", "this is a Loopable operator surface") + rows = _rows() + if tenant: + want = str(tenant).strip().lower() + rows = [r for r in rows if str(r.get("tenant") or "").lower() == want] + # ⛔ THE TIEBREAK IS THE STORE'S OWN ORDER, AND WITHOUT IT "NEWEST FIRST" IS FALSE FOR ANY TWO + # ROWS FILED IN THE SAME SECOND (wave-35 QA). `ts` is `int(time.time())` — SECOND resolution — + # and Python's sort is STABLE, so under `reverse=True` a tie keeps INSERTION order, which is + # oldest-first: the one direction this route promises never to return. It is not hypothetical + # and it is not a flake: `api_api`'s R8 row failed on it in the close-out battery, in the + # wave-35 QA battery and in a solo re-run, and the baseline recorded the true count anyway, so + # a REAL regression would have read as the known-flake line. + # ⚠ ENUMERATE OVER THE ALREADY-FILTERED LIST: `_rows()` appends, so a row's index IS its arrival + # order (the module docstring's "newest LAST in the store"), and `?tenant=` only drops rows, so + # the surviving indexes stay monotonic. Raising `ts` to sub-second resolution instead would + # break every id already minted as `fb__…`. + rows = [r for _, r in sorted(enumerate(rows), + key=lambda p: (int(p[1].get("ts") or 0), p[0]), reverse=True)] + out = {"rows": [dict(r) for r in rows], "total": len(rows), "capacity": MAX_ROWS, + "tenants": sorted({str(r.get("tenant") or "") for r in _rows() if r.get("tenant")})} + if len(_rows()) >= MAX_ROWS: + # R6's second sentence, aimed at the person who can act: submissions are being REFUSED right + # now, and the operator is the only one who can see that from here. + out["note"] = ("the feedback store is at capacity, so new submissions are being refused. " + "Archive what you have read") + return out diff --git a/api/routes_grid.py b/api/routes_grid.py index 6ecb9ae5218a9f681e81a330664cfb8bd55461dc..6c7609921247127cfa6dc7a7880db7a3fd45bbcd 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -726,6 +726,111 @@ def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "custome return out +#: ⭐⭐ THE BULK CELL DOOR'S OWN CEILING, and it is REPORTED rather than silently enforced. The +#: standing no-cap rule governs data read from a connected source; this is a WRITE of team-typed +#: values, so a bound is right — but R6's second sentence still binds, which is why exceeding it +#: is a 400 naming the number and the count, never a truncation. +#: Sized against the real job with headroom: the 2027 catalog is 1,397 rows. +MAX_BULK_ROWS = 10_000 + + +@router.post("/grid/bulk-cells") +def bulk_cells(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """`{scopeKey, rows: {"": {key: value}}}` → shared cells on many rows, ONE transaction. + + ⭐⭐ WHY THIS EXISTS AT ALL, because "we already have `/grid/events`" is the obvious objection. + That door is capped at `_MAX_EVENTS = 24` (it is sized for the client's resend window), so a + 1,397-row import is ~59 sequential POSTs against ONE JSON document — and this repo has a + MEASURED scar for exactly that: 18 writes against one document under the store's coalescing + single-flight landed **zero** while answering 200 eighteen times. Raising `_MAX_EVENTS` would + have widened the browser's own resend window to fix a script's problem. One door, one + transaction, is the honest shape. + + ⛔ TENANT-WIDE KEYS ONLY, AND THAT IS THE POINT RATHER THAN A LIMITATION. It writes through + `shared_overlay`, which every account in the tenant reads. A per-user bulk write would be a + contradiction: nobody imports 1,397 rows of the team's work so that one account can see it — + that is precisely the failure owner ruling R6 was made to avoid. A key that is not declared + `shared: true` in the canonical contract is REFUSED BY NAME. + + ⛔ ADMIN ONLY. This mutates data every account in the tenant reads, in bulk, in one call. + D-172 already records that the shared stratum has an asymmetric wall (anyone may create a + tenant-wide column, only the creator or an admin may delete one); a new door does not get to + inherit the loose half of an asymmetry somebody already flagged. + """ + import core.shared_overlay as shared_overlay + import modules.product_data as pd + import core.perm_scope as perm_scope + from routes_products import MODULE as PRODUCT_MODULE, scoped_pool + + scope = _scope_or_400((body or {}).get("scopeKey")) + if scope != "product": + # Fail-closed with the reason: the shared stratum is a PRODUCT-topic mechanism today. + raise err(400, "scope_not_bulk_writable", + f"bulk cell writes are available on the product database; '{scope}' has no " + f"tenant-wide cell stratum") + session.require(PRODUCT_MODULE) + if not session.admin: + raise err(403, "admin_only", + "a bulk write changes values every account in this workspace reads") + + rows = (body or {}).get("rows") + if not isinstance(rows, dict): + raise err(400, "bad_rows", "expected {rows: {\"\": {field: value}}}") + if len(rows) > MAX_BULK_ROWS: + raise err(400, "too_many_rows", + f"at most {MAX_BULK_ROWS} rows per request; this one carried {len(rows)}") + + pids, _team, _src, fields_base = scoped_pool(session) + allowed = {int(p) for p in pids} + shared_keys = set(pd.SHARED_KEYS()) + hidden = perm_scope.hidden_keys(session.user, PRODUCT_MODULE, fields_base) + + clean, unknown_pid, refused_keys = {}, [], {} + for raw_pid, values in rows.items(): + try: + pid = int(raw_pid) + except (TypeError, ValueError): + unknown_pid.append(str(raw_pid)[:40]) + continue + # ⚠ THE POOL IS THE WALL. `scoped_pool` is the same predicate the read door uses, so a + # caller cannot write a row they could not see — including a row in another BU. + if pid not in allowed: + unknown_pid.append(str(raw_pid)[:40]) + continue + keep = {} + for key, value in dict(values or {}).items(): + key = str(key) + if key not in shared_keys: + refused_keys.setdefault("not_shared", set()).add(key) + continue + if key in hidden: + refused_keys.setdefault("hidden_by_permissions", set()).add(key) + continue + if not isinstance(value, (str, int, float)) or isinstance(value, bool): + refused_keys.setdefault("unsupported_value", set()).add(key) + continue + keep[key] = value + if keep: + clean[pid] = keep + + written = shared_overlay.put_rows(pd.TABLE_KEY, clean, st=pd.TABLE_OPS.st) if clean else {} + out = { + "rows_written": len(written), + "cells_written": sum(len(v) for v in written.values()), + "rows_requested": len(rows), + } + # ⭐ R6's second sentence: what did NOT land, and why. A bulk door that reports only its + # successes is how 701 unmatched SKUs disappear quietly. + if unknown_pid: + out["rows_not_in_your_pool"] = {"count": len(unknown_pid), + "sample": sorted(unknown_pid)[:10]} + if refused_keys: + out["refused_fields"] = {reason: sorted(keys) + for reason, keys in refused_keys.items()} + return out + + @router.post("/grid/events") def grid_events_route(body: dict = Body(default=None), session: Session = Depends(require_session)): @@ -822,13 +927,30 @@ def grid_events_route(body: dict = Body(default=None), try: for one in events: eid = str(one.get("id") or "") if isinstance(one, dict) else "" + # ⭐⭐ D-291 — WHICH event was refused, not merely THAT something was. The handler + # appends to a shared list, so the refusals belonging to THIS event are exactly the + # ones that appeared across THIS call. Reading the list once after the loop would + # answer "the batch was refused" and leave the caller to guess which member, which is + # the same class of unfalsifiable answer the channel exists to end. + _before = len(ctx.out.refusals) rerender = grid_events.handle_one(one, ctx) - results.append({"id": eid, "rerender": bool(rerender)}) + row = {"id": eid, "rerender": bool(rerender)} + mine = ctx.out.refusals[_before:] + if mine: + # `refused` is the SHAPE a caller branches on; the first reason is the one that + # stopped this write (a handler returns at its first refusal). + row["refused"] = mine[0] + results.append(row) except grid_events.StoreUnavailable: raise err(503, "store_unavailable", "the tenant store is unavailable — none of your changes were saved") out = {"results": results, "rerender": any(r["rerender"] for r in results)} + if ctx.out.refusals: + # ⚠ ALSO AT THE TOP LEVEL, because a batch that was wholly refused must not read as a + # batch that wholly landed. A caller that only checks the envelope still learns something + # is wrong, and a caller that walks `results` learns exactly which member. + out["refusals"] = list(ctx.out.refusals) if ctx.out.doc is not None: # ⭐ C4 / W30-T27 — `docPayload` IS THE NAME THE CLIENT ALREADY DECLARES. `types.ts` has # carried `docPayload?: {pid, docId, name, mime, data_b64}` since C5, and `Documents.tsx` diff --git a/api/routes_keychain.py b/api/routes_keychain.py index aa051d5b975eeabd5ac67db60652ea04b47edc98..4ef61ca06030bf7abf2238ae1421cbde6686560d 100644 --- a/api/routes_keychain.py +++ b/api/routes_keychain.py @@ -296,6 +296,114 @@ def _rel_reconnect(rt): return True +# ══════════════════════════════════════════════════ W35-T45 / R11: THE ENV -> KEYCHAIN MIGRATION +# +# R11: *"Tenant #0's Odoo credential MIGRATES onto the keychain, with the environment kept as +# fallback."* The owner chose this over a read-only display row WITH THE MIGRATION RISK STATED, so +# the risk is what this block is mostly about. +# +# ⭐ WHAT WAS ALREADY TRUE, CHECKED BEFORE ANY OF IT WAS WRITTEN: the keychain-first-then-env +# RESOLVER has existed since the 2026-08-04 cutover. `harness/runtime.py::odoo_source` is already +# (1) a keychain `odoo` entry, (2) tenant #0's compiled env connector, (3) None for anybody else, +# and `visible_entries` already serves a non-admin the row WITHOUT a preview. So R11 is not "build +# a resolver" — it is "give tenant #0 the ROW", which is the only reason its Keychain page looks +# empty while its Odoo grids work. +# +# ⛔⛔ AND THE ONE REAL HAZARD IS NOT THE CREDENTIAL, IT IS THE PAUSE FLAG. `odoo_flag_key()` returns +# the first keychain `odoo` entry id when one exists and `ENV_ODOO_FLAG_KEY` ("odoo-env") otherwise — +# so CREATING THE ENTRY MOVES THE ADDRESS THE PAUSE FLAG LIVES AT. A tenant #0 that was paused under +# `odoo-env` would come back UNPAUSED, silently, at the first boot after this ships: the connector +# resumes pulling live Odoo because a migration changed which key the freeze was stored under. That +# is D-259's exact shape (a pause written under one key and read under another) and +# [[a-guard-bound-to-a-role-stops-guarding-when-the-role-moves]]. The flag is carried across in the +# SAME pass, and the carry is asserted. + + +def _env_odoo_fields(): + """The four env values `odoo_client` reads, or `(None, why)` when they are not all present. + + ⛔ ALL FOUR OR NOTHING, and this completeness check is load-bearing rather than defensive. + Creating the entry makes `odoo_source` resolve through branch 1 INSTEAD of the env — so a + PARTIAL migration would hand the connector `{url, db}` and no key and take tenant #0's Odoo + offline, on a deployment where it had been working. The env fallback cannot save it, because the + entry's existence is what turns the fallback off. + ⚠ The names are `odoo_client.py`'s own (`ODOO_URL`/`ODOO_DB`/`ODOO_USER`/`ODOO_API_KEY`) and the + field names are `harness/connectors/odoo.py`'s stored shape (`{url, db, user, api_key}`). Two + vocabularies meet here; nowhere else. + """ + want = (("url", "ODOO_URL"), ("db", "ODOO_DB"), + ("user", "ODOO_USER"), ("api_key", "ODOO_API_KEY")) + got = {field: (os.environ.get(env) or "").strip() for field, env in want} + missing = sorted(env for field, env in want if not got[field]) + if missing: + return None, (f"the environment is missing {', '.join(missing)}, and a partial credential " + f"would take this tenant's Odoo offline rather than migrate it") + return got, "" + + +def migrate_env_odoo(rt): + """R11 — put tenant #0's environment Odoo credential on its keychain, once. Returns a report. + + `{"done": bool, "entry": id|"", "carried_pause": bool, "why": str}` — `why` is filled on every + path including the skips, because "already migrated", "no keychain key on this deployment" and + "the env is incomplete" are three different operator actions. + + ⛔⛔ IT MUST RUN IN THE CONTAINER, WHICH IS WHY `main.py` CALLS IT AND NO SCRIPT DOES. D-195, + measured three times: a developer's CLI write to the tenant store is reverted by the running + Space within a minute (download-modify-upload, last-write-wins) — and **the write reports success + every time**, then a fresh read confirms it, and it is gone by the next poll. A CLI migration here + would be a dry run that lies, and the thing it would lie about is a credential. + + ⚠ FAIL-QUIET AND IDEMPOTENT. It runs on EVERY boot; the second one must be a no-op and a + third-party failure must not take the boot down. + """ + out = {"done": False, "entry": "", "carried_pause": False, "why": ""} + if not env_odoo_available(rt): + # Not tenant #0, or this deployment has no env Odoo at all. Both are normal states. + out["why"] = "this tenant has no environment Odoo credential to migrate" + return out + fields, why = _env_odoo_fields() + if not fields: + out["why"] = why + return out + # ⛔ READ THE PAUSE FLAG BEFORE THE WRITE. After the entry exists, `odoo_flag_key()` answers the + # NEW key and the old one is unreachable through the resolver — so the only moment this fact can + # be observed is now. [[undo-capture-before-the-write]] applied to a guard rather than to data. + try: + was_paused = bool(((rt.get(_CONNECTOR_FLAGS_KEY) or {}) + .get(_ENV_ODOO_FLAG_KEY) or {}).get("paused")) + except Exception: # noqa: BLE001 + was_paused = False + row, why = _kc().ensure_entry_of_type( + rt, "odoo", "Odoo (migrated from this deployment)", fields, "system") + if row is None: + out["why"] = why + return out + out["done"], out["entry"] = True, row["id"] + if was_paused: + # The freeze followed the credential. Without this the connector silently RESUMES pulling + # live Odoo at the first boot after the migration. + def _up(cur): + cur = cur if isinstance(cur, dict) else {} + entry = dict(cur.get(row["id"]) or {}) + entry["paused"] = True + entry["pausedBy"] = "system" + entry["pausedNote"] = ("carried over from the environment source when the credential " + "was migrated onto the keychain") + cur[row["id"]] = entry + return cur + + try: + rt.update(_CONNECTOR_FLAGS_KEY, _up, flush="sync") + out["carried_pause"] = True + except Exception as exc: # noqa: BLE001 + # ⛔ SAID OUT LOUD. A migration that moved the credential and lost the freeze is worse + # than one that did not run, so this is the one failure that must never be silent. + out["why"] = (f"the credential migrated but the PAUSE could not be carried over " + f"({type(exc).__name__}), so this tenant's Odoo is no longer frozen") + return out + + def _own_row(session, entry_id): """The visible row for `entry_id`, or a 404. ⛔ A 404 rather than a 403 for an entry the caller cannot see: telling a member that somebody else's personal credential EXISTS is the @@ -703,6 +811,31 @@ def odoo_config_put(body: dict = Body(default=None), if server_db: notes.append(_rewrite_server_db(session, server_db)) + # ⭐⭐ W33-T65 / W30-R6's SECOND SENTENCE — THE CADENCE IS SET AND ONLY PARTLY OBEYED, AND THE + # PERSON SETTING IT IS THE ONE WHO HAS TO BE TOLD. Measured, not guessed: + # · `main.py::_store_resync_loop` reads the interval from `sync_seconds(get_runtime( + # "royal-imports"))` — a HARDCODED slug — and then sleeps ONCE for the whole process. So + # for tenant #0 this control moves EVERYBODY's sync, and for every other tenant the value + # is stored, clamped, displayed and never read. + # · `manual` stores and reads back as `None`, and the loop has no branch on it: it sleeps a + # default 1800 s and syncs anyway. "Only when I ask" asks all the same. + # ⛔ NEITHER IS FIXABLE FROM THIS FILE — the loop lives in `main.py`, which this lane does not + # own — and shipping a setting that silently does nothing is the exact failure R6 names. So it + # is REPORTED here, at the moment of the change, with what it really controls. Delete these + # notes when the loop becomes per-tenant, not before. + if clean_every is not None: + if not rel.is_royal(session.tenant): + notes.append("this interval is saved, but the sync loop currently reads its schedule " + "from one workspace for the whole deployment — so it will not change how " + "often YOUR data refreshes until per-workspace scheduling ships") + else: + notes.append("this interval is saved and it is the one the deployment's sync loop " + "uses — it changes the refresh rate for every workspace on this " + "deployment, not only this one") + if clean_every == "manual": + notes.append("⚠ 'manual' does not yet stop the background sync: the loop has no " + "manual-only branch, so data still refreshes on the default interval") + out = odoo_config(session) return {**out, "notes": [n for n in notes if n]} @@ -783,6 +916,32 @@ def odoo_disconnect(session: Session = Depends(require_session)): cur["frozenBy"] = str(session.uname) return cur + # ⭐⭐ W33-T65 — THE FREEZE HAD A HOLE, AND THE NOTE BELOW WAS THE THING THAT MADE IT A DEFECT + # RATHER THAN A LIMIT. `rel.frozen` has exactly ONE consumer, `odoo_relational.refresh`, which + # materialises the EIGHT copied grids. The other two (`ut_odoo_order_lines`, + # `ut_odoo_gl_lines` — `READ_THROUGH_KEYS`) do not go through `refresh` at all: they read + # THROUGH the per-tenant DuckDB mirror, and the mirror is advanced by `datastore.sync_all`, + # which gates on the connector PAUSE flag and has never heard of `frozen`. So a disconnected + # workspace kept serving LIVE, still-moving rows in its two biggest grids while this route's + # own sentence promised *"nothing is being refreshed"*. + # + # ⛔ THE FIX IS TO MAKE THE SENTENCE TRUE, not to soften it. Disconnect now flips the pause + # flag on the RESOLVED source as well, which is the switch `sync_all` and `reconcile_deletes` + # actually read — so both halves of "frozen" mean the same thing. Two orderings are borrowed + # from `pause_connector` because it learned them the hard way: + # · the flag key is resolved BEFORE the credential is deleted — after the delete there is no + # resolved source left to name, and the flag would land under a key nothing reads (D-10). + # · the snapshot is captured BEFORE the flag flips, so there is a last-successful-sync to + # serve; a failed capture leaves the connector live rather than paused-with-nothing. + _, flag_key = _resolved_odoo_key(session.runtime) + snapshots = 0 + if flag_key: + try: + snapshots = save_pool_snapshots(session.runtime, taken_by=session.uname) + except Exception: # noqa: BLE001 + # A snapshot is a nicety; the freeze is the promise. Reported, never fatal. + snapshots = 0 + # ⚠ THE FLAG FIRST, THE CREDENTIAL SECOND. If the flag write fails, nothing has happened and # the connector is still live; if the delete failed AFTER the flag landed, the tenant is # frozen with an unused key, which is recoverable from the Keychains pane. The reverse order @@ -791,6 +950,19 @@ def odoo_disconnect(session: Session = Depends(require_session)): session.runtime.update(rel.CONFIG_KEY, _up, flush="sync") except Exception: raise err(503, "store_unavailable", "nothing was disconnected — try again") + + paused_mirror = False + if flag_key: + def _pause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": True} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _pause, flush="sync") + paused_mirror = True + except Exception: # noqa: BLE001 + paused_mirror = False + removed = "" if entry is not None: try: @@ -801,10 +973,28 @@ def odoo_disconnect(session: Session = Depends(require_session)): raise err(503, "store_unavailable", "the databases are frozen but the stored credential was not removed — " "delete it under Keychains") + + # ⛔ THE SENTENCE IS COMPOSED, NOT CONSTANT, because the two sources genuinely differ and the + # old fixed string was wrong about one of them. A tenant whose Odoo came from the DEPLOYMENT + # ENVIRONMENT has no credential for this route to remove — `.env` is the container's, not a + # tenant screen's — so it said "the key back" about a key it never held. R6's second sentence: + # the limit that cannot be removed is REPORTED, with what to do instead. + note = ("Your Odoo databases are frozen: every row and every column you had is still there and " + "still readable, and nothing is being refreshed.") + if not paused_mirror and flag_key: + note += (" ⚠ The live mirror could not be paused, so the two read-through databases " + "(order lines and GL lines) may keep advancing — pause the Odoo connector under " + "Keychains to stop them.") + note += (" Reconnecting adds the key back and resumes into the same databases." if entry + else " This workspace's Odoo credential comes from the deployment environment, so " + "there was no stored key to remove — the databases are frozen and Reconnect " + "resumes them into the same tables.") return {"frozen": True, "removedEntry": removed, - "note": "Your Odoo databases are frozen: every row and every column you had is still " - "there and still readable, and nothing is being refreshed. Reconnecting adds " - "the key back and resumes into the same databases."} + # ⚠ ON THE WIRE, so the client and a gate can both see which half happened. A boolean + # nobody returns is a guarantee nobody can check. + "pausedMirror": paused_mirror, "snapshots": snapshots, + "source": "keychain" if entry else "env", + "note": note} @router.post("/admin/connectors/odoo/reconnect") @@ -820,8 +1010,26 @@ def odoo_reconnect(session: Session = Depends(require_session)): _rel_reconnect(session.runtime) except Exception: raise err(503, "store_unavailable", "the change was not saved — try again") + # ⭐⭐ W33-T65 — AND THE PAUSE DISCONNECT SET, or the freeze would be one-way. Clearing only + # `frozen` restores the eight materialised grids and leaves the mirror pinned forever, so the + # two read-through grids would sit at the disconnect date while the panel said "connected + # again" — the same disagreement between the halves of "frozen", pointing the other way. + # ⚠ Resolved AFTER `_rel_reconnect`: a tenant reconnects by adding the key back FIRST, so the + # resolved source only exists again by this point. + _, flag_key = _resolved_odoo_key(session.runtime) + resumed = False + if flag_key: + def _unpause(cur): + cur = cur if isinstance(cur, dict) else {} + cur[str(flag_key)] = {"paused": False} + return cur + try: + session.runtime.update(_CONNECTOR_FLAGS_KEY, _unpause, flush="sync") + resumed = True + except Exception: # noqa: BLE001 + resumed = False connected = bool(_odoo_entry(session)) or env_odoo_available(session.runtime) - return {"frozen": False, "connected": connected, + return {"frozen": False, "connected": connected, "resumedMirror": resumed, "note": ("Odoo is connected again and the databases you already had will refresh in " "place." if connected else "The freeze is lifted, but there is no Odoo credential yet — add one under " diff --git a/api/routes_nav.py b/api/routes_nav.py index a0fc896c047d2a2912bcd7c178fabe71a73cc371..9ae358b988e5faa26ed55d6b1e25fbb3aaba7bf3 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -248,6 +248,183 @@ def _read_recents(runtime, uname, allowed): return out[:_MAX_RECENTS] +# ── VIEW READERS — shared with `routes_starred` (W35-T39/T42) ──────────────────────────────── +# +# ⛔⛔ W35-T43 (rulings R4/R7) — THE "MARK IMPORTANT" COUNTS ARE GONE FROM THIS ROUTE, and the +# block that computed them (`_important_counts`) went with them. It read one store bucket PER +# DATABASE on the route D-175 spent a whole wave reducing to a single read: MEASURED on a 13-database +# fixture, `GET /nav` performed **14 per-database view-bucket reads** and stamped `important` on +# **14** pages. It is **0 and 0** now. +# +# Those reads were bounded by a 2.5 s wall clock (`_IMPORTANT_BUDGET_S`) whose own note conceded the +# cost: twelve buckets are 6.2 ms WARM and **7,331 ms COLD**, so the budget existed to stop a cold +# container converting a slow success into a manufactured failure against `nav.ts`'s 20 s deadline. +# A budget that can be exhausted is a number that can be silently short, which is why it also had to +# publish a `degraded` entry. R7 removes the whole apparatus by moving the question to +# `GET /starred/counts`, called AFTER the page paints — where a slow answer costs a late badge +# instead of a late rail. Closes D-288 and D-289. +# +# ⚠ WHAT SURVIVES AND WHY: `_visible_views`, `_view_record_count` and `_granted_view_ids` are the +# READERS, and `routes_starred` calls all three. They were never the cost — the per-database store +# read was — and duplicating them into the new route would have put two answers behind one badge. + +#: Suffix `core.view_templates.workspace_key` appends. Stripping it is how a page key becomes the +#: TOPIC key the cohort bucket is named from — `customer_data` -> `customer_table_workspace` -> +#: `customer` -> `customer_cohorts`. Derived rather than re-listed on purpose: `_WS_KEYS` is +#: already the one place `customer_data`/`product_data` are mapped to their topic, and a second +#: copy here would be free to drift from it ([[one-question-two-normalizers]]). +_WS_SUFFIX = "_table_workspace" + + +def _visible_views(doc, uname, is_admin, granted_ids): + """Every view on ONE database that THIS caller can see, from an already-read workspace doc. + + ⛔ PER-CALLER, NOT TENANT-WIDE, and this is the half a server-side count is most likely to get + wrong. `_table_workspace` is `{username: {views, fields, overlays}, '__shared__': {...}}` + — one home per view, never both — so "how many views are marked important here" has a + DIFFERENT answer for every account. Measured on tenant #0: `leadership` has one marked view and + `admin` has none, on the same database. A tenant-wide count would put a number in the owner's + rail that no view sidebar they can open would ever add up to. + + ⚠ `table_store._may_see` is imported rather than re-expressed. It is private, and reaching for + it is still the right call: the alternative is `TableOps.shared_views`, which re-reads the whole + bucket per database (the N whole reads this block exists to avoid), and the only other option is + a second copy of a permission predicate. A wrong copy of `_may_see` widens what a user is told + exists; a private import cannot. + """ + import core.table_store as table_store + out = {} + for vid, view in ((doc.get(uname) or {}).get("views") or {}).items(): + if isinstance(view, dict): + out[str(vid)] = view + for vid, view in ((doc.get(table_store.SHARED_KEY) or {}).get("views") or {}).items(): + if isinstance(view, dict) and table_store._may_see(view, uname, is_admin): + out.setdefault(str(vid), view) + # The wave-21 named-user grants. The ids come from ONE tenant-wide bucket read once for the + # whole request; the RECORD is already in hand, in whichever stratum its owner keeps it. + if granted_ids: + for stratum, blob in doc.items(): + if stratum == uname or not isinstance(blob, dict): + continue + for vid, view in (blob.get("views") or {}).items(): + if str(vid) in granted_ids and isinstance(view, dict): + out.setdefault(str(vid), view) + return out + + +def _view_record_count(cfg, cohorts): + """How many RECORDS this view resolves to, or None when that cannot be answered for free. + + ⛔ `None` IS AN ANSWER AND IT IS THE IMPORTANT ONE. Two of the three shapes below are exact + because the view CARRIES its row set; the third — an ordinary filtered view — can only be + counted by running its filters over the records, and the records are the one thing this route + must never read (`D-175`/`D-185`: `/nav` is a rows-free projection, and reaching for `rows` + here raises by that projection's own contract). So a filtered view is reported as UNCOUNTED and + the database's `partial` flag says so, which is the whole of R6's second sentence applied to a + badge: a limit that cannot be removed is REPORTED with its cause, never papered over with a + number that is short by an unknown amount. + + ⚠ THIS IS ALSO WHY THE SERVER DOES NOT SIMPLY MIRROR THE CLIENT. `CustomerGrid::alertCounts` + counts a filtered view fine and gives up on a SERVER-WINDOWED one (`D-205`'s `Important 0+`); + this end is the exact inverse — it has no rows at all and no window either. The two are honest + about different halves, which is why `partial` had to be on the wire rather than derived. + """ + if not isinstance(cfg, dict): + return None + # A cohort-locked view IS its cohort: the lock and the id are the same fact (`grid_events` + # re-stamps it on every write), and a cohort's membership is a stored pid LIST, not a query. + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + n = cohorts.get(lock) + return int(n) if isinstance(n, int) else None + # A curated row set carries its own count. + pids = cfg.get("memberPids") + if isinstance(pids, list) and pids: + return len(pids) + return None + + +def _visible_database_keys(session): + """`(keys, enumerated)` — every DATABASE this session may open that HAS a view bucket. + + ⭐ ADDED BY W35-T39 because neither existing enumeration in this file answers this question: + + · `_placeable_top_keys` applies the ACCOUNT grant and `may_open`, and **not the TENANT + CATALOGUE** — correctly, because a folder placement is cosmetic and pruning one is loss. + A star scan cannot borrow that: it would open the view bucket of a database this workspace's + catalogue does not include. `nav()` applies the catalogue filter; that helper never has. + · `nav()`'s own `_page_keys` is assembled from the page DICTS it is building for the wire, so + it cannot be reached from another route without rebuilding the payload. + + So this is the KEY-SET question on its own, with the same three walls `nav()` applies in the same + order: the account grant (`perms.nav_pages`), the TENANT catalogue, and `may_open` for the + tenant's own databases. `view_templates.workspace_key` is the last filter and it is what makes + the answer honest for a caller about to read a view bucket: a module surface with no workspace + (`sales`, `ar`) is not a database and has no views to star. + + ⚠ THE `parent` CLAUSE BELOW IS A NO-OP TODAY AND IS KEPT ONLY TO MIRROR `nav()`. `perms.nav_pages` + excludes `group_only` rows and therefore **emits no `parent` at all** (its own docstring says so: + a dangling reference would be worse than a flat list), so `customer_data` arrives here FLAT even + though the registry gives it `parent: 'customers'`. Stated because the opposite is the obvious + reading — this ticket's first draft assumed the clause was excluding children and wrote a + docstring around a defect that does not exist. + + ⚠ `enumerated` is False when the `ut_` listing raised — the caller must not treat the set as + authoritative for `ut_` keys, exactly as `_placeable_top_keys` requires. + """ + import core.view_templates as view_templates + keys, enumerated = set(), True + tcfg = getattr(session.runtime.tenant, "config", None) or {} + tmods = tcfg.get("modules", "all") + allowed = None if tmods == "all" else {str(k) for k in (tmods or [])} + for p in (perms.nav_pages(session.user) or []): + key = str(p.get("key") or "") + if allowed is not None and key not in allowed \ + and str(p.get("parent") or "") not in allowed: + continue + if view_templates.workspace_key(key): + keys.add(key) + try: + import core.user_tables as user_tables + # The SAME `may_open`-filtered listing `nav()` renders from, over the ROWS-FREE projection + # (W32-T02) — so this costs ~0.1% of the document rather than a 28.6 MB deep copy. + for e in user_tables.nav_entries(viewer=session.uname, is_admin=session.admin, + st=session.runtime): + k = str(e.get("key") or "") + if k: + keys.add(k) + except Exception: + enumerated = False + return keys, enumerated + + +def _granted_view_ids(session): + """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`. + + ⭐ EXTRACTED (W35-T39) SO THE STAR AND THE COUNT SHARE ONE ANSWER. `_visible_views` above + takes this set as its third stratum, and it had exactly one caller (`_important_counts`) with + the derivation inline — which W35-T43 deletes. `routes_starred` needs the identical set to + answer "which views has this person starred", so the derivation moves here beside the reader + it feeds rather than being copied into a second file ([[one-evaluator-per-question]]). + + ⚠ THE ROLE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry naming + me", and `grid_events._granted_views` — the reader whose answer any consumer of this has to + agree with — then requires `role_for(...) in ('view','edit')`. Dropping that second test + would admit a view the sidebar does not list. One tenant-wide bucket, read once per call, so + it costs a dict lookup per id. + ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one. + """ + import core.shares as shares + try: + return {str(v) for v in + ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {}) + .get("view") or []) + if shares.role_for("view", str(v), session.uname, + st=session.runtime) in ("view", "edit")} + except Exception: + return set() + + @router.get("/nav") def nav(session: Session = Depends(require_session)): """`{pages: [{key,label,source?,chrome}], landing}`. @@ -336,7 +513,7 @@ def nav(session: Session = Depends(require_session)): # empty rather than leaving 200-[] ambiguous. return {"pages": [], "landing": None, "empty": "no_databases"} raise err(403, "no_surfaces", - "your account has no dashboards assigned — ask an administrator") + "your account has no dashboards assigned. Ask an administrator.") # WAVE 19 (R8 / C1): the tenant's name + icon overrides, merged LAST — after the registry # rows, after the tenant module filter, after the user tables. Merged HERE rather than # applied by the client for one reason: `label` is what every reader of this payload shows, @@ -350,6 +527,21 @@ def nav(session: Session = Depends(require_session)): # leak with no symptom until two tenants rename the same registry key. Copy the rows before # merging on the day that invariant changes. meta = _read_nav_meta(session.runtime) + # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. + # + # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS + # ORDER so a cold container would reach the databases somebody actually works in first. Every + # one of those was a correct answer to the wrong question: the block was `N` per-database store + # reads on the route D-175 spent a wave reducing to ONE, and no ordering makes an N-read block + # free. MEASURED on a 13-database fixture: **14 view-bucket reads and 14 `important` stamps per + # `/nav`** before this ticket, **0 and 0** after. + # + # R7 moves the question to `GET /starred/counts`, called AFTER the page paints. A slow answer + # there is a late badge; a slow answer here was a late RAIL. Closes D-288 and D-289. + # ⚠ `recents` STAYS and is still ONE read — it is the Home landing's own payload (C10/R7 of wave + # 23), and it was only computed this early so the deleted budget could spend itself in its order. + _page_keys = {str(p.get("key", "")) for p in pages} + recents = _read_recents(session.runtime, session.uname, _page_keys) # Wave 21 (C3): the definitions, once — `manage`/`canDelete` below answer from `createdBy`. # WAVE 27 (C9): `locked` answers from `recordMode`, off the same one read. # ⭐ W31-T10: that read is now the SAME one the merge above did — it used to be a second, @@ -398,6 +590,13 @@ def nav(session: Session = Depends(require_session)): p["locked"] = True elif session.admin: p["manage"] = True + # ⛔ W35-T43 — `important` IS NO LONGER STAMPED HERE. W34-T10 emitted it for every database + # this route could read, including `{marked: 0, counted: 0, partial: false}`, so a consumer + # never had to test for the key. R4 retires the count from the rail and the flyout entirely + # (A's W35-T07 is the client half) and R7 moves the numbers to `GET /starred/counts`. + # ⚠ `nav.ts::NavPage` still DECLARES `important` on the client until A's ticket lands; an + # absent key reads as `undefined` there, which is the same thing the optional field already + # meant for a non-database row. Flagged to A rather than assumed harmless. entry = meta.get(p.get("key")) if meta else None if not entry: continue @@ -422,8 +621,11 @@ def nav(session: Session = Depends(require_session)): # both doors finally agree on what a placeable key is. This block keeps using the assembled # list because it already holds it: re-enumerating here would be a second store read for an # answer sitting in a local variable. - recents = _read_recents(session.runtime, session.uname, - {str(p.get("key", "")) for p in pages}) + # + # ⚠ W34-T10 MOVED THE READ, NOT THE RULE. `recents` is now computed ABOVE the enrichment loop, + # because the important-count block spends its budget in RECENTS ORDER (see there). It is still + # ONE read of `nav_recents`, still pruned against the assembled page list, and it is used here + # unchanged — a second `_read_recents` call would be the extra store read this comment forbids. # ⭐ W31-T11 — TWO KINDS OF ABSENCE, NAMED SEPARATELY, and both keys are ALWAYS PRESENT. # `omitted` — this workspace's catalogue does not include these modules. Deliberate. # `degraded` — a part of this payload could not be read. NOT deliberate, and the shell says @@ -460,7 +662,7 @@ def nav_opened(body: dict = Body(default=None), raise err(400, "bad_request", "no page was named") if not session.runtime.available(): raise err(503, "store_unavailable", - "the tenant store is unavailable — nothing was recorded") + "the tenant store is unavailable. Nothing was recorded.") stamp, uname = _now(), session.uname def _up(data): @@ -484,7 +686,7 @@ def nav_opened(body: dict = Body(default=None), session.runtime.update(_NAV_RECENTS_KEY, _up, flush='async') except Exception: raise err(503, "store_unavailable", - "the tenant store refused the write — nothing was recorded") + "the tenant store refused the write. Nothing was recorded.") return {"key": key, "at": stamp} @@ -549,7 +751,7 @@ def save_nav_meta(body: dict = Body(default=None), if "name" in body: if not key.startswith("ut_"): raise err(400, "name_not_allowed", - "only a database you created can be renamed — this one's name comes " + "only a database you created can be renamed. This one's name comes " "from the module registry") name = " ".join(str(body.get("name") or "").split())[:_MAX_NAV_NAME] if not name: @@ -560,7 +762,7 @@ def save_nav_meta(body: dict = Body(default=None), if not session.runtime.available(): raise err(503, "store_unavailable", - "the tenant store is unavailable — nothing was saved") + "the tenant store is unavailable. Nothing was saved.") def _up(data): data = data if isinstance(data, dict) else {} @@ -583,7 +785,7 @@ def save_nav_meta(body: dict = Body(default=None), session.runtime.update(_NAV_META_KEY, _up) except Exception: raise err(503, "store_unavailable", - "the change was not saved — the store refused the write") + "the change was not saved: the store refused the write.") return {"key": key, "meta": _read_nav_meta(session.runtime).get(key, {})} @@ -608,7 +810,7 @@ def save_nav_prefs(body: dict = Body(default=None), clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated) if not session.runtime.available(): raise err(503, "store_unavailable", - "the tenant store is unavailable — nothing was saved") + "the tenant store is unavailable. Nothing was saved.") def _up(data): data = data if isinstance(data, dict) else {} @@ -622,7 +824,7 @@ def save_nav_prefs(body: dict = Body(default=None), session.runtime.update(_NAV_PREFS_KEY, _up) except Exception: raise err(503, "store_unavailable", - "the folder change was not saved — the store refused the write") + "the folder change was not saved: the store refused the write.") return {"prefs": clean} diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py index 6792dc58921b7e7674e769665985d87a44453d53..11790a31b91cda4ad49eb5883c5dcbbe67885f9f 100644 --- a/api/routes_odoo_tables.py +++ b/api/routes_odoo_tables.py @@ -841,7 +841,18 @@ def odoo_table_rows(table_key: str, # THE WALL FIRST, and it is the SAME one the materialised path uses — 404 for a key that does # not exist, 403 for one this session may not open. Reused rather than re-stated: a second # idea of "may this session open this database" is a permission bug waiting to happen. - defn = _defn_or_refuse(session, table_key) + # + # ⭐⭐ W33-T02 / D-213 — `defs_only=True`, AND THIS ROUTE IS THE CLEANEST OPT-IN ON THE BOARD. + # `defn` is read EXACTLY ONCE below, for `recordMode`; the fields come from `rel.TABLES`'s + # `mk_fields()` and the rows from `datastore.window`, so nothing here has ever touched + # `defn["rows"]`. That is what makes it safe by inspection rather than by argument — and it is + # why the same change must NOT be swept across this file: `odoo_tables_status` a few hundred + # lines down reads `table["rows"]` and its per-row `refreshed` stamps for the MATERIALISED + # grids, and a projection there raises `KeyError` on every one of them. + # ⚠ `mirror_stamp`'s bare `except: return ""` would convert exactly that raise into a silently + # BLANK `refreshed` column rather than a red — which is why the boundary is drawn here, at the + # one function that provably needs no row, instead of at the file. + defn = _defn_or_refuse(session, table_key, defs_only=True) fields = None for _bucket, key, _label, mk_fields in rel.TABLES: if key == table_key: diff --git a/api/routes_publish.py b/api/routes_publish.py new file mode 100644 index 0000000000000000000000000000000000000000..b0a8bbebf8820fd13b3bc96e8cf175a815430a99 --- /dev/null +++ b/api/routes_publish.py @@ -0,0 +1,917 @@ +"""routes_publish.py — PUBLISH AN INTERFACE VIEW AS A LINK (wave 33, owner item 8b, ruling R5). + +The owner, verbatim: *"Have the ability to publish interface, when a user's view is an interface +(e.g. Map or Catalog), we should have the ability to publish a link that can be either accessed +publicly or with a password that the user that share it can toggle."* + +R5, in five clauses, and every one of them is load-bearing: + * publish = a per-view secret token in a **server-only bucket**; + * a `public | password` toggle the SHARER owns, with a **hashed** passphrase beside it; + * an **unauthenticated** read-only route plus a `#/v/` client route; + * the publish surface **projects only the columns the view shows** — never the whole table; + * **creator-or-admin** may publish, and **revoke ROTATES** the token. + +⛔ THIS FILE COPIES `routes_forms.py`'S CONSTRUCTION ON PURPOSE, AND THE TICKET SAID TO. That door +has been public since wave 23 and carries scar tissue no fresh design would reproduce: ONE refusal +for every resolution failure (so the route is not an oracle for which tokens are real), a +constant-time compare, a STREAMING body bound rather than `Body(...)` (FastAPI reads and parses the +whole body before the handler's first line, and `content-length` is caller-supplied), a sliding +rate window rather than a fixed bucket, and an index that is a POINTER, never a permission. + +⚠ WHAT IS DELIBERATELY *NOT* SHARED WITH `routes_forms.py`: the bucket. A form token buys a WRITE +door into a table; a publish token buys a READ projection of one view. Folding them into one index +would make a single leaked string ambiguous about which of the two it opens, and would make +`_resolve` return an object whose capability depends on a field rather than on which door was +knocked. Two buckets, two resolvers, one shape. + +⚠ AND NOT SHARED WITH SHARING. **Three systems now answer some version of "who can see this"** and +they are not the same question (audit S-4): the grant registry drives *Shared with me*, +`table_store.is_shared` GATES opening a view inside the app, and a published link is a THIRD thing +— an unauthenticated read of a projection, bound to a secret rather than to an account. A published +link is NOT a grant: it creates no registry row, appears in nobody's *Shared with me*, and cannot +be revoked by removing a person, because there is no person. + + python verify_forms.py # this file's gate; section 6 onward +""" +import hashlib +import hmac +import os +import secrets +import time + +from fastapi import APIRouter, Depends, Request + +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The SERVER-ONLY index: `{token: {table, view, access, pw?, salt?, iter?, createdBy, createdAt}}`. +#: ⛔ IT IS NEVER `display.*`. Contract C1 puts two PRESENTATIONAL flags on the view spec +#: (`published`, `publishAccess`) precisely so the client has something to render, and the browser +#: writes that spec on every autosave — so anything secret living there would be echoed back to the +#: browser by construction. `aios_grid._clean_display`'s allowlist enforces the other half. +TOKENS_KEY = "publish_tokens" +TOKEN_BYTES = 24 + +#: Passphrase storage. PBKDF2-HMAC-SHA256 with a per-link salt. +#: ⚠ D-130 IS THE SCAR THIS AVOIDS: a form's "invited addresses" list is IDENTIFICATION — it says +#: who you claim to be and anyone may claim it. A passphrase is AUTHENTICATION. The difference is +#: not a stronger string, it is that the secret is never stored, never logged and never echoed. +PW_ITERATIONS = 240_000 +PW_SALT_BYTES = 16 +MIN_PASSPHRASE = 6 +MAX_PASSPHRASE = 128 + +#: v1 request protections, the same shape and the same numbers as the form door (contract C9), so +#: the two public routes cannot drift into different postures. ⚠ In-process, therefore PER WORKER. +RATE_WINDOW_S = 60 +RATE_PER_WINDOW = 30 +MAX_BODY_BYTES = 16 * 1024 + +_HITS: dict = {} + +#: The view kinds that may be published, i.e. R5's "interface". +#: ⚠ THE CLIENT'S SOURCE OF TRUTH IS `customer-grid/iconShapes.ts::MODE_GROUP` (wave 33 item 9 +#: moved `swipe` and `timeseries` into this group). This constant is the SERVER's copy and the two +#: are held in step by `verify_forms.py`, which parses `MODE_GROUP` and compares — because a +#: server list that silently drifts from the picker means a mode a user can create and cannot +#: publish, with nothing anywhere going red. A `grid` or `kanban` view is a re-shaping of a row +#: set; publishing one would be publishing the table, which is exactly what R5's projection clause +#: exists to prevent. +#: ⛔⛔ W33-T68 — `form` IS DELIBERATELY ABSENT, AND ITS ABSENCE IS THE FIX. +#: A `form` view's rows ARE the submissions people have sent it. Every other mode here re-shapes a +#: row set the publisher already curated; a form's row set is other people's answers, gathered under +#: an implicit promise that they go to the owner. Publishing one turned "share this interface" into +#: "serve the responses to anyone holding the link" — on the one unauthenticated door in the +#: product, with no wall between the link and the data. +#: ⚠ AND A FORM ALREADY HAS ITS OWN PUBLIC DOOR: `#/form/` through `routes_forms.py`, which +#: serves the BLANK form for submitting and never the stored rows. So this is not a capability +#: removed, it is a second door onto the same object that should never have existed beside the +#: first. Publishing a form to be filled in still works, at the URL that was always for it. +#: ⚠ `verify_forms.py` holds this list in step with the client's `MODE_GROUP`; the client must not +#: offer Publish on a form view, or the picker promises what this refuses. +PUBLISHABLE_MODES = ("map", "catalog", "swipe", "timeseries") + + +def _refuse(): + """THE ONE REFUSAL, for every way a published link can fail to resolve. + + Wrong token, revoked link, deleted view, deleted table, a view that stopped being an + interface, a wrong passphrase. ⛔ Callers must not branch a more specific message out of it: + the difference between "no such link" and "that link was revoked" tells an enumerator which + tokens are real, and the difference between "no such link" and "wrong passphrase" tells them + which links are worth guessing at. + """ + return err(403, "bad_publish_token", "that link is not valid") + + +def _same(a: str, b: str) -> bool: + """Constant-time. A `==` leaks a secret's prefix through timing, one character at a time.""" + return hmac.compare_digest(str(a or ""), str(b or "")) + + +def _client_ip(request: Request) -> str: + """⛔⛔ W33-T70 — `x-forwarded-for` IS GONE FROM THIS FUNCTION, AND THAT WAS THE WHOLE HOLE. + + The header is chosen by the caller. Keying a rate limit on it means an enumerator writes a new + value per request and every request lands in a fresh bucket: MEASURED at **100 of 100 admitted + through a 30-per-window ceiling**. A limiter with a caller-chosen key is not a limiter, and its + old docstring said the header was "a rate-limit key and NOTHING else" — which was exactly the + use it could not support. It is safe as a LOG field and as nothing else. + + The socket peer is the only thing here the caller cannot choose, so it is the key. + ⚠ AND BEHIND HF'S PROXY EVERY CALLER SHARES ONE PEER, which is why `_rate_ok` counts FAILURES + ONLY (see there). A per-peer ceiling over ALL traffic would be one global bucket, i.e. an alarm + that fires for everyone [[alarm-that-fires-for-everyone]] — the honest reading of a shared peer + is that we cannot separate callers, not that we should throttle them together. + """ + return request.client.host if request.client else "?" + + +def _rate_ok(key: str, now: float) -> bool: + """Is this caller UNDER the failure ceiling? Pure check — call `_note_failure` to count. + + A SLIDING window. A fixed bucket lets a caller spend a whole allowance at 11:59:59 and the + whole next one at 12:00:00 — i.e. double the limit, back to back, against a door whose entire + protection is that guessing a 24-byte token is slow. + + ⛔ W33-T70 — IT COUNTS FAILURES, NOT REQUESTS, and the split is what makes a shared proxy peer + survivable. A legitimate reader opens a link that RESOLVES, so they never touch the counter at + all; an enumerator produces nothing but misses. Counting every request under one shared peer + would have denied service to everybody the moment one guesser showed up — trading a token + oracle for an outage is not a fix. + """ + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + _HITS[key] = seen + return len(seen) < RATE_PER_WINDOW + + +def _note_failure(key: str, now: float) -> None: + """Record one failed resolution against this peer, and keep the table bounded.""" + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + + +#: A salt used ONLY to burn the same PBKDF2 a real verification would, when there is no stored hash +#: to check against. Its value is irrelevant; its COST is the point. +_DUMMY_SALT = b"\x00" * 16 + + +def _equalise_pw_cost(entry) -> None: + """⛔⛔ W33-T70 — SPEND THE PBKDF2 EVEN WHEN THERE IS NOTHING TO VERIFY. + + MEASURED before this existed: a wrong TOKEN answered in ~0.4 ms and a wrong PASSPHRASE in + ~82 ms — a **206x gap with zero overlap across 24 samples**. So the refusal's careful wording + ("that link is not valid", identical for both) was undone by the clock: anyone could sort real + tokens from fake ones by timing alone, then spend their guesses only on the real ones. The + docstring on `_refuse` describes exactly the leak the code then handed over for free. + + Called on every path that fails BEFORE a passphrase check would have happened, so the cheap + branch costs what the expensive one costs. `iter` is taken from the entry when there is one, so + a link minted under a different iteration count stays indistinguishable too. + """ + iterations = PW_ITERATIONS + if isinstance(entry, dict): + try: + iterations = int(entry.get("iter") or PW_ITERATIONS) + except (TypeError, ValueError): + iterations = PW_ITERATIONS + _hash_pw("", _DUMMY_SALT, iterations) + + +def _public_base() -> str: + return (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/") + + +def _link_of(token: str) -> str: + return f"{_public_base()}/#/v/{token}" + + +def _tokens(rt) -> dict: + """This tenant's publish index, always a dict.""" + try: + found = rt.get(TOKENS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _hash_pw(passphrase: str, salt: bytes, iterations: int = PW_ITERATIONS) -> str: + return hashlib.pbkdf2_hmac("sha256", str(passphrase).encode("utf-8"), + salt, int(iterations)).hex() + + +def _pw_ok(entry: dict, passphrase: str) -> bool: + """Constant-time verify against the stored digest. + + ⛔ RETURNS FALSE, NEVER RAISES, and never distinguishes "this link has no passphrase stored" + from "the passphrase is wrong" — a `password` link whose hash went missing must refuse, not + fall open. [[default-must-pass-its-own-guard]] + """ + stored, salt_hex = str(entry.get("pw") or ""), str(entry.get("salt") or "") + if not stored or not salt_hex: + return False + try: + salt = bytes.fromhex(salt_hex) + except ValueError: + return False + got = _hash_pw(passphrase, salt, int(entry.get("iter") or PW_ITERATIONS)) + return _same(got, stored) + + +def _find_view(rt, table_key: str, view_id: str): + """`(view, owner_username)` from the table's workspace bucket, or `(None, "")`. + + The bucket shape is `{username: {views: {id: view}}}` — the same walk `routes_forms._find_view` + does, and the reason a view is addressable by id ALONE here: the id is unique within the table, + while the owner is what we are trying to discover. + """ + if not table_key or not view_id: + return None, "" + try: + bucket = rt.get(f"{table_key}_table_workspace") or {} + except Exception: # noqa: BLE001 + return None, "" + if not isinstance(bucket, dict): + return None, "" + for owner, blob in bucket.items(): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + if isinstance(views, dict) and isinstance(views.get(view_id), dict): + return views[view_id], str(owner) + return None, "" + + +def _mode_of(view: dict) -> str: + """The view's display mode. Absent means `grid` — the stored default is "say nothing".""" + cfg = (view or {}).get("config") if isinstance(view, dict) else None + disp = (cfg or {}).get("display") if isinstance(cfg, dict) else None + return str((disp or {}).get("mode") or "grid") if isinstance(disp, dict) else "grid" + + +def _may_administer_view(session: Session, table_key: str, view_id: str): + """`(view, mode)` when this caller may publish THIS view, else raises. R5's creator-or-admin. + + ⛔ `session.uname` / `session.admin`, NOT `username` / `is_admin`. `Session` has no such + attributes, and reading one raises ABOVE this route's own `try:` — which is how D-107 arrived + as a bare plain-text 500 rather than as our JSON envelope. + """ + import core.table_store as table_store + + if not table_key.startswith("ut_"): + # The Odoo/registry grids are read-through mirrors with their own permission wall; a + # published projection of one would be a second, secret-gated door onto tenant data whose + # visibility the module gate is supposed to decide. + raise err(400, "not_a_database", "you can publish views on your own databases only") + # ⛔ A READ-THROUGH DATABASE CANNOT BE PUBLISHED, AND THE REFUSAL SAYS SO RATHER THAN MINTING + # A LINK THAT WILL NOT OPEN. `startswith("ut_")` admits `ut_odoo_*` and `ut_meta_*`, whose rows + # do not live in the tenant document at all — they are windowed out of the DuckDB mirror + # through a Session-bound path (`ut_assembly`, which 409s `window_required` on the big ones, + # D-174's lineage). An unauthenticated route has no session and therefore no such path, so a + # token minted here would resolve to a page that could never render. + # ⚠ THIS IS R6's SECOND SENTENCE, WHICH IS THE HALF THAT GETS DROPPED: a limit that genuinely + # cannot be removed must be REPORTED, with its cause, never silently enforced. Refusing at the + # MINT — where a person is standing in front of the answer — is the only place that reads as a + # sentence rather than as an empty page. + import core.user_tables as user_tables + + if user_tables.is_connected(table_key, st=session.runtime): + raise err(400, "connected_source", + "a database that reads through a connected source (Odoo, Meta Ads) cannot be " + "published as a public link: its rows are served from the tenant's mirror by a " + "signed-in request, and a public link has no session to serve them with") + view, owner = _find_view(session.runtime, table_key, view_id) + if not isinstance(view, dict): + raise err(404, "no_such_view", "that view no longer exists") + if not (owner == session.uname + or table_store._may_administer(view, session.uname, session.admin)): + raise err(403, "not_yours", "only this view's creator or an admin can publish it") + mode = _mode_of(view) + if mode not in PUBLISHABLE_MODES: + raise err(400, "not_an_interface", + "only an interface view (Map, Catalog, Swipe, Time-series, Form) can be " + "published as a link") + cfg = view.get("config") or {} + if cfg.get("cohortLock"): + raise err(400, "reader_scoped", + "this view is locked to a cohort, and a cohort's membership is resolved for " + "the person reading it — a public link has no reader, so the page would show " + "no rows at all. Publish a copy without the cohort lock.") + if _needs_a_reader(cfg.get("filters")): + raise err(400, "reader_scoped", + "this view filters on a cohort, a measure rule or a top-N slice, and each of " + "those is resolved for the person reading it — a public link has no reader, so " + "the page would show no rows at all. Publish a copy filtered on columns.") + return view, mode + + +#: How many rows a published page will serve. ⚠ R6 (no cap on connected-source data) does not +#: reach here twice over: a publishable database is `records_mutable`, i.e. the EDITABLE substrate +#: that keeps its bound (`user_tables.MAX_ROWS`), and a connected one is refused at the mint. What +#: R6's SECOND sentence does reach here is the reporting duty — a page that serves fewer rows than +#: the view has must SAY SO on the wire, with the cause, never just stop. +PUBLIC_ROW_CAP = 5000 + + +def _needs_a_reader(tree) -> bool: + """Does this filter tree contain a leaf that only a SIGNED-IN reader could resolve? + + ⛔ COHORTS, MEASURE RULES AND RANK SLICES ARE NOT PROPERTIES OF THE VIEW. Each is a SET the + host computes for the person asking — `filter_eval.EvalCtx`'s own docstring: *"each is an + answer a single ROW cannot compute … absent, the condition matches NOTHING rather than + everything"*. That default is right (it fails closed) and it is unusable here: an anonymous + page whose every row was silently filtered out is indistinguishable from a broken link, and + the reader has nobody to ask. Worse, `rank_sets` has NO server-side resolver anywhere in this + repo — `routes_alerts._evaluate` passes cohort/measure/today and nothing else — so a `topN` + view would serve zero rows on the server while showing twenty in the browser. + + ⭐ So such a view is refused AT THE MINT, where a person is standing in front of the answer. + That is R6's second sentence: a limit that genuinely cannot be removed is REPORTED with its + cause, never silently enforced. + + ⚠ THE PREDICATES ARE `filter_eval`'S OWN. Re-implementing "is this a cohort leaf?" here would + be a second evaluator that agrees today and drifts the day the leaf shape changes — and it + would drift SILENTLY, because the two answers only differ on views nobody has published yet. + [[one-evaluator-per-question]] + """ + from harness import filter_eval + + def walk(node) -> bool: + if isinstance(node, list): + return any(walk(n) for n in node) + if not isinstance(node, dict): + return False + if node.get("kind") == "group" or isinstance(node.get("children"), list): + return any(walk(n) for n in (node.get("children") or [])) + if filter_eval._is_cohort(node) or filter_eval._is_measure(node): + return True + return node.get("op") in filter_eval.RANK_OPS + + return walk(tree) + + +def _mirror_display(rt, table_key: str, view_id: str, published: bool, access: str = "password"): + """Keep contract C1's two PRESENTATIONAL flags on the stored view in step with this bucket. + + ⛔ WHY THIS EXISTS AT ALL, and it was found by a reviewer rather than by a gate: revoking a + link dropped the token and left `config.display.published: true` on the view, so the grid's + own UI would go on saying "published" about a link that no longer resolves. Two records of one + fact, and only one of them moving, is the [[flag-shipped-without-its-writer]] shape — here with + the writer present and the OTHER half forgotten. + + ⚠ THE BUCKET REMAINS THE TRUTH. These flags exist so the client has something to render + without asking; they are a MIRROR, never a source, and nothing in this module reads them back + to decide anything. `read_view_link` deliberately reports both so a drift is visible rather + than assumed away. + + ⚠ ONLY THE TWO LEGAL KEYS ARE WRITTEN, with E's fail-closed coercion reproduced exactly + (`aios_grid._clean_display`: `published: True` always carries a `publishAccess`, and anything + that is not the literal `public` stores as `password`) — so a value written here and a value + written by the browser cannot disagree. + """ + def _set(cur): + cur = dict(cur or {}) + for owner, blob in list(cur.items()): + views = (blob or {}).get("views") if isinstance(blob, dict) else None + view = views.get(view_id) if isinstance(views, dict) else None + if not isinstance(view, dict): + continue + cfg = dict(view.get("config") or {}) + disp = dict(cfg.get("display") or {}) + if not disp.get("mode"): + # No display block means no interface view; nothing here should invent one. + continue + if published: + disp["published"] = True + disp["publishAccess"] = "public" if access == "public" else "password" + else: + disp.pop("published", None) + disp.pop("publishAccess", None) + cfg["display"] = disp + cur[owner] = {**blob, "views": {**views, view_id: {**view, "config": cfg}}} + return cur + + try: + rt.update(f"{table_key}_table_workspace", _set, flush="sync") + except Exception: # noqa: BLE001 + # ⚠ SWALLOWED, and deliberately: the token bucket is the truth and it has already been + # written. A mirror that failed to update leaves the UI one refresh out of date, which is + # strictly better than a 500 on a publish that actually succeeded. + pass + + +def _entry_for(rt, table_key: str, view_id: str): + """`(token, entry)` for a view's existing link, or `(None, None)`.""" + for token, where in _tokens(rt).items(): + if (isinstance(where, dict) and where.get("table") == table_key + and where.get("view") == view_id): + return str(token), where + return None, None + + +def _state_of(token, entry) -> dict: + """The link's state as the SHARER may see it. Built key by key. + + ⛔ NO `**entry`. The stored blob carries `pw` and `salt`; a spread would put both on the wire + to the browser, which is the whole failure this file's bucket exists to avoid, and it would do + it silently the first time somebody added a field. + """ + if not token or not isinstance(entry, dict): + return {"published": False, "access": "password", "token": "", "url": ""} + return { + "published": True, + "access": "public" if entry.get("access") == "public" else "password", + "token": str(token), + "url": _link_of(str(token)), + # A boolean, never the digest and never the salt. + "hasPassphrase": bool(entry.get("pw")), + "createdBy": str(entry.get("createdBy") or ""), + } + + +def _resolve(token: str): + """`(runtime, tenant_slug, table_key, view, entry)` for a token, or None. + + ⛔ THE INDEX IS A POINTER, NEVER A PERMISSION. Holding a token means the sharer minted it for + THIS view; it does not mean the holder may read anything else, and nothing downstream of here + may widen the subject beyond the `(table, view)` pair the entry names. + + ⚠ IT WALKS EVERY TENANT, because an unauthenticated request carries no tenant. That is the + form door's shape too, and the reason `_same` is constant-time: the walk compares the caller's + string against every stored token in the deployment. + """ + from harness import runtime as _rt + + if not token or len(token) < 16: + return None + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + for stored, where in _tokens(rt).items(): + if not _same(str(stored), token) or not isinstance(where, dict): + continue + table_key = str(where.get("table") or "") + view, _owner = _find_view(rt, table_key, str(where.get("view") or "")) + # A view deleted, or re-saved as a grid, since the link was minted. Both answer the + # ONE refusal — "that link is not valid" — rather than explaining which. + if not table_key or not isinstance(view, dict): + return None + if _mode_of(view) not in PUBLISHABLE_MODES: + return None + return rt, slug, table_key, view, where + return None + + +def _coord(value, limit: float): + """A real coordinate, or `None`. THE WALL that lets a map publish without publishing a column. + + ⛔ A VALUE TEST, NEVER A NAME TEST. A field merely CALLED `lat` proves nothing — a verifier put + the string `CANARY-LAT-AAA` in one and watched it reach the wire under the first version of + this code. Anything that is not a finite number inside the earth's range is not a location and + does not travel. + ⚠ `bool` is rejected explicitly: `isinstance(True, int)` is True in Python, and `float(True)` + is `1.0` — a checkbox column named `lat` would otherwise publish as a point off the coast of + Ghana. + """ + if value is None or isinstance(value, bool): + return None + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if n == n and abs(n) <= limit else None + + +def _visible_keys(view: dict, fields: list) -> list: + """The columns this view SHOWS, in the view's own order. R5's projection clause, in one place. + + ⛔ `config.order` IS NOT THE ANSWER AND IS THE OBVIOUS WRONG ONE. `aios_grid._default_view_config` + builds `order` as `shown + hidden`, and `grid_events`' `view_upsert` APPENDS every remaining + field to it — so `order` is every column the table has, hidden ones included. `visible` is the + only allowlist that exists; there is no stored "hidden" key to subtract. + + ⚠ AND AN EMPTY `visible` IS NOT "NO COLUMNS". `CustomerGrid` falls back to the table's default + set when a saved view carries none, so a public page that read `[]` as an empty allowlist would + render blank — and one that read it as "all fields" would LEAK. The fallback is the same + predicate the default config uses (`field.default is not False`), taken from `aios_grid` rather + than restated here. + """ + import aios_grid + + by_key = {str(f.get("key")): f for f in fields if isinstance(f, dict)} + stored = [str(k) for k in (((view or {}).get("config") or {}).get("visible") or [])] + keep = [k for k in stored if k in by_key] + if keep: + return keep + # ⛔⛔ W33-T69 — A STALE `visible` MUST SERVE NOTHING, NOT THE TABLE DEFAULT. + # `delete_field` never prunes a view's stored `visible`, so a published view whose columns were + # later deleted and replaced arrives here with a NON-EMPTY `stored` of which nothing survives — + # and the fallback below then WIDENS the public payload to whatever the table declares by + # default. The publisher chose five columns; the anonymous reader gets the table's idea of + # sensible. That is a widening on the one unauthenticated door in the product. + # ⚠ THE DISTINCTION IS `stored` NON-EMPTY, NOT `keep` EMPTY. A view that never stored `visible` + # at all (a legacy publish, a view saved before the key existed) has no intent to honour and + # the default IS the right answer for it — that is what the fallback was written for. A view + # that stored five keys and has none left DID state an intent, and every column it named is + # gone: the honest answer is no columns, which renders as an empty published view rather than + # somebody else's data. + if stored: + return [] + try: + default_visible = (aios_grid._default_view_config(fields) or {}).get("visible") or [] + except Exception: # noqa: BLE001 + default_visible = [] + fallback = [str(k) for k in default_visible if str(k) in by_key] + return fallback or [str(f.get("key")) for f in fields if f.get("default") is not False] + + +def _public_view(rt, table_key: str, view: dict) -> dict: + """The ONLY bytes a valid token buys. Built KEY BY KEY — there is no `**row` in this function. + + ⛔ THE REASON THAT IS A RULE AND NOT A STYLE. `aios_grid.rows_from_pool` puts `pid`, `_created`, + `lat` and `lon` on EVERY row regardless of what the view shows, and the stored row dict carries + every column the table has. Serialising a row and deleting the fields we do not want inverts + the failure: a column added next wave is INCLUDED by default and nobody notices, whereas an + allowlist that has not learned about it merely omits it. `routes_forms._public_form` is the + shipped precedent and says the same thing about itself. + """ + import core.user_tables as user_tables + from harness import filter_eval + + defn = user_tables.get(table_key, st=rt) or {} + fields = [f for f in (defn.get("fields") or []) if isinstance(f, dict)] + by_key = {str(f.get("key")): f for f in fields} + keys = _visible_keys(view, fields) + cfg = (view or {}).get("config") or {} + + # The row pool: the table's own rows, narrowed to the fields it declares, exactly as + # `routes_tables.scoped_pool` builds it for a materialised table. ⚠ A read-through table has + # no rows here and is refused at the MINT, so this branch is the only one that can be reached. + rows_src = [] + for rid, row in (defn.get("rows") or {}).items(): + if not str(rid).isdigit(): + continue + r = {k: v for k, v in (row or {}).items() if k in by_key} + r["pid"] = int(rid) + rows_src.append(r) + rows_src.sort(key=lambda r: r["pid"]) + + # The view's own row selection, through the SHARED evaluator. `_needs_a_reader` has already + # refused anything this context could not answer, so an empty result here means the filter + # genuinely matches nothing — not that we failed to resolve it. + ctx = filter_eval.EvalCtx(today=time.strftime("%Y-%m-%d")) + keep = set(filter_eval.visible_pids(cfg.get("filters"), rows_src, fields, ctx, + member_pids=cfg.get("memberPids"))) + chosen = [r for r in rows_src if r.get("pid") in keep] + + # ⛔⛔ COORDINATES RIDE A MAP WHEN THEY ARE COORDINATES — NOT WHEN THEY ARE VISIBLE, AND NOT + # BECAUSE OF WHAT A COLUMN IS CALLED. Two verifiers, one from each side, are why this reads + # the way it does; the first fix I wrote was wrong and the second report proved it. + # + # ⚠ THE LEAK (verifier #1, driven): a HIDDEN field keyed `lat` holding the string + # `CANARY-LAT-AAA` came out on the wire, because the pair was emitted before the `keys` + # projection and the bypass keyed on the FIELD NAME rather than on the value being a + # coordinate. So a column called `lat` could carry anything — a note, an address — and publish + # it. That is the real defect. + # + # ⛔ MY FIRST FIX GATED ON VISIBILITY, AND IT BROKE THE FEATURE (verifier #2): hiding the raw + # decimals is the NORMAL way somebody builds a Map view — nobody wants `38.7223` in the column + # list — so gating on `visible` meant an ordinary map published a page with no map, under two + # messages that contradicted each other ("no rows carry a location" vs "this view hides its + # location columns"). + # + # ⭐ THE RULE THAT SATISFIES BOTH: publishing a MAP is publishing WHERE THE ROWS ARE — that is + # what the sharer chose — so a real coordinate rides whether or not its column is shown, and a + # value that is not a coordinate never rides at all. `_coord` is the whole wall, and it is a + # VALUE test, so no naming convention can smuggle anything past it. + # ⚠ It mirrors `PublishedView.MapPlot`'s own `coord()` deliberately: the client must not plot + # what the server would not send, and the server must not send what the client would discard. + # Two normalizers on one question is a smell [[one-question-two-normalizers]] — kept here + # because they sit on opposite sides of a trust boundary, where the server's copy is the wall + # and the client's is display hygiene. + mode = _mode_of(view) + plotted = 0 + if mode == "map": + for r in chosen: + if _coord(r.get("lat"), 90) is not None and _coord(r.get("lon"), 180) is not None: + plotted += 1 + + limits = [] + if mode == "map" and chosen and not plotted: + # R6's second sentence. A map with nothing on it must say WHY — and this says the true + # why, which is about the DATA, because visibility is no longer part of the answer. + limits.append({ + "subject": "map", "effect": "not_plotted", + "detail": f"none of these {len(chosen)} rows carry a usable location", + "recommendation": "add `lat` and `lon` values to the records, then reload this link", + }) + if len(chosen) > PUBLIC_ROW_CAP: + # R6's second sentence. A short page that does not say it is short is the silent + # truncation the rule is actually about. + limits.append({ + "subject": "rows", "effect": "windowed", + "detail": f"this view has {len(chosen)} rows and a published page serves the first " + f"{PUBLIC_ROW_CAP}", + "recommendation": "narrow the view's filters, or share it with named people instead " + "of publishing a link", + }) + chosen = chosen[:PUBLIC_ROW_CAP] + + # ⛔ THE DISPLAY REFS ARE INTERSECTED WITH `visible`, NOT UNIONED INTO IT. A Map that colours + # by a column the view HIDES would otherwise put that column's value on every public row — + # the projection leak, arriving through the renderer rather than through the column list. The + # fail-closed choice is to drop the ref and render the map without colour; a published page + # that is slightly plainer beats one that ships a hidden column. + disp_in = (cfg.get("display") or {}) + display = {"mode": mode} + for ref in ("dateField", "stackField", "titleField", "colorField", "sizeField"): + if disp_in.get(ref) in keys: + display[ref] = disp_in[ref] + + return { + "title": str((view or {}).get("name") or "")[:200], + "mode": display["mode"], + "display": display, + "columns": [{"key": k, + "label": str(by_key[k].get("label") or k), + "type": str(by_key[k].get("type") or "text"), + **({"options": [str(o) for o in by_key[k]["options"]][:200]} + if isinstance(by_key[k].get("options"), list) and by_key[k].get("options") + else {})} + for k in keys], + # KEY BY KEY. `pid` rides because the client needs a stable row identity to render a list; + # it is a row NUMBER within this table and names nothing outside it. + # ⛔⛔ `lat`/`lon` RIDE ONLY ON A MAP **AND ONLY WHEN THE VIEW SHOWS THEM** — and the second + # half was missing, which was a LEAK. Found by a verifier that drove this route with a + # hidden field keyed `lat` carrying the string `CANARY-LAT-AAA`, and watched it come out + # on the wire. + # + # The first version emitted the pair BEFORE the `keys` projection, so `_visible_keys` never + # gated it. On a `ut_*` table coordinates are not magic: `routes_tables.scoped_pool` builds + # its row as `{k: v for k, v in row.items() if k in field_keys}`, so a value only survives + # if the table DECLARES a field keyed `lat`/`lon` — i.e. they are ORDINARY COLUMNS, and a + # view can hide them like any other. Hiding them therefore has to work here, because on a + # published page **the projection is the only wall there is** (`ut_*` databases have no + # hidden-field closure behind it, `routes_shares.py`'s docstring). + # + # ⚠ The bypass was keyed on the FIELD NAME, never on the value being a coordinate, so it + # forwarded whatever a column called `lat` happened to hold — a string, a note, anything. + # ⚠ And the shipped gate could not see it: it asserted the key NAMES rode on a map and not + # on a catalog, over a fixture whose rows carried no `lat` key at all — so it pinned the + # names while both values were `None` [[gate-answers-the-wrong-question]]. + # KEY BY KEY. `pid` rides because the client needs a stable row identity; it is a row + # NUMBER within this table and names nothing outside it. `lat`/`lon` ride only on a map, + # and only when they PARSE as coordinates — see the block above for why that is the test. + "rows": [{"pid": r.get("pid"), + **({"lat": _coord(r.get("lat"), 90), "lon": _coord(r.get("lon"), 180)} + if mode == "map" + and _coord(r.get("lat"), 90) is not None + and _coord(r.get("lon"), 180) is not None else {}), + **{k: r.get(k) for k in keys}} for r in chosen], + "total": len(chosen), + **({"limits": limits} if limits else {}), + } + + +async def _bounded_body(request: Request) -> dict: + """The request body, read WITH A BOUND — never `Body(...)`, never `await request.body()`. + + FastAPI reads and JSON-parses the WHOLE body before the handler's first line runs, so a + declared model is not a bound at all; `content-length` is caller-supplied, so checking it is + not one either. Streaming with a running total is the actual bound. + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + import json + try: + parsed = json.loads(b"".join(chunks) or b"{}") + except ValueError: + raise err(400, "bad_request", "that request could not be read") + return parsed if isinstance(parsed, dict) else {} + + +# ── THE SHARER'S DOOR (authenticated, creator-or-admin) ────────────────────────────────────── +# +# ⚠ THE NOUN IS `/view-link`, DELIBERATELY NOT `/views/{...}/publish`, and it mirrors the form +# door's `/form-link` for the same reason: a literal path segment sitting beside a `/{token}` +# wildcard is resolved by DECLARATION ORDER, and a noun that cannot collide with a token has no +# order to get wrong. + + +@router.get("/view-link") +def read_view_link(topic: str = "", view: str = "", + session: Session = Depends(require_session)): + """This view's link state. ⛔ A GET MUST NOT MINT — opening the panel is not publishing.""" + v, _mode = _may_administer_view(session, str(topic or ""), str(view or "")) + token, entry = _entry_for(session.runtime, str(topic), str(view)) + out = _state_of(token, entry) + # The presentational flags contract C1 put on the view spec, echoed back so the client can + # tell whether the two agree. They are a MIRROR of this bucket, never its source. + disp = ((v.get("config") or {}).get("display") or {}) if isinstance(v, dict) else {} + out["displayPublished"] = disp.get("published") is True + return out + + +@router.post("/view-link") +async def mint_view_link(request: Request, session: Session = Depends(require_session)): + """Publish this view, or change its access. `{topic, view, access?, passphrase?, rotate?}`. + + IDEMPOTENT WITHOUT `rotate`: opening the panel twice, or switching public↔password, must not + invalidate a link somebody already sent. `rotate: true` mints a fresh token and the previous + one dies in the SAME write, so there is never a window where both open the view. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _v, _mode = _may_administer_view(session, topic, view) + + access = "public" if body.get("access") == "public" else "password" + raw_pw = body.get("passphrase") + passphrase = "" if raw_pw is None else str(raw_pw) + if len(passphrase) > MAX_PASSPHRASE: + raise err(400, "passphrase_too_long", + f"a passphrase can be at most {MAX_PASSPHRASE} characters") + + existing_token, existing = _entry_for(session.runtime, topic, view) + rotate = bool(body.get("rotate")) + fresh = secrets.token_urlsafe(TOKEN_BYTES) + + # ⛔ A `password` LINK MUST END UP WITH A HASH, and there are exactly two ways to have one: + # the caller supplied a passphrase now, or one was already stored and is being kept. Anything + # else is refused HERE rather than stored and refused later — a link that cannot be opened by + # anybody is not a safe default, it is a broken feature that reads as a permission bug. + if access == "password": + if passphrase and len(passphrase) < MIN_PASSPHRASE: + raise err(400, "passphrase_too_short", + f"a passphrase needs at least {MIN_PASSPHRASE} characters") + if not passphrase and not (existing or {}).get("pw"): + raise err(400, "passphrase_required", + "a password-protected link needs a passphrase") + + minted = {"token": existing_token or fresh} + + def _set(cur): + cur = dict(cur or {}) + prev = None + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + prev = dict(where) + cur.pop(stored, None) + if not rotate: + minted["token"] = str(stored) + if rotate or prev is None: + minted["token"] = fresh + entry = {"table": topic, "view": view, "access": access, + "createdBy": str((prev or {}).get("createdBy") or session.uname), + "createdAt": float((prev or {}).get("createdAt") or time.time())} + # ⛔ THE STORED PASSPHRASE SURVIVES A TRIP THROUGH `public`, and the first version DROPPED + # it — found by a verifier that traced the toggle rather than the happy path. Publishing + # as `public` skipped this block entirely, so the hash was destroyed while the TOKEN was + # kept; switching back to `password` then demanded a new passphrase, silently, while the + # panel's own sentence promised the opposite ("leave blank to keep the current one"). + # ⚠ Carrying it is inert, not lax: `_pw_ok` is consulted ONLY when `access == "password"` + # (`get_published`/`open_published` both test it first), and `_state_of` exposes a boolean, + # never the digest. A hash nobody can reach is not a secret in use — but a promise the UI + # makes and the store breaks is a defect either way, and the honest fix is to keep the + # promise rather than to reword it. + if passphrase: + salt = secrets.token_bytes(PW_SALT_BYTES) + entry["salt"] = salt.hex() + entry["iter"] = PW_ITERATIONS + entry["pw"] = _hash_pw(passphrase, salt) + elif (prev or {}).get("pw"): + # Carried key by key, so a future field on the entry is not silently inherited. + entry["salt"] = str((prev or {}).get("salt") or "") + entry["iter"] = int((prev or {}).get("iter") or PW_ITERATIONS) + entry["pw"] = str((prev or {}).get("pw") or "") + cur[minted["token"]] = entry + return cur + + session.runtime.update(TOKENS_KEY, _set, flush="sync") + _mirror_display(session.runtime, topic, view, True, access) + token, entry = _entry_for(session.runtime, topic, view) + # ⛔ NO FABRICATED FALLBACK ENTRY HERE. The first version answered + # `_state_of(token or minted["token"], entry or {"access": access, "pw": "x"})`, and that + # `"pw": "x"` would have reported `hasPassphrase: true` about a PUBLIC link if the read-back + # ever came back empty — a lie in the safe-looking direction, which is the kind that survives. + # If the write cannot be read back, say so; do not describe a state nobody verified. + if not entry: + raise err(503, "not_saved", + "the link was minted but could not be read back — reload and try again") + return _state_of(token, entry) + + +@router.delete("/view-link") +async def revoke_view_link(request: Request, session: Session = Depends(require_session)): + """Unpublish. `{topic, view}`. + + ⛔ REVOKE ROTATES — it does not merely unset a flag. The token STRING is dropped from the + index in this write, so the old link stops resolving immediately; and because a later publish + mints `secrets.token_urlsafe(24)` afresh, the revoked string can never come back. An + implementation that kept the token and flipped an `enabled` flag would leave the secret live + in the store, one bug away from working again, and would make "revoked" a property somebody + could forget to check on a code path added later. + """ + body = await _bounded_body(request) + topic, view = str(body.get("topic") or ""), str(body.get("view") or "") + _may_administer_view(session, topic, view) + + def _drop(cur): + cur = dict(cur or {}) + for stored, where in list(cur.items()): + if (isinstance(where, dict) and where.get("table") == topic + and where.get("view") == view): + cur.pop(stored, None) + return cur + + session.runtime.update(TOKENS_KEY, _drop, flush="sync") + # ⛔ AND THE MIRROR COMES DOWN IN THE SAME BREATH. Without this the grid goes on showing + # "published" about a link that no longer resolves — the half a reviewer caught. + _mirror_display(session.runtime, topic, view, False) + return {"published": False, "access": "password", "token": "", "url": ""} + + +# ── THE PUBLIC DOOR (no session — this is the whole feature) ───────────────────────────────── +# +# ⛔ NO `Depends(require_session)` ON EITHER ROUTE BELOW, DELIBERATELY. "Public" in this app is not +# a flag or an allow-list entry — `main.py` has no auth middleware and no exempt-path table; a +# route is public exactly by omitting the dependency. Which is why the two are kept together, +# under one banner, rather than filed beside the sharer's routes they resemble. + + +@router.get("/published/{token}") +def get_published(token: str, request: Request): + """The published view. Read-only, unauthenticated, projected to the view's own columns.""" + if not _rate_ok(_client_ip(request), time.time()): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — AN UNKNOWN TOKEN ANSWERS EXACTLY WHAT A LOCKED ONE ANSWERS. + # This used to `raise _refuse()`, and that 403 was a free oracle: one unauthenticated GET, + # no passphrase, no cost, told an enumerator whether a token was REAL. `_refuse`'s own + # docstring says the difference between "no such link" and "that link was revoked" must + # never be observable — and the route beside it published that difference in its status + # code. Sorting real tokens from fake ones is the whole of the work; once it is free, the + # passphrase is all that is left and it can be attacked offline-cheap. + # ⚠ SO THE UNKNOWN TOKEN GETS THE LOCKED SHAPE: `{"locked": true}` and nothing else — no + # title, no columns, no count, the same bytes a real password link returns before anyone + # has tried to open it. The guess then costs a POST with a passphrase, which is rate + # limited and PBKDF2-priced. A PUBLIC link still opens on this GET, which is what a public + # link is for; what stops being visible is which PASSWORD tokens exist. + _note_failure(_client_ip(request), time.time()) + _equalise_pw_cost(None) + return {"locked": True} + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password": + # ⛔ THE SHAPE OF THE PASSWORD ANSWER, and it is not a refusal. A locked link must render + # a passphrase prompt, so this 200 says "there is something here and it is locked" and + # NOTHING else — no title, no column names, no row count. A 403 here would be + # indistinguishable from a bad token, which is right for a WRONG passphrase and wrong for + # a link the holder has not tried to open yet. + return {"locked": True} + return _public_view(rt, table_key, view) + + +@router.post("/published/{token}") +async def open_published(token: str, request: Request): + """Open a password-protected link. `{passphrase}`. + + ⛔ A WRONG PASSPHRASE AND A WRONG TOKEN ANSWER THE SAME 403, from the same `_refuse`. If they + differed, the route would confirm which tokens are real to anybody willing to send one guess — + and a 24-byte token's entire protection is that it cannot be found by guessing. + ⚠ The passphrase is compared against a PBKDF2 digest with `hmac.compare_digest`, and is never + stored, logged or echoed. D-130's scar: a form's invited-address list is IDENTIFICATION and + anyone may claim an identity; this is AUTHENTICATION and is treated as one. + """ + now = time.time() + if not _rate_ok(_client_ip(request), now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + body = await _bounded_body(request) + found = _resolve(token) + if not found: + # ⛔⛔ W33-T70 — SPEND THE PBKDF2 ANYWAY. A wrong token skipped the hash entirely and + # answered in ~0.4 ms while a wrong passphrase paid ~82 ms: a 206x gap, zero overlap in 24 + # samples, and a clean separation of real tokens from fake ones for anyone with a stopwatch. + # Both arms now cost the same, so the identical 403 above is finally identical in practice + # rather than only in wording. + _note_failure(_client_ip(request), now) + _equalise_pw_cost(None) + raise _refuse() + rt, _slug, table_key, view, entry = found + if entry.get("access") == "password" and not _pw_ok(entry, str(body.get("passphrase") or "")): + _note_failure(_client_ip(request), now) + raise _refuse() + # ⚠ A link with NO passphrase must still pay, or "this token is public" is readable from the + # clock on a route whose whole job is to be uninformative. + if entry.get("access") != "password": + _equalise_pw_cost(entry) + return _public_view(rt, table_key, view) diff --git a/api/routes_query.py b/api/routes_query.py index 2cb64ab478432b3a3a7e3c962f632dc713065775..78b8523237645e799eb2122884afaf67ecb3d9c2 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -1,123 +1,76 @@ -"""routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors. - - POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING - POST /api/v1/query/save {question, scope, spec} -> the saved view - GET /api/v1/query -> the AI-built views this caller may open - DELETE /api/v1/query/{qid} -> forget one - -**The AI turns a question about ONE granted database into a view SPEC. It may only build what the -backend already supports** (R1), and the wall on which database it may name is the EXISTING one. - -⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50 -(`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored -YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers -`ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its -own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid -cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model -never writes SQL and never invents a field name — it only picks registry keys.* - -The real foundation is the view stack that already ships, and this module is `routes_templates.py` -with a model where the curated registry used to be: - - · the wall `routes_templates._target_or_refuse` (imported, never re-implemented) - · the column contract `core.view_templates.columns_named` / `missing_columns` - · the view skeleton `core.view_templates._view` (one definition of what a view is) - · the validators `aios_grid._clean_display` / `clean_filter_tree` - · the writer `core.table_store.make(ws).save_view(..., shared=False)` - -⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns -None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a -leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a -name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So -membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are -the vocabulary ORACLE, never the refusal. - -⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than -inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions -(`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions -with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec -answering a different question ("Revenue by Creator" against a database holding no revenue). -Changing refusal from an enum value to its own tool changed cerebras not at all and made groq -worse. -⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated -here because the first draft of this header claimed the opposite. A door that dies whenever one -free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator -and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is -unreachable the answer is a sentence and never a view. Which provider answered rides back in the -response so a wrong answer stays diagnosable (`_call_model`'s note). - -⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments -against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real -output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP -error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this -file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument -object), and a 400 body is READ before it is treated as transport. - -⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column -is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT -QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own -shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the -validated spec (never asked of the model, which would let it flatter its own output), and saving is -a second, explicit call. The person is the last validator, by design. +"""Assistant chat and Query-owned virtual artefacts (W33-T72 / C8). + +Query reads only the detached, permission-filtered snapshot from +``deps.assistant_read_scope``. It owns a separate per-user namespace for threads, messages, +citations and virtual views; it never opens or mutates a source workspace. """ +import copy +import csv import datetime as _dt import hashlib +import io import json import os import re +import uuid -from fastapi import APIRouter, Body, Depends +from fastapi import APIRouter, Body, Depends, Response -from deps import Session, err, require_session +from deps import (Session, assistant_read_scope, assistant_source_status, err, + require_session) router = APIRouter(prefix="/api/v1") -#: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself -#: is a real saved view in its target's own workspace — this is only the index, so listing the -#: Query module never has to read every granted table's workspace to find them. Same split -#: `core/alerts.py` draws between definitions and the inbox. -QUERY_KEY = 'query_views' - MAX_QUESTION = 500 MAX_VISIBLE = 24 -MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule) - -#: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a -#: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's -#: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces -#: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that -#: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the -#: `swipe`/`form` failure ("whole, correct and unreachable") in reverse. -QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list') - -#: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived -#: from a question about existing records. A reason per member, because "excluded" with no cause -#: is how a hold outlives its own justification. +MAX_ARTIFACTS = 200 +# ⭐⭐ R16 — THE THREAD IS REPLAYED, AND THESE TWO NUMBERS ARE THE WHOLE BOUND ON IT. +# The chat was STATELESS: `_call_model` built `[system, user(question)]` and the conversation on +# screen reached the model as nothing at all, so "what about the other one?" could not be answered +# and the assistant could not grill anybody about anything. Replay is bounded twice because the +# two limits fail differently: a turn COUNT stops a long chat costing more every turn, and a +# CHARACTER budget stops one pasted wall of text from being the whole context. Both count from the +# NEWEST end, because that is what a pronoun refers to. +MAX_HISTORY_TURNS = 12 +MAX_HISTORY_CHARS = 6000 +MAX_RATING_REASON = 200 +QUERY_RATINGS = ("up", "down") +MODEL_AUTO = "auto" +# ⛔⛔ WHY QUERY DECLARES THE ENV NAMES RATHER THAN READING `analyst.PROVIDERS`: MEASURED +# 2026-08-16, `import harness.analyst` costs **5 to 8 SECONDS**, warm, on this box (it pulls +# `harness.tools`, which pulls yaml and the skill recipes). `GET /query` is a LIST path that BOTH +# the Assistant and Query call on mount, so importing the ladder to answer "which models can this +# deployment actually call?" would put eight seconds on the first paint of the AI module. The map +# below is Query's own declaration, and it is deliberately the cheap half. +# ⚠ A SECOND DECLARATION IS A DIVERGENCE WAITING TO HAPPEN, so it is CHECKED rather than trusted: +# `_providers()` already holds the platform ladder in memory and reconciles there, and anything it +# finds is REPORTED into the payload instead of a provider silently never being offered. +# The ORDER is Query's own and is deliberate: cerebras first, because this path needs tool calling +# and cerebras carries this account's tool-capable model. +QUERY_PROVIDER_ENV = { + "cerebras": "CEREBRAS_API_KEY", + "groq": "GROQ_API_KEY", + "openrouter": "OPENROUTER_API_KEY", +} +QUERY_PROVIDER_ORDER = tuple(QUERY_PROVIDER_ENV) +# Filled the first time the platform ladder is loaded; empty means "not yet known", never "clean". +_LADDER_DRIFT = [] +QUERY_KINDS = ("grid", "chart", "calendar", "kanban", "timeseries", "map", "list") +QUERY_WORKSPACE_EVENTS = {"view_create", "view_upsert", "view_delete"} QUERY_EXCLUDED = { - 'form': "a form is a door records come IN through and shows none — a question about existing " - "records cannot describe one, and its spec carries emails and a public token", - 'catalog': "a catalog is a published artifact needing page authoring and product codes " - "(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows", - 'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two " - "product decisions a question does not contain", + "form": "Forms collect new data and are not a read-only Query artefact.", + "catalog": "Catalog is a source-native presentation that Query cannot safely mutate.", + "swipe": "Swipe is an interactive source-native presentation, not an Assistant output.", } - -#: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON -#: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client, -#: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct -#: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked -#: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract -#: with ZERO select columns, and every validator passed it. _MODE_REFS = { - 'kanban': [('stackField', ('select',), True)], - 'calendar': [('dateField', ('date',), True)], - 'timeseries': [('dateField', ('date',), True)], - 'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)], + "kanban": [("stackField", ("select",), True)], + "calendar": [("dateField", ("date",), True)], + "timeseries": [("dateField", ("date",), True)], + "map": [("colorField", ("select",), False), ("sizeField", ("int", "currency", "pct"), False)], } def _now_iso(): - """UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule).""" return _dt.datetime.now(_dt.timezone.utc).isoformat() @@ -126,89 +79,176 @@ def _grid(): return aios_grid -def _templates(): - import core.view_templates as view_templates - return view_templates +def _safe(value): + """Detach values before they enter a durable Query object or an API reply.""" + return json.loads(json.dumps(value, default=str)) + + +def _namespace_key(session): + """A tenant runtime has one isolated key for each user's Query-owned objects.""" + principal = f"{session.tenant}:{session.uname}".encode("utf-8") + return "query_user_" + hashlib.sha256(principal).hexdigest()[:24] + + +def _blank_state(): + return {"version": 1, "threads": {}, "messages": {}, "citations": {}, "views": {}} + + +def _state(session): + try: + raw = session.runtime.get(_namespace_key(session)) or {} + except Exception: + raw = {} + if not isinstance(raw, dict): + return _blank_state() + out = _blank_state() + for key in out: + if key == "version": + continue + if isinstance(raw.get(key), dict): + out[key] = copy.deepcopy(raw[key]) + return out + + +def _new_id(prefix): + return f"{prefix}_{uuid.uuid4().hex[:16]}" + +def model_choices(): + """Choices are stable even when one is not configured, so explicit means explicit.""" + return [MODEL_AUTO, *QUERY_PROVIDER_ORDER] -def _target(session, scope): - """`(fields, source)` for a database this session may open — or a refusal. - ⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same - predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module - grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's - instruction is explicit that the wall is re-used, never re-implemented beside itself. The - private name is deliberate: an alias would mean editing a file no lane's fence covers. +def model_status(): + """Per choice: can this deployment actually CALL it, and if not, why not. - It returns field KEYS; this door also needs each field's TYPE (for the prompt and for - `_MODE_REFS`), so the types are read from the same source the wall consulted — never from a - list assembled here. + ⭐⭐ THE SAME DEFECT THE SOURCE CHIPS WERE FIXED FOR IN WAVE 33, ONE CONTROL TO THE LEFT. + `permitted` is not `answerable` for a database; OFFERED is not CALLABLE for a model. The picker + listed every name in the ladder whether or not this deployment holds its key, so choosing one + spent a click and a turn to be told "the selected model is unavailable" by the server, which + knew before the click. The reason travels with the flag, exactly as `sources` does. + + ⚠ It names no environment variable. "This model is not configured on this deployment" is the + cause a tenant user can act on (ask an admin); the variable name is our deployment's business. """ - import routes_templates - keys, source = routes_templates._target_or_refuse(session, scope) - key = str(scope or '').strip() - defs = [] - if key.startswith('ut_'): - import core.user_tables as user_tables - defn = user_tables.get(key, st=session.runtime) or {} - by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')} - else: - g = _grid() - raw = g.product_fields() if key == 'product_data' else g.FIELDS - by_key = {str(f.get('key')): f for f in raw if f.get('key')} - for k in keys: - f = by_key.get(k) or {} - defs.append({'key': k, 'label': str(f.get('label') or k)[:80], - 'type': str(f.get('type') or 'text')}) - return defs, source - - -# ── the model ─────────────────────────────────────────────────────────────────────────────── - -#: This module's OWN provider order — cerebras first. See the header: the refusal is a model -#: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one. -#: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place -#: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too. -QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter') - - -def _providers(): + live = [name for name in QUERY_PROVIDER_ORDER if os.environ.get(QUERY_PROVIDER_ENV[name])] + rows = [{"model": MODEL_AUTO, "available": bool(live), + "reason": "" if live else "no model is configured on this deployment"}] + for name in QUERY_PROVIDER_ORDER: + rows.append({"model": name, "available": name in live, + "reason": "" if name in live + else "this model is not configured on this deployment"}) + # Anything the platform ladder carries that Query does not offer, once a chat has taught us. + rows.extend({"model": name, "available": False, + "reason": "the platform can reach this model but Query does not offer it yet"} + for name in _LADDER_DRIFT) + return rows + + +def _providers(model=MODEL_AUTO): + """Auto returns the permitted ladder; an explicit choice returns at most one provider.""" import harness.analyst as analyst - pin = os.environ.get("AIOS_QUERY_PROVIDER") + + requested = str(model or MODEL_AUTO).strip().lower() by_name = {p["name"]: p for p in analyst.PROVIDERS} - order = [pin] if pin else list(QUERY_PROVIDER_ORDER) - return [by_name[n] for n in order - if n in by_name and os.environ.get(by_name[n]["env"])] + # ⚠ THE RECONCILIATION, at the ONE point the platform ladder is already in memory, so it costs + # nothing. A provider added to `analyst.PROVIDERS` would otherwise simply never appear here, + # and an env name changed there would make Query's availability answer quietly wrong. Both are + # invisible failures; this turns them into a row a reader can see [[limit-with-no-enforcer]]. + _LADDER_DRIFT[:] = sorted( + set(by_name) - set(QUERY_PROVIDER_ENV) + | {name for name, row in by_name.items() + if name in QUERY_PROVIDER_ENV and row.get("env") != QUERY_PROVIDER_ENV[name]}) + names = list(QUERY_PROVIDER_ORDER) if requested == MODEL_AUTO else [requested] + return [by_name[name] for name in names + if name in by_name and os.environ.get(by_name[name]["env"])] + + +def _history(state, thread_id): + """The prior turns of THIS thread, oldest first, in the transport's own message shape. + + ⛔ SORTED BY `(createdAt, role)`, NOT BY `createdAt` ALONE. `_submit` stamps the user turn and + the assistant turn with the SAME `now`, so on timestamp alone the pair ties and their order is + whatever the store's dict iteration happens to give — which is insertion order today and is + not a guarantee anybody wrote down. A replayed conversation with the answers before the + questions is worse than no replay: it reads as coherent and is backwards. + + ⚠ Scoped to one thread by construction. Replaying another thread's turns would leak one chat's + subject into another's answer, which is the kind of wrong that looks like a good answer. + """ + thread_id = str(thread_id or "") + if not thread_id: + return [] + rows = [row for row in state["messages"].values() + if isinstance(row, dict) and str(row.get("threadId") or "") == thread_id] + rows.sort(key=lambda row: (str(row.get("createdAt") or ""), + 0 if row.get("role") == "user" else 1)) + turns = [] + for row in rows[-MAX_HISTORY_TURNS:]: + content = " ".join(str(row.get("content") or "").split()) + if not content: + continue + role = "assistant" if row.get("role") == "assistant" else "user" + turns.append({"role": role, "content": content}) + # ⭐⭐ R20's "self-improving", and it is the ONLY thing that makes a rating more than a + # flag shipped without its writer. A thumbs-down becomes what it actually was: a turn in + # which the reader said the answer was not useful. Modelled as a USER turn rather than by + # editing the assistant's own words, because that is what happened, and because rewriting + # a stored answer to steer the next one is how a transcript stops being a record. + if role == "assistant" and row.get("rating") == "down": + reason = " ".join(str(row.get("ratingReason") or "").split())[:MAX_RATING_REASON] + turns.append({"role": "user", + "content": f"That answer was not helpful. {reason}".strip()}) + kept, spent = [], 0 + for turn in reversed(turns): + spent += len(turn["content"]) + if spent > MAX_HISTORY_CHARS and kept: + break + kept.append(turn) + kept.reverse() + return kept def _spec_schema(field_keys): - """The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a - refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads - exactly like a transport failure.""" - ops = sorted(_grid().FILTER_OPS) col = {"type": "string", "enum": sorted(field_keys)} return { "type": "object", "properties": { - "kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"], - "description": "refused = this database cannot answer the question"}, - "name": {"type": "string", "description": "a short title for the view"}, - "refusal": {"type": "string", - "description": "when kind=refused: ONE plain sentence naming what is " - "missing"}, + "kind": {"type": "string", "enum": [*QUERY_KINDS, "refused"]}, + "name": {"type": "string"}, + "refusal": {"type": "string"}, "visible": {"type": "array", "items": col}, - "filters": {"type": "array", "items": { - "type": "object", - "properties": {"colId": col, "op": {"type": "string", "enum": ops}, - "value": {"type": "string"}}, - "required": ["colId", "op"]}}, + # ⭐⭐ 2026-08-15 (owner: *"if you can build the view, the AI assistant should also be + # able to do it by using our tools on the backend"*). `rhs` and `important` were the + # only two things a person could express through `view_upsert` and this tool could not. + # + # ⛔ WITHOUT `rhs` THE ASSISTANT CANNOT STATE AN ERROR-CATCHER AT ALL — the one shape + # item 8a named by hand (*"price and COGS not matching"*). Every one of the 20 + # `FILTER_OPS` was already reachable, because they all read `value`; comparing a column + # against ANOTHER COLUMN is a different member (`aios_grid._clean_rhs`, CG-9) and it was + # simply absent here, so the model had no way to ask for it and no way to be told why. + # `kind` is `field` ONLY: `measure` needs a window and `stat` needs the population + # vocabulary, neither of which this snapshot-shaped reader carries — offering them + # would be a control that lies, which is the same rule the source chips follow. + "filters": {"type": "array", "items": {"type": "object", "properties": { + "colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)}, + "value": {"type": "string"}, + "rhs": {"type": "object", "properties": { + "kind": {"type": "string", "enum": ["field"]}, "colId": col, + }, "required": ["kind", "colId"]}, + }, "required": ["colId", "op"]}}, + # A personal legibility mark, exactly as `grid_events.view_upsert` treats it (wave 32 + # R5/C4) — not a lock, and no second permission wall. + "important": {"type": "boolean"}, "filterConj": {"type": "string", "enum": ["and", "or"]}, - "sorts": {"type": "array", "items": { - "type": "object", - "properties": {"colId": col, - "dir": {"type": "string", "enum": ["asc", "desc"]}}, - "required": ["colId", "dir"]}}, + "sorts": {"type": "array", "items": {"type": "object", "properties": { + "colId": col, "dir": {"type": "string", "enum": ["asc", "desc"]}, + }, "required": ["colId", "dir"]}}, "groupBy": col, + "aggregation": {"type": "object", "properties": { + "op": {"type": "string", "enum": ["count", "sum", "avg", "min", "max"]}, + "field": col, + }, "required": ["op"]}, "stackField": col, "dateField": col, "colorField": col, @@ -218,476 +258,1205 @@ def _spec_schema(field_keys): } -def _system_prompt(fields): - """The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the - target's own columns) — the model is never told about a key this server cannot honour.""" - cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]} - for f in fields], separators=(",", ":")) - kinds = ", ".join(QUERY_KINDS) - ops = ", ".join(sorted(_grid().FILTER_OPS)) - selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"] - dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"] - return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you -never invent a column or a kind of view. - -THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys: -{cols} - -VIEW KINDS you may name, and nothing else: {kinds} -FILTER OPS you may name, and nothing else: {ops} -The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)} -The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)} - -RULES: -1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref - MUST be one of the keys above. -2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or - counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED, - and it must be a select column); something over time -> calendar or timeseries (dateField - REQUIRED, and it must be a date column); geography -> map. -3. `visible` is the columns the question is about, most important first — not every column. -4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column - this database does not have, a number it does not store, or a picture that is not in the kind - list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one - asked is the worst possible answer — worse than saying you cannot build it. When in doubt, - refuse. -5. Every `value` is a string.""" - - -def _call_model(question, fields, chat=None): - """`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder. - - `chat` is injectable so `verify_query.py` proves this door end to end with no key and no - tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline). - - ⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the - FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer - unanswerable questions with plausible specs. Falling through on a 429 is still the right call — - a Query module that dies whenever one free tier rate-limits is worse than one that answers with - a weaker model behind the same validator and the same human read-back — but *which* model - answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is - what happens when the whole ladder is unreachable, not when the best rung is. +def _system_prompt(snapshot): + fields = snapshot["fields"] + cols = json.dumps([{"key": field["key"], "label": field.get("label", field["key"]), + "type": field.get("type", "text")} for field in fields], separators=(",", ":")) + return f"""You are the assistant inside this product, talking with one person about exactly one +database they already have permission to read. Reply in plain prose, and keep it short. + +DATABASE: {snapshot['database']} +SNAPSHOT VERSION: {json.dumps(snapshot['source_version'], default=str)} +PERMISSION-FILTERED RECORD COUNT: {len(snapshot['records'])} +FIELDS: {cols} +VIEW KINDS: {", ".join(QUERY_KINDS)} + +You may answer in words alone, and you may ask ONE short question back when the request is +ambiguous: which field is meant, which period, whether they want every record or a narrower set. +Prefer asking over guessing whenever the answer would change what you build. Earlier turns of this +conversation are above; use them, and read a pronoun as referring to what was just discussed. + +Call build_view ONLY when the person wants to SEE records: a table, a chart, a board, a list. Do +not call it to explain something, to confirm something, or to ask your question. When you do call +it you may also write one sentence saying what you built. + +Inside build_view: choose visible fields, supported filters and an optional aggregation. Use count +for record counts; sum, avg, min and max require one numeric field. The server computes and cites +every number from this exact snapshot. Never write SQL, invent fields, or name another database. +Return kind=refused with one plain sentence if the database cannot answer. + +To compare one column against ANOTHER column rather than a typed value, give the filter an rhs of +{{"kind":"field","colId":""}} and omit value. That is how you express questions like +"priced below what it costs us". Both columns must be in the FIELDS list above. +Set important=true when the view is one somebody should be chased about: an error, a mismatch, or +money at risk. Leave it out otherwise. + +Write with ordinary punctuation. Never use an em dash or an en dash: use a comma, a colon or a full +stop instead.""" + + +_FAILED_GEN = re.compile(r"(\{.*?\})\s*", re.S) + + +def _spec_from_400(body): + try: + value = ((json.loads(body) or {}).get("error") or {}).get("failed_generation") or "" + raw = (_FAILED_GEN.search(str(value)) or [str(value).strip()])[1] + result = json.loads(raw) + return result if isinstance(result, dict) else None + except Exception: + return None + + +# ⚠ BUILT FROM `chr`, AND A UNICODE ESCAPE IS NOT ENOUGH. `web_prose` reads a Python +# string's VALUE off the AST, not its spelling in the source, so an escape and the character +# itself are the SAME finding to it. Composing the class at runtime means no string literal in +# this file holds a dash, which is true rather than merely quiet: the only dashes in this module +# are the ones being removed. +_DASH = "[" + chr(0x2014) + chr(0x2013) + "]" + + +def _no_dashes(text): + """R6 applied where it is the ONLY place it can be applied: the model's own words. + + ⛔⛔ THE PROMPT INSTRUCTION IS NOT ENFORCEMENT, AND THIS IS MEASURED, NOT ANTICIPATED. The + system prompt ends with *"Never use an em dash or an en dash"* and the very next live turn came + back with *"Which view type would you like—grid, chart, list, or another?"* (cerebras, + 2026-08-16). Model prose reaches the screen exactly as a string literal does, and `web_prose` + scans SOURCE, so it cannot see a dash that arrives at runtime: the sweep every lane is doing + this wave is undone by our own assistant unless it is undone here. + + ⚠ A DIGIT RANGE IS A DIFFERENT SENTENCE. "10–20" means "10 to 20"; rewriting it as + "10, 20" states two numbers where the model stated a span, which is a wrong answer rather than + a punctuation fix. It gets its own rule, first. + """ + text = str(text or "") + text = re.sub(rf"(?<=\d)\s*{_DASH}\s*(?=\d)", " to ", text) + text = re.sub(rf"\s*{_DASH}\s*(?=[,.;:!?])", "", text) # abutting punctuation: it just goes + text = re.sub(rf"(?<=[,;:])\s*{_DASH}\s*", " ", text) # already punctuated: one space + return re.sub(rf"\s*{_DASH}\s*", ", ", text) + + +def _said(raw): + """The model's own words: one line of whitespace, no dash, and empty means nothing was said.""" + return _no_dashes(" ".join(str(raw or "").split())).strip() or None + + +def _from_chat(answer, provider): + """An injected transport may answer with a SPEC (a dict) or with PROSE (a string). + + ⛔ THE BRANCH BELONGS HERE, NOT AT THE CALLER. `_validate`'s first line refuses anything that + is not a dict, with "the assistant did not answer with a view" — so a prose answer passed + straight through would arrive as a REFUSAL, styled as one, and every conversation check would + pass against the wrong path while looking green [[one-question-two-normalizers]]. + """ + if isinstance(answer, str): + return None, _said(answer), provider, None + return answer, None, provider, None + + +def _call_model(question, snapshot, model=MODEL_AUTO, chat=None, history=None, session=None): + """Return ``(spec, said, provider, reason)`` without any source-data fallback. + + ``said`` is what the assistant SAID: the whole answer when it did not build a view, and the + sentence beside the view when it built one and talked as well. ``reason`` is set only when + something went wrong, so a prose ANSWER and a transport FAILURE are distinguishable one layer + up rather than both arriving as a bare sentence. + + ⭐⭐ W35-T31 · CONTRACT C7 (R9) — ``session`` IS HERE ONLY SO THE METER CAN BE TOLD WHOSE CALL + THIS WAS, and PRD amendment A1 is why it is an argument rather than ambient state: a + ``ContextVar`` bound in ``deps.require_session`` reads back ``None`` inside this function on the + SAME thread, so a ledger built on one would have recorded nothing and shown a dashboard of + zeros indistinguishable from a quiet week. + + ⚠ OPTIONAL, DELIBERATELY. ``verify_query`` calls this function directly with an injected + ``chat`` and no session, and a required argument would have made every one of those calls a + signature change. An omitted session is COUNTED in ``usage_ledger.UNATTRIBUTED`` and reported + to an admin, never dropped. """ + requested = str(model or MODEL_AUTO).strip().lower() + if requested not in model_choices(): + return None, f"the selected model ({requested or model}) is unavailable", None, "model_unavailable" tools = [{"type": "function", "function": { - "name": "build_view", - "description": "Emit the view spec, or refuse.", - "parameters": _spec_schema([f["key"] for f in fields])}}] - messages = [{"role": "system", "content": _system_prompt(fields)}, + "name": "build_view", "description": "Emit a virtual-view spec or a refusal.", + "parameters": _spec_schema([field["key"] for field in snapshot["fields"]]), + }}] + messages = [{"role": "system", "content": _system_prompt(snapshot)}, + *(history or []), {"role": "user", "content": question}] if chat is not None: - return chat(messages, tools), None, "injected" - - provs = _providers() - if not provs: - # No key at all: say so. An AI feature that silently does nothing is indistinguishable - # from one that was never built ([[flag-shipped-without-its-writer]]). - return None, ("the assistant is not configured on this deployment, so a view cannot be " - "built from a question yet"), None + provider = requested if requested != MODEL_AUTO else "injected" + return _from_chat(chat(messages, tools), provider) + + providers = _providers(requested) + if not providers: + sentence = (f"the selected model ({requested}) is unavailable" if requested != MODEL_AUTO + else "the assistant is not configured on this deployment") + return None, sentence, None, "model_unavailable" if requested != MODEL_AUTO else "not_configured" + import requests + + import usage_ledger + + def _book(provider, body): + """⭐⭐ C7 — ONE LEDGER LINE PER PROVIDER RESPONSE THIS FUNCTION READS. + + ⛔ HERE, INSIDE THE LADDER, AND NOT AT THE RETURN. The ladder tries providers in order and + a failed one has already SPENT tokens at that vendor; booking only the winner would report + a week cheaper than it was, which is the cost-surprise R13 cited. Every 200 this loop reads + gets a line, including the one whose answer turned out to be empty. + ⚠ `tokens_from` returns `(None, None)` when the provider did not say, and `None` is carried + through rather than coerced to 0 — a call booked at zero is an unmeasured call presented as + a free one. `usage_ledger` counts it as UNMEASURED so the total reads as a floor. + ⚠ `record` never raises, by its own contract, so this cannot break the assistant. + """ + ins, outs = usage_ledger.tokens_from(body) + usage_ledger.record( + "assistant", provider["name"], provider["model"], ins, outs, + total=usage_ledger.total_from(body), + st=getattr(session, "runtime", None), user=getattr(session, "uname", "")) + last = None - for p in provs: + for provider in providers: try: - r = requests.post(p["url"], timeout=60, - headers={"Authorization": f"Bearer {os.environ[p['env']]}"}, - json={"model": p["model"], "messages": messages, "tools": tools, - "tool_choice": "required", "temperature": 0.1, - "max_tokens": 1200}) - except Exception as e: # network blip — try the next provider - last = f"{p['name']}: {type(e).__name__}" + response = requests.post( + provider["url"], timeout=60, + headers={"Authorization": f"Bearer {os.environ[provider['env']]}"}, + # ⛔⛔ `auto`, NOT `required`, AND THAT ONE WORD IS R16. Under `required` the model + # had to emit a `build_view` call for EVERY turn: it could not answer a question, + # could not ask one back, and could not decline to build. "The AI chat must TALK + # with the user, not only build queries, and decide for itself whether to create a + # query view" is unreachable with the tool forced, whatever the prompt says. + # ⚠ AND IT IS A LIVE VENDOR-BEHAVIOUR CHANGE THAT NO GATE HERE CAN SEE, because + # `verify_query` injects `chat` and never builds this body. `_spec_from_400` below + # exists precisely because providers differ in how they emit a forced call; under + # `auto` a provider may also decline to build for a view-shaped question. That is + # why this ticket's evidence is a real three-turn run against a real provider and + # not a green gate. + json={"model": provider["model"], "messages": messages, "tools": tools, + "tool_choice": "auto", "temperature": 0.1, "max_tokens": 1200}, + ) + except Exception as exc: + last = f"{provider['name']}: {type(exc).__name__}" continue - if r.status_code == 400: - # ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the - # schema and returns the generation it rejected. Reading it turns a "transport - # failure" back into the refusal it actually was. - spec = _spec_from_400(r.text) + if response.status_code == 400: + spec = _spec_from_400(response.text) if spec is not None: - return spec, None, p["name"] - last = f"{p['name']}: 400" + return spec, None, provider["name"], None + last = f"{provider['name']}: 400" continue - if r.status_code != 200: - last = f"{p['name']}: HTTP {r.status_code}" - continue # 429 included: the NEXT provider, not a sleep - try: - msg = r.json()["choices"][0]["message"] - calls = msg.get("tool_calls") or [] - if not calls: - last = f"{p['name']}: no tool call" - continue - return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"] - except Exception as e: - last = f"{p['name']}: unreadable answer ({type(e).__name__})" + if response.status_code != 200: + last = f"{provider['name']}: HTTP {response.status_code}" continue - # ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it - # is rate-limited and the rest are too, the answer is a sentence, never a fabricated view. - return None, ("the assistant could not be reached just now (" + (last or "no provider") + - ") — try the question again in a moment"), None - - -_FAILED_GEN = re.compile(r'(\{.*?\})\s*', re.S) - - -def _spec_from_400(body): - """The spec out of a `tool_use_failed` body's `failed_generation`, or None.""" - try: - err_obj = (json.loads(body) or {}).get("error") or {} - except Exception: - return None - gen = str(err_obj.get("failed_generation") or "") - if not gen: - return None - m = _FAILED_GEN.search(gen) - raw = m.group(1) if m else gen.strip() - try: - spec = json.loads(raw) - except Exception: - return None - return spec if isinstance(spec, dict) else None - + try: + body = response.json() + # C7: booked from the RAW body, before anything below can raise on its shape. A + # provider that answered 200 and billed for it has spent tokens whether or not this + # function can read what it said. + _book(provider, body) + answer = body["choices"][0]["message"] + said = _said(answer.get("content")) + calls = answer.get("tool_calls") or [] + if calls: + # The prose rides ALONG with the view when the model wrote both, so the reader gets + # a sentence instead of `_explain`'s machine description of its own output. + spec = json.loads(calls[0]["function"].get("arguments") or "{}") + return spec, said, provider["name"], None + if said: + return None, said, provider["name"], None + # ⚠ Neither a call nor a word is a FAILED turn, not a silent one: falling through to + # the next provider is right, and swallowing it as an empty answer would show the + # reader a blank reply [[empty-answer-vs-unfinished-answer]]. + last = f"{provider['name']}: an empty answer" + except Exception as exc: + last = f"{provider['name']}: unreadable answer ({type(exc).__name__})" + if requested != MODEL_AUTO: + return None, f"the selected model ({requested}) is unavailable", None, "model_unavailable" + return (None, "the assistant could not be reached just now (" + (last or "no provider") + ")", + None, "provider_unreachable") -# ── validation: the REFUSAL this contract promises ─────────────────────────────────────────── def _validate(spec, fields): - """`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set. - - Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners - as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and - never a broken view. - """ - g = _grid() + """Return a cleaned, source-independent virtual-view config or a named refusal.""" if not isinstance(spec, dict): return None, "the assistant did not answer with a view", "no_spec" - by_key = {f['key']: f['type'] for f in fields} + by_key = {str(field.get("key")): str(field.get("type") or "text") for field in fields} keys = set(by_key) - kind = spec.get('kind') - - if kind == 'refused': - return None, (str(spec.get('refusal') or '').strip() - or "this database cannot answer that question"), "model_refused" - if kind in QUERY_EXCLUDED: - return None, (f"a {kind} view cannot be built from a question — " - + QUERY_EXCLUDED[kind]), "unsupported_kind" - if kind not in QUERY_KINDS: - # ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name - # promising something else. - return None, (f"{kind!r} is not a kind of view this product can build" - if kind else "the assistant did not name a kind of view"), "unsupported_kind" - - named = set(spec.get('visible') or ()) - for s in spec.get('sorts') or (): - if isinstance(s, dict) and s.get('colId'): - named.add(str(s['colId'])) - for f in spec.get('filters') or (): - if isinstance(f, dict) and f.get('colId'): - named.add(str(f['colId'])) - if spec.get('groupBy'): - named.add(str(spec['groupBy'])) - for ref in ('stackField', 'dateField', 'colorField', 'sizeField'): - if spec.get(ref): - named.add(str(spec[ref])) + kind = spec.get("kind") + if kind == "refused": + # Model-authored, so it faces the same R6 wall the model's chat prose does. + return None, _said(spec.get("refusal")) or "this database cannot answer that question", "model_refused" + if kind in QUERY_EXCLUDED or kind not in QUERY_KINDS: + return None, "that kind of view cannot be built from this question", "unsupported_kind" + + named = set(spec.get("visible") or ()) + for item in spec.get("filters") or (): + if isinstance(item, dict) and item.get("colId"): + named.add(str(item["colId"])) + # ⛔ THE RIGHT-HAND COLUMN IS A COLUMN AND MUST FACE THE SAME `missing` CHECK. Collecting + # only the left side is what makes D-229 possible one layer down: `clean_filter_tree` does + # NOT drop a leaf whose field-rhs names a column that does not exist — it keeps the leaf, + # strips the `rhs`, blanks the value, and `filter_sql.is_rule_active` then reports the rule + # INACTIVE. An inactive rule narrows nothing, so "margin under 10%" would come back as a + # view listing the ENTIRE catalogue under an error-catcher's name, with nothing red. + # Naming it here turns that into the ordinary "this database does not have: X" refusal. + rhs = item.get("rhs") if isinstance(item, dict) else None + if isinstance(rhs, dict) and rhs.get("colId"): + named.add(str(rhs["colId"])) + for item in spec.get("sorts") or (): + if isinstance(item, dict) and item.get("colId"): + named.add(str(item["colId"])) + for key in ("groupBy", "stackField", "dateField", "colorField", "sizeField"): + if spec.get(key): + named.add(str(spec[key])) + raw_aggregation = spec.get("aggregation") or {"op": "count"} + if isinstance(raw_aggregation, dict) and raw_aggregation.get("field"): + named.add(str(raw_aggregation["field"])) missing = sorted(named - keys) if missing: - # NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader - # this is the wrong database for this question. - return None, ("this database does not have the columns that answer needs: " - + ", ".join(missing)), "unknown_columns" + return None, "this database does not have: " + ", ".join(missing), "unknown_columns" - visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE] + visible = [str(key) for key in (spec.get("visible") or ()) if str(key) in keys][:MAX_VISIBLE] if not visible: - # An empty projection is not a view. `_default_view_config` would fill one in, which is - # how a question about two columns becomes a 285-column wall. - return None, ("that question did not name anything to show — try naming the columns you " - "want to see"), "no_columns" - - # ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX - # column the way the grid does" — a set row lands in several buckets, so a sum means something - # else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket - # keying, which is in no lane's fence this wave (booked, with that reason). What IS in this - # lane's power is refusing to GENERATE the broken case: the AI must only build what the backend - # and the renderer already support, which is R1's own sentence. Delete this guard when D-137 - # closes — the refusal names the debt so the reader knows it is a limit, not a rule. - if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'): - return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds " - f"several values at once, or a tick box, does not bucket the way a chart " - f"needs. Try a table grouped by it instead"), "chart_key_unsupported" - - for ref, families, required in _MODE_REFS.get(kind, ()): - val = spec.get(ref) - if val and by_key.get(val) not in families: - return None, (f"a {kind} view needs {ref[:-5]} to be a " - f"{' or '.join(families)} column, and {val!r} is a " - f"{by_key.get(val)} column"), "wrong_ref_type" - if required and not val: - return None, (f"a {kind} view needs a {' or '.join(families)} column and this " - f"database has none that fits"), "missing_ref" - - raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)] - filters = g.clean_filter_tree(raw_filters, keys) + return None, "that question did not name any fields to show", "no_columns" + raw_filters = [item for item in (spec.get("filters") or ()) if isinstance(item, dict)] + filters = _grid().clean_filter_tree(raw_filters, keys) if len(filters) != len(raw_filters): - # The belt caught something membership did not — an unknown op. Refuse rather than save a - # view whose condition was silently dropped (the widening this file exists to prevent). - return None, ("part of that filter is not something this product can express — try " - "asking it more simply"), "filter_dropped" - - display = {'mode': kind} - for ref, _families, _req in _MODE_REFS.get(kind, ()): - if spec.get(ref): - display[ref] = spec[ref] - cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None - if kind != 'grid' and not cleaned_display: - return None, (f"this product could not build a {kind} view from that question" - ), "display_dropped" - - clean = { - 'kind': kind, - 'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query', - 'visible': visible, - 'filters': filters, - 'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and', - 'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'} - for s in (spec.get('sorts') or ()) if isinstance(s, dict) - and s.get('colId') in keys][:3], - 'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None, - 'display': cleaned_display, + return None, "part of that filter is unsupported", "filter_dropped" + # ⛔⛔ A SECOND, NARROWER CHECK, AND THE LENGTH CHECK ABOVE CANNOT DO ITS JOB (D-229). + # A dropped leaf changes the COUNT; a stripped `rhs` does not — the leaf survives, so + # `len(filters) == len(raw_filters)` and the refusal above never fires. The failure is + # therefore silent in exactly the direction that matters: the condition stops narrowing and + # the view answers with every record. Assert the member survived, per leaf. + # ⚠ The `named` pass above already refuses an rhs naming a column this database lacks, so + # reaching here means something ELSE stripped it (a type the comparand cannot take, a future + # `_clean_rhs` rule). Both doors, because the two catch different causes and the cost of + # missing this one is a wrong answer that looks right. + for sent, kept in zip(raw_filters, filters): + if sent.get("rhs") and not kept.get("rhs"): + return None, "that column cannot be compared against another column", "rhs_dropped" + + aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {} + op = str(aggregation.get("op") or "").lower() + field = aggregation.get("field") + if op not in {"count", "sum", "avg", "min", "max"}: + return None, "the assistant gave an unsupported aggregation", "bad_aggregation" + if op == "count": + field = None + elif field not in keys or by_key.get(field) not in {"int", "currency", "pct"}: + return None, "that aggregation needs one visible numeric field", "bad_aggregation" + + display = {"mode": kind} + for ref, families, required in _MODE_REFS.get(kind, ()): + value = spec.get(ref) + if required and not value: + return None, f"a {kind} view needs {ref}", "missing_ref" + if value and by_key.get(value) not in families: + return None, f"{ref} has the wrong field type", "wrong_ref_type" + if value: + display[ref] = value + cleaned_display = _grid()._clean_display(display, keys) if kind != "grid" else None + if kind != "grid" and not cleaned_display: + return None, f"this product could not build a {kind} view", "display_dropped" + + return { + "kind": kind, + # The view NAME is model-authored too, and it is the string that ends up in the rail, in + # the flyout and on the artefact card. R6 reaches it here or nowhere. + "name": (_said(spec.get("name")) or "Query")[:60], + "visible": visible, + "filters": filters, + "filterConj": "or" if spec.get("filterConj") == "or" else "and", + "sorts": [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} + for item in (spec.get("sorts") or []) if isinstance(item, dict) + and item.get("colId") in keys][:3], + "groupBy": spec.get("groupBy") if spec.get("groupBy") in keys else None, + "aggregation": {"op": op, "field": field}, + "display": cleaned_display, + # ⚠ `is True`, not truthy, and UNCONDITIONAL — the same two rules `grid_events.view_upsert` + # follows for this key. `is True` so a model emitting the string "false" does not mark a + # view; unconditional so the mark is REMOVABLE rather than a flag that can be set and never + # cleared (a key written only when present leaves a stored `true` alive forever). + "important": spec.get("important") is True, + }, None, None + + +def _explain(view, fields): + label = {field["key"]: str(field.get("label") or field["key"]) for field in fields} + visible = ", ".join(label.get(key, key) for key in view["visible"][:6]) + result = f"{view['kind']} view of {visible}" + if view["filters"]: + result += "; filtered records only" + if view.get("groupBy"): + result += f"; grouped by {label.get(view['groupBy'], view['groupBy'])}" + agg = view["aggregation"] + if agg["op"] != "count": + result += f"; {agg['op']} of {label.get(agg['field'], agg['field'])}" + return result + "." + + +def _said_fallback(artifact): + """What a build turn SAYS when the provider volunteered no sentence of its own. + + ⚠ MEASURED, 2026-08-16, cerebras: a view-shaped question comes back as a tool call with + `content: null`. So this is the sentence a person reads on MOST build turns, not a rare + fallback, and R16 ("the AI chat must TALK with the user") is decided here rather than in the + prompt. `_explain` used to be it, and it is a receipt for our own output: *"grid view of + Company, Owner, Deal value; filtered records only."* It keeps its real job ON THE ARTEFACT, + where it labels the thing it describes. + """ + numeric = artifact.get("numeric") or {} + value = numeric.get("value") + if value is None: + return f"I built {artifact['name']}." + if isinstance(value, float) and value.is_integer(): + value = int(value) + number = f"{value:,}" if isinstance(value, (int, float)) else str(value) + label = " ".join(str(numeric.get("label") or "Records").split()) + return f"I built {artifact['name']}. {label}: {number}." + + +def _numeric_result(view, records): + aggregation = view["aggregation"] + if aggregation["op"] == "count": + return {"label": "Matching records", "value": len(records), "contributing_record_count": len(records)} + values = [] + for record in records: + try: + value = record.get(aggregation["field"]) + if value is not None and not isinstance(value, bool): + values.append(float(value)) + except (TypeError, ValueError): + continue + if not values: + return {"label": aggregation["op"], "value": None, "contributing_record_count": 0} + op = aggregation["op"] + value = {"sum": sum(values), "avg": sum(values) / len(values), "min": min(values), "max": max(values)}[op] + return {"label": f"{op.title()} of {aggregation['field']}", "value": value, + "contributing_record_count": len(values)} + + +def _referenced_fields(view): + """The citation names every source field that affected the displayed result.""" + out = list(view.get("visible") or ()) + for node in view.get("filters") or (): + if isinstance(node, dict) and node.get("colId"): + out.append(str(node["colId"])) + for node in view.get("sorts") or (): + if isinstance(node, dict) and node.get("colId"): + out.append(str(node["colId"])) + for key in ("groupBy",): + if view.get(key): + out.append(str(view[key])) + aggregation = view.get("aggregation") or {} + if aggregation.get("field"): + out.append(str(aggregation["field"])) + return list(dict.fromkeys(out)) + + +def _effective_filters(snapshot, view): + """Keep the source request and generated-view predicates distinct in provenance.""" + return { + "source": _safe(snapshot.get("filters")), + "view": {"conj": view.get("filterConj", "and"), + "nodes": _safe(view.get("filters") or [])}, } - return clean, None, None -def _explain(clean, fields): - """A plain-language read-back of what the view WILL do — DERIVED from the validated spec. +def _view_records(snapshot, view): + """Apply the exact validated virtual-view filter before calculating a cited number.""" + nodes = view.get("filters") or [] + if not nodes: + return list(snapshot["records"]) + from harness import filter_eval + tree = {"conj": view.get("filterConj", "and"), "nodes": nodes} + return [row for row in snapshot["records"] + if filter_eval.matches(tree, row, snapshot["fields"])] - ⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which - is precisely the failure this sentence exists to expose: a spec whose every column is real and - which answers a different question is invisible to validation, so the only thing that catches - it is a person reading an honest description of the spec that was actually accepted. - """ - label = {f['key']: f['label'] for f in fields} - ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least', - 'lt': 'is under', 'lte': 'is at most', 'contains': 'contains', - 'doesNotContain': 'does not contain', 'isEmpty': 'is empty', - 'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within', - 'topN': 'is in the top', 'bottomN': 'is in the bottom'} - kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board', - 'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'} - parts = [f"{kinds.get(clean['kind'], clean['kind'])} of " - + ", ".join(label.get(k, k) for k in clean['visible'][:6]) - + (f" and {len(clean['visible']) - 6} more columns" - if len(clean['visible']) > 6 else "")] - if clean['filters']: - joiner = " or " if clean['filterConj'] == 'or' else " and " - parts.append("showing only records where " + joiner.join( - f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}" - + (f" {f['value']}" if f.get('value') else "") - for f in clean['filters'] if isinstance(f, dict) and f.get('colId'))) - else: - parts.append("showing every record") - if clean['groupBy']: - parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}") - if clean['sorts']: - s = clean['sorts'][0] - parts.append(f"sorted by {label.get(s['colId'], s['colId'])}" - + (" (highest first)" if s['dir'] == 'desc' else " (lowest first)")) - for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'), - ('colorField', 'coloured by'), ('sizeField', 'sized by')): - val = (clean.get('display') or {}).get(ref) - if val: - parts.append(f"{word} {label.get(val, val)}") - return ", ".join(parts) + "." - - -# ── the registry ──────────────────────────────────────────────────────────────────────────── - -def _qid(scope, question): - """PINNED per (database, question), the `view_templates` discipline: asking the same thing - twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by - appending a number, so an unpinned id fills the store with numbered near-duplicates.""" - h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8")) - return "q_" + h.hexdigest()[:12] - - -def _registry(session): - try: - recs = session.runtime.get(QUERY_KEY) or {} - except Exception: - return {} - return recs if isinstance(recs, dict) else {} + +def _citation(citation_id, snapshot, view, numeric, view_id): + return { + "id": citation_id, + "href": f"#/query?view={view_id}&citation={citation_id}", + "database": snapshot["database"], + "snapshot": {"kind": snapshot["source_kind"], "version": _safe(snapshot["source_version"])}, + "fields": [field["key"] for field in snapshot["fields"] + if field.get("key") in set(_referenced_fields(view))], + "filters": _effective_filters(snapshot, view), + "permission_scope_applied": snapshot["permission_scope_applied"], + "aggregation": _safe(view["aggregation"]), + "contributing_record_count": numeric["contributing_record_count"], + "retrieved_at": snapshot["retrieved_at"], + } + + +def _citation_complete(citation): + """A numeric result is not publishable without complete provenance.""" + required = {"database", "snapshot", "fields", "filters", "aggregation", + "contributing_record_count", "retrieved_at", "href"} + snapshot = citation.get("snapshot") if isinstance(citation, dict) else None + return (isinstance(citation, dict) and required <= set(citation) + and bool(citation["database"]) and isinstance(snapshot, dict) + and "version" in snapshot and bool(citation["retrieved_at"]) + and isinstance(citation["fields"], list) + and isinstance(citation["aggregation"], dict)) + + +def _public_view(view): + source = view["source"] + # ⭐⭐ W35-T25 · CONTRACT C4 — `edited` IS DERIVED, NEVER STORED, and that is deliberate. + # A stored boolean beside two specs is a third source of truth that can disagree with both; + # the only honest answer to "has this been changed" is "compare it". It also means a Revert + # that restores the spec clears the badge by construction rather than by remembering to. + original = view.get("original_spec") + return { + "id": view["id"], "viewId": view["id"], "scope": source["database"], + "name": view["name"], "description": str(view.get("description") or ""), + "kind": view["view"]["kind"], "question": view["question"], + "explain": view["explain"], "threadId": view["threadId"], "createdAt": view["createdAt"], + "virtual": True, "source": _safe(source), "view": _safe(view["view"]), + "citationIds": list(view["citationIds"]), "numeric": _safe(view["numeric"]), + # ⚠ AN ARTEFACT MADE BEFORE THIS WAVE HAS NO ORIGINAL, so it reports `edited: false` and + # sends no `original_spec` — and the client shows neither the badge nor Revert. C4 is + # explicit that this is the right answer: a Revert with nothing to revert to is worse + # than an absent one, and it is the case an NC in `verify_query` covers. + "original_spec": _safe(original) if isinstance(original, dict) else None, + "edited": bool(isinstance(original, dict) and _safe(view["view"]) != _safe(original)), + } -def _mine(session, rec): - return session.admin or str(rec.get('createdBy') or '') == session.uname +def _source_still_permitted(session, source, permitted=None): + """A persisted artefact never outlives the caller's current data permission. + ⚠ `permitted` is the BATCH answer from `deps.assistant_source_status` — one rows-free + resolution for every source at once, rather than one whole-document read per saved view. That + function's docstring carries the measurement. The single-source path below stays for callers + holding exactly one artefact (the workspace-event door), and asks the same helper. -#: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE -#: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving -#: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set -#: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the -#: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door -#: would then refuse is a control that lies" — and it is the same predicate twice, published once. -BUILTIN_SCOPES = ('customer_data', 'product_data') + ⛔ `permitted`, NEVER `answerable`. A source whose rows are served through the connector mirror + cannot be ASKED and can still be SEEN: the artefact was built from a snapshot that was legal + when it was taken, and hiding it because the reader can no longer make a NEW one would read as + deletion. The two verdicts are separate for that reason. + """ + database = (source or {}).get("database") if isinstance(source, dict) else None + if not database: + return False + if permitted is None: + permitted = {key for key, row in assistant_source_status(session, [database]).items() + if row["permitted"]} + return database in permitted + + +def _public_state(state, session): + # ONE rows-free batch answers both questions: which sources this caller may still see (which + # artefacts stay listed) and which of them can actually be asked (which chips are live). + status = assistant_source_status(session) + for row in state["views"].values(): + key = (row.get("source") or {}).get("database") if isinstance(row.get("source"), dict) else None + if key and key not in status: + # An artefact whose source is no longer enumerable still gets a verdict rather than a + # KeyError — it resolves to "not permitted" and the artefact drops out, which is the + # same answer the per-view wall gave. + status.update(assistant_source_status(session, [key])) + permitted = {key for key, row in status.items() if row["permitted"]} + allowed_views = [row for row in state["views"].values() + if _source_still_permitted(session, row.get("source"), permitted)] + allowed_ids = {row["id"] for row in allowed_views} + allowed_citations = {citation_id for row in allowed_views + for citation_id in row.get("citationIds") or []} + threads = sorted(state["threads"].values(), key=lambda row: row.get("updatedAt", ""), reverse=True) + messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", "")) + views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True) + messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids] + return {"threads": _safe(threads), "messages": _safe(messages), "views": views, + "citations": _safe([row for key, row in state["citations"].items() + if key in allowed_citations]), "models": model_choices(), + # The model picker's own data, the same shape `sources` uses for databases. + "modelStatus": model_status(), + # The chip row's own data: a source this caller holds but cannot ask, and WHY. + # ⭐⭐ D-276 — `or row.get("visible")`. A source the caller can open elsewhere in the + # product but the assistant cannot read is now LISTED with its cause, instead of + # vanishing from a picker that shows every other database they hold. It arrives with + # `answerable: False` and a `reason`, which is the SAME shape the mirror-served grids + # already use, so the chip row needs no new state to render it. + # ⛔ `permitted` STILL GATES ARTEFACTS — `allowed_views` above is unchanged. This widens + # what is DESCRIBED, never what can be read or kept. + "sources": _safe([{"database": key, "answerable": row["answerable"], + "reason": row["reason"]} + for key, row in sorted(status.items()) + if row["permitted"] or row.get("visible")])} @router.get("/query") def list_queries(session: Session = Depends(require_session)): - """The AI-built views this caller may open: their own, still-granted, still-existing. - - ⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn - and a database deleted after a view was saved, and the Query module is chrome: it may render - nothing the server would not grant today (`nav.ts`'s chrome law). + return _public_state(_state(session), session) + + +# ⭐⭐ W35-T25 · CONTRACT C4 (owner item 4 / R3) — WHICH SPEC MEMBERS AN EDIT MAY MOVE. +# +# ⛔ AN ALLOW-LIST, AND IT IS THE SECURITY BOUNDARY OF THIS WHOLE TICKET. The client sends a +# `SavedView`, a shape it composes itself, and merging it wholesale would let a caller rewrite +# `kind` (which decides the renderer and the citation's own claim), `aggregation` (the number this +# artefact CITED) or `name`. R3 opens the view SPEC: *"filters, sort, group, visible columns, +# widths"*, plus the row height and column order that carry them. Nothing else. +# +# ⚠ `aggregation` IS DELIBERATELY ABSENT even though a person can change it on an ordinary grid. +# The artefact's citation records `contributing_record_count` and an op computed from THIS +# aggregation; letting an edit move it would leave a cited number describing a calculation the +# artefact no longer performs, which is the one thing every provenance rule in this module exists +# to prevent. +# +# ⚠ A COMMENT, NOT A BARE MODULE-LEVEL STRING. `verify_prose` reads a free-floating string literal +# as candidate copy, so writing this as a `"""..."""` above the constant put an em dash into the +# gate's own census as a 296th finding. A `#` block is out of scope by construction. +QUERY_EDITABLE_SPEC = ("visible", "order", "widths", "filters", "filterConj", "sorts", + "groupBy", "rowHeightMode", "frozenCount", "colorBy", "display") + + +def _clean_spec_edit(config, spec, fields): + """Merge a client view config onto the artefact's spec, cleaned against ITS OWN fields. + + ⛔ THE FIELD KEYS COME FROM THE STORED SOURCE, never from the request. The artefact carries + the snapshot's field list, so this needs no database read and cannot be widened by a caller + naming a column the snapshot did not have. """ - import routes_templates - out = [] - for qid, rec in sorted(_registry(session).items(), - key=lambda kv: str((kv[1] or {}).get('createdAt') or ''), - reverse=True): - if not isinstance(rec, dict) or not _mine(session, rec): + keys = {str(field.get("key")) for field in (fields or []) if isinstance(field, dict)} + out = copy.deepcopy(spec) if isinstance(spec, dict) else {} + if not isinstance(config, dict): + return out + for member in QUERY_EDITABLE_SPEC: + if member not in config: continue - try: - routes_templates._target_or_refuse(session, rec.get('scope')) - except Exception: - continue # no longer granted, or gone - out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'), - "name": rec.get('name'), "kind": rec.get('kind'), - "question": rec.get('question'), "explain": rec.get('explain'), - "createdAt": rec.get('createdAt')}) - return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)} - - -@router.post("/query/build") -def build_query(body: dict = Body(default=None), - session: Session = Depends(require_session)): - """A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.** - - A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a - successful answer to the person asking, and an error envelope would paint the error page R1 - forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an - over-long question, a database this session may not open. - - ⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took - `chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument - as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and - `POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on - `"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly — - which is exactly the shape of a test seam that becomes an input. The seam now lives on the - module-level `_build` below: the gate calls that, and the wire cannot. + value = config[member] + if member in ("visible", "order"): + cleaned = [str(key) for key in value if str(key) in keys] if isinstance(value, list) else [] + # ⚠ An EMPTY visible list is refused rather than stored: `_validate` already treats + # "no columns" as a refusal at creation, and a grid showing nothing is not an edit + # somebody meant to make. + if member == "visible" and not cleaned: + continue + out[member] = cleaned[:MAX_VISIBLE] + elif member == "widths": + out[member] = {str(key): int(width) for key, width in value.items() + if str(key) in keys and isinstance(width, (int, float)) + and 0 < float(width) <= 2000} if isinstance(value, dict) else {} + elif member == "filters": + out[member] = _grid().clean_filter_tree( + [item for item in value if isinstance(item, dict)], keys) if isinstance(value, list) else [] + elif member == "filterConj": + out[member] = "or" if value == "or" else "and" + elif member == "sorts": + out[member] = [{"colId": item["colId"], "dir": "desc" if item.get("dir") == "desc" else "asc"} + for item in value if isinstance(item, dict) and item.get("colId") in keys][:3] \ + if isinstance(value, list) else [] + elif member in ("groupBy", "colorBy"): + out[member] = value if value in keys else None + elif member == "rowHeightMode": + out[member] = value if value in ("short", "medium", "tall", "extra") else None + elif member == "frozenCount": + out[member] = max(0, min(6, int(value))) if isinstance(value, (int, float)) else 0 + elif member == "display": + out[member] = _grid()._clean_display(value, keys) if isinstance(value, dict) else out.get("display") + return out + + +@router.post("/query/{qid}/events") +def mutate_query_workspace(qid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """The only grid-mutation transport for a virtual Query workspace. + + ⭐⭐ W35-T25 (owner item 4 / R2) — `view_upsert` ON THE ARTEFACT ITSELF IS NOW ACCEPTED. It + used to 409 `query_workspace_immutable` for everything but a delete, on the reading that an AI + artefact is a snapshot. R2 replaces that reading: the SPEC is the reader's to shape, the + CITATION and the numbers behind it are not. `QUERY_EDITABLE_SPEC` above is where that line is + drawn, and `original_spec` is what makes the change undoable. + + Creating a SECOND view inside the workspace is still refused — an artefact holds exactly one — + and deleting the caller's personal artefact still removes it. The binding key, not a + client-supplied source scope, identifies it. """ + qid = str(qid) body = body if isinstance(body, dict) else {} - return _build(body.get("question"), body.get("scope"), session) - + binding = body.get("workspaceBinding") + event = body.get("event") + if not (isinstance(binding, dict) and binding.get("kind") == "query" + and str(binding.get("key") or "") == qid): + raise err(400, "query_workspace_mismatch", + "the Query workspace binding must name this virtual artefact") + if not isinstance(event, dict) or str(event.get("type") or "") not in QUERY_WORKSPACE_EVENTS: + raise err(400, "unsupported_query_workspace_event", + "Query accepts only view_create, view_upsert, or view_delete events") + + state = _state(session) + view = state["views"].get(qid) + if view is None or not _source_still_permitted(session, view.get("source")): + # A caller cannot use an opaque Query key to learn about a revoked artefact. + raise err(404, "unknown_query", "that Query artefact does not exist") + + event_type = str(event["type"]) + if event_type == "view_upsert": + sent = event.get("view") + sent = sent if isinstance(sent, dict) else {} + # ⛔ THE ID IS CHECKED THE SAME WAY THE DELETE'S IS. An upsert naming a different view is + # a create wearing an update's name, and a Query workspace holds exactly one view. + if str(sent.get("id") or "") != qid: + raise err(400, "query_workspace_mismatch", + "a Query view edit must name the same virtual artefact as its binding") + spec = _clean_spec_edit(sent.get("config"), view.get("view"), (view.get("source") or {}).get("fields")) + + def apply_edit(raw): + current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() + row = (current.get("views") or {}).get(qid) + if not isinstance(row, dict): + return current + # ⚠ BACKFILLED HERE, and only when absent: an artefact created before this wave has no + # original, and the FIRST edit is the last moment its pre-edit spec still exists. Not + # backfilling would leave it permanently unrevertable; backfilling unconditionally + # would make Revert restore the latest edit. + if not isinstance(row.get("original_spec"), dict): + row["original_spec"] = _safe(row.get("view")) + row["view"] = _safe(spec) + current["views"][qid] = row + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was saved") + session.runtime.update(_namespace_key(session), apply_edit, flush="async") + return {"workspaceBinding": {"kind": "query", "key": qid}, "event": event_type, + "view": _public_view(apply_edit(_state(session))["views"][qid])} + + if event_type != "view_delete": + raise err(409, "query_workspace_immutable", + "a Query workspace holds one view, so it cannot take another") + if str(event.get("viewId") or "") != qid: + raise err(400, "query_workspace_mismatch", + "a Query delete must name the same virtual artefact as its binding") + + deleted = delete_query(qid, session) + return {"workspaceBinding": {"kind": "query", "key": qid}, + "event": event_type, **deleted} + + +@router.post("/query/{qid}/revert") +def revert_query(qid: str, session: Session = Depends(require_session)): + """⭐⭐ CONTRACT C4 (R3) — restore the AI's original SPEC, and only the spec. + + ⛔ WHAT THIS DOES NOT DO, which the client's confirm says out loud BEFORE it acts: it does not + undo anything the reader changed in the SOURCE database. A Query view is live now, so a cell + edit made through it is a real write to a real record — reverting a view's filters cannot and + must not walk those back. R3 is explicit that the dialog states this before it acts, because a + Revert that silently leaves data changed is worse than one that never offered. + + ⚠ 409, not 404, when there is no original: the artefact exists and is readable, and the caller + asked for something that does not exist FOR IT. A 404 would say the artefact is gone. + """ + qid = str(qid) + # ⚠ `_artifact_or_404` returns `(state, row)`, not the row. Read it as a pair. + _state_now, view = _artifact_or_404(session, qid) + original = view.get("original_spec") + if not isinstance(original, dict): + raise err(409, "no_original_spec", + "this view was created before the assistant kept an original, so there is " + "nothing to revert to") + + def restore(raw): + current = copy.deepcopy(raw) if isinstance(raw, dict) else _blank_state() + row = (current.get("views") or {}).get(qid) + if not isinstance(row, dict): + return current + row["view"] = _safe(row.get("original_spec")) + current["views"][qid] = row + return current -def _build(question, scope, session, chat=None): - """`build_query`'s body, with the model injectable — see that route's note on why the seam is - here and not on the signature FastAPI reads.""" - question = ' '.join(str(question or "").split()) + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was reverted") + session.runtime.update(_namespace_key(session), restore, flush="async") + return _public_view(restore(_state(session))["views"][qid]) + + +def _sources(body, target, session): + raw = body.get("sources") if isinstance(body, dict) else None + sources = [str(item).strip() for item in raw] if isinstance(raw, list) else [] + sources = list(dict.fromkeys(item for item in sources if item)) + if target not in sources: + sources.insert(0, target) + # Source chips are a permissioned selection, not untrusted labels remembered in a thread. + # The model still receives ONLY `target`'s snapshot below; these calls do not pass a second + # source to it and D's helper refuses cache misses, mirrors and unshared databases. + for source in sources: + assistant_read_scope(session, source, fields=[], filters=None) + return sources + + +def _submit(body, session, chat=None): + body = body if isinstance(body, dict) else {} + question = " ".join(str(body.get("question") or "").split()) + target = str(body.get("database") or body.get("scope") or "").strip() + selected_model = str(body.get("model") or MODEL_AUTO).strip().lower() if not question: raise err(400, "bad_request", "no question was asked") if len(question) > MAX_QUESTION: - raise err(400, "question_too_long", - f"a question must be under {MAX_QUESTION} characters") - fields, _source = _target(session, scope) - if not fields: - raise err(400, "no_columns", "that database has no columns to build a view from") - - spec, transport, provider = _call_model(question, fields, chat=chat) - if transport: - return {"spec": None, "explain": None, "refused": transport, "provider": provider} - clean, refusal, code = _validate(spec, fields) - if refusal: - return {"spec": None, "explain": None, "refused": refusal, "reason": code, - "provider": provider} - return {"spec": clean, "explain": _explain(clean, fields), - "refused": None, "scope": scope, - "id": _qid(scope, question), "provider": provider} - - -@router.post("/query/save") -def save_query(body: dict = Body(default=None), session: Session = Depends(require_session)): - """Save an accepted spec as the CALLING USER's own view, and index it under Query. - - ⛔ THE SPEC IS RE-VALIDATED HERE, against the target's contract as it is NOW. The build door - wrote nothing, so this is the first write and it is the only place authorisation and validity - have to hold together — and a client is not a wall. An unsupported kind or an unknown column - is a **400 with a named code** on this door (the caller sent it), where the same fact is a - plain sentence on `build` (the model produced it). - """ - body = body if isinstance(body, dict) else {} - scope = str(body.get("scope") or '').strip() - question = ' '.join(str(body.get("question") or "").split())[:MAX_QUESTION] - fields, _source = _target(session, scope) - clean, refusal, code = _validate(body.get("spec"), fields) - if refusal: - raise err(400, code or "bad_spec", refusal) + raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters") + if not target: + raise err(400, "bad_request", "one target database must be selected") + if selected_model not in model_choices(): + raise err(400, "unknown_model", "that model is not available in Query") + + # This call happens before model selection and is the only permitted source-data read. + snapshot = assistant_read_scope(session, target, fields=None, filters=body.get("filters")) + sources = _sources(body, target, session) + state = _state(session) + thread_id = str(body.get("threadId") or "").strip() + if thread_id and thread_id not in state["threads"]: + raise err(404, "unknown_thread", "that chat does not exist") + if not thread_id: + thread_id = _new_id("thread") + now = _now_iso() + user_message_id = _new_id("message") + assistant_message_id = _new_id("message") + # R16: the turns already on screen go with the question. `state` was read above, BEFORE this + # turn's own messages exist, so the replay is strictly the prior conversation. + # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s + # docstring for why it is an argument and not ambient state (PRD amendment A1). + spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat, + history=_history(state, thread_id), session=session) + view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason) + + artifact = None + citation = None + if view is not None: + if len(state["views"]) >= MAX_ARTIFACTS: + raise err(400, "query_limit", f"you already have {len(state['views'])} Query artefacts") + view_id = _new_id("query") + citation_id = _new_id("citation") + numeric = _numeric_result(view, _view_records(snapshot, view)) + citation = _citation(citation_id, snapshot, view, numeric, view_id) + if not _citation_complete(citation): + raise RuntimeError("Query refused to persist an incomplete numeric citation") + source = {"database": snapshot["database"], "label": snapshot["label"], + "source_kind": snapshot["source_kind"], "source_version": _safe(snapshot["source_version"]), + "retrieved_at": snapshot["retrieved_at"], + "fields": _safe(snapshot["fields"]), "filters": _safe(snapshot["filters"]), + "permission_scope_applied": snapshot["permission_scope_applied"]} + artifact = {"id": view_id, "threadId": thread_id, "name": view["name"], "description": "", + "question": question, + "explain": _explain(view, snapshot["fields"]), "createdAt": now, "source": source, + "view": _safe(view), "citationIds": [citation_id], "numeric": numeric, + # ⭐⭐ W35-T25 · CONTRACT C4 (R3) — THE AI'S OWN SPEC, WRITTEN ONCE, HERE. + # R2 makes a Query view editable, so `view` moves from now on. This is the copy + # "Revert to AI original" restores, and the thing `edited` is measured against. + # ⛔ WRITTEN AT CREATE AND NOWHERE ELSE. Re-stamping it on any later write would + # make Revert restore the most recent edit — a Revert that reverts to nothing, + # which C4 names as worse than no Revert at all. + # ⚠ `_safe(view)` twice, not the same object twice: `view` is mutable and a + # shared reference would let an edit rewrite the original through the alias. + "original_spec": _safe(view), + "model": provider, "requestedModel": selected_model} + + assistant_message = { + "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now, + "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"), + "targetDatabase": target, "requestedModel": selected_model, "model": provider, + "reason": refusal_code, "viewId": artifact["id"] if artifact else None, + "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None, + } + user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now, + "content": question, "sources": sources, "targetDatabase": target, + "requestedModel": selected_model} + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + thread = current["threads"].get(thread_id) or {"id": thread_id, "createdAt": now} + # ⚠ THE FIRST QUESTION NAMES THE CHAT, and R16 is what makes that matter. While every turn + # was a fresh one-shot the title could only be the last thing asked; now that a thread is a + # conversation, retitling it on every follow-up renames the history entry out from under + # the reader, and "and the other one?" is a useless name for anything. + thread.update({"updatedAt": now, "title": thread.get("title") or question[:80], "sources": sources, + "model": selected_model, "activeViewId": artifact["id"] if artifact else thread.get("activeViewId")}) + current["threads"][thread_id] = thread + current["messages"][user_message_id] = user_message + current["messages"][assistant_message_id] = assistant_message + if artifact: + current["views"][artifact["id"]] = artifact + current["citations"][citation["id"]] = citation + return current + if not session.runtime.available(): - raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved") - - vt = _templates() - ws_key = vt.workspace_key(scope) - if not ws_key: - raise err(404, "unknown_table", "that database does not exist") - qid = _qid(scope, question) if question else _qid(scope, clean['name']) - - # ⛔ THE SKELETON IS `view_templates._view`, NOT A SECOND ONE. "A view assembled from a - # DIFFERENT skeleton is a second definition of what a view is — and the two drift on the day - # one of them gains a key." It also puts the mode at `config.display.mode`, which is the key - # the grid actually reads (`config.displayMode` was the first draft, and nothing reads it). - payload = vt._view(qid, clean['name'], f"Built by the assistant from: {question}"[:300], - clean['visible'], mode=clean['kind'], - sorts=clean['sorts'], filters=clean['filters'], - group_by=clean['groupBy'], conj=clean['filterConj']) - if clean.get('display'): - payload['config']['display'] = dict(clean['display']) - payload['createdBy'] = session.uname - - # ⛔ THE CAP REFUSES; IT DOES NOT EVICT — and the first draft evicted. Dropping the oldest entry - # would leave its saved VIEW behind in the table's own workspace: "stranded in the target's view - # list with no way back to the question that made it", which is the thing `delete_query` below - # exists to prevent. And an eviction nobody was told about is W30/R6's second sentence broken — - # a limit that cannot be removed must be REPORTED with its cause. So the 201st is a refusal - # naming the number and the way out, checked BEFORE the view is written. - existing = _registry(session) - if qid not in existing and len(existing) >= MAX_REGISTRY: - raise err(400, "query_limit", - f"you already have {len(existing)} saved questions, which is the limit " - f"({MAX_REGISTRY}). Delete one you no longer need and save this again.") + raise err(503, "store_unavailable", "the tenant store is unavailable; no chat was saved") + session.runtime.update(_namespace_key(session), update, flush="async") + return {"thread": _safe(update(state)["threads"][thread_id]), "userMessage": _safe(user_message), + "message": _safe(assistant_message), + "view": _public_view(artifact) if artifact else None, "citations": [citation] if citation else []} + + +@router.post("/query/chat") +def submit_chat(body: dict = Body(default=None), session: Session = Depends(require_session)): + return _submit(body, session) - import core.table_store as table_store - ops = table_store.make(ws_key, st=session.runtime) - saved = ops.save_view(session.uname, payload, shared=False) - entry = {"scope": scope, "viewId": qid, "name": saved.get('name') or clean['name'], - "kind": clean['kind'], "question": question, - "explain": _explain(clean, fields), "createdBy": session.uname, - "createdAt": _now_iso()} +@router.delete("/query/threads/{tid}") +def delete_thread(tid: str, session: Session = Depends(require_session)): + """Remove one of the caller's own chats, and only the chat. - def _up(data): - data = data if isinstance(data, dict) else {} - data[qid] = entry - return data + ⛔ THE ARTEFACTS SURVIVE, AND THAT IS THE POINT. A Query view is a durable object in its own + right — it appears in Query's rail, other people's links can point at it, and the owner's + instruction was that AI views LIVE THERE rather than inside the conversation that happened to + produce them. Cascading the delete would make tidying up a chat silently destroy work. Views + are deleted from Query's own rail, one at a time, by `delete_query` below. - session.runtime.update(QUERY_KEY, _up, flush='async') - return {"id": qid, **entry} + ⚠ It exists because the history had no prune. `delete_query` removed a view and left its + thread, so the panel could only ever grow — a navigation you cannot manage is half a + navigation, and a person clearing test chats found that out first. + """ + tid = str(tid) + state = _state(session) + if tid not in state["threads"]: + raise err(404, "unknown_thread", "that chat does not exist") + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + current["threads"].pop(tid, None) + for message_id in [key for key, row in current["messages"].items() + if (row or {}).get("threadId") == tid]: + current["messages"].pop(message_id, None) + # ⚠ THE VIEWS' `threadId` IS LEFT EXACTLY AS IT WAS, dangling. Blanking it looks tidier and + # is a data-loss bug: `queryApi.saved()` refuses a row with an empty `threadId`, so every + # artefact built in the deleted chat would silently vanish from Query's rail — the exact + # "tidying up destroys work" outcome this door is written not to have + # [[a-record-can-outlive-its-subject]]. Nothing resolves the id but the chat panel, which + # only ever looks up the thread it is showing. + return current + + session.runtime.update(_namespace_key(session), update, flush="async") + return {"deleted": tid} @router.delete("/query/{qid}") def delete_query(qid: str, session: Session = Depends(require_session)): - """Forget an AI-built view — the index entry AND the view itself. + qid = str(qid) + state = _state(session) + if qid not in state["views"]: + raise err(404, "unknown_query", "that Query artefact does not exist") + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + view = current["views"].pop(qid, None) + for citation_id in (view or {}).get("citationIds") or []: + current["citations"].pop(citation_id, None) + return current + + session.runtime.update(_namespace_key(session), update, flush="async") + return {"deleted": qid} + + +# ══ R20 — A QUERY VIEW HAS EVERY FUNCTION A DATABASE VIEW HAS ═══════════════════════════════════ +# +# ⛔⛔ THE DECISION THIS TICKET ASKED FOR, WRITTEN DOWN RATHER THAN IMPLIED, because it is the line +# every later reader will need and there is no other place it exists. +# +# **IMMUTABLE, and unchanged by R20:** the generated SPEC and its provenance. `view` (kind, visible, +# filters, sorts, aggregation, display, important), `source` (the snapshot, its version, its +# retrieval time, the permission scope), `question`, `citationIds`, `numeric`. A cited number is +# only worth citing if the thing it was computed from cannot be edited underneath it, so +# `mutate_query_workspace` still answers 409 `query_workspace_immutable` for `view_create` and +# `view_upsert`, and `QUERY_MUTATION_POLICY` on the client is untouched. +# +# **OPENED:** `name` and `description`. They are LABELS, not the answer: changing them cannot make +# a citation wrong. Plus a server-side DUPLICATE (the client never supplies a spec, so a copy is a +# copy) and an EXPORT, which is a read. +# +# ⛔ AND THE REASON THESE ARE NAMED DOORS RATHER THAN AN OPENED `view_upsert`, which is what the +# ticket's `how:` first suggested: the client refuses `view_upsert` on a Query binding LOCALLY, +# before any request leaves the browser (`queryPreview.ts::routeQueryViewMutation`). Opening the +# server there would give one question two answers, and the client's is the one a person +# experiences. A named door has exactly one answer, and "opening a route widens every field it +# carries unless the cleaner is explicit" is answered by the allow-lists below rather than by hope. + +QUERY_NOTE_MAX = 400 + + +def _artifact_or_404(session, qid, state=None): + """The one wall every artefact door goes through: it exists AND this caller may still see it.""" + state = _state(session) if state is None else state + row = state["views"].get(str(qid)) + if row is None or not _source_still_permitted(session, row.get("source")): + # A caller cannot use an opaque Query key to learn about a revoked artefact. + raise err(404, "unknown_query", "that Query artefact does not exist") + return state, row + + +def _copy_name(name, taken): + """`X copy`, then `X copy 2`, so duplicating twice does not make two rows with one name.""" + base = f"{str(name or 'Query')[:52]} copy" + if base not in taken: + return base[:60] + index = 2 + while f"{base} {index}" in taken and index < 99: + index += 1 + return f"{base} {index}"[:60] + + +@router.patch("/query/{qid}") +def rename_query(qid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Rename a Query view, or give it a description. Nothing else, by construction. + + ⚠ THE TEXT IS THE READER'S, SO IT IS NOT SWEPT FOR DASHES. R6 governs the copy WE write; a note + somebody typed about their own view is theirs, and rewriting its punctuation would be the + product editing a person's words. Model-authored strings are a different case and are swept + where they are produced (`_said`). + """ + body = body if isinstance(body, dict) else {} + state, row = _artifact_or_404(session, qid) + # An ALLOW-LIST, read key by key. A body carrying `view` or `source` changes neither. + patch = {} + if "name" in body: + patch["name"] = " ".join(str(body.get("name") or "").split())[:60] or row.get("name") or "Query" + if "description" in body: + patch["description"] = " ".join(str(body.get("description") or "").split())[:QUERY_NOTE_MAX] + if not patch: + raise err(400, "bad_request", "a Query view takes a new name or a new description") + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + if str(qid) in current["views"]: + current["views"][str(qid)].update(patch) + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was renamed") + session.runtime.update(_namespace_key(session), update, flush="async") + return _public_view(update(state)["views"][str(qid)]) + + +@router.post("/query/{qid}/duplicate") +def duplicate_query(qid: str, session: Session = Depends(require_session)): + """A second artefact with the same spec and its own identity. + + ⛔ THE CITATIONS ARE COPIED, NOT SHARED. `delete_query` pops an artefact's citations by id, so a + copy pointing at the original's citation ids would lose its provenance the moment the original + was deleted, and a number with no citation is the one thing this module refuses to persist. + """ + state, row = _artifact_or_404(session, qid) + if len(state["views"]) >= MAX_ARTIFACTS: + # R6's second sentence: a cap that cannot be lifted here names its cause and the remedy. + raise err(400, "query_limit", + f"you already have {len(state['views'])} Query views, which is the most one " + f"account can hold. Delete one to make room for this copy.") + new_id = _new_id("query") + copied = copy.deepcopy(row) + copied["id"] = new_id + copied["name"] = _copy_name(row.get("name"), {r.get("name") for r in state["views"].values()}) + copied["createdAt"] = _now_iso() + citations = {} + for citation_id in row.get("citationIds") or []: + source = state["citations"].get(citation_id) + if not source: + continue + fresh_id = _new_id("citation") + citation = copy.deepcopy(source) + citation["id"] = fresh_id + citation["href"] = f"#/query?view={new_id}&citation={fresh_id}" + citations[fresh_id] = citation + copied["citationIds"] = list(citations) + + def update(raw): + current = _blank_state() + if isinstance(raw, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw.get(key), dict): + current[key] = copy.deepcopy(raw[key]) + current["views"][new_id] = copied + current["citations"].update(citations) + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was copied") + session.runtime.update(_namespace_key(session), update, flush="async") + return _public_view(copied) + + +# ⛔ A REGISTRY KEY AND A GRID SCOPE ARE DIFFERENT SPELLINGS OF ONE DATABASE, and this product has +# already been bitten by treating them as the same: a `VIEW_OPEN` emit whose topic did not match was +# dropped SILENTLY, 200 OK, and an alert was filed against the wrong database. `customer_data` is +# what the assistant reads; `customer` is what the grid writes. `routes_grid._scope_or_400` refuses +# an unknown scope rather than defaulting it, so this map has to be right rather than nearly right. +_GRID_SCOPE = {"customer_data": "customer", "product_data": "product"} +# The keys a Query spec and a database view actually share. `aggregation` is deliberately absent: +# a database view has no aggregation of its own, which is why the reply below NAMES it as dropped +# instead of letting a "sum of value" answer land as a plain list of rows. +_SAVEABLE = ("visible", "filters", "filterConj", "sorts", "groupBy", "display", "important") + + +def _grid_scope(database): + key = str(database or "").strip() + return _GRID_SCOPE.get(key) or (key if key.startswith("ut_") else "") + + +@router.post("/query/{qid}/save-to-database") +def save_query_to_database(qid: str, session: Session = Depends(require_session)): + """R20: copy this answer into the database's OWN view list, where an ordinary view lives. + + ⛔⛔ `view_upsert` ANSWERS 200 WHILE WRITING NOTHING, IN TWO DIFFERENT WAYS, AND THE WIRE CANNOT + TELL YOU WHICH. `rerender: False` is the documented SUCCESS shape for this event, and it is + also what several silent-refusal branches return: a malformed payload, a missing id or name, an + id that belongs to a view the caller cannot SEE (the wave-9 shared-view guard, which makes + pinned ids effectively tenant-scoped), and a shared view they may see but not edit. So this + door does not believe the response. It READS THE STORE BACK and refuses out loud if the view is + not there. That read is same-process, which is what makes it valid despite `flush="async"`: + the write is visible through the cache long before it is durable. + + ⭐ ONE REQUEST, ONE EVENT. Eighteen POSTs against one JSON document under a coalescing + single-flight once landed ZERO while answering 200 eighteen times; the fix is a batch, never a + retry loop, which treats the symptom and doubles the races. + + ⚠ A FRESH ID, NEVER THE ARTEFACT'S. Reusing the Query id would be a guessable id in a shared + bucket, which is the exact door the wave-9 guard exists to shut. + """ + from routes_grid import grid_events_route + + state, row = _artifact_or_404(session, qid) + scope = _grid_scope((row.get("source") or {}).get("database")) + if not scope: + raise err(409, "unsaveable_source", + "this answer's database does not have a view list to save into.") + spec = row.get("view") or {} + config = {key: copy.deepcopy(spec[key]) for key in _SAVEABLE if spec.get(key) is not None} + config["important"] = spec.get("important") is True + view_id = _new_id("view") + name = " ".join(str(row.get("name") or "Query").split())[:120] or "Query" + + grid_events_route({"scopeKey": scope, "events": [{ + "id": _new_id("event"), "type": "view_upsert", + "view": {"id": view_id, "name": name, "config": config}, + }]}, session) + + document = session.runtime.get(f"{scope}_table_workspace") or {} + landed = (((document.get(session.uname) or {}).get("views") or {}).get(view_id) + if isinstance(document, dict) else None) + if not isinstance(landed, dict): + raise err(409, "view_not_saved", + "the database did not accept this view, so nothing was saved. Open the database " + "and check you can still add a view there.") + + # R6's second sentence, in the payload: what the database's own validator would not take is + # NAMED rather than quietly missing from a view that then looks complete. + stored = landed.get("config") or {} + dropped = [] + if str((spec.get("aggregation") or {}).get("op") or "count") != "count": + dropped.append("the aggregation, which a database view does not carry") + if len(stored.get("filters") or []) != len(config.get("filters") or []): + dropped.append("some of the conditions") + for key in ("visible", "sorts", "groupBy", "display"): + if config.get(key) and not stored.get(key): + dropped.append(key) + return {"scope": scope, "viewId": view_id, "name": landed.get("name") or name, + "dropped": dropped} + + +@router.post("/query/messages/{mid}/rating") +def rate_message(mid: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """R20's thumbs, and the ONE thing that stops them being decoration. + + ⛔ A RATING THAT IS WRITTEN AND NEVER READ IS A FLAG SHIPPED WITHOUT ITS WRITER. This one is + read: `_history` turns a thumbs-down into a user turn saying the answer was not helpful, with + the reason if one was given, so the next question in the same thread is answered against that. + The scope is honest and narrow: it steers THIS conversation. It does not train anything, does + not cross threads, and does not cross accounts. + + ⚠ Rating is idempotent and CLEARABLE (`rating: null`), because a mis-click that cannot be taken + back is worse than no control at all, and a flag that can be set and never cleared is the same + defect `important` was fixed for. + """ + body = body if isinstance(body, dict) else {} + mid = str(mid) + raw = body.get("rating") + rating = str(raw or "").strip().lower() + if raw is not None and rating not in QUERY_RATINGS: + raise err(400, "bad_rating", "a rating is up, down, or nothing at all") + state = _state(session) + row = state["messages"].get(mid) + if row is None: + raise err(404, "unknown_message", "that message does not exist") + if row.get("role") != "assistant": + raise err(400, "bad_rating", "only an answer can be rated") + reason = " ".join(str(body.get("reason") or "").split())[:MAX_RATING_REASON] + + def update(raw_state): + current = _blank_state() + if isinstance(raw_state, dict): + for key in ("threads", "messages", "citations", "views"): + if isinstance(raw_state.get(key), dict): + current[key] = copy.deepcopy(raw_state[key]) + message = current["messages"].get(mid) + if message is not None: + message["rating"] = rating if raw is not None else None + # A reason belongs to a thumbs-down; clearing the rating clears it with them. + message["ratingReason"] = reason if (raw is not None and rating == "down") else "" + return current + + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable; nothing was recorded") + session.runtime.update(_namespace_key(session), update, flush="async") + return _safe(update(state)["messages"][mid]) + + +@router.get("/query/{qid}/export") +def export_query(qid: str, session: Session = Depends(require_session)): + """The artefact's own rows and columns, as CSV. - Leaving the view behind would strand it in the target's view list with no way back to the - question that made it ([[a-record-can-outlive-its-subject]]). + ⚠ THE ARTEFACT IS FROZEN; ITS DATA IS NOT, and the difference is worth stating because the two + are easy to conflate. This re-reads the source through the SAME permission wall that built the + artefact and applies the view's own stored predicate, so the file matches what the grid is + showing right now, not the snapshot the stored `numeric` was computed from. That is the useful + answer: a person exports what they are looking at. """ - rec = _registry(session).get(str(qid)) - if not isinstance(rec, dict) or not _mine(session, rec): - raise err(404, "unknown_query", "that view does not exist") - _target(session, rec.get('scope')) # the wall, again, on the write path - vt = _templates() - ws_key = vt.workspace_key(rec.get('scope')) - if ws_key and session.runtime.available(): - import core.table_store as table_store - table_store.make(ws_key, st=session.runtime).delete_view( - session.uname, rec.get('viewId') or str(qid)) - - def _up(data): - data = data if isinstance(data, dict) else {} - data.pop(str(qid), None) - return data - - session.runtime.update(QUERY_KEY, _up, flush='async') - return {"deleted": str(qid)} + state, row = _artifact_or_404(session, qid) + source = row.get("source") or {} + # ⛔ THIS PASSES A STORED OUTPUT BACK INTO AN INPUT SLOT, WHICH IS WORTH STATING RATHER THAN + # HIDING. `source["filters"]` is `deps.assistant_read_scope`'s OWN return value, and that value + # is `validate_assistant_filter(filters, visible)` — the VALIDATED form of whatever the caller + # asked for. MEASURED: `queryApi.submitChat` sends no `filters` key at all, so in production + # this is `None` on every artefact and the round trip never happens. + # ⚠ It is sent anyway, and deliberately, because the alternative FAILS OPEN. If a caller ever + # does narrow a source, dropping the filter here would export MORE rows than the artefact was + # built from, silently. Sending it means a stored filter that cannot be re-validated comes back + # as a NAMED 400 (`assistant_bad_filter`) instead — the fail-closed direction. The gate asserts + # what this door PASSES rather than what the fixture echoes back, because a double built from + # the producer would make this argument valid by construction. + snapshot = assistant_read_scope(session, source.get("database"), fields=None, + filters=source.get("filters") or None) + labels = {field["key"]: str(field.get("label") or field["key"]) for field in snapshot["fields"]} + columns = [key for key in (row["view"].get("visible") or ()) if key in labels] + if not columns: + raise err(409, "nothing_to_export", + "none of this view's columns exist in the database any more, so there is " + "nothing to export. Ask the assistant to build it again.") + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + writer.writerow([labels[key] for key in columns]) + for record in _view_records(snapshot, row["view"]): + writer.writerow(["" if record.get(key) is None else record.get(key) for key in columns]) + stem = re.sub(r"[^A-Za-z0-9]+", "_", str(row.get("name") or "query")).strip("_") or "query" + return Response( + content=buffer.getvalue(), media_type="text/csv; charset=utf-8", + # ⚠ ASCII-only filename: a header carrying a non-ASCII byte is refused by some servers and + # silently mangled by others, and the stem above comes from a model-authored name. + headers={"Content-Disposition": f'attachment; filename="{stem[:60]}.csv"'}) diff --git a/api/routes_records.py b/api/routes_records.py index b30aa3103388460dd961a26609faa6f8a6af2bfc..ef0bb2c8153db1e40b2656a715f545d80b1fe22b 100644 --- a/api/routes_records.py +++ b/api/routes_records.py @@ -15,8 +15,12 @@ one here: customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant product `routes_products.scoped_pool` behind the `product_data` grant - ut_ `routes_tables.scoped_pool`, whose `_defn_or_refuse` IS the wall - (404 unknown / 403 not yours — a user table has no module grant) + ut_ `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall + (404 unknown / 403 not yours — a user table has no module grant). + ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every + ROW to derive a pid set this module discards, and RAISES 409 on a + read-through grid past one window, which is why the record drawer painted + an error page on `ut_odoo_gl_lines`. ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning @@ -49,31 +53,74 @@ def _scope_or_400(raw): def _pool_or_refuse(session: Session, scope: str): """The pids this session may attach comments to ON THIS DATABASE — grant wall included. - Returns the frozenset. Raises the topic's own 403/404/503, so a caller who may not open the - surface never learns anything about the row they asked about. + Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503, + so a caller who may not open the surface never learns anything about the row they asked about. + + ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past + one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns + `frozenset()` for both — see the branch below. + ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that + do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py` + that REPLACES this function with its own lambda. A gate's test double is a caller + ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a + bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no + failing name. Change the shape here and that double changes with it. """ if scope == "product": from routes_products import MODULE as PRODUCT_MODULE, scoped_pool session.require(PRODUCT_MODULE) pids, _team, _rows, _fields = scoped_pool(session) - return pids + # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes + # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised + # a `TypeError` — a 500 on every product comment — while both other branches worked. A + # return shape is a contract even when the function is private. + return (pids, False) if scope.startswith("ut_"): - # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pool` IS + # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the # rows routes do. - from routes_tables import scoped_pool - - pids, _rows, _fields, _defn = scoped_pool(session, scope) - return pids + # + # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG. + # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on + # a read-through grid larger than one window it RAISES `409 window_required`. So opening the + # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel + # that renders comments about ONE row it already has. `scoped_pids` answers the identical + # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes + # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace` + # used to become openable on those two grids. + from routes_tables import scoped_pids + + limits = [] + pids, _fields, _defn = scoped_pids(session, scope, limits=limits) + # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is + # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a + # database with no rows and for a read-through grid too big to enumerate — it distinguishes + # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move + # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which + # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]). + # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database + # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the + # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside + # `scoped_pids` and answered 404/403. The pid set was only ever an existence check. + return (pids, bool(limits)) from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids session.require(CUSTOMER_MODULE) - return frozenset(allowed_pids(session)) + return (frozenset(allowed_pids(session)), False) def _in_book(pid, session, scope): - if pid not in _pool_or_refuse(session, scope): + pids, unbounded = _pool_or_refuse(session, scope) + if unbounded: + # The table wall passed and the row set is larger than this process will enumerate. Said + # out loud rather than silently admitting: R6's second sentence is that a limit which + # cannot be removed gets REPORTED, and this is the one place the report has no envelope to + # ride in. + print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — " + f"admitting on the table wall alone, per D-72") + return + if pid not in pids: # 403, not 404: the record may exist, but this session may not inspect it. raise err(403, "out_of_scope", "that record is not in your book") diff --git a/api/routes_shares.py b/api/routes_shares.py index 7a96392eba03033ca450a2eb4a970663603fcc9c..62278f5c62b12e8fbc3a8d25d03ffd9705fc8bd5 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -336,8 +336,16 @@ def _notify_new_grantees(session, kind, oid, before, after): # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED # from there rather than typed again — one vocabulary, one owner. + # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY + # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and + # are honestly named by their machine; a SHARE has a real person, and only this call + # site knows who. ⛔ It is passed as its OWN field rather than recovered from the + # `detail` prose below: a sender parsed out of " shared this with you" breaks + # the first time the sentence is reworded, silently, in the header + # [[grep-output-is-not-source]]. The prose stays as the body; this is the From. alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id, - detail=f"{sharer} shared this with you", st=session.runtime) + detail=f"{sharer} shared this with you", actor=sharer, + st=session.runtime) except Exception: # noqa: BLE001 return diff --git a/api/routes_slack.py b/api/routes_slack.py new file mode 100644 index 0000000000000000000000000000000000000000..ce7c6c960ce3941d1a8cfdbb68ac4cc9342997b9 --- /dev/null +++ b/api/routes_slack.py @@ -0,0 +1,756 @@ +"""routes_slack.py — MANAGE AGENT: a bot per Slack channel, walled by the engine that walls a +person. (wave 33, owner item 10 · `W33-T38` / `W33-T39` / `W33-T67`.) + +Owner, verbatim (2026-08-14): *"Expand Slack we need to be able to configure an AI bot PER channel +whose permissions we can toggle based on Fields etc of the databases, exactly like how we toggle +permissioning per user."* + +──────────────────────────────────────────────────────────────────────────────────────────────── +⛔⛔ D-84 SAID THIS WAS "adapted from OpenTag" AND THAT PREMISE IS FALSE (PRD amendment A1). + +`PENDING.md` D-84 has named OpenTag as the model for per-Slack-channel permissioning since +2026-08-08. **OpenTag has no permission model to adapt.** `app/runtime-host.ts` is +`identifyUser: () => OPENTAG_SERVICE_USER` — ONE constant identity for every request — and its own +`sender-context.ts` says the Slack sender is *"informational only… not gating access."* Its +authorization ceiling is whole-MCP-server, decided once at process boot from env vars. It is a +live specimen of identity that LOOKS like a permission and is not. + +`core/perm_scope.py` is already strictly more expressive: per database, per row, per FIELD, per +principal. So item 10 is OUR engine projected onto a CHANNEL principal — see `agent_principal` +below, which is the whole projection and is nine lines. Study: +`.claude/wiki/research/opentag-adoption.md` (licence gate: MIT, PASSED). + +What OpenTag genuinely contributes is the WRITE-APPROVAL interceptor (`W33-T67`, `approval_card` +below) and nothing else. Its TRANSPORT is AVOID wholesale — hosted CopilotKit Intelligence, a +second Node process, a persistent outbound socket and each tenant's Slack credentials held by a +third party, against ONE process on the HF free tier. The door here is `routes_forms.py`'s proven +unauthenticated-public shape plus Slack's signing-secret check. + +──────────────────────────────────────────────────────────────────────────────────────────────── +⚠ A CHANNEL IS A PRINCIPAL, NOT A PERSON. The wall binds to the Slack conversation id, so every +human in that channel reads through the same rules. That is the honest reading of "a bot per +channel" and the pane says it out loud, because the alternative is an admin assuming per-person +scoping from a screen that looks exactly like the per-person one. + +⚠ AND THE IDENTITY IS THE CHANNEL ID, NEVER ITS NAME. A channel can be renamed; its id cannot. +A wall bound to `#sales` silently re-points the day somebody renames the channel. +""" +import hmac +import json +import os +import secrets +import time +from datetime import datetime, timezone + +from fastapi import APIRouter, Body, Depends, Request + +from deps import Session, err, require_session +from routes_admin import admin_gate + +router = APIRouter(prefix="/api/v1") + +#: The tenant's channel agents: `{id: record}`. A per-tenant bucket, so it rides +#: `runtime.store_key`'s prefix and never lands in tenant #0's namespace. +AGENTS_KEY = "slack_agents" + +#: Pending mutations awaiting a human's click (`W33-T67`). Server-only, per tenant. +PENDING_KEY = "slack_pending_writes" + +#: How long an approval card stays clickable. An approval is a statement about the world as it was +#: when the card was rendered; an hour later the values it names may no longer be the values it +#: would write. Expiring is the honest behaviour, and an expired card says so rather than 403ing. +APPROVAL_TTL_S = 15 * 60 + +#: Slack rejects a replayed request older than 5 minutes; so do we, before any signature work. +SLACK_MAX_SKEW_S = 60 * 5 + +MAX_BODY_BYTES = 64 * 1024 + +#: ⛔ ROW-CAPPED BECAUSE AN APPROVER WHO CANNOT READ THE CARD CANNOT APPROVE IT. Copied +#: deliberately from OpenTag's `agent/write_confirmation.py`, which caps for the same reason: +#: past a certain length Slack collapses a message behind "Show more", and a human clicking +#: Approve on a card whose tail they never saw is worse than no card at all. +APPROVAL_MAX_ROWS = 12 + + +# ── the tenant's agent records ─────────────────────────────────────────────────────────────── +def _agents(rt): + """`{id: record}` for one tenant. `{}` on any failure — an unreadable bucket must degrade to + "this tenant has no agents", never to a 500 on the settings pane.""" + try: + found = rt.get(AGENTS_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def _clean_agent(raw, prior=None): + """One stored record, built KEY BY KEY. + + ⛔ Never `dict(raw)` and never `**raw`. This record is the SUBJECT of a permission decision; + a client-supplied key landing in it is a client-supplied input to `perm_scope`. The same rule + `routes_forms._public_form` follows on the way out, applied here on the way in. + """ + prior = prior if isinstance(prior, dict) else {} + out = { + "id": str(prior.get("id") or raw.get("id") or ""), + "channel": str(raw.get("channel") or prior.get("channel") or "").strip()[:64], + "channelName": str(raw.get("channelName") or prior.get("channelName") or "").strip()[:80], + "label": " ".join(str(raw.get("label") or prior.get("label") or "").split())[:80], + "active": bool(raw.get("active", prior.get("active", True))), + # ⛔ DEFAULT TRUE, AND `is not False` RATHER THAN TRUTHINESS. A record written before this + # key existed must read as "approval required": the one direction a default here is + # allowed to be wrong in is the safe one, because an unattended write cannot be undone by + # revoking the bot afterwards [[default-must-pass-its-own-guard]]. + "writeApproval": raw.get("writeApproval", prior.get("writeApproval", True)) is not False, + # The permission block. Same shape, same validator, same engine as a user's. + "perms": prior.get("perms") if isinstance(prior.get("perms"), dict) else {}, + "perms_v": int(prior.get("perms_v") or 0), + "createdBy": str(prior.get("createdBy") or raw.get("createdBy") or ""), + "createdAt": str(prior.get("createdAt") or raw.get("createdAt") or ""), + } + return out + + +def _put_agent(session, agent_id, mutate): + """Read-modify-write ONE agent under the tenant's bucket, synchronously. + + `flush="sync"` because the client re-reads the list immediately after every write here, and an + async flush would let the refetch win the race and paint the pre-write state — the + [[refetch-eats-its-own-write]] shape. + """ + def _set(cur): + cur = dict(cur or {}) + rec = cur.get(agent_id) if isinstance(cur.get(agent_id), dict) else None + nxt = mutate(rec) + if nxt is None: + cur.pop(agent_id, None) + else: + cur[agent_id] = nxt + return cur + + session.runtime.update(AGENTS_KEY, _set, flush="sync") + + +# ── ⭐⭐ THE PROJECTION — THE WHOLE OF "the same engine that permissions a user" ─────────────── +def agent_principal(agent): + """A channel agent, as a PRINCIPAL `core/perm_scope` already understands. + + ⭐⭐ THIS IS THE ENTIRE POINT OF THE TICKET AND IT IS NINE LINES. `perm_scope` takes a user + RECORD — a plain dict with `perms`, `perms_v` and `role` — not a User object and not a session. + So a channel agent carrying a `perms` block simply IS such a record, and `may_access`, + `visible_fields`, `hidden_keys` and `apply_row_scope` work on it unmodified. There is no second + engine, no parallel vocabulary, and nothing about a channel to keep in step with anything about + a person. That is what the owner's *"exactly like how we toggle permissioning per user"* means + when it is true in code rather than in appearance. + + ⛔ `role` IS HARDCODED NON-ADMIN AND MUST STAY SO. `perm_scope.may_access` returns True + unconditionally for `role == 'admin'` — the break-glass clause that keeps the owner out of a + locked store. A bot has no such emergency and no hands: an agent record that could carry + `role: 'admin'` would be one stored key away from bypassing every wall on this page. The key is + not copied from the record; it is written here, every time. + + ⛔ AND `perms_v` IS FORCED TO THE CURRENT VERSION, which makes an UNDECLARED database DENY + (C-PERM amendment 4). Without it a fresh agent would fall through to `perms.may_open`'s legacy + grant wall and read `modules: 'all'`-shaped defaults — i.e. a brand-new bot would start with + access to everything. Fail-closed from the first byte. + """ + import core.perm_scope as perm_scope + return { + "username": f"slack:{(agent or {}).get('channel') or '?'}", + "role": "agent", + "perms": (agent or {}).get("perms") or {}, + "perms_v": perm_scope.PERMS_VERSION, + # No `bus`, no `agent`: a channel is not a business unit and not a salesperson. Absent + # rather than 'all' — `perms.allowed_bu_labels` narrows on absence, which is the direction + # this must fail in. + } + + +def agent_may_read(agent, module): + """May this channel open `module` at all?""" + import core.perm_scope as perm_scope + if not (agent or {}).get("active", True): + return False + return bool(perm_scope.may_access(agent_principal(agent), module)) + + +def agent_visible_fields(agent, fields, module): + """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets, + so a formula over a hidden column cannot leak it back through arithmetic.""" + import core.perm_scope as perm_scope + return perm_scope.visible_fields(fields, agent_principal(agent), module) + + +def agent_rows(agent, rows, module, fields, ctx=None): + """The rows this channel may receive — `permits()`, so an unanswerable permanent filter DENIES + rather than being ignored.""" + import core.perm_scope as perm_scope + p = agent_principal(agent) + hide = perm_scope.hidden_keys(p, module, fields) + kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx) + # Both wires, never one: the field list and the row payload are separate, and stripping only + # the first leaves the value sitting in the second where anyone can read it. + return [perm_scope.strip_row(r, hide) for r in kept] + + +# ── the credential ─────────────────────────────────────────────────────────────────────────── +def slack_creds(rt): + """This TENANT's Slack credentials from its own keychain, or None. + + ⛔ NO ENVIRONMENT FALLBACK, DELIBERATELY, AND THIS IS D-202's LESSON APPLIED BEFORE IT + HAPPENS AGAIN. Meta Ads shipped loading end-to-end and was still not a connector, because + `keychain.meta_creds` had no caller and the token came from the owner's `.env` — making it a + tenant-#0 FACT indistinguishable from a screenshot. `keychain.meta_creds`' own docstring spells + out the rule: handing the environment's credential to a tenant whose admin has not stored one + is the leak the resolver exists to prevent. So a tenant with no entry gets None and the pane + SAYS so. + + ⚠ Reads through `list_entries` + `read_fields` (both public) rather than `keychain._first_creds` + (private, and in a file outside this lane's fence). Same deterministic rule: entries are + id-sorted, so "first" is stable across reads rather than dict-order luck. + """ + try: + import core.keychain as keychain + for row in keychain.list_entries(rt): + if str(row.get("type") or "") != "slack": + continue + fields = keychain.read_fields(rt, row.get("id")) + if isinstance(fields, dict) and fields.get("signing_secret"): + return fields + except Exception: # noqa: BLE001 + # A locked or unreadable keychain reads as "not configured" for the PANE, which is honest; + # the signature check below fails closed regardless, so this cannot widen anything. + return None + return None + + +# ── the authenticated doors (the Manage agent surface) ─────────────────────────────────────── +def _summary(agent, modules): + """One sentence per agent for the list row — computed here, because the alternative is one + round trip per row to fill one cell (the same reason `AdminUser.access` exists).""" + governed = [m for m in modules if m.get("enforced")] + perms = agent.get("perms") or {} + open_n = sum(1 for m in governed if (perms.get(m["key"]) or {}).get("access")) + if not governed: + return "" + if open_n == 0: + return "No access" + restricted = sum(1 for m in governed + if (perms.get(m["key"]) or {}).get("access") + and ((perms.get(m["key"]) or {}).get("filter") + or (perms.get(m["key"]) or {}).get("hiddenFields"))) + head = (f"All {open_n} database{'' if open_n == 1 else 's'}" if open_n == len(governed) + else f"{open_n} of {len(governed)} databases") + return f"{head}, {restricted} restricted" if restricted else head + + +@router.get("/agents") +def list_channel_agents(session: Session = Depends(admin_gate)): + """This tenant's channel agents, plus whether Slack is reachable at all.""" + import routes_admin + modules = routes_admin._perm_modules(session) + rows = [] + for aid, a in sorted(_agents(session.runtime).items()): + if not isinstance(a, dict): + continue + rows.append({"id": aid, "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", + "label": a.get("label") or "", "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, modules)}) + return {"agents": rows, "configured": slack_creds(session.runtime) is not None} + + +@router.post("/agents") +def create_channel_agent(body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + channel = str(body.get("channel") or "").strip() + if not channel: + raise err(400, "no_channel", "a Slack channel ID is required") + existing = _agents(session.runtime) + # ⚠ ONE AGENT PER CHANNEL. Two records for one conversation would mean two walls, and nothing + # anywhere decides which one applies — the [[one-question-two-normalizers]] shape, in the one + # place where the two answers are "may read" and "may not". + for a in existing.values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + raise err(409, "channel_taken", + "that channel already has an agent — edit it instead of adding a second") + aid = secrets.token_urlsafe(9) + rec = _clean_agent(dict(body, id=aid, createdBy=session.uname, + createdAt=datetime.now(timezone.utc).isoformat(timespec="seconds"))) + _put_agent(session, aid, lambda _prior: rec) + fresh = _agents(session.runtime).get(aid) + if not isinstance(fresh, dict): + # The store took the write and did not record it. A 200 here would tell an administrator + # the agent exists when it does not. + raise err(503, "store_unavailable", "the agent was NOT created") + return {"agent": {"id": aid, "channel": rec["channel"], "channelName": rec["channelName"], + "label": rec["label"], "active": rec["active"], + "writeApproval": rec["writeApproval"], "summary": "No access"}} + + +@router.get("/agents/{agent_id}") +def get_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + """The agent PLUS the permission envelope — the SAME shape `get_perms` answers with. + + ⭐ ONE PAYLOAD SHAPE FOR TWO ROOMS. `permsModel.parsePermsPayload` parses this, and + `settings/ModulePermsList` renders it, because they are literally the same client code. A + second envelope here would mean a second parser and a second set of defaults within a wave. + """ + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) + if not isinstance(a, dict): + raise err(404, "no_such_agent", "no agent with that id") + modules = routes_admin._perm_modules(session) + enforced = [m["key"] for m in modules if m.get("enforced")] + stored = a.get("perms") or {} + principal = agent_principal(a) + import core.perm_scope as perm_scope + perms_out = {} + for k in enforced: + e = stored.get(k) + perms_out[k] = e if isinstance(e, dict) else { + "access": bool(perm_scope.may_access(principal, k)), "filter": None, + "hiddenFields": []} + return {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "perms": perms_out, + # ⛔ ALWAYS THE CURRENT VERSION, because `agent_principal` forces it: the editor must + # render "undeclared = denied", not the legacy-grant reading it would show at 0. + "perms_v": perm_scope.PERMS_VERSION, + # A bot is NEVER an admin — see `agent_principal`. Sent so the editor never paints + # the "Everything (admin)" state for a principal that cannot have it. + "is_admin": False, + "modules": modules, + "fields_by_module": {k: routes_admin._module_fields(k) for k in enforced}} + + +@router.put("/agents/{agent_id}/perms") +def put_channel_agent_perms(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + """Replace this agent's permission block wholesale — the same validator a user's block gets.""" + import routes_admin + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + if "perms" not in body: + raise err(400, "empty_patch", "no perms to save") + mods = routes_admin._perm_modules(session) + # ⭐ THE SAME `_clean_perms`, NOT A COPY OF IT. It refuses a filter this module cannot + # evaluate, refuses a hiddenFields key that names nothing, refuses an unenforced `ut_*` key, + # and refuses a BU condition the pushdown cannot read. Every one of those is exactly as true + # for a channel as for a person, and a second validator here is a second place for one of + # them to go missing. + cleaned = routes_admin._clean_perms( + body.get("perms"), + enforced_keys={m["key"] for m in mods if m.get("enforced")}, + listed_keys={m["key"] for m in mods}) or {} + + import core.perm_scope as perm_scope + + def _mut(prior): + if not isinstance(prior, dict): + return None + nxt = dict(prior) + nxt["perms"] = cleaned + nxt["perms_v"] = perm_scope.PERMS_VERSION + return nxt + + _put_agent(session, str(agent_id), _mut) + fresh = _agents(session.runtime).get(str(agent_id)) or {} + if (fresh.get("perms") or {}) != cleaned: + # The one direction this must never fail in: telling an administrator the wall is up when + # it is not. + raise err(503, "store_unavailable", "the permissions were not saved — the agent is UNCHANGED") + return {"id": str(agent_id), "perms": fresh.get("perms") or {}, + "perms_v": int(fresh.get("perms_v") or 0)} + + +@router.patch("/agents/{agent_id}") +def patch_channel_agent(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(admin_gate)): + body = body if isinstance(body, dict) else {} + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + #: ⛔ `perms` IS NOT PATCHABLE HERE. It has its own door with its own validator; accepting it + #: on the metadata route would be a second, unvalidated way to write the wall. + allowed = {k: v for k, v in body.items() + if k in ("label", "active", "writeApproval", "channelName")} + if not allowed: + raise err(400, "empty_patch", "nothing to change") + + def _mut(prior): + return _clean_agent(allowed, prior=prior) if isinstance(prior, dict) else None + + _put_agent(session, str(agent_id), _mut) + import routes_admin + a = _agents(session.runtime).get(str(agent_id)) or {} + return {"agent": {"id": str(agent_id), "channel": a.get("channel") or "", + "channelName": a.get("channelName") or "", "label": a.get("label") or "", + "active": a.get("active", True) is not False, + "writeApproval": a.get("writeApproval", True) is not False, + "summary": _summary(a, routes_admin._perm_modules(session))}} + + +@router.delete("/agents/{agent_id}") +def delete_channel_agent(agent_id: str, session: Session = Depends(admin_gate)): + if not isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(404, "no_such_agent", "no agent with that id") + _put_agent(session, str(agent_id), lambda _prior: None) + if isinstance(_agents(session.runtime).get(str(agent_id)), dict): + raise err(503, "store_unavailable", "the agent was NOT removed") + return {"ok": True} + + +# ── ⭐ W33-T67: THE WRITE-APPROVAL GATE ─────────────────────────────────────────────────────── +def _is_empty(value): + """Is this value ABSENT, as opposed to falsy? + + ⛔ COPIED EXACTLY FROM OpenTag's `agent/write_confirmation.py`, and the exactness is the point: + `0` and `False` are VALUES a human must see on the card. Setting a price to 0 or a flag to + False is precisely the kind of write somebody needs to approve, and a naive `if not value` + drops both rows silently — the approver then reads a card that does not describe the write + they are approving. The MIT licence gate for this borrowing is recorded in + `.claude/wiki/research/opentag-adoption.md`. + """ + return value is None or (isinstance(value, str) and value.strip() == "") + + +def approval_card(action, values, agent=None): + """The card a human reads before a bot writes. `{action, rows, truncated, note}`. + + ⛔ ROW-CAPPED, and the cap is a SAFETY property rather than a layout one: past a certain + length Slack collapses a message behind "Show more", and **an approver who cannot read the + card cannot approve it.** When rows are dropped the card SAYS how many — a truncation nobody + was told about is the violation, not the truncation (wave-30 R6's second sentence). + """ + rows = [{"field": str(k), "value": v} + for k, v in (values or {}).items() if not _is_empty(v)] + rows.sort(key=lambda r: r["field"]) + shown, dropped = rows[:APPROVAL_MAX_ROWS], max(0, len(rows) - APPROVAL_MAX_ROWS) + return { + "action": str(action or "")[:80], + "rows": shown, + "truncated": dropped, + "note": (f"{dropped} more field(s) not shown — open the record to review them before " + f"approving." if dropped else ""), + "agent": (agent or {}).get("label") or (agent or {}).get("channel") or "", + } + + +def intercept_write(agent, action, values, stash): + """THE INTERCEPTOR. Returns `("commit", None)` or `("await_approval", card)`. + + ⭐ The one thing worth taking from OpenTag: a MUTATING tool call is halted, rendered as a card + naming the action and its values, and committed only on an explicit human accept. Everything + else in that repo — the transport, the runtime, the identity model — is AVOID (amendment A1). + + ⛔ FAIL-CLOSED ON A MISSING FLAG: `is not False`, so a record written before `writeApproval` + existed requires approval. The unsafe direction here is not recoverable — revoking a bot does + not un-write what it wrote. + """ + if (agent or {}).get("writeApproval", True) is not False: + card = approval_card(action, values, agent) + stash(card) + return "await_approval", card + return "commit", None + + +def _pending(rt): + try: + found = rt.get(PENDING_KEY) or {} + except Exception: # noqa: BLE001 + return {} + return found if isinstance(found, dict) else {} + + +def stash_pending(rt, agent, action, values, now=None): + """Store one awaiting-approval mutation and return its token. + + The token is what rides the card's button, so it is `secrets.token_urlsafe` and not the + action's id: a guessable value on an unauthenticated door is a way to approve somebody else's + write. Server-only bucket, per tenant, exactly as `routes_forms.TOKENS_KEY` is. + """ + token = secrets.token_urlsafe(18) + row = {"at": float(now if now is not None else time.time()), + "channel": str((agent or {}).get("channel") or ""), + "action": str(action or "")[:80], + "values": values if isinstance(values, dict) else {}} + rt.update(PENDING_KEY, lambda cur: {**(cur or {}), token: row}, flush="sync") + return token + + +#: `{action kind: fn(rt, row) -> None}`. Registered by whatever can actually perform a mutation. +#: +#: ⛔⛔ EMPTY IN PRODUCTION TODAY, AND SAYING SO IS THE POINT. `commit_pending` below is the ONE +#: place an approved write may be performed, so it is the place a negative control can bite — but +#: a gate is only as honest as what it guards. Until something registers here, an approved card +#: performs NOTHING and `commit_pending` returns `no_committer` rather than pretending. That is a +#: declared absence, not a silent one [[flag-shipped-without-its-writer]]. +_COMMITTERS = {} + + +def register_committer(kind, fn): + """Declare who may perform an approved mutation of `kind`. Idempotent by key.""" + _COMMITTERS[str(kind)] = fn + + +def commit_pending(rt, token, approved, now=None): + """Resolve ONE approval card. Returns one of + `expired` · `unknown` · `rejected` · `no_committer` · `committed` · `failed`. + + ⛔⛔ THIS IS THE WRITE GATE, AND IT IS A SEPARATE FUNCTION FROM THE ROUTE ON PURPOSE. T67's + `done-when` asks for an NC proving a REJECTED card writes nothing. A control aimed at the + route could not bite while the route had no write in it at all — it would stay green with the + guard deleted, because there would be nothing for the guard to be protecting anything from + ([[gate-can-report-green-on-nothing]], and a verifier caught exactly that on this ticket's + first draft). Putting the commit behind ONE named seam gives the control a real subject: the + gate registers a recorder, and `rejected` must leave it untouched while `approve` must call it + exactly once. + + ⛔ THE TOKEN IS CONSUMED BEFORE EITHER BRANCH, and before any committer runs. A card that + survives its own click is a replayable write, and "approve twice" must never mean "write + twice". The delete is `flush="sync"` for the same reason. + """ + now = float(now if now is not None else time.time()) + row = _pending(rt).get(str(token)) + if not isinstance(row, dict): + return "unknown" + # Consume FIRST — before the TTL verdict, before the approve/reject branch, before any write. + rt.update(PENDING_KEY, + lambda cur: {k: v for k, v in (cur or {}).items() if k != str(token)}, + flush="sync") + if now - float(row.get("at") or 0) > APPROVAL_TTL_S: + return "expired" + if not approved: + # ⛔ NOTHING BELOW THIS LINE RUNS FOR A REJECTED CARD. The whole gate is this return. + return "rejected" + fn = _COMMITTERS.get(str(row.get("action") or "")) + if fn is None: + return "no_committer" + try: + fn(rt, row) + except Exception: # noqa: BLE001 + # A committer that raises must not read as a commit. The card is already consumed, so the + # honest report is that it failed, and the human asks again. + return "failed" + return "committed" + + +# ── the UNAUTHENTICATED Slack door ─────────────────────────────────────────────────────────── +#: Sliding window, keyed on the SLACK TEAM, never the client IP. +#: +#: ⛔ WHY NOT THE IP, WHICH IS WHAT `routes_forms` DOES. Every Slack event arrives from Slack's own +#: infrastructure, so an IP window is one shared bucket for every workspace on the platform: one +#: busy tenant would spend the allowance for all of them, and the symptom would be another +#: tenant's bot going quiet. The team id is the closest thing to the actual noisy party. +#: ⚠ It is attacker-CONTROLLED before the signature is checked, so the window is applied AFTER +#: verification, never before — an unsigned request is refused on cost grounds anyway (a signature +#: check is a single HMAC). +RATE_WINDOW_S, RATE_PER_WINDOW = 60, 60 +_HITS: dict = {} + + +def _rate_ok(key, now): + seen = [t for t in _HITS.get(key, ()) if now - t < RATE_WINDOW_S] + seen.append(now) + _HITS[key] = seen + if len(_HITS) > 4096: + for k in [k for k, v in _HITS.items() if not v or now - v[-1] > RATE_WINDOW_S]: + _HITS.pop(k, None) + return len(seen) <= RATE_PER_WINDOW + + +async def _raw_body(request): + """The body as RAW BYTES, capped. + + ⛔⛔ RAW, NOT PARSED, AND THAT IS NOT A STYLE CHOICE. Slack signs the literal string + `v0:{timestamp}:{body}` byte for byte. `routes_forms._bounded_body` — the function this is + otherwise copied from — returns only the parsed dict and DISCARDS the bytes, so a verbatim + copy of it could not verify a Slack signature at all: re-serialising the dict changes + whitespace and key order and the HMAC no longer matches. The bug would look like "Slack + signatures are always invalid", which is indistinguishable from a wrong secret. + + ⚠ And it streams rather than calling `request.body()`: FastAPI would otherwise have buffered + the whole body before the handler's first line, so a content-length check inside the handler + caps nothing (`routes_forms`' own scar, recorded in its docstring). + """ + size, chunks = 0, [] + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BODY_BYTES: + raise err(413, "body_too_large", "that request is too large") + chunks.append(chunk) + return b"".join(chunks) + + +def _refuse(): + """ONE refusal for every reason. No oracle. + + A wrong signature, a stale timestamp, an unknown team, an unconfigured tenant and a channel + with no agent all answer identically — otherwise the door tells an unauthenticated caller + which tenants exist and which channels are configured, which is the question it was built to + not answer. `routes_forms._refuse` takes the same posture for the same reason. + """ + return err(403, "bad_request", "that request could not be verified") + + +def _verify(secret, timestamp, raw, signature, now): + """Slack's v0 signature, with the replay window checked FIRST.""" + try: + ts = int(str(timestamp or "0")) + except ValueError: + return False + if abs(now - ts) > SLACK_MAX_SKEW_S: + return False + base = b"v0:" + str(ts).encode("ascii") + b":" + (raw or b"") + import hashlib + expected = "v0=" + hmac.new(str(secret).encode("utf-8"), base, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, str(signature or "")) + + +def _resolve_tenant(raw, timestamp, signature, now): + """`(slug, runtime, creds)` for the tenant whose signing secret verifies this request. + + ⛔ THE TENANT IS DECIDED BY THE SIGNATURE, NEVER BY THE PAYLOAD. A body-supplied `team_id` + would let an unauthenticated caller name the tenant it wants to be — the exact widening + `deps._user_for`'s tenant-equality check closes on the authenticated side. So every configured + tenant's secret is tried and the one that VERIFIES names the tenant. + + ⚠ NO EARLY `break` ON A FAILED CANDIDATE and no per-tenant error: the loop's cost must not + depend on which tenant matched. `routes_forms._resolve` walks tenants the same way, for the + same reason — a public door has no session to ask. + """ + from harness import runtime as _rt + hit = None + try: + slugs = list(_rt.known_tenants()) + except Exception: # noqa: BLE001 + return None, None, None + for slug in slugs: + try: + rt = _rt.get_runtime(slug) + except Exception: # noqa: BLE001 + continue + creds = slack_creds(rt) + if not creds: + continue + if _verify(creds.get("signing_secret"), timestamp, raw, signature, now) and hit is None: + hit = (slug, rt, creds) + return hit if hit else (None, None, None) + + +@router.post("/slack/events") +async def slack_events(request: Request): + """Slack's Events API. UNAUTHENTICATED BY CONSTRUCTION — no `Depends(require_session)`. + + ⭐ Built like `routes_forms.py`'s public door and not like OpenTag's: one process, no vendor, + no second runtime, no persistent outbound socket, and this tenant's Slack secret never leaves + this deployment (amendment A1). + """ + now = time.time() + raw = await _raw_body(request) + ts = request.headers.get("x-slack-request-timestamp") + sig = request.headers.get("x-slack-signature") + slug, rt, _creds = _resolve_tenant(raw, ts, sig, now) + if not rt: + raise _refuse() + + try: + payload = json.loads(raw or b"{}") + except ValueError: + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + + # Slack's one-time endpoint handshake. Answered ONLY after the signature verified — an + # unsigned challenge echo would confirm the endpoint exists to anyone who probes it. + if payload.get("type") == "url_verification": + return {"challenge": str(payload.get("challenge") or "")[:512]} + + team = str(payload.get("team_id") or slug) + if not _rate_ok(f"{slug}:{team}", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + event = payload.get("event") if isinstance(payload.get("event"), dict) else {} + channel = str(event.get("channel") or "") + agent = None + for a in _agents(rt).values(): + if isinstance(a, dict) and str(a.get("channel") or "") == channel: + agent = a + break + # ⛔ NO AGENT, OR AN INACTIVE ONE, IS SILENCE — a 200 with no action. Not a 403: Slack retries + # a non-2xx up to three times, so refusing here would turn "this channel is not configured" + # into three refusals per message, and the retry storm would be the only visible symptom. + # ⚠ And it must not say WHICH — a distinguishable answer tells an unauthenticated caller which + # channels this workspace has configured. + if not isinstance(agent, dict) or agent.get("active", True) is False: + return {"ok": True} + + # ⛔⛔ THE ANSWERING HALF IS NOT BUILT YET, AND SAYING SO IS THE POINT. What ships here is the + # DOOR and the WALL: a verified request, resolved to a tenant by signature, matched to a + # channel agent whose `perms` block is a `perm_scope` principal (`agent_principal`). Composing + # a reply means an LLM call on the cheap-first ladder plus the read path, and a half-built + # answerer that returns something plausible is worse than one that returns nothing. + # Booked rather than faked [[flag-shipped-without-its-writer]]. + return {"ok": True} + + +@router.post("/slack/interact") +async def slack_interact(request: Request): + """The approval card's button (`W33-T67`). UNAUTHENTICATED, signature-verified, same shape.""" + now = time.time() + raw = await _raw_body(request) + slug, rt, _creds = _resolve_tenant(raw, request.headers.get("x-slack-request-timestamp"), + request.headers.get("x-slack-signature"), now) + if not rt: + raise _refuse() + # Slack posts interactions as `application/x-www-form-urlencoded` with a `payload=` field. + from urllib.parse import parse_qs + try: + form = parse_qs(raw.decode("utf-8")) + payload = json.loads((form.get("payload") or ["{}"])[0]) + except Exception: # noqa: BLE001 + raise _refuse() + if not isinstance(payload, dict): + raise _refuse() + if not _rate_ok(f"{slug}:interact", now): + raise err(429, "too_many_requests", "too many requests — wait a moment and try again") + + actions = payload.get("actions") if isinstance(payload.get("actions"), list) else [] + choice = (actions[0] if actions and isinstance(actions[0], dict) else {}) + token = str(choice.get("value") or "") + # ⭐ ANYTHING THAT IS NOT LITERALLY "approve" IS A REJECT. Fail-closed on a malformed, + # truncated or unknown action id: the direction that can be wrong here without anyone + # noticing is the one that writes. + approved = str(choice.get("action_id") or "") == "approve" + # ⛔ THE ROUTE DOES NOT DECIDE — `commit_pending` does, and it is the ONE seam a negative + # control can aim at. Consumption, the TTL, the reject branch and the committer lookup all + # live there; this handler only turns its verdict into a sentence. + verdict = commit_pending(rt, token, approved, now=now) + return {"text": { + # An unknown, already-used or expired token is not an error: a person clicking a stale + # card should be told it is stale, not shown a failure. + "unknown": "That approval is no longer available. Ask the bot again.", + "expired": "That approval expired before it was answered. Ask the bot again.", + "rejected": "Rejected. Nothing was changed.", + # ⚠ HONEST, not reassuring: the answering half that would register a committer is booked, + # so an approved card today performs nothing and says exactly that rather than "Approved." + "no_committer": "Approved — but this workspace has nothing configured to carry it out yet.", + "failed": "That change could not be completed. Nothing was saved; ask the bot again.", + "committed": "Approved.", + }.get(verdict, "That approval is no longer available. Ask the bot again.")} + + +@router.get("/slack/health") +def slack_health(session: Session = Depends(require_session)): + """Is Slack configured for THIS tenant? Session-gated, and it answers about one tenant only — + enumerating the others would answer a question about our customer list.""" + creds = slack_creds(session.runtime) + return {"configured": creds is not None, + "agents": len(_agents(session.runtime)), + "hasBotToken": bool((creds or {}).get("bot_token"))} diff --git a/api/routes_starred.py b/api/routes_starred.py new file mode 100644 index 0000000000000000000000000000000000000000..c017e94c6788da13f98e01f1246786ea172a9c88 --- /dev/null +++ b/api/routes_starred.py @@ -0,0 +1,828 @@ +"""routes_starred.py — WAVE 35 (rulings R4/R5/R6/R7, contracts C2/C3/C5): THE STAR. + +Owner items 8, 9 and 12. **Star and "mark important" are ONE idea and ONE flag** (R4): on screen +the word is Star, and starring an object lands it on Home and in the new Starred module. Four +kinds are starrable — `database`, `agent`, `query`, `view` — and RECORDS get their own door +(C5, `/starred/records`), because a record's identity is `(database, pid)` rather than a bare id. + +⛔⛔ THE ONE DESIGN RULE THAT DECIDES EVERYTHING BELOW: **A VIEW'S STAR IS NOT STORED HERE.** +C2 is explicit — `kind: "view"` writes the EXISTING `config.important` in that database's own +`_table_workspace`, and never a second store. R4 says star and mark-important are one flag; +two stores for one flag is two flags that can disagree, and the badge inside the database and the +row on Home would then answer differently about the same view. So this file keeps THREE id lists +(`databases`, `agents`, `queries`) in its own per-user bucket, and for views it is a READER and a +WRITER of somebody else's bucket rather than an owner of anything. + +⚠ WHICH IS WHY `GET /starred` IS NOT FREE, AND EVERY CALLER MUST KNOW. Finding the starred views +means reading each visible database's view bucket — the same N reads `routes_nav._important_counts` +did before W35-T43 deleted it. Measured on tenant #0 (2026-08-16, recorded on that constant): +twelve buckets cost **6.2 ms WARM in total** and **7,331 ms of first fetch COLD**. So the scan is +budgeted, it is spent in the caller's own RECENTS order (a mark lives on a database somebody works +in), and a database it could not reach is named in `degraded` rather than quietly carrying no star. +That is R6's second sentence applied to a list: a limit that cannot be removed is REPORTED with its +cause, never silently enforced. **Call this AFTER first paint.** + +⚠ NO INDEX, DELIBERATELY. The obvious fix for the scan is a `starred.views` id list written beside +the three above. That is the second store C2 forbids: the moment the flag and the index can +disagree — a view deleted, a view unmarked from inside the database, a store write that lands on one +and not the other — the product has two answers to one question and no way to tell which is right. +The scan is the price of one flag, and it is paid off the critical path. +""" +import time + +from fastapi import APIRouter, Body, Depends + +import routes_nav +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + +#: The per-user bucket holding the three ID-LIST kinds. `{username: {databases, agents, queries}}` +#: +#: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about +#: the object, so a tenant-wide star would put one person's shortlist on everybody's Home. +_STARRED_KEY = "starred" + +#: The four kinds C2 declares. `view` is the odd one out — see the header. +KINDS = ("database", "agent", "query", "view") + +#: The three kinds this file's own bucket holds, and the payload key each answers under. +_LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"} + +#: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with +#: the cause and the action (unstar something), rather than being accepted and truncated. A star +#: silently dropped is a star the user believes they set. +MAX_PER_KIND = 200 + +#: An id is a store key, a view id or an automation id — all of which are short. Bounded so a +#: hostile client cannot grow one user's document with one very long string. +_MAX_ID = 120 + +#: The wall-clock ceiling on the WHOLE view scan, checked between databases — the same number and +#: the same reason as the `_IMPORTANT_BUDGET_S` this replaces. It bounds the FIRST request after a +#: container starts; warm it is not reached. +VIEW_SCAN_BUDGET_S = 2.5 + + +def _clean_id(raw): + """One id, or "" — the only shape this file stores.""" + return str(raw or "").strip()[:_MAX_ID] + + +def _clean_list(raw): + """A stored id list, validated on the way OUT as well as in. + + Re-validating a read is not redundant: the bucket outlives this code and an older build (or a + hand edit) could have written a shape this one does not accept. Prune, never invent — the + posture `_clean_nav_prefs` takes one file over. + """ + if not isinstance(raw, list): + return [] + out, seen = [], set() + for item in raw[:MAX_PER_KIND]: + got = _clean_id(item) + if got and got not in seen: + seen.add(got) + out.append(got) + return out + + +def _read_lists(session): + """This caller's three id lists. A store blip answers empty rather than taking the page down.""" + try: + mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {} + except Exception: + return {name: [] for name in _LIST_KINDS.values()} + if not isinstance(mine, dict): + return {name: [] for name in _LIST_KINDS.values()} + return {name: _clean_list(mine.get(name)) for name in _LIST_KINDS.values()} + + +def _database_keys(session): + """`(keys, enumerated)` — every database this session may open that has a view bucket. + + ⚠ `routes_nav._visible_database_keys` IS REUSED RATHER THAN RE-DERIVED, and that is not + tidiness: it merges the compiled registry (behind the account grant AND the tenant catalogue) + with `user_tables.nav_entries`, the `may_open`-filtered listing over the rows-free projection. + Writing a second enumeration here is how the folder-drop defect happened (wave 27, item 6): one + door knew about `ut_*` keys and the other did not, and the disagreeing door answered 200 while + storing nothing. + ⛔ AND IT IS NOT `_placeable_top_keys`, WHICH NEVER APPLIES THE TENANT CATALOGUE — right for a + cosmetic folder placement, wrong for a scan that then opens each database's view bucket. See + that helper's own note for the second difference (module surfaces have no views to star). + """ + return routes_nav._visible_database_keys(session) + + +def _scan_order(session, keys): + """`keys`, RECENTS-FIRST — the order the scan budget is spent in. + + ⛔ THE ORDER IS LOAD-BEARING AND W34-T10 LEARNED IT THE EXPENSIVE WAY. A budget spent in + whatever order the keys happen to arrive is a lottery: measured cold on tenant #0, that block + spent all of its first draft's budget on ten EMPTY Odoo buckets and was cut off two rows before + `customer_data`, which holds the only marked view in the tenant. The feature would have been + correct and shown nothing. A star lives on a database somebody works in, and `nav_recents` is + exactly that list, newest first. + """ + recents = routes_nav._read_recents(session.runtime, session.uname, keys) + order = [r["key"] for r in recents] + seen = set(order) + order += [k for k in sorted(keys) if k not in seen] + return order + + +def _view_docs(session, order): + """`({key: workspace_doc}, unread)` — each database's view bucket, PROJECTED, within budget. + + ⛔ PROJECTED, and the two dropped keys are what makes this affordable: `overlays` is per-record + cell values and `fields` is the schema stratum, neither of which says anything about a view. On + `customer_data` they are most of the bucket. + ⚠ `unread` is a store blip OR the budget — reported, never a confident absence. + """ + import core.view_templates as view_templates + docs, unread = {}, [] + started = time.perf_counter() + for key in order: + ws_key = view_templates.workspace_key(key) + if not ws_key: + continue # a module surface or a folder head, not a table at all + if time.perf_counter() - started > VIEW_SCAN_BUDGET_S: + unread.append(key) + continue + try: + docs[key] = session.runtime.get_projection(ws_key, + drop=("overlays", "fields")) or {} + except Exception: + unread.append(key) + return docs, unread + + +def _refresh_one(session, cache, key): + """Re-read ONE database's bucket inside a cache the caller already holds. + + ⛔ THIS IS WHY A WRITE MAY REUSE A SCAN AT ALL. The star write changes exactly one bucket, so + every OTHER document in the cache is still true — but the one just written is stale by + definition, and serving it back would answer the POST with the flag's OLD value. That is a lost + write wearing the face of a failed read ([[refetch-eats-its-own-write]] in reverse), and it is + the trap that makes "just reuse the cache" wrong without this function. + """ + import core.view_templates as view_templates + order, docs, unread = cache + ws_key = view_templates.workspace_key(key) + if not ws_key: + return cache + try: + docs[key] = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {} + except Exception: + docs.pop(key, None) + if key not in unread: + unread.append(key) + return (order, docs, unread) + + +def _scan_starred_views(session, keys=None, cache=None): + """`(rows, unread)` — every view this caller has starred, newest-worked database first. + + A row is `{"id", "database", "name", "config"}`. ⚠ `config` is carried so the COUNTS route does + not have to read every bucket a second time: `_view_docs` is the expensive half of this file + (7,331 ms cold for twelve buckets) and two consumers asking the same question twice per request + is the 1+N shape D-175 spent a wave removing from `/nav`. `_starred_views` below drops it for the + wire. + + ⛔ `cache` IS `(order, docs, unread)` FROM AN EARLIER PASS IN THE SAME REQUEST, and it exists + because the view-star WRITE otherwise scanned every bucket TWICE: once in `_find_view` to locate + the view, then again to build the answer. That is the same 1+N shape, in the same file, on the + same expensive read — introduced an hour after it was removed one function away. A caller that + has written must `_refresh_one` the database it wrote before passing its cache. + """ + if cache is not None: + order, docs, unread = cache + else: + if keys is None: + keys, _ = _database_keys(session) + order = _scan_order(session, keys) + docs, unread = _view_docs(session, order) + granted = routes_nav._granted_view_ids(session) + out = [] + for key in order: + doc = docs.get(key) + if doc is None: + continue + for vid, view in routes_nav._visible_views(doc, session.uname, session.admin, + granted).items(): + cfg = view.get("config") or {} + if cfg.get("important") is True: + out.append({"id": vid, "database": key, + "name": str(view.get("name") or vid)[:120], "config": cfg}) + return out, unread + + +def _starred_views(session, keys=None, cache=None): + """The C2 wire rows — `{"id", "database", "name"}`, with the config dropped. + + ⭐ `name` is a SUPERSET of C2, published to B as `NOTE E-1`: the scan already holds the record, + so the name is free, and without it Home would render a starred view as a bare id. + ⛔ THE CONFIG DOES NOT GO ON THE WIRE. It carries `memberPids` (a curated view's whole pid list) + and every filter leaf, which is somebody's row set travelling to a client that asked for a label. + """ + rows, unread = _scan_starred_views(session, keys, cache=cache) + return ([{"id": r["id"], "database": r["database"], "name": r["name"]} for r in rows], + unread) + + +def _query_names(session, qids): + """`({qid: name}, resolved)` for the caller's own Query artefacts. + + ⭐ ANSWERS `ASK B-1 (2)`. C2 declares `queries: [qid]` and there is no view/query id -> label map + anywhere in the client, so Home and Starred would render a starred Query as a bare id. B's only + alternative was `GET /query`, which drags threads, messages, citations and every saved view + across the wire to render two strings. + + ⚠ IT READS LANE C's BUCKET AND CALLS LANE C's HELPERS, WITHOUT EDITING LANE C's FILE. Query + state is ONE small per-user document (`query_user_`: threads, messages, citations, views — + no grid rows), and `_namespace_key`/`_state` are pure functions of the session. Re-deriving the + key here would be a second spelling of a hash, which is the one thing guaranteed to drift. + + ⛔ `resolved` IS FALSE WHEN THE READ FAILED, and the caller must not prune on it — a store blip + would otherwise delete somebody's whole starred-query list. Same posture as + `_clean_nav_prefs(keep_unknown_ut=...)`. + """ + if not qids: + return {}, True + try: + import routes_query + views = (routes_query._state(session) or {}).get("views") or {} + except Exception: + return {}, False + out = {} + for qid in qids: + row = views.get(qid) + if isinstance(row, dict): + out[qid] = str(row.get("name") or qid)[:120] + return out, True + + +def _payload(session, keys=None, cache=None): + """The C2 answer: the three id lists, the starred views, and what could not be read. + + ⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s + `omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a + consumer forgets to test for. + ⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`. + """ + lists = _read_lists(session) + if keys is None: + keys, _ = _database_keys(session) + # ⚠ PRUNED ON READ, NEVER ON WRITE — `_read_recents`' posture, and both halves are deliberate. + # A database whose grant was revoked must stop being offered without anybody running a + # migration, and must come back if the grant does. A key that is not visible therefore reaches + # the store and never reaches a screen. + # ⚠ ONLY `databases` is pruned. An agent id or a query id would each cost another store read to + # validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can + # disclose nothing they did not already know, and B/C render only objects they can already see + # (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap. + lists["databases"] = [k for k in lists["databases"] if k in keys] + # ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers + # "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a + # store blip that returned no artefacts must not delete the whole starred-query list. + qnames, qresolved = _query_names(session, lists["queries"]) + if qresolved: + lists["queries"] = [q for q in lists["queries"] if q in qnames] + views, unread = _starred_views(session, keys, cache=cache) + out = {"databases": lists["databases"], "agents": lists["agents"], + "queries": lists["queries"], "queryNames": qnames, + "views": views, "degraded": unread} + if unread: + # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list. + out["note"] = (f"{len(unread)} database(s) could not be read in time, so a view you " + f"starred there is missing from this list. Open one of them once and the " + f"next load will include it") + return out + + +@router.get("/starred") +def starred(session: Session = Depends(require_session)): + """C2 — everything this caller has starred. + + ⚠ NOT ON `/nav`'s PATH, and R7 is the ruling: `/nav` is a rows-free projection whose budget was + already the subject of D-175/D-185/D-288. This route is called AFTER Home/Starred paint. + """ + return _payload(session) + + +def _find_view(session, vid, database=""): + """`(key, ws_key, stratum, view, cache)` for a view this caller can SEE, else None. + + `stratum` is the username whose block holds it, or `table_store.SHARED_KEY`. `cache` is the + `(order, docs, unread)` this call paid for — but **only when it scanned everything**. With a + `database` hint it read ONE bucket, which is not a complete answer for the caller's whole + starred list, so it hands back None and the caller pays for a full pass. Handing back a partial + cache would silently drop every starred view on every OTHER database. + + ⚠ `database` IS AN OPTIONAL FAST PATH (a superset of C2, published to B and C): the client + almost always knows which database the view belongs to, and naming it turns an N-bucket scan + into one read. Absent, the scan runs — because C2's body carries only `kind`/`id`/`on`, and a + door that REQUIRED the hint would break the contract two lanes are building against. + """ + import core.table_store as table_store + import core.view_templates as view_templates + + keys, _ = _database_keys(session) + if database: + # An unknown or invisible key must not become a wider scan. It answers "not found", which + # is the same answer an id nobody holds gets — see the write door's note on why. + if database not in keys: + return None + order, cache = [database], None + else: + order = _scan_order(session, keys) + docs, unread = _view_docs(session, order) + if not database: + cache = (order, docs, unread) + granted = routes_nav._granted_view_ids(session) + for key in order: + doc = docs.get(key) + if doc is None: + continue + if str(vid) not in routes_nav._visible_views(doc, session.uname, session.admin, granted): + continue + ws_key = view_templates.workspace_key(key) + # WHICH stratum holds it decides the WALL, so it is resolved here rather than guessed. + # Personal first: a caller's own block is the common case and needs no grant. + if isinstance(doc.get(session.uname), dict) and \ + str(vid) in ((doc[session.uname].get("views") or {})): + return (key, ws_key, session.uname, doc[session.uname]["views"][str(vid)], cache) + shared = (doc.get(table_store.SHARED_KEY) or {}).get("views") or {} + if str(vid) in shared: + return (key, ws_key, table_store.SHARED_KEY, shared[str(vid)], cache) + for stratum, blob in doc.items(): + if stratum == session.uname or not isinstance(blob, dict): + continue + if str(vid) in (blob.get("views") or {}): + return (key, ws_key, str(stratum), blob["views"][str(vid)], cache) + return None + + +def _may_star_view(session, stratum, view): + """May this caller WRITE the star on this view? Fail-closed. + + ⛔ A VIEW ID THE CALLER CANNOT WRITE MUST REFUSE **LOUDLY**, and this is a standing scar: + `view_upsert` answers **200 while writing nothing** when the id belongs to a view the caller + cannot see, so a pinned id is effectively tenant-scoped and every symptom of it is silent + ([[a-view-id-another-user-holds-refuses-silently]]). This function is the reason the write door + below raises instead of shrugging. + + THREE HOMES, THREE WALLS: + · the caller's OWN stratum — theirs, no further question; + · `__shared__` — `table_store._may_edit`, which is the predicate the view door itself uses; + · another user's stratum, reached by a wave-21 grant — the grant must be `edit` or `owner`. + ⚠ A `view` ROLE IS NOT A WRITE GRANT. `_granted_view_ids` admits both roles because it + answers "can they SEE it"; this answers "may they CHANGE it", and collapsing the two would + let a read-only grantee mark somebody else's shared view. + """ + import core.shares as shares + import core.table_store as table_store + if stratum == session.uname: + return True + if stratum == table_store.SHARED_KEY: + return bool(table_store._may_edit(view, session.uname, session.admin)) + return shares.role_for("view", str((view or {}).get("id") or ""), session.uname, + is_admin=session.admin, + st=session.runtime) in ("edit", "owner") + + +def _write_view_star(session, ws_key, stratum, vid, on): + """Set `config.important` IN PLACE, in the view's own bucket. Returns True when it landed. + + ⛔⛔ IN PLACE, AND **NOT** THROUGH `grid_events.view_upsert` OR `table_store.save_view` — three + separate reasons, each of which has cost this repo something: + + 1. `view_upsert` REBUILDS `config` from a literal allowlist and filters `memberPids` against + `allowed_pids` and `cohortLock` against `cohort_ids`. Routing a flag toggle through it + means a cohort view whose pool this request did not assemble comes back with its curated + row set pruned — "the 40 accounts we agreed to call" quietly becoming fewer, under the + same name, with nothing going red. + 2. `table_store.save_view` re-runs `_unique_name` against every view name in the document. + A star has no business re-allocating a name; a collision would RENAME the starred view. + 3. Both are "write the whole view" doors. This writes one boolean. + + ⚠ SET UNCONDITIONALLY, never "only when True" — `grid_events` records the same rule beside its + own `important` line: a key written only when present means unstarring is silently a no-op and + the stored `true` survives, i.e. a mark you can set and never clear. + ⚠ `bool(on)`, so a JSON `"false"` cannot become a star. + ⚠ FLUSH IS THE DEFAULT (sync). This is a deliberate one-at-a-time act like a folder rename, not + the autosave hot path `table_store` uses `async` for — and a coalesced write is exactly what + made eighteen 200s land zero rows in wave 33. + """ + landed = {"ok": False} + + def _up(data): + data = data if isinstance(data, dict) else {} + block = data.get(stratum) + if not isinstance(block, dict): + return data + views = block.get("views") + if not isinstance(views, dict) or str(vid) not in views: + return data + view = views[str(vid)] + if not isinstance(view, dict): + return data + cfg = view.get("config") + if not isinstance(cfg, dict): + cfg = {} + view["config"] = cfg + cfg["important"] = bool(on) + landed["ok"] = True + return data + + session.runtime.update(ws_key, _up) + return landed["ok"] + + +@router.post("/starred") +def set_starred(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """C2 — star or unstar ONE object, and answer with the caller's whole new list. + + Body: `{"kind": "database"|"agent"|"query"|"view", "id": str, "on": bool}`, plus an OPTIONAL + `"database"` when `kind` is `view` (the fast path — see `_find_view`). + + ⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled + (T15): an ack would leave the browser as the only place that knows what the new state is. + """ + body = body if isinstance(body, dict) else {} + kind = str(body.get("kind") or "").strip() + oid = _clean_id(body.get("id")) + if kind not in KINDS: + raise err(400, "bad_request", + f"kind must be one of {', '.join(KINDS)}") + if not oid: + raise err(400, "bad_request", "no object was named") + if "on" not in body or not isinstance(body.get("on"), bool): + # Explicit rather than defaulted: a toggle whose default is "star" turns a malformed + # unstar into a star, which is the one direction a user cannot undo by repeating it. + raise err(400, "bad_request", "`on` must be true or false") + on = bool(body["on"]) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + + if kind == "view": + found = _find_view(session, oid, _clean_id(body.get("database"))) + if not found: + # ⛔ 404 FOR BOTH "no such view" AND "not yours to see", deliberately: a distinct answer + # would turn this route into an oracle that sorts real view ids from invented ones. + raise err(404, "unknown_view", "that view is not available") + db_key, ws_key, stratum, view, cache = found + if not _may_star_view(session, stratum, view): + raise err(403, "forbidden", + "that view belongs to somebody else and you have read access only") + try: + landed = _write_view_star(session, ws_key, stratum, oid, on) + except Exception: + raise err(503, "store_unavailable", + "the star was not saved: the store refused the write.") + if not landed: + # The view moved between the read and the write (deleted, shared, unshared). Loud, + # because a 200 over a write that changed nothing is the defect this file's own + # `_may_star_view` note is about. + raise err(409, "view_moved", + "that view changed while the star was being saved. Try again") + # ⛔ REUSE THE SCAN, REFRESH THE ONE BUCKET WE WROTE. Without this the POST scans every view + # bucket TWICE — once to find the view, once to build the answer — which is the 1+N shape + # this file removed from `/starred/counts` an hour earlier, reintroduced by the write path. + # ⚠ `_refresh_one` is not optional: the written document is stale by definition and serving + # it back would answer the POST with the flag's OLD value. + if cache is not None: + cache = _refresh_one(session, cache, db_key) + return _payload(session, cache=cache) + + field = _LIST_KINDS[kind] + uname = session.uname + # ⛔ THE CAP IS ENFORCED INSIDE THE TRANSACTION, NOT BY A READ BEFORE IT, AND THAT IS NOT + # PEDANTRY. A pre-flight `len(current) >= MAX_PER_KIND` check is advisory: two concurrent stars + # both pass it, both append, and the document ends up at MAX+1 — where `_clean_list`'s + # `raw[:MAX_PER_KIND]` then drops one **on the next read**. That is a star the user set, saw + # confirmed, and cannot find, which is exactly the silent truncation R6's second sentence + # forbids. Refusing inside the read-modify-write makes the ceiling true rather than likely. + # ⚠ IT RECORDS THE REFUSAL RATHER THAN RAISING FROM INSIDE `_up`: an exception thrown through + # `store.update` would surface as this route's own 503 "the store refused the write", which is a + # different and wrong story about a limit the caller can act on. + refused = {"full": False} + + def _up(data): + data = data if isinstance(data, dict) else {} + mine = dict(data.get(uname) or {}) if isinstance(data.get(uname), dict) else {} + ids = _clean_list(mine.get(field)) + if on: + if oid not in ids: + if len(ids) >= MAX_PER_KIND: + refused["full"] = True + return data + ids.append(oid) + else: + ids = [i for i in ids if i != oid] + if ids: + mine[field] = ids + else: + mine.pop(field, None) + # A user with nothing starred is REMOVED rather than stored empty — a bucket that + # accumulates `{}` per account is a document that grows forever and says nothing. + if mine: + data[uname] = mine + else: + data.pop(uname, None) + return data + + try: + session.runtime.update(_STARRED_KEY, _up) + except Exception: + raise err(503, "store_unavailable", + "the star was not saved: the store refused the write.") + if refused["full"]: + # R6's second sentence: the cause and the action, never a 200 over a star that was dropped. + raise err(400, "starred_full", + f"you have starred the maximum of {MAX_PER_KIND} items of this kind. " + f"Unstar one to make room") + return _payload(session) + + +# ══════════════════════════════════════════════════════════════════ STARRED COUNTS (R7, C3) +# +# ⭐ R7: *"Row counts come from a NEW on-demand endpoint called after Home/Starred paint, never from +# `/nav`. A view whose count is not free shows the CAUSE, never an invented number."* +# +# ⛔ SO EVERY VID LANDS IN EXACTLY ONE OF `counts` AND `notes`, AND THAT IS THE CONTRACT (C3). A vid +# in neither is a client rendering nothing with no idea why; a vid in both is two answers to one +# question. `notes` is not a fallback for "we did not get round to it" — it is the CAUSE, in a +# sentence, with what to do about it, which is R6's second sentence applied to a badge. + + +def _cohort_sizes(session, key, cache): + """`{cohort_id: size}` for THIS caller's own cohorts on `key`'s topic. Memoised per request. + + ⛔ THIS CALLER'S OWN COHORTS ONLY, and the consequence is deliberate: a SHARED view locked to a + cohort somebody else owns finds no id here, so it is reported UNCOUNTED with that cause rather + than counted out of a stratum this session cannot see. Widening the read would make the number + disclose the SIZE of another person's private list, which is a leak wearing a bug fix. + ⚠ Read through `session.runtime`, NOT `modules.cohort`'s module-level helpers: those call + `core.store` directly and carry no tenant namespace. The MODULE is asked for the bucket NAME (it + owns that rule) and this route does the reading, which is the only tenant-correct combination. + """ + import core.view_templates as view_templates + import modules.cohort as cohort_mod + ws_key = view_templates.workspace_key(key) or "" + suffix = "_table_workspace" + scope = ws_key[:-len(suffix)] if ws_key.endswith(suffix) else key + if scope in cache: + return cache[scope] + try: + bucket = session.runtime.get(cohort_mod.key_for(scope)) or {} + mine = bucket.get(session.uname) or {} + cache[scope] = {str(cid): len(c.get("members") or []) + for cid, c in mine.items() if isinstance(c, dict)} + except Exception: + cache[scope] = {} + return cache[scope] + + +def _count_note(cfg, cohorts): + """WHY this view's size is not free, as one sentence naming a cause AND a next step. + + ⛔ R6's SECOND SENTENCE IS THE SPEC HERE, and a bare dash is the thing it forbids. D-253 was + booked once already for a limit whose explanation was hover-only, so this has to be a real + sentence a client can put in a `title` AND an `aria-label`. + ⚠ THREE DIFFERENT CAUSES, not one generic apology. A view locked to somebody else's list, a + filtered view and a whole-table view are three different facts, and only one of them is + something the reader can act on. + """ + cfg = cfg if isinstance(cfg, dict) else {} + lock = str(cfg.get("cohortLock") or "").strip() + if lock: + return ("This view is locked to a list owned by somebody else, so its size is not " + "something your account can read. Ask whoever shared it.") + if cfg.get("filters"): + return ("Counting a filtered view means reading every record, which this page does not " + "do. Open the database and the number is beside the view.") + return ("This view covers the whole database, so its size is the record count. Open the " + "database to see it.") + + +@router.get("/starred/counts") +def starred_counts(session: Session = Depends(require_session)): + """C3 — `{"counts": {vid: int}, "notes": {vid: str}, "degraded": [key]}`. + + ⛔ CALLED AFTER THE PAGE PAINTS, NEVER FROM `/nav` (R7). `/nav` is a rows-free projection whose + budget was the subject of D-175, D-185, D-288 and D-289; W35-T43 takes the last of that off it, + and putting a counting pass back on the nav's path would undo the whole point. + + ⚠ THE COUNTER IS `routes_nav._view_record_count`, CALLED — not a second implementation. It is the + one function that knows which shapes are free (a cohort lock resolves to a stored member list; a + curated `memberPids` carries its own length) and returns None for everything else. A second + counter here would be free to disagree with the badge inside the database. + """ + rows, unread = _scan_starred_views(session) + cache, counts, notes = {}, {}, {} + for row in rows: + cfg, key, vid = row["config"], row["database"], row["id"] + sizes = _cohort_sizes(session, key, cache) + n = routes_nav._view_record_count(cfg, sizes) + # ⛔ EXACTLY ONE OF THE TWO MAPS (C3). The `if/else` is what makes that structural rather + # than a rule somebody has to remember: there is no path that writes both and none that + # writes neither. + if isinstance(n, int): + counts[vid] = n + else: + notes[vid] = _count_note(cfg, sizes) + return {"counts": counts, "notes": notes, "degraded": unread} + + +# ══════════════════════════════════════════════════════════════════ RECORD STARS (R5/R6, C5) +# +# ⭐ R5: *"Records get their own star, feeding an undeletable 'Starred records' view beside +# 'All records' on every database."* R6: *"Record stars are PER USER, in the same per-username +# stratum views already live in."* +# +# ⛔⛔ THE ID LIST IS ITS OWN KEY IN THAT STRATUM AND IT DOES NOT TRAVEL THROUGH THE VIEW DOOR. +# D-170, measured: on a read-through grid a VIEW WRITE answers **409 `window_required`** — +# `/grid/events` -> `routes_tables.ut_write_ctx` -> `scoped_pids` -> `scoped_pool` -> +# `routes_odoo_tables.whole_pool` raises `TooBigToMaterialise`, because the shared read/write +# assembly insists on the pid set. `ut_odoo_order_lines` (255,286 rows) and `ut_odoo_gl_lines` +# (971,034) serve rows through a window correctly and refuse every view-config write. A record star +# stored inside a view record would therefore be IMPOSSIBLE on exactly the two biggest databases in +# the tenant. This writes one key in the workspace bucket and touches no pid set, so it answers 200 +# there. +# +# ⚠ AND FOR THE SAME REASON IT DOES NOT VALIDATE THE ID AGAINST THE ROW SET. Checking that a pid +# exists means materialising the pool, which is the 409 again. `/nav/opened` takes the identical +# posture and says why: the WRITE is cheap and unvalidated, the READ is what prunes — a client +# intersects `memberPids` with the rows it can see, exactly as a cohort already does +# (`aios_grid.workspace_wire` drops absent members and reports `missing`). + +#: The per-user key inside `_table_workspace`. BESIDE `views`, never inside one (R6). +_RECORDS_KEY = "starredRecords" + +#: The pinned id of the projected view. ⚠ C's rail pins this too (W35-T30) — it is a WIRE constant, +#: so it is exported rather than spelled twice. +STARRED_VIEW_ID = "starred-records" +STARRED_VIEW_NAME = "Starred records" + +#: A bound per (user, database). Reported on overflow, never silently enforced. +MAX_STARRED_RECORDS = 500 + + +def _record_ids(doc, uname): + """This caller's starred record ids on ONE database, validated on the way out. + + ⛔ INTS, AND A NON-DIGIT ID IS NOT STORED. Every record identity in this product is a digit + string — a pool `pid` is an int and a `ut_*` row key is `str(record_id).isdigit()` — and the + CLIENT's view engine reads `memberPids` as numbers (a cohort's members are ints). A mixed list + would make the projection match nothing on the rows it was built for, silently. + """ + raw = (doc.get(uname) or {}).get(_RECORDS_KEY) if isinstance(doc.get(uname), dict) else None + if not isinstance(raw, list): + return [] + out, seen = [], set() + for item in raw[:MAX_STARRED_RECORDS]: + try: + pid = int(item) + except (TypeError, ValueError): + continue + if pid not in seen: + seen.add(pid) + out.append(pid) + return out + + +def records_view(ids): + """The **"Starred records"** view, as a SavedView — or None when nothing is starred. + + ⭐ THE SERVER HANDS C THE WHOLE OBJECT (C5) rather than a bare id list, so the rail cannot grow + a second idea of what this view is. `kind: 'system'` and `locked: True` are what make it + undeletable, the same two facts that protect `all-customers`. + + ⛔ `memberPids` IS THE WHOLE MECHANISM: it makes this a CURATED ROW SET, which is the one shape + `routes_nav._view_record_count` can count for FREE — no rows read, no filter run. That is why C3 + can put a real number beside it and why R7's counts endpoint costs nothing here. + ⚠ None WHEN EMPTY, not an empty view: a view listing zero records under a name promising some is + the shape D-229 was booked for. C shows the row only when there is something in it. + """ + pids = [int(p) for p in (ids or [])] + if not pids: + return None + return { + "id": STARRED_VIEW_ID, + "name": STARRED_VIEW_NAME, + "kind": "system", + "locked": True, + "note": "The records you starred on this database. Only you can see this list.", + "config": {"filters": [], "filterConj": "and", "sorts": [], "groupBy": None, + "colorBy": None, "rowHeightMode": "short", "order": [], "visible": [], + "widths": {}, "memberPids": pids}, + } + + +def _database_or_refuse(session, key): + """The workspace bucket for a database this caller may OPEN, or a refusal. + + ⚠ THE SPLIT IS `routes_nav.nav_schema`'s, COPIED IN SHAPE RATHER THAN INVENTED: a `ut_*` + database is walled by `user_tables.may_open` (its creator or an admin or a grantee), and a + built-in module by `session.require`, which would 403 every `ut_` key because a user table is + deliberately not a module. + """ + import core.view_templates as view_templates + ws_key = view_templates.workspace_key(key) + if not ws_key: + raise err(404, "unknown_database", "that database is not available") + if key.startswith("ut_"): + import core.user_tables as user_tables + if not user_tables.may_open(key, session.uname, session.admin, st=session.runtime): + raise err(403, "forbidden", "that database belongs to another user") + else: + session.require(key) + return ws_key + + +def _records_payload(session, key, ws_key): + try: + doc = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {} + except Exception: + doc = {} + ids = _record_ids(doc, session.uname) + return {"database": key, "ids": ids, "count": len(ids), "view": records_view(ids)} + + +@router.get("/starred/records") +def starred_records(database: str = "", session: Session = Depends(require_session)): + """C5 — the record ids THIS caller starred on ONE database, plus the view to render. + + ⚠ ONE DATABASE PER CALL, deliberately. Answering "every database" would be the N-bucket scan + the view half already pays for, on a route a grid calls every time it opens. + """ + key = _clean_id(database) + if not key: + raise err(400, "bad_request", "no database was named") + return _records_payload(session, key, _database_or_refuse(session, key)) + + +@router.post("/starred/records") +def set_starred_record(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """C5 — star or unstar ONE record. Body `{"database": key, "id": str, "on": bool}`. + + ⛔ PER USER, IN THE PER-USERNAME STRATUM, AND IT MUST NEVER FALL BACK TO A TENANT-WIDE LIST + (R6). One person's stars becoming everyone's is the failure mode a shared column has here — a + custom field on these grids is written through a per-username stratum anyway, so a "shared" + answer would be a deliberate widening, not a shortcut. + """ + body = body if isinstance(body, dict) else {} + key = _clean_id(body.get("database")) + raw_id = _clean_id(body.get("id")) + if not key: + raise err(400, "bad_request", "no database was named") + if not raw_id.isdigit(): + # Named rather than coerced: see `_record_ids` on why a non-numeric id cannot be stored. + raise err(400, "bad_request", "a record id is a number") + if "on" not in body or not isinstance(body.get("on"), bool): + raise err(400, "bad_request", "`on` must be true or false") + pid, on = int(raw_id), bool(body["on"]) + ws_key = _database_or_refuse(session, key) + if not session.runtime.available(): + raise err(503, "store_unavailable", + "the tenant store is unavailable. Nothing was saved.") + current = _records_payload(session, key, ws_key)["ids"] + if on and pid not in current and len(current) >= MAX_STARRED_RECORDS: + raise err(400, "starred_full", + f"you have starred the maximum of {MAX_STARRED_RECORDS} records on this " + f"database. Unstar one to make room") + uname = session.uname + + def _up(data): + data = data if isinstance(data, dict) else {} + block = data.get(uname) + block = dict(block) if isinstance(block, dict) else {} + ids = _record_ids({uname: block}, uname) + if on: + if pid not in ids: + ids.append(pid) + else: + ids = [i for i in ids if i != pid] + if ids: + block[_RECORDS_KEY] = ids + else: + block.pop(_RECORDS_KEY, None) + # ⚠ THE STRATUM IS KEPT WHEN IT STILL HOLDS ANYTHING ELSE. Popping a username whose views + # and overlays live in the same block would delete somebody's whole workspace to clear one + # star — which is why this writes `block` back rather than replacing it. + if block: + data[uname] = block + else: + data.pop(uname, None) + return data + + try: + session.runtime.update(ws_key, _up) + except Exception: + raise err(503, "store_unavailable", + "the star was not saved: the store refused the write.") + return _records_payload(session, key, ws_key) diff --git a/api/routes_statements.py b/api/routes_statements.py index fb608f0da2cfb35c3dccd25d431893e6c8a1b141..771ac0a8a1b1326942145be58c132991bd3e66aa 100644 --- a/api/routes_statements.py +++ b/api/routes_statements.py @@ -31,6 +31,10 @@ from fastapi import APIRouter, Body, Depends from deps import Session, err from routes_admin import admin_gate +# W35-T35 (C8): the tenant predicate, held once. See `_royal_only` for why the import runs this +# way round. `main.py` already imports both modules, so this adds no load. +import automation_engine as engine + router = APIRouter(prefix="/api/v1") #: Cache the Odoo follow-up pull briefly. The Streamlit page used `@st.cache_data(ttl=1800)`; the @@ -48,8 +52,17 @@ def _royal_only(session: Session) -> Session: """⛔ See the module docstring, guardrail 3. The send client is env-credentialed, so it is tenant #0's and only tenant #0's. Refuse for anyone else rather than send as the wrong company. - Keyed on the runtime, never on a request field: a tenant is a property of the SESSION.""" - if getattr(session.runtime, "key", None) != "royal-imports": + Keyed on the runtime, never on a request field: a tenant is a property of the SESSION. + + ⭐⭐ WAVE 35 · T35 / CONTRACT C8 — THE TEST ITSELF NOW LIVES IN ONE PLACE. Statements became an + agent step this wave, so the automation engine has to answer the same question at two more + doors (which tenants see the action, which tenants may store it). C8's words are "the predicate + is imported, never re-expressed", and this is the direction that import can run: the engine + imports no route module and no FastAPI, so reaching `_royal_only` FROM it would drag `deps` and + `routes_admin` into a route-free module. The engine holds the boolean; this holds the refusal. + ⚠ WHAT STAYS HERE IS THE HTTP SHAPE, and that is deliberate — a 404 rather than a 403, so the + surface is invisible rather than forbidden. The engine must not know about status codes.""" + if not engine.is_statement_tenant(session.runtime): raise err(404, "not_found", "statements are not configured for this workspace") return session @@ -91,7 +104,9 @@ def statements(refresh: int = 0, session: Session = Depends(_gate)): "replyTo": cs.REPLY_TO, "company": cs.COMPANY}, "templates": {"subject": cs.DEFAULT_SUBJECT, "intro": cs.DEFAULT_INTRO, "footer": cs.DEFAULT_FOOTER}, - "tiers": ["A-Urgent", "B-Active", "C-Light", "Monitor"], + # W35-T35: ONE literal, read from the engine, so the sender's worklist and a statements + # agent's tier filter cannot come to mean different things. + "tiers": list(engine.STATEMENT_TIERS), } @@ -166,3 +181,89 @@ def send(body: dict = Body(default=None), session: Session = Depends(_gate)): failed.append({"customer": name, "error": str(e)[:200]}) return {"sent": sent, "failed": failed, "skipped": skipped, "safeMode": bool(cs.SAFE_MODE), "test": bool(override)} + + +# --------------------------------------------------------------------------------------------- +# WAVE 35 · T36 / CONTRACT C8 / OWNER RULING R10 — THE AGENT'S PARKED BATCH. +# +# ⛔⛔ THE AGENT ASSEMBLES; A PERSON CLICKS SEND. `automation_engine.run_statements` renders a batch +# and parks it on the automation definition, and it imports no mail path at all. THIS is the only +# door that releases one, and it is deliberately built out of the parts already here: +# · `_gate` — `admin_gate` THEN `_royal_only`, byte-for-byte the dependency the other three +# endpoints use. Not a copy of the checks: the same object. +# · `cs.queue_statement` — the same call `send()` above makes, so SAFE_MODE's allow-list refusal +# happens in the DATA LAYER, where no route, payload or UI can reach around it. +# ⇒ Nothing about the guardrails is re-expressed here, which is what makes "unchanged" checkable. +# +# ⚠ WHY THE BATCH IS RE-RENDERED FROM THE LIVE WORKLIST RATHER THAN SENT AS STORED. The parked +# `html` is what a person APPROVED and is what the review screen shows; but a balance can move +# between the parking and the click, and mailing a figure we know to be stale is worse than mailing +# a fresh one. So the parked batch decides WHO and WITH WHAT WORDS, and the live row decides the +# NUMBERS — the same split `preview` already makes. A customer who has left the worklist entirely +# (they paid) is reported as skipped rather than invoiced. + +@router.get("/admin/statements/agent/{auto_id}") +def agent_batch(auto_id: str, session: Session = Depends(_gate)): + """What is parked and waiting for a click, for ONE agent. `null` when nothing is.""" + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + if not isinstance(pending, dict): + return {"pending": None} + # ⚠ The rendered `html` is NOT returned in the list — a 200-statement batch of rendered mail is + # megabytes, and the review screen needs the count and the names to decide. `preview` already + # renders ONE on demand. + return {"pending": { + "ts": pending.get("ts"), "count": int(pending.get("count") or 0), + "notes": list(pending.get("notes") or []), + "items": [{k: v for k, v in it.items() if k != "html"} + for it in (pending.get("items") or [])], + "safeMode": bool(_cs().SAFE_MODE)}} + + +@router.post("/admin/statements/agent/{auto_id}/send") +def agent_send(auto_id: str, body: dict = Body(default=None), + session: Session = Depends(_gate)): + """Release a parked batch. THE CLICK R10 REQUIRES — nothing else in the system calls this. + + ⛔ IT REFUSES AN EMPTY OR MISSING BATCH rather than answering 200 with nothing sent: a send + that reports success and mails nobody is the `view_upsert` failure mode (200 OK, zero writes) + arriving on the one route in this system that talks to a mail server. + """ + cs = _cs() + defn = (engine.all_definitions(session.runtime) or {}).get(str(auto_id)) or {} + pending = defn.get("pendingStatements") + items = list((pending or {}).get("items") or []) if isinstance(pending, dict) else [] + if not items: + raise err(404, "no_batch", "there is nothing parked for this agent to send") + # ⚠ An explicit subset is allowed (a person unticking a customer on the review screen) but it + # may only NARROW the parked batch. A name that was never parked cannot be introduced by the + # payload, or the review stops being what authorised the send. + want = {str(n) for n in (body or {}).get("customers") or []} + if want: + items = [it for it in items if str(it.get("customer")) in want] + if not items: + raise err(400, "bad_request", "none of those customers are in the parked batch") + rows, _ = _rows() + sent, failed, skipped = [], [], [] + for it in items: + name = str(it.get("customer") or "") + row = _find(rows, name) + if row is None: + skipped.append({"customer": name, + "reason": "no longer on the collection list, so nothing is owed"}) + continue + try: + # ⛔ THE SAME DATA-LAYER CALL `send()` MAKES. SafeModeBlocked is raised INSIDE + # `queue_statement`, so the guardrail cannot be argued with from here. + mid = cs.queue_statement(cs.Odoo(), row, + str(it.get("subject") or "") or cs.DEFAULT_SUBJECT, + cs.DEFAULT_INTRO, cs.DEFAULT_FOOTER) + sent.append({"customer": name, "to": row.get("Email"), "mailId": mid}) + except Exception as e: # noqa: BLE001 + failed.append({"customer": name, "error": str(e)[:200]}) + # ⚠ CLEARED ONLY WHEN NOTHING IS LEFT TO RETRY. A batch dropped while some of it failed would + # lose the list of who still needs a statement, and nobody would know to look. + if sent and not failed: + engine.clear_statements(session.runtime, auto_id) + return {"sent": sent, "failed": failed, "skipped": skipped, + "safeMode": bool(cs.SAFE_MODE), "cleared": bool(sent and not failed)} diff --git a/api/routes_tables.py b/api/routes_tables.py index 9c10a00bcb6bebef5a419438c739891c9817f849..58340d8dc9ff90f31a7bd3cf5c5c9b8ab3d4b967 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -105,7 +105,7 @@ def _refresh_relations(session): name=f"rel-refresh:{tenant or 'default'}").start() -def _defn_or_refuse(session, table_key, st=None): +def _defn_or_refuse(session, table_key, st=None, defs_only=False): """The per-table wall: 404 for a key that does not exist, 403 for one this session may not open. 404-before-403 leaks nothing useful — ut keys are guessable slugs, and 'exists but not yours' is exactly what may_open is for. @@ -113,11 +113,30 @@ def _defn_or_refuse(session, table_key, st=None): ⭐ `st` LETS A CALLER LEND THE SNAPSHOT IT IS ALREADY HOLDING (W31 QA), the same `lend()` the nav and `/tables` use since W31-T10/T12. The WALL is untouched — the same `may_open`, asked about the same table — it is simply not re-reading a 28.5 MB document to ask it. + + ⭐⭐ W33-T01 (D-213) — `defs_only=True` MAKES THAT READ A PROJECTION, AND IT IS OPT-IN PER CALL + SITE ON PURPOSE. This wall asks two questions ("does the key exist", "may this session open + it") and neither has ever read a row, but three of its ten callers go on to read `rows` OFF THE + DEFINITION THIS RETURNS — so a blanket swap would turn `scoped_pool`, `scoped_pids` and + `table_footprint` into `KeyError`s on every materialised table. The flag is therefore the + CALLER's claim about what it will do next, not a global setting; each opt-in below is annotated + with why it can make that claim ([[reuse-and-delete-are-hypotheses]]). + + ⛔ **`defs_only` IS FOR READS.** The six write walls (`_records_or_refuse`, `patch_shared_cell`, + `delete_shared_field`, `patch_table`, `delete_table`, `import_rows`) deliberately do NOT pass + it: a projected snapshot must never reach a post-write read-back (contract C5), and the + pre-write lend note below is the same argument one layer down. + + ⚠ **WHAT COMES BACK IS A `store._Projected`, WHICH IS THE EVIDENCE, NOT AN IMPLEMENTATION + DETAIL.** `all_defs` falls back to the whole read on ANY failure, so "the answer was right" + proves nothing about which read produced it. `core.store.is_projected(defn)` is how a gate + asserts the fast path was TAKEN. """ ut = _ut() # ⭐⭐ W31 QA — THE SWEEP, AND IT IS ONE LINE BECAUSE IT IS DONE HERE RATHER THAN PER ROUTE. # Owner: *"this is just one case, I need you to check and apply the fix everywhere too."* - # This guard has FOURTEEN call sites and cost TWO whole-document deep copies at every one + # This guard has TEN direct call sites (counted 2026-08-14; the note said FOURTEEN, which was + # the route count, not the caller count) and cost TWO whole-document deep copies at every one # (`get`, then `may_open`) — on tenant #0 that is 2 x 28.5 MB (D-185) to answer two questions # about ONE table. Lending here fixes every caller at once, including the eight routes the # sweep enumerated (`PATCH /shared/{pid}` · `DELETE /shared/fields/{k}` · `GET|POST /rows` · @@ -128,7 +147,8 @@ def _defn_or_refuse(session, table_key, st=None): # would hand that stale snapshot to `patch_row`'s and `add_row`'s post-write read-back, which # is [[refetch-eats-its-own-write]] with the sign flipped — a write that reports the value it # replaced. The remaining in-route reads are deliberately untouched and booked instead. - st = st if st is not None else ut.lend(session.runtime) + if st is None: + st = ut.lend_defs(session.runtime) if defs_only else ut.lend(session.runtime) defn = ut.get(table_key, st=st) if not defn: raise err(404, "unknown_table", "that database does not exist") @@ -145,7 +165,7 @@ def _records_or_refuse(session, table_key, st=None): defn = _defn_or_refuse(session, table_key, st=st) if not _ut().records_mutable(table_key, st=st): raise err(403, "records_read_only", - "records in this automation-owned database are read-only — add Instagram " + "records in this automation-owned database are read-only. Add Instagram " "handles in a Profile database and let enrichment populate this database") return defn @@ -234,15 +254,21 @@ _PID_SCOPE_LIMIT = { } -def scoped_pool(session: Session, table_key: str): +def scoped_pool(session: Session, table_key: str, st=None): """`(pids, rows_src, fields_base, defn)` — THE USER-TABLE WALL, on its own. + ⭐ W33-T03 (D-214) — `st` LETS A CALLER LEND THE DOCUMENT IT IS ALREADY HOLDING, exactly as + `_defn_or_refuse` has since W31 QA. It is passed straight through to that wall and nowhere + else, so the permission question is answered by the same code against the same document. + ⛔ READ CALLERS ONLY. A lend is a PRE-WRITE snapshot; handing one to a path that writes and then + reads back is [[refetch-eats-its-own-write]] with the sign flipped (contract C5). + `routes_products.scoped_pool`'s sibling, extracted for the same reason (wave 19, item 12): a caller that only needs "which rows of this database may this session touch" — record comments — must ask through `_defn_or_refuse` (404/403, the whole wall) rather than growing a second idea of what a user table's pool is. """ - defn = _defn_or_refuse(session, table_key) + defn = _defn_or_refuse(session, table_key, st=st) fields_base = [dict(f) for f in (defn.get("fields") or [])] field_keys = {f["key"] for f in fields_base} # ⭐⭐ W30-T31 / D-87 — A READ-THROUGH DATABASE HAS NO ROWS HERE, SO THEY COME FROM THE MIRROR. @@ -350,7 +376,7 @@ def delete_shared_field(table_key: str, field_key: str, owner = str(defn.get("createdBy") or "") if not session.admin and owner != session.uname: raise err(403, "forbidden", - f"a tenant-wide column can be removed by its creator or an admin — this one " + f"a tenant-wide column can be removed by its creator or an admin. This one " f"was added by {owner or 'somebody else'}, and dropping it would delete the " f"value for every account") dropped = shared_overlay.drop_field(table_key, str(field_key), st=session.runtime) @@ -375,8 +401,14 @@ def scoped_pids(session: Session, table_key: str, limits=None): middle. The row WALL is unchanged — a narrower or wider set here would be a permission change, and this is a performance change. - ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs two whole-document - deep copies (`ut.get` then `may_open`), which is D-87 and W30-T31. This removes the row pass. + ⚠ It does NOT make the write cheap on its own: `_defn_or_refuse` still costs a whole-document + read, which is D-87 and W30-T31. This removes the row pass. + ⭐ CORRECTED 2026-08-14 (W33-T01): that sentence said **two** deep copies (`ut.get` then + `may_open`) and had been stale since W31 QA taught the wall to `lend()` — the two questions + have shared ONE read since `routes_tables.py`'s lend line. And as of this ticket the read is a + PROJECTION on the read-through arm, so the sentence is now true only of the materialised one. + Booked because a stale performance note is how a wave re-fixes something twice + ([[stale-baseline-unreadable-deltas]]). ⭐⭐ W31-T20 / D-174 — `limits` IS AN OUT-PARAMETER, AND IT IS THE POINT OF THE TICKET. Pass a list and this function APPENDS R6's sentence to it when the pid set could not be resolved (a @@ -386,7 +418,20 @@ def scoped_pids(session: Session, table_key: str, limits=None): write is expected to carry the sentence through. Omitting the list means the caller accepts an unannounced empty scope, which is only ever right for a caller that does not use the pids. """ - defn = _defn_or_refuse(session, table_key) + # ⭐⭐ W33-T01 / D-213 — THE DATABASE-SWITCH PATH, AND IT STARTS ON A PROJECTION. + # + # `GET /workspace?scope=` reaches here through `ut_assembly(with_rows=False)` + # (`routes_grid.py`'s ut_ branch), which is what a person is waiting for when they click a + # database in the nav flyout: 1.8-7.3 s live for a 3-6 KB payload, of which one whole-document + # read is ~703 ms warm and 20.6 s cold. This function reads `fields` and (below) `readThrough` + # off the definition — no row — so the WALL can be answered from the 0.1% projection. + # + # ⛔ IT IS THE TRAP ON THIS BOARD, SO IT IS SAID TWICE: this function is NAMED and DOCUMENTED + # as the rows-free twin of `scoped_pool` and the materialised arm below still reads `rows`. + # The opt-in is therefore CONDITIONAL, and the condition is `materialises`, which reads + # `readThrough` — a definition key, safe under the projection, and already lent the defn so it + # costs no read of its own. + defn = _defn_or_refuse(session, table_key, defs_only=True) fields_base = [dict(f) for f in (defn.get("fields") or [])] # ⚠ W30-T31: on a read-through database the stored `rows` is `{}` by construction, so the # comprehension below would answer an EMPTY pid set — and the promise this function makes is @@ -407,8 +452,25 @@ def scoped_pids(session: Session, table_key: str, limits=None): return frozenset(), fields_base, defn except RuntimeError as e: raise err(503, "store_not_ready", str(e)) - pids = frozenset(int(rid) for rid in (defn.get("rows") or {}) if str(rid).isdigit()) - return pids, fields_base, defn + # ⛔⛔ MATERIALISED: THE PID SET *IS* `rows`, SO THIS ARM TAKES THE WHOLE READ — the projection + # above cannot serve it and would raise rather than answer `{}` (that is the whole design of + # `_Projected`). The wall has already passed on the projected document, so this re-reads the + # DEFINITION and does not re-ask `may_open`: re-walling would be a second, differently-shaped + # answer to a question already answered, which is how two ideas of ownership got into this file + # once before (see `may_open`'s own note in `core/user_tables.py`). + # + # ⚠ THE HONEST COST, STATED RATHER THAN BURIED: a materialised table now pays the projection + # PLUS the whole read — ~1.4 ms on top of ~703 ms on tenant #0, i.e. 0.2%. The pid set, the + # wall and the returned shape are byte-for-byte what they were; only tenant #0's ten + # read-through databases (every one of them, which is why the switch was slow) skip the big + # read entirely. + whole = _ut().get(table_key, st=session.runtime) + if whole is None: + # Between the wall and here the table was deleted by another request. Same refusal the + # wall gives, rather than an empty pid set nobody can distinguish from an empty table. + raise err(404, "unknown_table", "that database does not exist") + pids = frozenset(int(rid) for rid in (whole.get("rows") or {}) if str(rid).isdigit()) + return pids, fields_base, whole def ut_write_ctx(session: Session, table_key: str): @@ -441,7 +503,7 @@ def ut_write_ctx(session: Session, table_key: str): def ut_assembly(session: Session, table_key: str, storage_key: str = "", - consume_corrections: bool = True, with_rows: bool = True): + consume_corrections: bool = True, with_rows: bool = True, st=None): """The user-table mirror of `grid_assembly` / `product_assembly` — SAME g-dict keys, so `/workspace` and the events route consume any of the three interchangeably. @@ -470,7 +532,9 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", limits = [] if with_rows: - pids, rows_src, fields_base, defn = scoped_pool(session, table_key) + # ⭐ W33-T03 (D-214): `st` is a caller's LEND, threaded to the wall and nowhere else. Only + # a READ route passes one — see `scoped_pool`'s own note and contract C5. + pids, rows_src, fields_base, defn = scoped_pool(session, table_key, st=st) else: pids, fields_base, defn = scoped_pids(session, table_key, limits=limits) rows_src = [] @@ -671,17 +735,17 @@ def create_table(body: dict = Body(default=None), raise err(400, "bad_label", "give the database a name") if not session.runtime.available(): raise err(503, "store_unavailable", - "the tenant store is unavailable — nothing was created") + "the tenant store is unavailable. Nothing was created") source = body.get("source") try: key = ut.create(label, session.uname, fields=body.get("fields"), source=source, st=session.runtime) except Exception: raise err(503, "store_unavailable", - "the tenant store refused the write — nothing was created") + "the tenant store refused the write. Nothing was created") if not key: raise err(400, "refused", - f"could not create it — the name may be empty or this tenant already has " + f"could not create it. The name may be empty or this tenant already has " f"{ut.MAX_TABLES} databases") return {"key": key} @@ -713,8 +777,11 @@ def table_footprint(table_key: str, session: Session = Depends(require_session)) Counts drill to the SAME buckets `user_tables.delete` cleans; a dialog listing categories without numbers would break [[no-unverifiable-aggregates]] at the scariest moment. Walled - like the delete itself: only someone who could delete may case the joint.""" - defn = _defn_or_refuse(session, table_key) + like the delete itself: only someone who could delete may case the joint. + + ⭐ W33-T01 / D-213: the wall and `createdBy`/`fields` come off the PROJECTION; only the row + COUNT needs the whole document, and only on a materialised table.""" + defn = _defn_or_refuse(session, table_key, defs_only=True) if not (session.admin or defn.get("createdBy") == session.uname): raise err(403, "forbidden", "only the database's creator or an admin can delete it") s = session.runtime @@ -737,7 +804,19 @@ def table_footprint(table_key: str, session: Session = Depends(require_session)) auto.append({"id": str(aid), "name": d.get("name") or str(aid)}) except Exception: pass - return {"rows": len(defn.get("rows") or {}), "fields": fields, "views": len(views), + # ⛔ THE ROW COUNT IS THE ONE FIELD THAT NEEDS THE WHOLE DOCUMENT, and it needs it only where + # the rows are actually stored here. A read-through database keeps `rows: {}` by construction, + # so `len(...)` answered **0** for it before this change and answers 0 now — identical, and the + # projection is not what makes it wrong. + # ⚠ 0 IS A WRONG NUMBER FOR A READ-THROUGH GRID and always was (`ut_odoo_gl_lines` would say 0 + # in a dialog headed "what dies with this database"). Booked rather than fixed here: this + # ticket is a read-path change and correcting it means asking the mirror for a `count(*)` + # inside a confirm dialog. See the `PENDING:` line in `mailbox/A.md`. + if _ut().materialises(table_key, st=s, defn=defn): + rows = len((_ut().get(table_key, st=s) or {}).get("rows") or {}) + else: + rows = 0 + return {"rows": rows, "fields": fields, "views": len(views), "sharedUsers": len(g.get("entries") or []), "automations": sorted(auto, key=lambda a: a["name"].lower())} @@ -766,7 +845,7 @@ def delete_table(table_key: str, session: Session = Depends(require_session)): try: _ut().delete(table_key, st=session.runtime) except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") + raise err(503, "store_unavailable", "the delete did not land. Try again") return {"ok": True} @@ -856,7 +935,17 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): import aios_grid _ig_forward(session) - g = ut_assembly(session, table_key) + # ⭐⭐ W33-T03 / D-214 — ONE READ OF THE TENANT DOCUMENT FOR THE WHOLE REQUEST, MEASURED. + # This route asked for it FOUR times: the wall (via `scoped_pool`), then `limit_report` TWICE + # (`row_limit` calls `materialises` and then `is_connected`, and each takes its own whole copy), + # then `records_mutable` at the envelope. Each is ~703 ms warm on tenant #0, and all four ask + # about the SAME document in the SAME request. The lend is the fix W31 QA already built for + # `_defn_or_refuse`; this threads it through the three sites that never got it. + # ⚠ SAFE HERE FOR THE SAME REASON IT IS SAFE THERE: this is a pure READ route. A lend is a + # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the + # value it replaced (contract C5). + _lent = _ut().lend(session.runtime) + g = ut_assembly(session, table_key, st=_lent) # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge # this line does not perform since the route was written (owner item 3, 2026-08-09: # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*). @@ -904,13 +993,16 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): # so a tenant using `json` for a short config sees no change at all. rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], _thin_json(g["fields"], merged, table_key), derived=g["derived"]) + # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/ + # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`. + _stamp_ai_states(table_key, g["fields"], rows, st=_lent) # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this # database says so here, with its cause and the recommendation, rather than waiting to be # discovered as a refused paste. `None` for a connected source, and an EMPTY LIST is the # honest answer for a table nothing limits — never an absent key, which a client cannot tell # apart from an older server. - _report = _ut().limit_report(table_key, st=session.runtime) + _report = _ut().limit_report(table_key, st=_lent) # ⭐ C4 / D-138 — THE DOCUMENTS PRODUCER. Absent since EXIT-6 deleted `app.py`, which was the # only thing that ever set this key; the write door never stopped working and every client # half is complete, but all six `onDoc*` handlers read `payload?.docs ? … : undefined`, so a @@ -924,7 +1016,49 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): "identity": {"pid": "pid"}, "scope": {"table": table_key, "rowCount": len(rows)}, "limits": [_report] if _report else [], - "recordsMutable": _ut().records_mutable(table_key, st=session.runtime)} + "recordsMutable": _ut().records_mutable(table_key, st=_lent)} + + +#: The per-cell provenance a row carries on the wire, one key per enrichment column. +#: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic. +#: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new +#: argument at `CustomerGrid`'s call site, both in another lane's fence, to reach the two surfaces +#: that must paint it. The row already carries `_created`, `lat` and `lon` for exactly this +#: reason, so every reader already tolerates keys that are not columns, and both surfaces hold the +#: row already. ⚠ COLLISION-PROOF BY CONSTRUCTION: `_clean_field` strips leading underscores off +#: every field key, so no column can ever be called `_ai_*`. +AI_STATE_PREFIX = "_ai_" + + +def _stamp_ai_states(table_key, fields, rows, st=None): + """Add `_ai_` to each row for every `ai_enrich` column. Mutates and returns `rows`. + + ⛔ A PROJECTION, NOT THE MARK SET. The stratum holds a hash, a model, a timestamp, a token + count and an error per cell; a browser needs ONE WORD to paint a state, and shipping the rest + would grow this payload by a dict per enriched cell for data no reader reads. The vocabulary + is `agent`/`human`/`stale`/`error` (`api/ai_enrich.py::cell_state`), which is also what the + RUNNER obeys, so the badge and the behaviour cannot disagree about whose cell it is. + + ⚠ AN ABSENT KEY MEANS `empty`, and only non-empty states are stamped: a table with no + enrichment column is untouched, and a freshly created column adds nothing until something + runs. ⛔ `stale` is DERIVED here rather than stored, so it is computed against TODAY'S row + instead of against whatever was true when the value was written. + """ + cols = [f for f in (fields or []) if str(f.get("type") or "") == "ai_enrich"] + if not cols: + return rows + import ai_enrich as _ae + for field in cols: + col = str(field.get("key") or "") + marks = _ut().ai_enrich_marks(table_key, col, st=st) + cfg = field.get("aiEnrich") if isinstance(field.get("aiEnrich"), dict) else {} + for row in (rows or []): + if not isinstance(row, dict): + continue + state = _ae.cell_state(marks.get(str(row.get("pid"))), cfg, row, col) + if state != "empty": + row[AI_STATE_PREFIX + col] = state + return rows @router.get("/tables/{table_key}/rows/{pid}/fields/{fkey}") @@ -973,7 +1107,7 @@ def add_row(table_key: str, body: dict = Body(default=None), rid = ut.add_row(table_key, values, session.uname, st=session.runtime, rid=(body or {}).get("rid")) except Exception: - raise err(503, "store_unavailable", "the row was not saved — the store refused") + raise err(503, "store_unavailable", "the row was not saved. The store refused") if rid is None: # C3 (wave 25): `add_row` also refuses a profile cell that is not a handle, so the cap # sentence alone would misdirect — the reader would go and count rows. Ask the same @@ -983,11 +1117,11 @@ def add_row(table_key: str, body: dict = Body(default=None), _h, ok = ut.normalize_profile(values[pf["key"]], pf["profile"].get("source")) if not ok: raise err(400, "refused", - f"{str(values[pf['key']])[:80]!r} is not an Instagram profile — " + f"{str(values[pf['key']])[:80]!r} is not an Instagram profile. " f"{pf.get('label') or pf['key']!r} takes a handle (@name) or a " f"profile link (instagram.com/name)") raise err(400, "refused", - f"row refused — the table may be at its {ut.MAX_ROWS}-row cap") + f"row refused. The table may be at its {ut.MAX_ROWS}-row cap") _refresh_relations(session) return {"rid": rid, "pid": int(rid)} @@ -1041,14 +1175,14 @@ def import_rows(table_key: str, body: dict = Body(default=None), for key, value in row.items(): why = ut.cell_type_refusal(by_key[key], value) if why: - raise err(400, "bad_value", f"row {index + 1}: {why} — nothing was imported") + raise err(400, "bad_value", f"row {index + 1}: {why}. Nothing was imported") try: made = ut.add_rows(table_key, rows_in, session.uname, st=session.runtime) except Exception: - raise err(503, "store_unavailable", "nothing was imported — the store refused") + raise err(503, "store_unavailable", "nothing was imported. The store refused") if made is None: raise err(400, "refused", - f"nothing was imported — {len(rows_in)} rows would take this database past " + f"nothing was imported. {len(rows_in)} rows would take this database past " f"its {ut.MAX_ROWS}-row cap, or a profile column rejected a value") _refresh_relations(session) return {"imported": len(made), "pids": [int(r) for r in made]} @@ -1068,12 +1202,18 @@ def import_rows(table_key: str, body: dict = Body(default=None), # as the rename (item 6a): two places to look, and the surfaces disagreed silently. def _field_or_refuse(session, table_key, fkey=""): - """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule.""" - defn = _defn_or_refuse(session, table_key) + """The schema wall. `_defn_or_refuse` first (404/403 on the table), then the per-field rule. + + ⭐ W33-T01 / D-213: definitions only. Everything read off `defn` here is `fields` and + `createdBy`; the returned value is DISCARDED by all three callers (they re-fetch what they + write through the `user_tables` write doors), so no projected snapshot survives into a write. + ⚠ It is a SCHEMA wall on a write route, not a row-write wall — the distinction contract C5 + draws is about the snapshot reaching a read-BACK, and this one does not escape the function.""" + defn = _defn_or_refuse(session, table_key, defs_only=True) ut = _ut() if not ut.is_user_table(table_key, st=session.runtime): raise err(400, "not_a_user_table", - "only a user-created database has an editable schema — a connected source " + "only a user-created database has an editable schema. A connected source " "owns its own columns") if fkey and not ut.may_edit_field(table_key, fkey, session.uname, session.admin, st=session.runtime): @@ -1126,12 +1266,41 @@ def add_field(table_key: str, body: dict = Body(default=None), # field relational", which is the question that was always meant. if field.get("type") in ("link", "rollup"): _refresh_relations(session) - return {"field": field} + return _with_dropped(ut, {"field": field}, body) + + +def _with_dropped(ut, out, body): + """Attach the NAMED list of config keys the validator did not keep (wave 34, R13 / W34-T51). + + ⛔⛔ THIS EXISTS BECAUSE THE VALIDATOR HAS NO ERROR CHANNEL AND CANNOT GROW ONE. Every bag + cleaner in `core/user_tables.py` returns `dict | None` and drops unknown keys in silence, and + `verify_fields_contract` asserts that they do -- so the drop is correct and the SILENCE is the + defect. T51's contract is that an unknown config key is dropped **and named**, so the naming + rides the response beside the accepted field rather than inside the validator. + + ⚠ OMITTED WHEN EMPTY, deliberately: an always-present `dropped: []` teaches every reader to + ignore the key, which is how a report stops being read before it stops being true. + """ + dropped = ut.ai_enrich_dropped_keys((body or {}).get("aiEnrich")) + if dropped: + out = dict(out) + out["dropped"] = dropped + return out def _refusal_sentence(ut, session, body, table_key="", fkey=""): """Why was this column refused? The specific reason when we can name one, the general list otherwise — never a specific-sounding guess.""" + # ⭐ WAVE-34 (R13): the enrichment column's own sentence, named BEFORE the automation bag + # below. `_clean_field` DERIVES `field.automation` for this kind, so a refused enrichment + # column would otherwise be explained by the flow law -- "pick a flow, or make this an + # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user + # never touched. + if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": + bag = (body or {}).get("aiEnrich") + if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): + return ("an AI enrichment column needs a prompt. It is the only thing that can " + "produce a value here, so a column without one would stay empty forever") # ⭐ C3 (wave 25, R7): the one-profile-per-table refusal NAMES THE EXISTING COLUMN, which is # what the contract asks for and what makes it actionable — "at most one" sends the reader # hunting through a 40-column schema for a flag they cannot see from the header. @@ -1143,11 +1312,11 @@ def _refusal_sentence(ut, session, body, table_key="", fkey=""): # real mistake was the column type sends them to fix the wrong thing, which is the # misdirection D-46 closed one door over. if str((body or {}).get("type") or "text").strip().lower() != "text": - return ("a profile column is a flag on an ordinary TEXT column — it validates what " + return ("a profile column is a flag on an ordinary TEXT column. It validates what " "is typed into it, which it can only do for text") existing = ut.profile_field(table_key, st=session.runtime) if table_key else None if existing and existing.get("key") != str(fkey): - return (f"this database already has a profile column — " + return (f"this database already has a profile column: " f"{existing.get('label') or existing.get('key')!r}. A database has at most " f"one, so the automation knows which handle to enrich; edit that column, or " f"take the flag off it first") @@ -1155,16 +1324,16 @@ def _refusal_sentence(ut, session, body, table_key="", fkey=""): if isinstance(bag, dict): flow = str(bag.get("flowId") or "").strip() if not flow: - return ("an automation column has to name the automation that fills it — pick a " + return ("an automation column has to name the automation that fills it. Pick a " "flow, or make this an ordinary column") if not ut.flow_bound(bag, st=session.runtime): return (f"this column names automation {flow!r}, which does not exist in this " - f"workspace — it may have been deleted; pick a flow that is still there") + f"workspace. It may have been deleted; pick a flow that is still there") kind = str((body or {}).get("type") or "").strip() if kind and kind not in ut.UT_FIELD_TYPES: return (f"{kind!r} is not a column type here (types: " f"{', '.join(sorted(ut.UT_FIELD_TYPES))})") - return (f"the column was refused — check the name and type, or the table may be at its " + return (f"the column was refused. Check the name and type, or the table may be at its " f"{ut.MAX_FIELDS}-column cap (types: {', '.join(sorted(ut.UT_FIELD_TYPES))})") @@ -1187,7 +1356,7 @@ def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), try: migrated = ut.rename_choice_values(table_key, fkey, renames, st=session.runtime) except Exception: - raise err(503, "store_unavailable", "the rename did not land — try again") + raise err(503, "store_unavailable", "the rename did not land. Try again") # The per-user workspace strata and any view filter naming the old value are the OTHER # half of C-RENAME and belong to `core.table_store`. Called only if it is there: an # enumerator's mirror waits for its counterpart rather than guessing at its shape, and a @@ -1214,7 +1383,85 @@ def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), out = {"field": field} if migrated is not None: out["migrated"] = migrated - return out + return _with_dropped(ut, out, body) + + +def _fire_on_change(table_key, pid, changed, session): + """Run any `on_change` enrichment column whose prompt names a cell that just moved. + + ⛔ ONE DEFINITION READ FOR THE WHOLE WRITE, and that is the point rather than an optimisation. + `automation_engine.grid_hook` calls `all_definitions(st)` once PER EVENT, which turns a + 20,000-row import into 20,000 whole-document reads on the single process this product runs + (`D-134`, and `W34-T54`'s own `how:` says not to rebuild it). `on_change_fields` is pure and + takes the definition, so this reads once and asks about every column. + + ⛔ AND IT NEVER FAILS THE WRITE. The cell edit has already succeeded and been acknowledged; + an enrichment that could not run is a missing value, not a lost edit, and the run's own report + carries the reason. ⚠ It is also deliberately SYNCHRONOUS and bounded to this one row: a + fan-out here would put a vendor call on the critical path of every keystroke-commit. + """ + import ai_enrich as _ae + + try: + defn = _ut().get(table_key, st=session.runtime) or {} + wanted = _ae.on_change_fields(defn, changed.keys()) + for field in wanted: + # ⭐ W35-T41 / C7 — `user` is the usage ledger's attribution. An on-change run is still + # somebody's edit spending somebody's tokens, so it is booked against the person who + # typed rather than left unattributed. + _ae.run_field(table_key, field["key"], st=session.runtime, rows=[str(pid)], + user=session.uname) + except Exception: # noqa: BLE001 + pass + + +@router.post("/tables/{table_key}/fields/{fkey}/enrich") +def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Run an AI enrichment column (wave 34, owner ruling R13). Returns the run's own REPORT. + + ⛔ THIS DOOR SPENDS MONEY, so it rides the same wall every other schema write rides + (`_field_or_refuse`) rather than a looser one of its own. A read-only viewer cannot bill the + tenant by opening a grid. + + `{"rows": ["3"]}` is a MANUAL run of exactly those records: a person asked, in front of the + value being replaced, so it skips the `overwrite` policy. An ABSENT `rows` is the automatic + plan, where the policy and the never-overwrite-a-human law both apply. The two are one + function with one flag, not two runners. + + ⚠ THE REPORT IS THE PRODUCT, not a status code. It carries `filled`, `failed`, `skipped` by + reason, `tokens` spent, the provider, per-row errors, and `limit` (R6's second sentence: a + ceiling that stopped the run names its cause and a remedy). A 200 with `filled: 0` and a + populated `skipped` is a correct, informative answer, and the client must render it rather + than treat it as success. + """ + _field_or_refuse(session, table_key, fkey) + import ai_enrich as _ae + + rows = (body or {}).get("rows") + if rows is not None and not isinstance(rows, list): + raise err(400, "bad_rows", "`rows` must be a list of record ids, or absent to run the " + "rows this column's own settings choose") + # The caller's permitted pool, the same one the row doors use. A named row outside it is + # dropped rather than refused: a stale client naming a record that has been deleted or moved + # out of scope should not fail a run over the rows it can legitimately fill. + if rows is not None: + allowed = {str(p) for p in scoped_pids(session, table_key)[0]} + rows = [str(r) for r in rows if str(r) in allowed] + # ⭐ `W34-T54`'s bulk menu is this one field: "Rows never filled" sends `blank`, "All rows" + # sends `always`. Anything else falls back to the column's own saved policy rather than to a + # default, so a typo cannot quietly widen what a run touches. + report = _ae.run_field(table_key, fkey, st=session.runtime, rows=rows, + policy=str((body or {}).get("scope") or "") or None, + # A named row set through THIS door is a person asking. + manual=rows is not None, + # ⭐ W35-T41 / C7 — the usage ledger's attribution. + user=session.uname) + if report.get("problem"): + # A run that could not start at all is not a 200: nothing was attempted, nothing was + # spent, and the reason is actionable (no provider configured, or the wrong column). + raise err(400, "enrich_refused", str(report["problem"])) + return report @router.delete("/tables/{table_key}/fields/{fkey}") @@ -1222,7 +1469,7 @@ def delete_field(table_key: str, fkey: str, session: Session = Depends(require_s _field_or_refuse(session, table_key, fkey) if not _ut().delete_field(table_key, fkey, st=session.runtime): raise err(400, "refused", - "that column could not be removed — a database must keep at least one") + "that column could not be removed. A database must keep at least one") _refresh_relations(session) return {"deleted": fkey} @@ -1247,7 +1494,7 @@ def delete_row(table_key: str, rid: str, session: Session = Depends(require_sess try: ok = _ut().delete_row(table_key, rid, st=lent) except Exception: - raise err(503, "store_unavailable", "the delete did not land — try again") + raise err(503, "store_unavailable", "the delete did not land. Try again") if not ok: raise err(400, "refused", "rows can only be deleted from user-created databases") _refresh_relations(session) @@ -1280,7 +1527,7 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), "pid": pid, "updates": updates}, ctx) except grid_events.StoreUnavailable: raise err(503, "store_unavailable", - "the tenant store is unavailable — your change was not saved") + "the tenant store is unavailable. Your change was not saved") # ⚠ THE READ-BACK IS THE DEFINITION ROW, AND ON A `ut_` SCOPE THAT IS THE WHOLE OF IT. # # ⛔ CORRECTED, wave-29 T22 (owner item 2a): this note used to say "THE READ-BACK SPANS BOTH @@ -1319,6 +1566,22 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), return False refused = sorted(k for k in updates if not _took(k)) + # ⭐⭐ WAVE-34 (R13) — THE HUMAN-EDIT STAMP, AND IT HAS TO HAPPEN HERE. "Did a person write + # this cell?" is not recoverable from the value afterwards, so the only place to record it is + # the door a person writes through. `ai_enrich_may_write` then refuses to let any automatic + # run overwrite it, whatever the column's `overwrite` policy says. + # ⚠ STAMPED FROM THE CELLS THAT ACTUALLY TOOK, never from what was asked: marking a refused + # write `human` would freeze a cell against the agent on the strength of an edit that never + # landed. `note_human_edit` filters to the enrichment columns itself and is a no-op otherwise. + took = {k: v for k, v in accepted.items() if k not in refused} + if took: + try: + _ut().note_human_edit(table_key, took, pid, st=session.runtime) + except Exception: # noqa: BLE001 + # Provenance is metadata about a write that has already succeeded. Failing the + # request here would tell the user their edit was lost when it was not. + pass + _fire_on_change(table_key, pid, took, session) out = {"pid": pid, "updates": accepted} if refused: out["refused"] = refused diff --git a/api/routes_usage.py b/api/routes_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..ae770793c62ea1ec928315c1bdd1b62ef2346abb --- /dev/null +++ b/api/routes_usage.py @@ -0,0 +1,51 @@ +"""routes_usage.py — WAVE 35 (ruling R9, contract C7): `GET /api/v1/usage`, the AI meter. + +Owner item 13's second door. R9: the dashboard REPORTS weekly usage per surface against an +allowance and does **not** cut anyone off this wave; over the allowance the product still works and +the bar is red. + +⛔ THIS ROUTE ENFORCES NOTHING AND MUST NOT START. It is the only reader of `usage_ledger`, and the +ledger has no ceiling in it (see that module's header). A `403 over_allowance` here would turn a +reporting feature into a wall the owner explicitly did not ask for this wave. + +⚠ TWO SCOPES, AND THE WIDER ONE IS ADMIN-ONLY. `?scope=me` (the default) is the caller's own week; +`?scope=tenant` folds every account and is refused for a non-admin — how much AI another employee +used is not a fact this product hands out sideways. +""" +from fastapi import APIRouter, Depends + +import usage_ledger +from deps import Session, err, require_session + +router = APIRouter(prefix="/api/v1") + + +@router.get("/usage") +def usage(scope: str = "me", session: Session = Depends(require_session)): + """`{week, resets, allowance, total, over, calls, unmeasured, scope, surfaces: [...]}`. + + Every one of the four surfaces is present whether or not it was used — see + `usage_ledger.usage`'s note on why a missing row and a quiet week must not look the same. + """ + wanted = str(scope or "me").strip().lower() + if wanted not in ("me", "tenant"): + raise err(400, "bad_request", "scope must be me or tenant") + if wanted == "tenant" and not session.admin: + raise err(403, "forbidden", + "only an administrator can see the whole workspace's AI usage") + out = usage_ledger.usage(session.runtime, session.uname, + tenant_wide=(wanted == "tenant")) + if session.admin and usage_ledger.UNATTRIBUTED["calls"]: + # ⛔ THE METER REPORTS ITS OWN BLIND SPOT (R6's second sentence). A call `record()` could not + # attribute — no store handle reached it, or the store refused — is absent from every number + # above, and an operator has to be able to see that rather than infer it. + # ⚠ ADMIN ONLY, and labelled as a PROCESS figure rather than a tenant one: the counter is a + # module global in this container, so it spans every tenant this process has served and is + # not a fact about the workspace being asked about. + out["unattributed"] = { + "calls": usage_ledger.UNATTRIBUTED["calls"], + "surfaces": dict(usage_ledger.UNATTRIBUTED["surfaces"]), + "note": ("these AI calls reached no ledger target in this container and are missing " + "from every figure above. It is a wiring gap, not usage"), + } + return out diff --git a/api/usage_ledger.py b/api/usage_ledger.py new file mode 100644 index 0000000000000000000000000000000000000000..4d42c06bc5bf7e6f06bddf11fbc908bf05a8be34 --- /dev/null +++ b/api/usage_ledger.py @@ -0,0 +1,281 @@ +"""usage_ledger.py — WAVE 35 (ruling R9, contract C7): ONE METER FOR EVERY AI SURFACE. + +R9: *"ONE meter for every AI surface. The dashboard REPORTS weekly usage against an allowance and +does NOT cut anyone off this wave; over the allowance it still works, shown red."* + +⛔ SO THIS FILE MEASURES AND NEVER ENFORCES. There is no ceiling here, no refusal, and no caller of +this module may branch on its answer. The one limit the product does enforce is per COLUMN and lives +where it was built (`ai_enrich.ceiling_report`); this is the tenant-wide REPORT, and mixing the two +would make an over-allowance week silently stop somebody's enrichment run. + +**THE FOUR SURFACES.** `assistant` (a person asking the Assistant a question) · `field_agent` (an AI +enrichment column filling cells) · `ai_review` (a model deciding a review stage) · `automation_draft` +(a sentence becoming a draft flow). They are the four LLM entry points the product has, and the gate +in `verify_api.py` DERIVES them from the code rather than reading this list, so a fifth entry point +is caught by the gate and not by somebody remembering to edit a tuple. + +⛔⛔ **`record()` MUST BE HANDED ITS TARGET, AND THAT IS A MEASUREMENT, NOT A STYLE CHOICE.** The +obvious design is to bind `(runtime, username)` into a `contextvars.ContextVar` inside +`deps.require_session` — every session-authed request passes through it — so `record()` could take +C7's five arguments and resolve the rest itself. **It does not work here and it fails SILENTLY.** +Measured 2026-08-17 with a FastAPI `TestClient`: a `ContextVar` set inside a sync dependency reads +back `None` in the handler *and in every function the handler calls*, on the SAME OS thread (17820 +both sides) — anyio copies the context per callable, so a `.set()` in the dependency's copy is +invisible to the endpoint's. An async endpoint reads `None` too. A ledger built that way would have +recorded nothing, answered no error, and shown a dashboard of zeros that looks exactly like a quiet +week ([[flag-shipped-without-its-writer]]). + +⚠ THEREFORE `st` AND `user` ARE KEYWORD ARGUMENTS AFTER C7'S FIVE, and a call that omits them is +COUNTED as unattributed and REPORTED rather than dropped — R6's second sentence applied to the meter +itself. A surface whose caller cannot reach a store handle is a gap somebody has to close; a meter +that hides the gap is worse than one that admits it. +""" +from __future__ import annotations + +import os +import time + +#: The store bucket. Small by construction — counters only, never a per-call log: a log would grow +#: without bound on a route that runs per row, and nothing in R9 asks a question a log answers. +STORE_KEY = "ai_usage" + +#: The four AI surfaces (C7). ⚠ A surface this list does not name is REFUSED rather than stored under +#: whatever string arrived, because a typo'd surface is a meter that reads low forever with a second +#: row nobody looks at. +SURFACES = ("assistant", "field_agent", "ai_review", "automation_draft") + +#: The weekly allowance the dashboard measures against (R9). ⛔ IT IS A REPORTING LINE, NOT A GATE. +#: Env-tunable so a deployment can set a real number without a release. +WEEKLY_ALLOWANCE = int(os.environ.get("AIOS_AI_WEEKLY_TOKENS") or 1_000_000) + +#: How many weeks stay in the bucket. A meter is about this week and the trend behind it; keeping +#: every week forever turns a counter document into an append-only log by another route. +MAX_WEEKS = 8 + +#: Calls this PROCESS could not attribute to a store and a user — see the header. Reported to an +#: admin by `GET /usage`, never silently zero. +UNATTRIBUTED = {"calls": 0, "surfaces": {}} + + +def week_of(when=None): + """The ISO week this instant belongs to, as `"2026-W33"`. + + ⚠ UTC, EXPLICITLY, and never a naive local stamp. `routes_nav` records the same correction for + its own stamps: a naive local time written on a UTC host and parsed by a non-UTC browser misfiles + into the wrong bucket with nothing to go red. `%G`/`%V` are the ISO year and ISO week, which is + the only pair that agrees with itself across a year boundary (`%Y-%W` does not). + """ + return time.strftime("%G-W%V", time.gmtime(when if when is not None else time.time())) + + +def resets_at(when=None): + """The UTC date the CURRENT week's counters roll over, as `"YYYY-MM-DD"`. + + R9 asks the page to state when the allowance resets, so the server answers it — a client that + computed "next Monday" itself would be a second implementation of the week boundary, and the two + would disagree for anybody whose clock is not UTC ([[one-question-two-normalizers]]). + """ + now = when if when is not None else time.time() + tm = time.gmtime(now) + # `tm_wday` is 0 for Monday, so days-until-next-Monday is 7 minus however far in we are. + return time.strftime("%Y-%m-%d", time.gmtime(now + (7 - tm.tm_wday) * 86400)) + + +def tokens_from(body): + """`(tokens_in, tokens_out)` for one provider response. Either may be None. + + ⛔ `None` MEANS THE PROVIDER DID NOT SAY, AND IT IS NOT ZERO. A ledger that books an unmeasured + call at zero reports a cheaper week than happened, which is exactly the cost-surprise complaint + R13 cited when the first token accounting was built. `ai_enrich._usage_tokens` has said this + since wave 34; this is that reader SPLIT, because C7 asks for input and output separately. + + ⚠ THE TOTAL-ONLY SHAPE IS REAL AND IS NOT A SPLIT. Some providers answer only `total_tokens`. + That is reported by `record(total=...)` rather than by inventing a split, because halving a + total would put two numbers on a dashboard that were never measured. + """ + usage = (body or {}).get("usage") + if not isinstance(usage, dict): + return (None, None) + + def _int(*keys): + for key in keys: + got = usage.get(key) + if isinstance(got, int) and not isinstance(got, bool): + return got + return None + + return (_int("prompt_tokens", "input_tokens"), + _int("completion_tokens", "output_tokens")) + + +def total_from(body): + """One call's TOTAL tokens, or None — the shape a caller with no use for the split wants. + + ⚠ Prefers the provider's own `total_tokens` over a sum, because a provider that reports both may + count cached or reasoning tokens in the total and in neither half. + """ + usage = (body or {}).get("usage") + if isinstance(usage, dict): + for key in ("total_tokens", "totalTokens"): + got = usage.get(key) + if isinstance(got, int) and not isinstance(got, bool): + return got + ins, outs = tokens_from(body) + if ins is None and outs is None: + return None + return (ins or 0) + (outs or 0) + + +def _int_or_none(value): + if isinstance(value, int) and not isinstance(value, bool): + return max(0, value) + return None + + +def record(surface, provider, model, tokens_in=None, tokens_out=None, *, + total=None, calls=1, st=None, user=""): + """Book ONE AI call. Returns True when the line landed durably, False when it could not. + + C7's signature is the five positional parameters; everything after `*` is this module's own + extension and each one has a reason: + + · `total` — for a caller that knows the total and not the split (`ai_enrich.run_field` + accumulates a per-run total through an `_ask` contract a gate injects against, and changing + that contract to carry a split would break the injected shape for no gain here). + · `calls` — so a RUN over many rows books one line with its real call count instead of one + store write per row. + · `st` / `user` — the target. Required in practice; see this module's header for the measured + reason a contextvar cannot supply them. + + ⛔ IT NEVER RAISES. A meter that can break the feature it measures is worse than no meter: a + store blip during an enrichment run would abort the run and lose the cells. + ⛔ AND IT NEVER SILENTLY SUCCEEDS. A call it cannot attribute increments `UNATTRIBUTED`, which + `GET /usage` shows an admin. + """ + key = str(surface or "").strip() + if key not in SURFACES: + # Refused rather than stored: a surface nobody named is a row nobody reads, and the meter + # would read low forever with no sign of why. + return False + ins, outs = _int_or_none(tokens_in), _int_or_none(tokens_out) + tot = _int_or_none(total) + if tot is None and (ins is not None or outs is not None): + tot = (ins or 0) + (outs or 0) + n = max(0, int(calls) if isinstance(calls, int) and not isinstance(calls, bool) else 1) + if not n: + return False + uname = str(user or "").strip().lower() + if st is None: + UNATTRIBUTED["calls"] += n + UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n + return False + week = week_of() + prov, mdl = str(provider or "")[:40], str(model or "")[:80] + + def _up(data): + data = data if isinstance(data, dict) else {} + weeks = {w: v for w, v in data.items() if isinstance(v, dict)} + row = ((weeks.setdefault(week, {}) + .setdefault(uname or "-", {})) + .setdefault(key, {})) + row["calls"] = int(row.get("calls") or 0) + n + if tot is None: + # ⛔ "the call happened, the tokens are unknown" IS A STATE, and it has to be + # representable or the meter quietly under-reports. The dashboard shows it as a count of + # unmeasured calls beside the token total. + row["unmeasured"] = int(row.get("unmeasured") or 0) + n + else: + row["tokens"] = int(row.get("tokens") or 0) + tot + if ins is not None: + row["tokens_in"] = int(row.get("tokens_in") or 0) + ins + if outs is not None: + row["tokens_out"] = int(row.get("tokens_out") or 0) + outs + if prov: + row["provider"] = prov + if mdl: + row["model"] = mdl + # Oldest weeks first to go. Sorting ISO week strings sorts chronologically by construction, + # which is the second reason `%G-W%V` is the format rather than a prettier one. + for stale in sorted(weeks)[:-MAX_WEEKS]: + weeks.pop(stale, None) + return weeks + + try: + # ⚠ `flush='async'` — the coalescing mode ([[store-async-flush]]). This is the hot path of + # every AI surface in the product (an enrichment run books one line per RUN, but the + # assistant books one per question), and a blocking hub upload per call would put a network + # round trip inside the answer a person is waiting for. The mutation applies to the + # in-process cache immediately, so `GET /usage` reads its own writes. + st.update(STORE_KEY, _up, flush="async") + return True + except Exception: # noqa: BLE001 + UNATTRIBUTED["calls"] += n + UNATTRIBUTED["surfaces"][key] = UNATTRIBUTED["surfaces"].get(key, 0) + n + return False + + +def _blank_row(): + return {"calls": 0, "tokens": 0, "tokens_in": 0, "tokens_out": 0, "unmeasured": 0} + + +def _fold(rows): + out = _blank_row() + for row in rows: + if not isinstance(row, dict): + continue + for field in ("calls", "tokens", "tokens_in", "tokens_out", "unmeasured"): + got = row.get(field) + if isinstance(got, int) and not isinstance(got, bool): + out[field] += max(0, got) + return out + + +def usage(st, user="", *, tenant_wide=False, week=None): + """This week's meter. `{week, resets, allowance, total, over, surfaces: [...], ...}`. + + `tenant_wide=True` folds every account in the tenant — the admin view. Otherwise only `user`. + + ⛔ EVERY SURFACE IS PRESENT WHETHER OR NOT IT WAS USED, at zero. A surface a consumer has to test + for is a surface a consumer forgets to test for, and R9 says ONE meter for every AI surface: a + missing row and a quiet week would render identically, and only one of them is true. + ⛔ NOTHING HERE IS ESTIMATED. A surface with no ledger line reports zero, never a guess + ([[no-unverifiable-aggregates]]). + """ + wk = str(week or week_of()) + try: + stored = st.get(STORE_KEY) or {} + except Exception: # noqa: BLE001 + stored = {} + by_week = stored.get(wk) if isinstance(stored, dict) else {} + by_week = by_week if isinstance(by_week, dict) else {} + uname = str(user or "").strip().lower() + if tenant_wide: + buckets = [b for b in by_week.values() if isinstance(b, dict)] + else: + buckets = [by_week.get(uname or "-")] if isinstance(by_week.get(uname or "-"), dict) else [] + surfaces = [] + for key in SURFACES: + folded = _fold([b.get(key) for b in buckets]) + folded["surface"] = key + surfaces.append(folded) + total = sum(s["tokens"] for s in surfaces) + unmeasured = sum(s["unmeasured"] for s in surfaces) + out = { + "week": wk, + "resets": resets_at(), + "allowance": WEEKLY_ALLOWANCE, + "total": total, + # R9: over the allowance the product STILL WORKS and the bar goes red. This flag is the + # colour, never a wall — nothing in the API reads it. + "over": bool(WEEKLY_ALLOWANCE and total > WEEKLY_ALLOWANCE), + "calls": sum(s["calls"] for s in surfaces), + "unmeasured": unmeasured, + "scope": "tenant" if tenant_wide else "me", + "surfaces": surfaces, + } + if unmeasured: + # R6's second sentence: the number is short by an unknown amount and the payload says so + # rather than presenting a confident total. + out["note"] = (f"{unmeasured} call(s) this week did not report a token count, so the total " + f"above is a floor. The provider decides whether to report usage") + return out diff --git a/api/web_agent.py b/api/web_agent.py index 1de71471fbc7c97fc3ddae43d39529794cf39253..00d58c51c0871457876ccf81c697deac757729fe 100644 --- a/api/web_agent.py +++ b/api/web_agent.py @@ -1,9 +1,10 @@ """web_agent.py — THE SEAM between an automation step and a browser that runs somewhere else. -CONTRACT C5 (wave 31, ruling R10 / D-51). E ships this; **C mounts it** in -`automation_engine.py`'s action dispatch for `web_read` and adds the `verify_wiring` row — -⛔ E cannot verify its own mounting, and an unmounted runner is a whole, correct, UNREACHABLE -feature, which is exactly how five wave-29 features shipped behind green gates. +CONTRACT C5 (wave 31, ruling R10 / D-51). E shipped this and C mounted it in +`automation_engine.py`'s action dispatch; both halves landed and the `verify_wiring` row exists. +⚠ That sentence was written in the FUTURE TENSE as a wave-31 hand-off and stayed that way for +three waves, which is the smaller cousin of the correction below: a fence note describes one +wave's division of labour and reads, afterwards, as a description of the code. result, error = web_agent.run_step(step, ctx) # -> (dict|None, str) @@ -30,22 +31,39 @@ foreground step of an automation RUN and far too slow for a route a person is wa cost is per JOB, not per step — `run_plan` sends a whole flow's steps in ONE job for that reason. ──────────────────────────────────────────────────────────────────────────────────────────────── -⛔ THREE THINGS ARE TRUE OF PRODUCTION TODAY AND ARE NOT DEFECTS IN THIS FILE. `capability()` -reports each as a sentence, and `verify_web_agent.py` holds each to a check, so none of them can -be discovered by a customer instead of by us: - - 1. THE SPACE'S TOKEN CANNOT PAY FOR A JOB. `deploy_web.py` pushes `HF_TOKEN`, which is scoped to - the org `royal-imports`; HF answers **402 `Pre-paid credit balance is insufficient`** there - and 0 jobs have ever run under it. `AIOS_HF_TOKEN` (user `fsanyoto`, 100 jobs) works and is - NOT pushed. Measured 2026-08-12. - 2. THE SPACE DOES NOT SHIP THE JOB SCRIPT. `deploy_web.py` uploads `api/*.py` and - `platform/{core,modules,harness}`; `jobs/` is in no manifest. And `check_upload()` walks - IMPORTS, so it structurally cannot see a file opened by PATH — its own comments say so, about - `aios_grid_fields.json`, which crashed a Space for exactly this reason. - 3. `routes_web_agent.py` IS NOT MOUNTED. `main.py` is another lane's file this wave. - -Each needs one line somebody else owns. Until then this module answers with a sentence rather -than a silence, which is the whole of R6's second half. +⛔⛔ WAVE 34 · W34-T44 — THIS BLOCK LISTED "THREE THINGS TRUE OF PRODUCTION TODAY". ALL THREE WERE +FALSE BY THE TIME ANYBODY READ THEM, AND THIS DOCSTRING IS WHERE THE ERROR CAME FROM. + +It said the Space token 402s, that `deploy_web.py` never uploads `jobs/`, and that +`routes_web_agent.py` is not mounted — so the whole web-action family was inert in production. +**Four later documents repeated it** (the wave-34 PRD's scouted index, the plan-time scout's +report, `W34-T44`'s own `how:`, and `run_gates.ps1`'s annotation telling a deploy to expect +`web_agent` at 48/50), and every one of them traced back here. Re-checked line by line, against +the files themselves, on 2026-08-16: + + 1. THE TOKEN IS PUSHED. `deploy_web.py:1128` adds `AIOS_HF_TOKEN` to the Space's secrets when it + is set, and `ENV_TOKEN` below prefers it over `HF_TOKEN`, so the org token that 402s is never + the one reached. The 402 itself is still true OF THAT TOKEN and is why the order matters. + 2. THE JOB SCRIPT SHIPS. `deploy_web.py::_jobs_files` (:413) collects `jobs/` and :867 uploads + it. The `check_upload()` caveat stands as a caveat: it walks IMPORTS and cannot see a file + opened by PATH, so this upload is covered by the manifest, never by that check. + 3. IT IS MOUNTED. `main.py:264`, `app.include_router(routes_web_agent.router)` — and + `verify_web_agent.py` asserts the route is in the SERVED set (`app.openapi()["paths"]`), + which is green. + +⭐ MEASURED END TO END the same day, from a dev box: `capability()` = ready True / "configured" / +`AIOS_HF_TOKEN` / playwright 1.62.0, and a real `web_read` of `https://example.com h1` submitted in +3.8 s, was alive at 21.4 s and returned `value "Example Domain"`, status 200, 106 ms in-browser, +24.5 s wall clock cold. +⚠ STILL UNPROVEN, so nobody over-corrects in the other direction: that run was NOT on the deployed +Space, which carries the secret only if the last deploy read this `.env`. Proving it needs a click +on staging, and until then "it works" means "it works from a box with the token". + +⛔ THE STANDING LESSON, kept here because this file is where it started: **a docstring stating +facts about PRODUCTION has no expiry and nothing re-reads it when the world changes.** Three +sentences about somebody else's file outlived all three of those files' fixes, and were quoted as +current by four documents and a ticket. `capability()` answers the same question in under a second +and returns TODAY's answer; prefer calling it to reading this. """ import base64 import hashlib @@ -223,7 +241,7 @@ def _sentence_for(exc, namespace): f"point this deployment at one that has it ({ENV_NAMESPACE} / {ENV_TOKEN[0]}). " f"Nothing was read.") if status in (401, 403): - return (f"Hugging Face refused the browser job as unauthorised for {where} — this " + return (f"Hugging Face refused the browser job as unauthorised for {where}. This " f"deployment's token does not carry `job.write` on that account. Point it at an " f"account the token owns ({ENV_NAMESPACE}), or configure a token that owns this " f"one ({ENV_TOKEN[0]}). Nothing was read.") @@ -524,7 +542,7 @@ def _await(api, job_id, namespace, started, log): break if state in ("COMPLETED", "ERROR", "CANCELED"): return None, (f"The browser job {job_id} ended {state} without returning a result" - f"{' — it never started printing' if not booted else ''}. " + f"{'. It never started printing' if not booted else ''}. " f"Nothing was read.") if not booted and waited > START_TIMEOUT: # The submitted-but-never-started case, named as itself. @@ -534,7 +552,7 @@ def _await(api, job_id, namespace, started, log): pass return None, (f"The browser job {job_id} was accepted but had not started after " f"{int(START_TIMEOUT)}s (it stayed {state or 'queued'}), so it was " - f"cancelled. Nothing was read — try again, or the platform is busy.") + f"cancelled. Nothing was read. Try again, or the platform is busy.") if waited > RUN_TIMEOUT: try: api.cancel_job(job_id=job_id, namespace=namespace) diff --git a/jobs/web_agent_job.py b/jobs/web_agent_job.py index 43396a0e4907fa63b56988428df4ab5e41df2c6b..358deb37cc3c541a1e1dee8da0771a3b10e41b2f 100644 --- a/jobs/web_agent_job.py +++ b/jobs/web_agent_job.py @@ -113,6 +113,21 @@ INSTALL_DEPS = os.environ.get("WEB_AGENT_INSTALL_DEPS", "1") != "0" INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0" MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload MAX_STEPS = 20 # matches the engine's MAX_ACTIONS — one flow, one job +#: ⛔ D-193 — HOW OLD A CHECKPOINT MAY BE AND STILL BE SERVED, in hours. **Deliberately SHORT.** +#: The key is content-addressed (`runId` + a hash of the steps), so a scheduled automation with a +#: stable run id hits the same key every time it runs; with no ceiling it would be served the same +#: stored page for as long as the file existed, which is a measurement of a web page presented with +#: today's date on it. Six hours makes a same-morning retry cheap and a next-day run honest. +#: ⚠ A CEILING, NOT A CACHE POLICY. The point of the checkpoint is that a RETRY of a job that +#: already paid for a fetch does not pay again; that window is minutes, not days. +try: + CHECKPOINT_MAX_AGE_H = float(os.environ.get("WEB_AGENT_CHECKPOINT_MAX_AGE_H", "6") or 6) +except (TypeError, ValueError): + CHECKPOINT_MAX_AGE_H = 6.0 +#: How many expired `runs/*.json` one job may delete, in ONE commit. ⛔ Bounded because the repo +#: already spends a commit per run against HF's 256-commits/hr budget, and a sweep that spent one +#: commit per FILE would be a denial of service on the store it is tidying. +CHECKPOINT_SWEEP_MAX = 200 def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None: @@ -162,6 +177,8 @@ def _checkpoint(blob: str, payload: dict) -> None: repo_id=CHECKPOINT_REPO, repo_type="dataset", commit_message=f"web-agent run {run_id}") say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}") + # ⚠ AFTER the write, never before: the sweep is housekeeping and the result is the job. + _sweep_expired() except Exception as exc: # noqa: BLE001 — a boundary # ⛔ CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can # fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not @@ -171,8 +188,94 @@ def _checkpoint(blob: str, payload: dict) -> None: f"the stdout frame still carries the result") +def _too_old(payload: dict) -> str: + """"" if this checkpoint may still be served, else the SENTENCE saying why it may not. + + ⛔ D-193 — A RESUME THAT IS A YEAR OLD IS NOT A RESUME, IT IS A STALE ANSWER SERVED AS FRESH. + The key is deliberately content-addressed (`runId` + a hash of the steps), so a scheduled + automation reusing a stable run id resumed the SAME STORED PAGE for as long as the file + existed. `finishedAt` was written by `_emit` from the first day and read by nothing. + ⚠ AN UNREADABLE OR ABSENT TIMESTAMP IS TREATED AS TOO OLD, not as fresh. A checkpoint written + before this field existed carries no date, and "no date" must not be the one input that gets + served forever — fail toward re-fetching, which costs a job, rather than toward a wrong answer, + which costs a number nobody can explain. + """ + stamp = str(payload.get("finishedAt") or "") + if not stamp: + return "it carries no finishedAt, so its age cannot be established" + try: + when = datetime.fromisoformat(stamp) + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + except Exception: # noqa: BLE001 + return f"its finishedAt ({stamp[:40]}) is not a date this job can read" + hours = (datetime.now(timezone.utc) - when).total_seconds() / 3600.0 + if hours > CHECKPOINT_MAX_AGE_H: + return (f"it finished {hours:.1f} h ago and the ceiling is {CHECKPOINT_MAX_AGE_H} h " + f"(WEB_AGENT_CHECKPOINT_MAX_AGE_H)") + return "" + + +def _sweep_expired() -> None: + """Delete checkpoints past the ceiling, ONE COMMIT, at most `CHECKPOINT_SWEEP_MAX` of them. + + ⛔ D-193's SECOND CLAUSE, and it is a different problem from the first. Ageing out the READ + stops a stale answer being served; it does not stop the repo growing one file per run forever. + ⛔ ONE COMMIT FOR THE WHOLE SWEEP. `upload_file`/`delete_file` each spend a commit, and this + repo already spends one per run against HF's 256-per-hour budget — a per-file sweep would + exhaust the budget it exists to protect, and the failure would land on the next run's WRITE. + `create_commit` with a list of delete operations is one commit whatever the count. + ⚠ AGE IS READ FROM THE LISTING, not by downloading each file. `last_commit` is what the API + gives cheaply; downloading N files to read a timestamp inside them would make a tidy-up cost + more than the runs it is tidying. + ⚠ REPORT-DON'T-RAISE, like `_checkpoint`: this runs AFTER the result is in hand, and a + housekeeping failure must never destroy a result that has already been paid for. + """ + if not (CHECKPOINT_REPO and HF_TOKEN): + return + try: + from huggingface_hub import CommitOperationDelete, HfApi + api = HfApi(token=HF_TOKEN) + cutoff = datetime.now(timezone.utc).timestamp() - CHECKPOINT_MAX_AGE_H * 3600.0 + doomed = [] + for info in api.list_repo_tree(CHECKPOINT_REPO, repo_type="dataset", path_in_repo="runs", + expand=True, recursive=False): + path = getattr(info, "path", "") + when = getattr(getattr(info, "last_commit", None), "date", None) + if not path.endswith(".json") or when is None: + continue + if when.timestamp() < cutoff: + doomed.append(path) + if len(doomed) >= CHECKPOINT_SWEEP_MAX: + break + if not doomed: + return + api.create_commit(repo_id=CHECKPOINT_REPO, repo_type="dataset", + operations=[CommitOperationDelete(path_in_repo=p) for p in doomed], + commit_message=f"sweep {len(doomed)} checkpoints past " + f"{CHECKPOINT_MAX_AGE_H} h") + say(f"CHECKPOINT swept {len(doomed)} expired run(s) in one commit") + except Exception as exc: # noqa: BLE001 — a boundary + say(f"CHECKPOINT sweep skipped ({type(exc).__name__}: {str(exc)[:160]}) — " + f"the result is unaffected") + + def _resume(run_id: str): - """A prior run's result, if this exact run already finished. None otherwise.""" + """A prior run's SUCCESSFUL and RECENT result, if one exists. None otherwise. + + ⛔ D-199 — A FAILED RUN'S CHECKPOINT USED TO REPLAY FOREVER. `_fail` checkpoints too + (`_fail` → `_emit` → `_checkpoint`), so the stored envelope for a failure is exactly as + truthy as one for a success; the caller tested `if prior:` and nothing ever looked at `ok`. + Under a stable `runId` a failed fetch could then never be re-attempted — it replayed the + stored failure unchanged, with `resumedFromCheckpoint: True` on it. The defect was LATENT + only because no deployment set `WEB_AGENT_REPO`, i.e. it armed itself on the day somebody + switched checkpointing on, which is the day nobody would be reading this function. + ⚠ THE TEST BELONGS HERE, NOT AT THE CALL SITE. `_resume` is the one place that decides what a + resumable checkpoint IS; a caller-side `if prior and prior.get("ok")` would be a second + definition, and the next caller would be written from the first one. + ⚠ A REFUSED CHECKPOINT SAYS SO IN THE LOG. Silently re-running looks identical to never + having had a checkpoint, and those two need to be distinguishable at 3am. + """ if not (CHECKPOINT_REPO and HF_TOKEN and run_id): return None try: @@ -181,9 +284,21 @@ def _resume(run_id: str): filename=f"runs/{run_id}.json", token=HF_TOKEN, force_download=True) with open(path, encoding="utf-8") as fh: - return json.load(fh) + prior = json.load(fh) except Exception: # noqa: BLE001 return None # absent is the normal case + if not isinstance(prior, dict): + say(f"RESUME refused — runs/{run_id}.json is not an object; re-running") + return None + if not prior.get("ok"): + say(f"RESUME refused — the stored run FAILED " + f"({str(prior.get('error') or 'no reason recorded')[:120]}); re-running") + return None + stale = _too_old(prior) + if stale: + say(f"RESUME refused — {stale}; re-running") + return None + return prior # ── the plan ──────────────────────────────────────────────────────────────────────────────── diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 5728254c4029ef2e883da439dfd1f9757c83e5f0..80b23e386a224df3567eb29d551b7eaa22c9f2cd 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -120,9 +120,20 @@ def _round(v): #: surfaces that CREATE them are the user-table databases, where a relation between two tables is #: a thing that exists. On the Odoo-backed Customer/Product grids there is no second user table to #: point at, so the column menu there simply never offers one. +#: ⭐⭐ WAVE-34 (owner ruling R13) — `ai_enrich` JOINS, and it had to join HERE in the same change +#: that put it in `UT_FIELD_TYPES`, not a ticket later. The wave planned a SERVER-FIRST landing on +#: the reasoning that a kind the client does not offer is invisible while a kind the server refuses +#: deletes a column. That reasoning is sound and the conclusion was still wrong, because THREE +#: parity gates chain over these sets and none of them permits a partial landing: +#: `verify_api` W18-UT `UT_FIELD_TYPES - CUSTOM_FIELD_TYPES == set()` (this line) +#: `aw_fields_contract` §5 `CREATABLE_TYPES == CUSTOM_FIELD_TYPES`, EXACT set equality +#: `types.ts` `CREATABLE_TYPES: readonly FieldType[]`, so the union must carry it too +#: Measured live by lane B at 17:09: `api_api` went 969/969 to 968/969 the moment the kind entered +#: `core/user_tables.py` alone, printing `got {'ai_enrich'} want set()`. The comments on +#: `json`, `link`/`rollup` and `code` below all say the same thing in their own words. CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date", "checkbox", "phone", "email", "url", "rating", "created_time", "formula", - "automation", "image", "json", "link", "rollup", "code"} + "automation", "image", "json", "link", "rollup", "code", "ai_enrich"} #: ⭐ WAVE-27 item 13 (owner ruling R13) — the `code` field's LANGUAGES. #: @@ -1344,6 +1355,13 @@ def _clean_display(raw, valid_keys): dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY — never the whole array, because losing one column should not cost the user a dashboard they spent time building. + + W33-T45 (contract C1, ruling R5) adds `published` + `publishAccess`. ⚠ THE HALF OF THE ROUND + TRIP THIS FUNCTION CANNOT ENFORCE: `customer-grid/types.ts::cleanDisplay` is a SECOND + normalizer, in the browser, which rebuilds the config key by key on every autosave and drops + anything it does not name. A key accepted here and unknown there dies on the next column + resize — silently, because the client then POSTs the stripped config and the host REPLACES + the stored one. "The host accepts it" is half a round trip; that file is the other half. """ if not isinstance(raw, dict): return None @@ -1522,6 +1540,28 @@ def _clean_display(raw, valid_keys): form = _clean_form(raw.get('form'), valid_keys) if form: out['form'] = form + # ── ⭐⭐ W33-T45 / CONTRACT C1 / RULING R5 (owner item 8b): IS THIS INTERFACE PUBLISHED ──── + # + # ⛔ TWO KEYS LIVE HERE AND TWO DELIBERATELY DO NOT. The `published` flag and the sharer's + # `public | password` choice are DISPLAY state — the view says what it is, the client renders + # a badge from it, and it travels with the view like every other key in this dict. The SECRET + # TOKEN and the PASSPHRASE HASH do not: they live in the server-only bucket, exactly as + # `routes_forms.py`'s `TOKENS_KEY` holds the form token. `config.display` is echoed back to + # every user who can open the view, so a token in here is a token published to the audience + # the password was meant to exclude. `_clean_form` above carries the same rule and the same + # reason; this is the second door, not a new one. + # + # ⛔ AND THE COERCION IS FAIL-CLOSED, WHICH IS WHY THIS IS NOT A BARE ALLOWLIST. A plain + # allowlist drops an unrecognised `publishAccess` and KEEPS `published: True` — leaving a + # published view with no stated access, i.e. a third state neither the ruling nor the client + # has a meaning for, on the one key where guessing wrong publishes a tenant's data to the + # open internet. So: a published view ALWAYS carries an access, and anything that is not the + # literal `'public'` reads as `'password'`. The unpublished case stores nothing at all — + # absent means unpublished, and a `published: False` would be the second way to say nothing + # that `kanbanClamp` and `mode: 'grid'` are both here to forbid. + if raw.get('published') is True: + out['published'] = True + out['publishAccess'] = 'public' if raw.get('publishAccess') == 'public' else 'password' return out diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index d52218949811a431b8d60c2cae68fe1b0d12cb35..ab13e52200c16ed629d388103bf5eb197a27e67f 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -10,12 +10,25 @@ "default": true, "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." }, + { + "key": "partner_id", + "label": "Odoo ID", + "type": "int", + "source": "odoo", + "derived": true, + "default": false, + "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." + }, { "key": "odoo_status", "label": "Odoo record", "type": "status", "source": "odoo", "default": false, + "options": [ + "Active", + "Archived" + ], "description": "Whether this customer still exists in Odoo. Archived means deleted there." }, { @@ -458,6 +471,63 @@ "type": "select", "source": "odoo", "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller." + }, + { + "key": "needs_pricing", + "label": "Needs pricing", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the NEEDS PRICING tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "march_pricelist", + "label": "March pricelist", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the March Pricelist tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "price_changes", + "label": "Price changes", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the Price Changes tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "closeouts", + "label": "Closeouts", + "type": "select", + "source": "overlay", + "default": false, + "options": [ + "Yes" + ], + "shared": true, + "description": "Team-maintained. A SKU carries “Yes” when it appears on the Closeouts tab of the 2027 catalog workbook; no value means it is not on that list. Shared with everyone in the workspace — the whole point of R8 is that the TEAM's work lands in the system, not one importer's private column." + }, + { + "key": "notes", + "label": "Notes", + "type": "text", + "source": "overlay", + "default": false, + "shared": true, + "description": "Team-maintained. The 2027 workbook's own non-Odoo columns (Product Description, Packing) folded into one field. Shared with everyone in the workspace." } ] } diff --git a/platform/core/alerts.py b/platform/core/alerts.py index c7b9ac070259e1351acb2b1c3ef2d2be065f8423..b8cd80e417fa849ce11a35f653f4f1a199104152 100644 --- a/platform/core/alerts.py +++ b/platform/core/alerts.py @@ -209,7 +209,7 @@ def _queue(rec, pids, labels, st=None): _st(st).update(NOTIFICATIONS_KEY, _apply, flush='async') -def notify(owner, label, *, topic='automation', key='', row_id='', detail='', st=None): +def notify(owner, label, *, topic='automation', key='', row_id='', detail='', actor='', st=None): """⭐ WAVE 23 (C6) — put ONE notification in a named user's inbox, for a producer that is not a view-alert evaluation. @@ -222,6 +222,24 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', st The inbox shape is UNCHANGED, deliberately: `AlertsPane` reads one list, and a second row shape would mean a client that must branch on where a notification came from. `alertId` carries the producer's key so a click-through can route (W23-W5). + + ⭐⭐ W33-T28 (`ASK C-14`) — `actor` IS THE ONLY THING THAT KNOWS WHO. Owner item: *"the Inbox + reads like email"*, and a verifier reading the carried wave-32 surface found the SENDER position + occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you", i.e. a + CATEGORY where mail puts a who — with no sender field anywhere on the wire, in the model or in + the markup. An alert and an automation have no person behind them and are honestly named by + their machine; **a SHARE has a real person, and only the PRODUCER knows their name.** By the time + the read path sees the row that fact is gone, so it has to be carried at write time. + + ⛔ ADDITIVE, AND THAT IS THE WHOLE DESIGN. No existing caller changes, no stored row changes + shape, and every reader already tolerates the key's absence — so an inbox written before this + line reads exactly as it did. The alternative C rejected is worth recording because it is the + tempting one: recovering the name by REGEXING the body prose (`" shared this with you"`). + A sender parsed out of a sentence breaks the first time the sentence is reworded, and it breaks + SILENTLY, in the header — [[one-question-two-normalizers]] with the second normaliser being + English. + ⚠ Blank is a legitimate value and means *"no person did this"*, NOT "unknown". The client falls + back BY KIND rather than to an empty From column, so an automation never borrows a human's name. """ owner = str(owner or '').strip() if not owner or not label: @@ -230,7 +248,8 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', st item = {'id': f"{key or topic}:{row_id or 'n'}:{at}", 'alertId': str(key or ''), 'viewId': '', 'topic': str(topic), 'rowId': str(row_id or ''), 'label': str(detail or label), - 'alertLabel': str(label), 'at': at, 'read': False} + 'alertLabel': str(label), 'at': at, 'read': False, + 'actor': str(actor or '')} def _apply(data): box = list(data.get(owner) or []) diff --git a/platform/core/data_binding.py b/platform/core/data_binding.py new file mode 100644 index 0000000000000000000000000000000000000000..45e592b90f80ac433c9ce3210fbd8e4a103e6ce0 --- /dev/null +++ b/platform/core/data_binding.py @@ -0,0 +1,270 @@ +"""WHICH STORE THIS DEPLOYMENT IS ALLOWED TO WRITE — the fail-closed binding (D-315 / D-298). + +⛔⛔ THE SENTENCE THIS FILE EXISTS TO MAKE IMPOSSIBLE. Owner, 2026-08-18, verbatim: *"this is so +critical that whenever we do staging, we don't fuck up anyone's data. EVER."* It had already +happened once: `fsanyoto/loopable` became STAGING on 2026-08-14 having never been given a store of +its own, so two different builds wrote tenant #0's real business data. Compose that with D-305 +(the store is last-write-wins at whole-document granularity) and a routine staging deploy does not +merely add test rows — it can erase a real user's views, fields and typed cell values. + +⛔ A DEPLOY FLAG IS NOT THE FIX; THE FLAG IS THE THING THAT ALREADY FAILED. The isolation that +existed was bound to a ROLE ("the staging Space"), and a guard bound to a role stops guarding the +moment the role moves — which these two Spaces have done twice. So the binding here is on the +container's OWN IDENTITY, which no deploy can forget to pass and no role swap can invalidate. + +⭐⭐ THE RULE, AND IT CLASSIFIES THE DEPLOYMENT RATHER THAN THE REPO: + + 1. the PRODUCTION deployment (identity in `PRODUCTION_SPACES`) -> may write anything + 2. any other deployment writing a KNOWN production store -> REFUSED + 3. any other deployment writing a repo on its OWN allowlist -> allowed + 4. anything else -> REFUSED + +⛔ WHY NOT CLASSIFY THE REPO, which is the design everybody reaches for first. A DENYLIST of +production stores fails OPEN for the next tenant nobody remembered to add. A SUFFIX RULE +(`*-staging-data`) pins a SPELLING rather than a claim — see [[gate-pins-a-spelling-not-a-claim]] — +and a store named without the suffix is unprotected while reading as protected. Rule 4 is the one +that survives a tenant nobody has provisioned yet: an unknown repo is refused, so a store this +deployment was never handed cannot be written by accident. + +⭐⭐ AND RULE 4 IS NOT THEORETICAL — IT IS THE HALF OF D-315 THE REGISTER ROW DID NOT KNOW ABOUT. +Measured 2026-08-18 by reading both control planes: `royal-imports/cfo-os-staging-data`'s own +`tenants.json` is byte-identical to live's and names **`royal-imports/aios-nurilab-data`, +`aios-gtmlab-data` and `aios-loopable-data`** — the three other tenants' REAL stores. So +`--data-repo=` only ever isolated tenant #0; `harness/runtime.py`'s `store.for_repo(repo)` path +resolved three PRODUCTION repos out of the staging store and wrote them, and no flag has ever +touched that path. The exposure is FOUR production stores, not one. + +⚠ READS ARE NOT REFUSED, AND THAT IS A DELIBERATE, STATED SCOPE DECISION. `Store.get` is lenient +by contract: a refused read would hand back `{}`, a caller would paint an empty grid, and the whole +Space would read as "no data" rather than as "refused" — the exact +[[empty-answer-vs-unfinished-answer]] shape this codebase has shipped before. A refused WRITE is +loud (503, with the reason). So staging still SEES production data when pointed at it; it can no +longer CHANGE it. Narrowing the read is a separate decision and is booked, not smuggled in here. + +⚠ ABSENT IDENTITY FAILS CLOSED, AND THE FAILURE LOOKS LIKE AN OUTAGE. If a container exposes none +of the identity variables, it is treated as "not the production deployment" and every write to a +production store refuses. That is the safe direction, but on a live Space it would read as a total +product outage — which is why `describe()` reports the RAW observations and not just the verdict: +per D-160 the Space environment cannot be read from outside, so the container has to say what it +saw. One deploy then answers "does SPACE_ID exist here", instead of three. +""" +import os + +#: The deployment that OWNS production data. A frozenset rather than a string because the product +#: has had two public addresses before and will again. +#: +#: ⚠ THIS SET IS DUPLICATED IN `aios-web/deploy_web.py` AS `PUBLIC_LIVE`, on purpose and with a +#: gate: `deploy_web` is a standalone CLI that must run without the platform package importable, so +#: it cannot import this. `verify_store_binding.py::section_constants` asserts the two literals +#: agree — a constant two features share is a constant that drifts ([[constant-two-features-share]]). +PRODUCTION_SPACES = frozenset({'fsanyoto/runloopable'}) + +#: The stores that hold REAL customer business data. Enumerated 2026-08-18 from the live control +#: plane (`royal-imports/cfo-os-data::tenants.json`), not guessed. +#: +#: ⛔ THIS LIST IS RULE 2, NOT THE WHOLE GUARD. It exists to make a DELIBERATE mis-pointing loud — +#: `OS_DATA_REPO=royal-imports/cfo-os-data` typed on a laptop, or a production repo added to a +#: sandbox allowlist by hand. Rule 4 is what covers a tenant provisioned after this line was +#: written, which is why the file does not depend on this staying complete. +PRODUCTION_STORES = frozenset({ + 'royal-imports/cfo-os-data', # tenant #0 — Royal Imports + 'royal-imports/aios-nurilab-data', # Nurilab + 'royal-imports/aios-gtmlab-data', # GTM Lab + 'royal-imports/aios-loopable-data', # Loopable +}) + +#: The historical default of `OS_DATA_REPO`. Kept as a name so the resolution below can say what it +#: is refusing to assume, rather than repeating a literal. +PRODUCTION_DEFAULT_STORE = 'royal-imports/cfo-os-data' + +#: The env vars a Hugging Face Space container sets to describe itself, in the order they are +#: trusted. `SPACE_ID` is the canonical one; the pair is the documented fallback. Read as a LIST so +#: `describe()` can report every one of them and what it held. +_IDENTITY_VARS = ('SPACE_ID', 'SPACE_AUTHOR_NAME', 'SPACE_REPO_NAME', 'SPACE_HOST') + +#: The env a NON-production deployment uses to name the stores it may write, beyond `OS_DATA_REPO`. +#: Comma-separated. +#: +#: ⭐ THE POLARITY IS THE POINT, and it is what keeps trap 1 satisfied. Forgetting to set this +#: makes a deployment write LESS, never more — so an omitted deploy argument is a refusal, not a +#: silent grant. That is the exact inverse of `--data-repo`, whose omission printed +#: "OS_DATA_REPO unchanged" and left a Space on production. +_SANDBOX_STORES_VAR = 'AIOS_SANDBOX_STORES' + +#: The local (no-identity) escape hatch. Present ⇒ writes to production stores are permitted from a +#: machine with no Space identity, AND every commit this process makes is stamped distinguishably +#: (see `local_commit_suffix`). Deliberately does NOT widen a Space: a container that HAS an +#: identity and is not production is refused with no override, because that is the case the owner +#: said "EVER" about. +_LOCAL_OPT_IN_VAR = 'AIOS_ALLOW_PRODUCTION_DATA' + + +class StoreWriteRefused(RuntimeError): + """A write this deployment is not allowed to make. + + ⛔ DELIBERATELY NOT A SUBCLASS OF `grid_events.StoreUnavailable`, and the reason is the routes. + Several of them carry `except StoreUnavailable:` blocks that DEGRADE to a session-scoped + fallback workspace — correct for an outage, catastrophic for a refusal, because the user would + be told the change was saved somewhere. A distinct type falls through all of them to the + app-level handler, which answers 503 with the reason and never claims a save. + """ + + +def _env(name): + return str(os.environ.get(name) or '').strip() + + +def deployment_id(): + """This container's own identity, or `''` when it is not a Space (i.e. local dev). + + ⚠ NOT DERIVED FROM ANY DEPLOY ARGUMENT. That is the whole design: `SPACE_ID` is set by the + platform inside the container, so it cannot be forgotten, mistyped, or left behind by a Space + that changed role without being redeployed. + """ + sid = _env('SPACE_ID') + if sid: + return sid + author, repo = _env('SPACE_AUTHOR_NAME'), _env('SPACE_REPO_NAME') + if author and repo: + return f'{author}/{repo}' + return '' + + +def is_production_deployment(): + """True only on the deployment that owns production data. Unknown ⇒ False (fail closed).""" + return deployment_id() in PRODUCTION_SPACES + + +def is_space(): + """True when this process is running inside a Space at all. + + The distinction matters exactly once: a Space that is not production gets NO override, while a + laptop does. `deployment_id()` conflates them into `''`, so ask separately. + """ + return bool(deployment_id()) + + +def sandbox_allowlist(): + """The repos a non-production deployment may write: `OS_DATA_REPO` + `AIOS_SANDBOX_STORES`.""" + allowed = {r.strip() for r in _env(_SANDBOX_STORES_VAR).split(',') if r.strip()} + own = _env('OS_DATA_REPO') + if own: + allowed.add(own) + return frozenset(allowed) + + +def local_override(): + """True when a machine with no Space identity has explicitly opted in to production data.""" + return (not is_space()) and _env(_LOCAL_OPT_IN_VAR) == '1' + + +def default_store(): + """What `OS_DATA_REPO` resolves to when it is unset. + + ⛔⛔ FACT (1) OF D-315 DIES HERE: `core/store.py` used to read + `os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')`, so a Space that was never given + the key — every NEW Space, by construction — came up bound to tenant #0's real business data. + The default now exists ONLY on the deployment that owns it. + + ⚠ IT STILL RETURNS THE PRODUCTION ID ELSEWHERE, RATHER THAN `None` OR `''`, AND THAT IS + DELIBERATE. A `None` repo would make `Store.__init__` — which runs at IMPORT time via + `_DEFAULT = for_repo(REPO)` — raise or bind something nonsensical, killing the app and every + gate on any non-production machine. Returning the real id keeps binding total and lets + `write_refusal` refuse the WRITE with a message that names the actual problem. Reads still + work, so a developer sees a populated app that will not let them damage it. + """ + return PRODUCTION_DEFAULT_STORE + + +def write_refusal(repo): + """`None` if this deployment may write `repo`, else the sentence explaining why not. + + ⭐ RETURNS A REASON RATHER THAN A BOOL, because [[report-the-cause-before-you-fix-it]]: a + silent refusal makes every theory about it unfalsifiable, and this one fires on a path where + the operator cannot read the container's environment (D-160). + """ + rid = str(repo or '').strip() + if not rid: + return ('no store repo is bound. Refusing to guess: an unnamed store is how a deployment ' + 'ends up on somebody else\'s data.') + if is_production_deployment(): + return None # rule 1 — the live Space owns its data + if rid in PRODUCTION_STORES: # rule 2 — never, from anywhere else + if local_override(): + return None + who = deployment_id() or 'this machine (no Space identity)' + if is_space(): + return (f'{who} is not the production deployment, so it may not write {rid}, which ' + f'holds real customer data. Point this Space at its own store with ' + f'--data-repo=, or add it to {_SANDBOX_STORES_VAR}. There is no override for ' + f'a Space: a build that is not live never writes live data.') + return (f'{who} may not write {rid}, which holds real customer data. Set OS_DATA_REPO to a ' + f'sandbox store, or set {_LOCAL_OPT_IN_VAR}=1 to opt in deliberately (every commit ' + f'is then stamped as a local write so it can be told apart afterwards).') + allow = sandbox_allowlist() # rule 3 — what it was explicitly handed + if rid in allow: + return None + who = deployment_id() or 'this machine (no Space identity)' + listed = ', '.join(sorted(allow)) or '(nothing)' + return (f'{who} was not given {rid}. A non-production deployment may write only the stores it ' + f'was handed: {listed}. Add it to {_SANDBOX_STORES_VAR} if that is intended.') + + +def check_write(repo, operation='write'): + """Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door.""" + reason = write_refusal(repo) + if reason: + raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}') + + +def local_commit_suffix(): + """A marker appended to commit messages made under the local override, else `''`. + + ⭐ THIS IS THE SECOND HALF OF D-298's EXIT CONDITION, satisfied rather than argued around: *"a + local run either cannot reach the live store without an explicit opt-in, OR stamps its commits + distinguishably"*. The measured complaint was that a laptop's commits were titled + `update user_tables` — byte-identical to what the live Space writes every few minutes — so a + local run and production were indistinguishable in the store's git history, which is the ONLY + audit trail this product has. Both clauses now hold: the opt-in is required AND it stamps. + """ + if not local_override(): + return '' + import getpass # noqa: PLC0415 — only on the opt-in path + import socket # noqa: PLC0415 + try: + who = getpass.getuser() + except Exception: # noqa: BLE001 + who = 'unknown' + try: + host = socket.gethostname() + except Exception: # noqa: BLE001 + host = 'unknown' + return f' [LOCAL DEV {who}@{host}]' + + +def describe(repo=None): + """What this container sees about itself — the payload behind the status field. + + ⛔ IT REPORTS THE RAW OBSERVATIONS, NOT ONLY THE VERDICT, and that is the point rather than + debug noise. D-160 established that a Space's environment cannot be read from outside, so + "is staging on the right store?" is a question only the container can answer. A verdict alone + ("production: false") cannot distinguish "SPACE_ID says staging" from "SPACE_ID does not exist + on this platform at all" — two states with the same verdict and completely different fixes. + Reporting the variables makes ONE deploy answer it. + + ⚠ Carries no secret: a Space id and a dataset repo id are public names. The DATA is not, and + none of it is here. The field is session-gated anyway (see `routes_admin.settings`). + """ + rid = str(repo or '').strip() + return { + 'deployment': deployment_id(), + 'production': is_production_deployment(), + 'repo': rid, + 'writable': write_refusal(rid) is None if rid else False, + 'refusal': write_refusal(rid) if rid else None, + 'localOverride': local_override(), + 'allowlist': sorted(sandbox_allowlist()), + # The raw half. `''` means "the variable is absent", which is itself the answer on a + # platform that does not set it. + 'observed': {name: _env(name) for name in _IDENTITY_VARS}, + } diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 9a384301f9c0c65de31d5a9ad8e6c9f61e78cdea..b4b78bdc3ea98f45b86fbf5032c1bfd708894d4e 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -40,7 +40,12 @@ down, and this note is the marker for that work. import datetime as dt import json as _json_ev # wave 23 (C7): the json field's parse wall from dataclasses import dataclass, field -from typing import MutableMapping, Optional +# ⚠ ALIASED ON PURPOSE. `field` is imported above for the dataclasses that need it, and this file +# uses `field` as a LOCAL VARIABLE in a dozen handlers (`field = {'key': key, …}`). A +# `field(default_factory=list)` written inside one of those functions would resolve to the local +# dict and raise. The alias is unshadowable, so D-291's channel cannot be broken by a rename. +from dataclasses import field as dataclass_field +from typing import List, MutableMapping, Optional import core.store as store import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates @@ -92,6 +97,21 @@ class EventResult: rerender: bool = False doc: Optional[dict] = None # doc_fetch's answer — the bridge has no response channel toast: Optional[str] = None # add_to_list's confirmation + #: ⭐⭐ D-291 — WHY A WRITE WAS DECLINED. Until 2026-08-17 this door had NO disclosure channel + #: at all: `field_upsert` carries EIGHT distinct `return False` refusals and every one of them + #: answered `{"results":[{"id":"…","rerender":false}]}` — byte-identical to a success. + #: ⛔ THE COST WAS NOT THE MISSING MESSAGE, IT WAS THAT EVERY THEORY BECAME UNFALSIFIABLE. + #: D-275 (a custom field on the product grid answers 200 and is absent from a fresh read) sat + #: open across TWO waves because no amount of reading the source can say WHICH of the eight + #: fired on a given request — and the register's own advice is that reporting the cause IS the + #: diagnostic ([[report-the-cause-before-you-fix-it]]). D-276 is the precedent: shipping the + #: report first produced, in one deploy, a measurement that REFUTED a clean by-elimination + #: argument about the cause. + #: ⚠ SAFE BY CONSTRUCTION, and this is the check to re-run if a new refusal is added: every + #: entry describes the caller's OWN submitted event. It never names another user, another + #: tenant, or a row the caller cannot see — the failure mode D-276's `visible` flag exists to + #: avoid one layer up. + refusals: List[dict] = dataclass_field(default_factory=list) @dataclass @@ -579,6 +599,39 @@ def _known_usernames(): # ------------------------------------------------------------------ THE SEAM +#: ⭐⭐ D-291's WRITER. Every refusal in `field_upsert` goes through here so the caller learns WHY +#: instead of receiving a success-shaped answer. +#: +#: ⛔ IT RETURNS **True**, NOT False, AND THAT HALF IS A SECOND DEFECT THIS CLOSES. `handle_one`'s +#: return value means *"this page must re-render from server state"* — so a refusal that returned +#: False left the browser's OPTIMISTIC column on screen indefinitely. The user typed a column, saw +#: it appear, and it was never stored; only a reload revealed it had never existed. The file already +#: states the rule for the one refusal that got it right (`refused_perms`): *"the client's optimistic +#: copy is a lie that must be repainted"*. Now every refusal obeys it. +#: +#: `code` is a stable machine token (the client may branch on it); `message` is the sentence a +#: person reads. ⚠ The message must name what the CALLER can change — a refusal a user cannot act +#: on is only marginally better than silence. +def _refuse(ctx, code, message, key=''): + """Record a NAMED refusal on the result and demand a repaint. Always returns True.""" + try: + out = getattr(ctx, 'out', None) + if out is not None and hasattr(out, 'refusals'): + if len(out.refusals) < 24: # one per event in the window; never unbounded + out.refusals.append({ + 'event': 'field_upsert', + 'key': str(key or '')[:80], + 'code': str(code)[:40], + 'message': str(message)[:300], + }) + except Exception: # noqa: BLE001 + # ⛔ REPORTING MUST NEVER BE THE THING THAT BREAKS THE WRITE PATH. A refusal that raises + # while explaining itself would turn a declined field into a 500 — strictly worse than the + # silence this replaces. + pass + return True + + def handle_events(events, ctx): """The event LOG entry point → one EventResult. The `[-24:]` window lives here. @@ -963,8 +1016,30 @@ def handle_one(event, ctx): # broken). Role `edit` (or admin) writes THROUGH to the owner's record — config only, # never identity or reach: `createdBy`/`permissions` are pinned from the stored record. # Role `view` (or no grant) refuses with a toast instead of forking. + # + # ⛔⛔ WAVE 33 · T13 (owner item 3) — A PINNED PER-USER ID IS NOT A FOREIGN VIEW, AND + # TREATING IT AS ONE IS A CROSS-USER WRITE THAT REPORTS SUCCESS. + # + # `find_view` scans EVERY personal stratum and returns the FIRST match. `all-customers` + # and `tpl_overview` are PINNED ids that every user of a table has their own copy of — + # so the moment ANY colleague autosaves one (a column resize is enough), the wall reads + # "somebody else owns this id" for everybody else. MEASURED, on this exact path, after + # one colleague resized a column on the Overview: + # · a NON-ADMIN marking it is REFUSED, with a toast blaming sharing — a view nobody + # ever shared, on their own database; + # · an ADMIN marking it answers 200 and writes the mark into the OTHER USER'S + # stratum. The admin's own workspace still serves the un-marked view, so the mark + # "does not work"; the colleague silently gains a mark they never made. + # That is item 3's second mechanism, and it is a residency bug wearing a sharing bug's + # error message. + # + # ⚠ THIS FUNCTION ALREADY STATES THE RULE THE WALL BREAKS, seventy lines up: "A SYSTEM + # VIEW IS PER-USER STATE AND MUST NEVER BE SHARED", enforced there by pinning + # `perms = {'edit': 'personal'}`. A per-user view can therefore never BE a grant, which + # is the only thing this wall exists to protect — so exempting it narrows nothing. + _pinned_per_user = (_kind == 'system' or view_id in ('all-customers', _ag_overview_id())) _save_as = uname - if _store_of(ctx).available(): + if _store_of(ctx).available() and not _pinned_per_user: _foreign = _tops(ctx).find_view(view_id) if _foreign and _foreign[0] != uname: _o_uname, _stored = _foreign @@ -1077,7 +1152,8 @@ def handle_one(event, ctx): if kind == 'field_upsert': raw = event.get('field') if not isinstance(raw, dict): - return False + return _refuse(ctx, 'malformed', + 'the event carried no field object') key = str(raw.get('key') or '')[:80] import aios_grid as _ag2 if key.startswith(_ag2.MEASURE_FIELD_PREFIX): @@ -1086,7 +1162,10 @@ def handle_one(event, ctx): # measure this user could also filter by. Fail closed on anything else. field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer}) if field is None: - return False + return _refuse( + ctx, 'measure_not_offered', + 'this measure is not one you can filter by on this page, so a column ' + 'cannot be built from it', key) fmt = _ag2._clean_format(raw.get('format'), field.get('type')) if fmt: field['format'] = fmt @@ -1138,13 +1217,19 @@ def handle_one(event, ctx): # second, private, non-computing copy here would be a second definition of the same # column. A refusal repaints, so a stale client says something rather than nothing. if ftype in ('link', 'rollup'): - return False + return _refuse( + ctx, 'wrong_door', + 'a link or rollup column belongs to the table schema, not to a personal ' + 'overlay - create it from the Fields panel instead', key) readonly = ftype in _ag2.READONLY_CUSTOM_TYPES # The editable family must SAY it is overlay (the client's own contract); the # read-only pair (formula / created_time) is stored under the cohort column's # mechanism instead, so no cell write can ever be accepted for it. if not readonly and raw.get('source') != 'overlay': - return False + return _refuse( + ctx, 'source_not_overlay', + "an editable column must declare source 'overlay'; this one declared " + '%r' % (raw.get('source'),), key) field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120], 'type': ftype, 'source': 'odoo' if readonly else 'overlay', 'default': True, 'custom': True, @@ -1170,7 +1255,10 @@ def handle_one(event, ctx): if ftype in ('select', 'multiselect'): field['options'] = _ag2._clean_options(raw.get('options')) if not field['options']: - return False # a select with no choices can never hold a value + # a select with no choices can never hold a value + return _refuse( + ctx, 'no_options', + 'a select column needs at least one choice', key) field.update(_ag2._choice_appearance(raw, field['options'])) if ftype == 'rating': field['max'] = _ag2._clean_rating_max(raw.get('max')) @@ -1179,7 +1267,13 @@ def handle_one(event, ctx): # alone so a later-deleted ref blanks the CELLS, not the column. formula = _ag2._clean_formula(raw.get('formula'), valid_keys) if formula is None: - return False + # ⭐ THE ONE D-275 IS MOST LIKELY TO BE. `valid_keys` is THIS page's column + # set, so a formula naming a column that exists on another grid is refused + # here and was, until now, indistinguishable from a save. + return _refuse( + ctx, 'formula_refs', + 'the formula is empty or names a column this table does not have', + key) field['formula'] = formula if ftype == 'automation': # C5-AUTOFIELD (wave 18): persist the validated config bag. No `return False` @@ -1199,7 +1293,9 @@ def handle_one(event, ctx): auto = _ag2._clean_automation(raw.get('automation'), valid_keys, flow_ids=_flow_ids) if auto is None: - return False + return _refuse( + ctx, 'automation_flow', + 'the automation this column names does not exist', key) field['automation'] = auto # ⭐ WAVE-27 item 13 (R13): the code column's language. OPTIONAL, like the # `automation` bag above and unlike `formula` — an unconfigured code column is a @@ -1216,7 +1312,14 @@ def handle_one(event, ctx): if fmt: field['format'] = fmt else: - return False + # ⛔ THE FALL-THROUGH, AND IT IS THE WIDEST OF THE EIGHT: the key is neither a base + # contract field (a note/format override) nor a `custom_`/`measure_` key, so there is + # no stratum it could be written to. A client sending an unprefixed new key lands + # here — and until D-291 that answered exactly like a save. + return _refuse( + ctx, 'unknown_key', + "a new column's key must start with 'custom_'; a base column can only take a " + 'note or a format override', key) # Wave-5 item 1 — createdBy + permissions on the CREATED strata (custom_/measure_). # `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the # client never supplies it, so a browser cannot claim someone else's field. A diff --git a/platform/core/keychain.py b/platform/core/keychain.py index c68126e22926bfee4f477fccfd0cd6d4b3712d0e..3bb97e549e64da02aeed59cce8f6e2af373c7bae 100644 --- a/platform/core/keychain.py +++ b/platform/core/keychain.py @@ -166,6 +166,52 @@ def read_fields(rt, entry_id): return None +def first_entry_of_type(rt, etype): + """The metadata row of the FIRST entry of `etype`, or None. Never decrypts anything. + + ⭐ W35-T45 — the same "first, by insertion-id sort" rule `_first_creds` resolves on, exposed as + an IDENTITY question rather than a credential one. `ensure_entry_of_type` needs to know whether + an entry exists WITHOUT unlocking it, and `list_entries` + a hand-rolled `next()` at each caller + is how two places come to disagree about which entry is "the" one. + """ + return next((e for e in list_entries(rt) if e.get('type') == str(etype or '')), None) + + +def ensure_entry_of_type(rt, etype, label, fields, username): + """Create ONE entry of `etype` only if the tenant has none. `(row, why)`; `row` is None on skip. + + ⭐ W35-T45 / R11 — the primitive tenant #0's env-to-keychain migration is built from, and the + reason it lives here is that only this module knows what "already has one" means: `odoo_creds` + resolves to the FIRST entry of a type, so a second one would be stored, invisible in every + resolver, and impossible to tell from the one that serves. + + ⛔ IDEMPOTENT BY TYPE, NOT BY A FLAG. A migration that keys idempotency on a marker it writes + itself is one lost marker away from running twice ([[a-migration-that-runs-on-the-next-write]]); + keying it on the thing it would create cannot double-apply, whatever else fails. + + ⛔ AND IT REFUSES ON A LOCKED KEYCHAIN RATHER THAN STORING A SECRET IN THE CLEAR. `add_entry` + would raise `KeychainLocked`; this answers with a REASON instead, because "there is no Fernet key + on this deployment" is an operator fact to report, not an exception to propagate out of a boot + thread where nobody reads it. + + ⚠ `why` IS RETURNED EVEN ON SUCCESS-BY-SKIP. "Already migrated" and "could not migrate" are + different operator actions and a bare None cannot tell them apart. + """ + etype = str(etype or '').strip().lower() + if etype not in ENTRY_TYPES: + return None, f'{etype!r} is not a keychain entry type' + existing = first_entry_of_type(rt, etype) + if existing: + return None, f'this tenant already has a {etype} entry ({existing["id"]})' + if not unlocked(): + return None, ('this deployment has no keychain key, and a credential must never be stored ' + 'in the clear') + try: + return add_entry(rt, label, etype, fields, username), '' + except Exception as exc: # noqa: BLE001 + return None, f'{type(exc).__name__}: {exc}' + + def odoo_creds(rt): """R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls back to the environment). Deterministic order = insertion-id sort, so 'first' is stable.""" diff --git a/platform/core/perm_scope.py b/platform/core/perm_scope.py index d19d3ddabf761f7ba09d73b0a03b77471a9c0532..3944187b70b630f840161399751e6f7fabb9a979 100644 --- a/platform/core/perm_scope.py +++ b/platform/core/perm_scope.py @@ -87,6 +87,22 @@ def may_access(user, module): return perms.may_open(user, module) # legacy record: the old grant wall +def assistant_entry(user, module): + """The explicit database grant an Assistant snapshot may rely on, else ``None``. + + The interactive application retains an admin break-glass path and a temporary legacy-grant + compatibility path. Neither is an answer at the Assistant's app-stored data boundary: + that reader must be able to name the migrated grant that admitted a database. In particular, + a store-outage admin identity with no ``perms`` document is not an unresolved permission that + may be widened into data access. + """ + e = entry(user, module) + if (not is_migrated(user) or not isinstance(e, dict) + or not bool(e.get('access', True)) or not may_access(user, module)): + return None + return e + + # ── FIELDS ─────────────────────────────────────────────────────────────────────────────────── def hidden_keys(user, module, fields): """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5). @@ -143,6 +159,37 @@ def visible_fields(fields, user, module): if not (isinstance(f, dict) and f.get('key') in hide)] +def assistant_visible_fields(fields, user, module): + """Visible closure under one explicit Assistant grant, including for an admin caller.""" + e = assistant_entry(user, module) + if e is None: + return [] + # `hidden_keys` deliberately preserves the interactive admin break-glass behaviour. The + # Assistant uses its explicit grant instead, but shares the same transitive formula closure. + hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k} + if not hidden: + return list(fields or ()) + + refs = {} + for field in fields or (): + if not isinstance(field, dict) or not field.get('key'): + continue + expr = field.get('formula') + if isinstance(expr, str) and expr: + refs[field['key']] = {match.strip() for match in _FORMULA_REF.findall(expr) + if match.strip()} + for _ in range(12): + grew = False + for key, deps in refs.items(): + if key not in hidden and deps & hidden: + hidden.add(key) + grew = True + if not grew: + break + return [field for field in (fields or ()) + if not (isinstance(field, dict) and field.get('key') in hidden)] + + def strip_row(row, hide): """Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run per row: the field list and the row payload are two different wires, and stripping only the @@ -170,6 +217,89 @@ def apply_row_scope(rows, user, module, fields, ctx=None): return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)] +def assistant_apply_row_scope(rows, user, module, fields, ctx=None): + """Apply an explicit Assistant grant's permanent filter without an admin bypass.""" + e = assistant_entry(user, module) + if e is None: + return [] + tree = e.get('filter') + if not tree: + return list(rows or ()) + from harness import filter_eval as fe + return [row for row in (rows or ()) if fe.permits(tree, row, fields, ctx)] + + +def validate_assistant_filter(tree, fields): + """Return a strict, detached Assistant filter tree or raise ``ValueError``. + + The display filter cleaner is intentionally permissive: it drops a stale column or leaves an + inactive condition alone so an old saved view can still open. An Assistant data reader cannot + inherit that behaviour — dropping a predicate turns a request for a subset into a wider read. + This validator therefore admits only visible field operands that the existing evaluator can + answer from one stored row. Cohort, measure and rank conditions need separate materialised + set resolvers and are refused here rather than guessed. + """ + if tree is None: + return None + from harness import filter_eval as fe + + by_key = {str(field.get('key')): field for field in (fields or ()) + if isinstance(field, dict) and field.get('key')} + if not by_key: + raise ValueError('assistant filter has no visible field contract') + + def _leaf(raw): + if not isinstance(raw, dict): + raise ValueError('assistant filter leaf must be an object') + allowed = {'id', 'colId', 'op', 'value', 'value2', 'rhs'} + if set(raw) - allowed: + raise ValueError('assistant filter carries an unsupported operand') + col = raw.get('colId') + op = raw.get('op') + if not isinstance(col, str) or col not in by_key: + raise ValueError('assistant filter names an unknown or hidden field') + if (not isinstance(op, str) or op not in fe.FILTER_OPS or op in fe.RANK_OPS + or op in {'between', 'within'} or col == fe.COHORT_FIELD): + raise ValueError('assistant filter uses an unsupported operator') + rhs = raw.get('rhs') + if rhs is not None: + if (not isinstance(rhs, dict) or rhs.get('kind') != 'field' + or set(rhs) - {'kind', 'colId'} + or not isinstance(rhs.get('colId'), str) + or rhs['colId'] not in by_key): + raise ValueError('assistant filter names an unknown or hidden right-hand field') + out = {name: raw[name] for name in ('id', 'colId', 'op', 'value', 'value2') + if name in raw} + if rhs is not None: + out['rhs'] = {'kind': rhs.get('kind'), 'colId': rhs['colId']} + if not fe.is_rule_active(out, by_key): + raise ValueError('assistant filter is inactive or unanswerable') + return out + + def _node(raw): + if not isinstance(raw, dict): + raise ValueError('assistant filter node must be an object') + if 'children' not in raw: + return _leaf(raw) + if set(raw) - {'conj', 'children'}: + raise ValueError('assistant filter group carries an unsupported operand') + children = raw.get('children') + if raw.get('conj') not in ('and', 'or') or not isinstance(children, list) or not children: + raise ValueError('assistant filter group must be a non-empty and/or group') + return {'conj': raw['conj'], 'children': [_node(child) for child in children]} + + if isinstance(tree, list): + if not tree: + raise ValueError('assistant filter list must not be empty') + return {'conj': 'and', 'nodes': [_node(node) for node in tree]} + if not isinstance(tree, dict) or set(tree) - {'conj', 'nodes'}: + raise ValueError('assistant filters must be a tree with nodes') + nodes = tree.get('nodes') + if tree.get('conj') not in ('and', 'or') or not isinstance(nodes, list) or not nodes: + raise ValueError('assistant filter tree must be a non-empty and/or tree') + return {'conj': tree['conj'], 'nodes': [_node(node) for node in nodes]} + + # ── THE PUSHDOWN ───────────────────────────────────────────────────────────────────────────── #: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly #: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission diff --git a/platform/core/perms.py b/platform/core/perms.py index f1f234422737ee5f548bbe22daf91238a4830634..0c26d8098d989330c5259c53943a17fead2087dc 100644 --- a/platform/core/perms.py +++ b/platform/core/perms.py @@ -199,6 +199,116 @@ def nav_pages(user): return out +# ------------------------------------------------------------------ governable modules (C3) +#: ⭐⭐ WAVE 33, OWNER ITEM 11 — *"its an error that ALL fucking tenant can see both Customer and +#: Product database under Manage User, this is only for Royal Imports tenant. Make sure per tenant +#: configuration always work, this is my fourth time I flag it."* +#: +#: The cause was two lines in `routes_admin.py`: `_PERM_MODULES = ("customer_data","product_data")` +#: and a `_MODULE_LABELS` dict beside it. `get_perms` built its answer from the literal and never +#: touched `session.runtime`, so the permission editor was TENANT-BLIND BY CONSTRUCTION — every +#: tenant was handed tenant #0's two grid modules whether or not it is provisioned for either. +#: +#: ⛔ WHY THE DERIVATION LIVES HERE AND NOT AT THE ROUTE. `routes_nav.py::nav` already applies +#: exactly this rule and has since wave 18; the perms route is the ONE surface that skipped it. +#: A second copy of the rule at a second call site is how the two came apart in the first place, +#: so the rule is stated once, as a pure function of (runtime, catalogue), and both the route and +#: `verify_perm_scope` ask THIS. +def tenant_modules(runtime): + """The registry keys this TENANT is provisioned for. None means "every key". + + ⛔ FAIL-CLOSED ON THE EMPTY LIST, and that is not the obvious reading. `routes_admin`'s own + docstring records two shipped fail-OPENs of exactly this shape (`modules: []` and + `bus: []` both read as UNRESTRICTED), so ONLY the literal `'all'` — or an absent + key, which is how the compiled-in tenant #0 record is written — means every module. A + bucket-provisioned tenant gets `{"modules": rec.get("modules", [])}` + (`harness/runtime.py::_builder_from_record`), so `[]` there means the tenant was provisioned + for NOTHING and must be told so, never handed the catalogue. + + Legacy keys are migrated on read for the same reason `allowed_modules` does it: a tenant + record written before a rename must not silently lose the module it names. + """ + cfg = getattr(getattr(runtime, 'tenant', None), 'config', None) or {} + raw = cfg.get('modules', 'all') + if raw is None or raw == 'all': + return None + if isinstance(raw, str): + raw = [raw] + return {_LEGACY_KEYS.get(str(k).strip(), str(k).strip()) + for k in raw if str(k or '').strip()} + + +def tenant_has_module(runtime, key): + """Is `key` in this tenant's provisioned set? PARENT-AWARE, like `may_open`. + + `routes_nav` keeps a page whose `parent` is granted even when the child key is not, and a + governable list that disagreed with the nav would offer an admin a wall for a database the + tenant cannot open — or hide one it can. + """ + allowed = tenant_modules(runtime) + if allowed is None or key in allowed: + return True + parent = registry.BY_KEY.get(key, {}).get('parent') + return parent in allowed if parent else False + + +def tenant_governable_modules(runtime, topics, ut_entries=()): + """The databases the permission editor may render for THIS tenant, in stable order. + + ⚠ NAMED `tenant_*` TO STAY DISTINCT FROM `registry.governable_modules()`, which lane E + published under contract C3 and which this consumes. They answer different questions — + registry's is "which topics are governable at all, platform-wide"; this one is "which + databases does THIS tenant govern" — and two same-named functions one import apart is the + [[one-question-two-normalizers]] shape waiting to happen. + + `topics` — the registry-backed grid modules the editor can build pickers for, as + `[{'key': ..., 'label'?: ...}]`, i.e. `registry.governable_modules()` narrowed + to the topics the caller has a field-schema arm for. The CALLER owns that + narrowing because the schema behind each arm is code + (`routes_admin::_module_fields`), not a policy list. Tenant-blind on the way + in; this function is the only place the tenant is applied. + `ut_entries` — this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape. + + Returns `[{'key', 'label', 'enforced'}]`. + + ⛔⛔ `enforced` IS THE HONEST HALF AND IT IS NOT DECORATION. `perm_scope` walls the REGISTRY + topics — `grid_assembly` calls `may_access`, `visible_fields` and `apply_row_scope` on every + read. It is never consulted for a `ut_*` database: `routes_tables`' visibility is + `user_tables.may_open` (creator, admin, or a `core.shares` grant) and there is no module + grant in that path at all. So a stored `ut_*` wall would be INERT — the editor would say + DENY, the table routes would keep serving, and nothing anywhere would say so. That is the + same class as the empty-dropdown defect (wave 26 item 24): a control that answers when it + cannot. The database is LISTED, because the owner asked to see every database; the wall is + declared unenforced, and `routes_admin::_clean_perms` REFUSES to store one against it rather + than accepting a rule nobody applies. Arming `perm_scope` over `ut_*` is a change to + `routes_tables`/`user_tables` (lane A's fence) — booked, not faked. + """ + out = [] + for t in topics or (): + key = str((t or {}).get('key') or '').strip() + if not key or not tenant_has_module(runtime, key): + continue + # The LABEL comes from the registry, never from a dict beside the list. `_MODULE_LABELS` + # was a hardcoded copy of two registry labels, so wave 33's R2 relabel ("Odoo customers" / + # "Odoo products") would have moved one of them and left the permission editor showing the + # old name with nothing to notice. + label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key) + out.append({'key': key, 'label': label, 'enforced': True}) + seen = {r['key'] for r in out} + for e in ut_entries or (): + key = str((e or {}).get('key') or '').strip() + if not key or key in seen: + continue + seen.add(key) + out.append({'key': key, 'label': (e.get('label') or key), 'enforced': False}) + return out + + +def enforced_module_keys(modules): + """The subset of `governable_modules`' answer a permission block may actually be stored for.""" + return {m['key'] for m in (modules or ()) if m.get('enforced')} + + def landing_page(user): """Where a restricted user lands: the first MAIN-chrome page in their grant. None for unrestricted users (the client lands on the Analyst, as the host does). diff --git a/platform/core/registry.py b/platform/core/registry.py index 97a571af0bd97600f4f4ce1080506c6ceff1556f..d038770a0f7833ea467cd48949d4f6248a8bc65f 100644 --- a/platform/core/registry.py +++ b/platform/core/registry.py @@ -22,6 +22,22 @@ Fields: 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement + Collections + Analyst/plumbing): products, assortment, financial, management, pricing, inventory, vendor, warehouse, returns, outreach, complexity, health. + subject set -> the REAL-WORLD POPULATION this row presents, as a source-qualified id + ('odoo:res.partner'). ⭐ W33-T41 / item 12a: it is what makes "two databases present + the same thing" a statement code can refuse, and it exists because NO OTHER FIELD + CAN CARRY THAT TEST. Not the key (`product_data` and `ut_odoo_products` are different + keys by construction — that is the bug, not the check), and NOT THE LABEL: the owner's + own complaint is "why do we have 'Odoo products' ALREADY with the current Products + database?", i.e. the twin was already carrying the label the legacy row is about to + take (W33-T42). Between T42 and T44 the two rows legitimately share a label, so a + label-keyed check would refuse the rename that fixes the bug. + ⚠ A row without `subject` claims nothing and conflicts with nothing — silence is not + a claim, so adding this field cannot retro-refuse anything already shipped. + topic set -> this row is a GOVERNABLE DATABASE and this is the word the rest of the codebase + already calls it by (`routes_grid._SCOPES`, `routes_alerts._TOPICS`, `scope=` on the + grid routes). ⭐ W33-T46 / contract C3: it is what `governable_modules()` keys on, and + it exists so `routes_admin`'s permission list can be DERIVED from the registry instead + of being the hardcoded two-element tuple that made every tenant see Customer + Product. note one-line scope note shown in the UI / docs """ # WAVE-9 I8 — `'source'`: which connected system a module's data comes from. @@ -41,7 +57,7 @@ REGISTRY = [ # briefing still read it, and pages_sales.py survives UNREGISTERED as the Y1 envelope's # template + verify_api's fixture. {'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True, - 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs — by BU. Retired wave 16: ' + 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs. By BU. Retired wave 16: ' 'rebuild as grid chart/dashboard views (compare series, KPI deltas, tables).'}, # ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer # dashboard completely, i want to redesign it, exactly with the backend we have". So the @@ -65,7 +81,14 @@ REGISTRY = [ # myday->customer_list was done. The STORE keys are deliberately NOT renamed: # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them # would orphan every saved view. Label-only changes need none of that machinery. - {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', + # LABEL renamed 'Customer' -> 'Odoo customers' (owner item 12, 2026-08-14 / W33-T42): the + # naming convention of every other database from this connector ("Odoo orders", "Odoo vendor + # bills"). ⭐ STILL LABEL-ONLY, and the note above is the precedent — the KEY does not move, so + # no grant, saved view, cohort, formula, shared column or `?page=` deep link is touched. + # ⛔ `nav_meta` could not have done this: `routes_nav.py` raises `400 name_not_allowed` for any + # key that does not start with `ut_`, so a tenant rename cannot reach a registry module at all. + {'key': 'customer_data', 'label': 'Odoo customers', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', + 'subject': 'odoo:res.partner', 'topic': 'customer', 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer ' 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship ' 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'}, @@ -81,7 +104,7 @@ REGISTRY = [ # _LEGACY_KEYS maps cohort→customer_data so grants + ?page= deep links land on the grid # that now hosts the cohorts. {'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', 'archived': True, - 'note': 'Hand-curated, unchanging customer lists — folded into the Customer rail as ' + 'note': 'Hand-curated, unchanging customer lists. Folded into the Customer rail as ' 'locked views (wave 16). The set is fixed: it changes only when someone adds ' 'or removes a member. Open them from Customer > Views > Cohorts.'}, # ARCHIVED wave 16 (owner item 7, R11, 2026-08-02) beside Sales. The agent DRAWER and the @@ -101,13 +124,16 @@ REGISTRY = [ 'group_only': True, 'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is ' 'not part of this family.'}, - {'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family', + # LABEL renamed 'Product' -> 'Odoo products' (owner item 12, 2026-08-14 / W33-T42) — see the + # customer_data note above; label-only, the key stays `product_data`. + {'key': 'product_data', 'label': 'Odoo products', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family', + 'subject': 'odoo:product.product', 'topic': 'product', 'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) ' 'stock columns, with saved views and custom fields. Identity = the SKU code.'}, {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True, - 'note': 'SKU movers, zombies, velocity, coverage, drawers — by BU (sales-derived).'}, + 'note': 'SKU movers, zombies, velocity, coverage, drawers. By BU (sales-derived).'}, {'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True, - 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness — by BU.'}, + 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness. By BU.'}, {'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True, 'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'}, {'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True, @@ -122,7 +148,7 @@ REGISTRY = [ 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 13: "We should be able to replace ' 'Procurement completely, and add it as part of the Product database"). The buy ' 'list is now a saved VIEW on the Product grid, filtered on a FORMULA field over ' - 'the supplier/lead-time columns and the demand measure — the owner\'s ruling R3: ' + 'the supplier/lead-time columns and the demand measure. The owner\'s ruling R3: ' '"Buy list is just a View, with a Filter from a Formula field that taps into ' 'Metrics Fields... the 8-month demand baseline etc. is just math in a Formula ' 'field." The ROW STAYS validate_only: `modules/procurement.py` still owns the ' @@ -136,7 +162,7 @@ REGISTRY = [ 'source': 'Odoo', 'api_surface': False, 'admin_only': True, 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 15: "Collections should also be ' 'entirely replicable as just a View under Customer"). The worklist is the shared ' - '"Collections" view on the Customer grid — same numbers, from this module\'s own ' + '"Collections" view on the Customer grid. Same numbers, from this module\'s own ' 'reconciled blocks (ar_open/ar_overdue/ar_exposure/days_to_pay + the four aging ' 'buckets). The ROW STAYS validate_only so validate.py keeps running ar.validate(), ' 'which is the proof those columns rest on; archiving it would have skipped the ' @@ -145,7 +171,7 @@ REGISTRY = [ 'sender (app._collections_statements -> modules/collections_send), the ONE ' 'sanctioned Odoo writer, which the same ruling says stays untouched. So the row ' 'keeps a Streamlit page for ADMINS ONLY (`admin_only`) and leaves the API payload ' - 'entirely (`api_surface: False`) — no "Collections" in the React nav, no second ' + 'entirely (`api_surface: False`). No "Collections" in the React nav, no second ' 'worklist, and the biweekly send keeps its door.'}, {'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True, 'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'}, @@ -154,7 +180,7 @@ REGISTRY = [ {'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True, 'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'}, {'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True, - 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE — fix the rule, not the rep.'}, + 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE. Fix the rule, not the rep.'}, {'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True, 'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'}, {'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True, @@ -167,29 +193,37 @@ REGISTRY = [ 'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'}, {'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True, 'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE ' - '(owner 2026-07-23): every discrepancy / potential-error marker lives here — ' + '(owner 2026-07-23): every discrepancy / potential-error marker lives here. ' 'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-' - 'order — so operational workflows stay clean for doing the work.'}, + 'order. So operational workflows stay clean for doing the work.'}, {'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True, 'nav': False, 'validate_only': True, - 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) — row kept ONLY so ' + 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing). Row kept ONLY so ' 'validate.py keeps running the semantic-layer contracts (the Analyst grounding ' - 'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does — delete ' + 'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete ' 'them") made that literal: `validate_only` takes it out of the account menu too, ' 'which is the last place it was still visible. The proof survives; the door does not.'}, - {'key': 'automation', 'label': 'Automation', 'brand': False, 'hq': True, 'validate': False, + {'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False, 'nav': True, - 'note': 'Wave 18 (C-AUTONAV): the Automation surface — scheduled jobs that create and ' - 'refresh user databases (website scrape-to-DB, the Instagram field). React-only ' - 'surface (no PAGE_FUNCS entry, the no-new-Streamlit rule); admins hold it via ' - '"all", other users need the explicit grant — fail-closed default.'}, + 'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases ' + '(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS ' + 'entry, the no-new-Streamlit rule); admins hold it via "all", other users need the ' + 'explicit grant, fail-closed default. ' + '⭐ WAVE 34 · R12. THE LABEL IS "Agents" AND THE KEY IS STILL `automation`. The ' + 'owner ruled the module becomes Agent: this is our agent deployment surface going ' + 'forward. Only the LABEL moved, because the key is a permission scope, a nav ' + 'placement key, a grant name and the `#/automation` route; renaming it would revoke ' + 'every stored grant and 404 every bookmark to buy a caption. ⚠ The plural is not a ' + 'slip: R15 and the wave PRD both spell the rail row "Agents", and the degraded ' + 'fallback row in Shell.tsx hard-codes that spelling, so a singular here would paint ' + 'two different words in one list depending on whether /nav answered.'}, {'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False, 'nav': False, # sidebar sentinel (account group) — owner IA 2026-07-12 'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis ' 'toggle (Orders vs Invoiced) moved here from the sidebar.'}, {'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False, 'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py — live LLM cost) - 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer — answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'}, + 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer. Answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'}, ] BY_KEY = {m['key']: m for m in REGISTRY} @@ -197,3 +231,90 @@ BY_KEY = {m['key']: m for m in REGISTRY} def brand_filterable(key) -> bool: return bool(BY_KEY.get(key, {}).get('brand')) + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner: *"why do we have 'Odoo products' already with the current Products database? The +# Unique ID is redundant, make sure this never happens. One unique ID per database always."* +# +# ⛔ THE CHECK HAD TO GO WHERE THE BUG WAS MADE, AND THAT IS NOT WHERE ANYONE WOULD LOOK. +# `core.user_tables.create` — the UI door — HAS had a collision loop all along: it slugs the +# label to a key and appends `_2 … _49` until the key is free. It never fired here for two +# reasons, and both matter. (1) It compares KEYS inside ONE tenant's `user_tables` document, +# and `customer_data` / `product_data` are compiled REGISTRY rows that never enter that +# document at all — the two halves of the duplicate live in different namespaces and only meet +# in `routes_nav.py::nav`, where the registry pages and `user_tables.nav_entries` are merged +# for display. (2) The connector path never calls `create()`: `odoo_relational. +# _ensure_table_inplace` writes `cur[key] = {...}` straight into the document. So a check bolted +# onto `create()` would have been green through the entire life of this defect. +# +# ⚠ THE SUFFIX LOOP IS ALSO WHY A RELABEL ALONE WOULD HAVE MADE THIS WORSE, MECHANICALLY: a +# naive "just rename Product to Odoo products" mints `ut_odoo_products_2` — the owner's +# redundant id again, from the fix for it. +# +#: subject -> the ONE registry key entitled to present it. Derived, never typed twice. +SUBJECT_CLAIMS = {m['subject']: m['key'] for m in REGISTRY if m.get('subject')} + + +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# ⭐⭐ W33-T46 / CONTRACT C3 — THE GOVERNABLE MODULE LIST IS DERIVED, NOT A LITERAL +# ═════════════════════════════════════════════════════════════════════════════════════════════ +# +# The owner, on his FOURTH flag of the same thing: *"its an error that ALL fucking tenant can see +# both Customer and Product database under Manage User, this is only for Royal Imports tenant. +# Make sure per tenant configuration always work."* +# +# ⛔ THE CAUSE IS A TWO-ELEMENT TUPLE. `routes_admin._PERM_MODULES = ("customer_data", +# "product_data")` — a compiled literal that `get_perms` builds its whole list from without ever +# touching `session.runtime`. It is tenant-blind BY CONSTRUCTION, which is why no amount of +# per-tenant work elsewhere ever fixed it, and why the answer is not "filter the tuple" but +# "delete the tuple": a second hand-typed copy of a fact the registry already holds is a thing +# that CAN go stale, and this one did, four times. +# +# ⚠ THE LINE BETWEEN THE TWO HALVES, stated because both sides need it: this map is +# REGISTRY-SIDE and TENANT-BLIND — it answers *"which compiled modules are governable, and what +# are they called"*, nothing more. Applying the tenant's own `config["modules"]` filter and +# merging that tenant's `ut_*` databases is the CALLER's half (`routes_admin`, D's fence), which +# is exactly what `routes_nav.py` already does correctly and `routes_admin` never did. +def governable_modules(): + """`{topic: {'key','label','subject'}}` — every registry row that is a governable DATABASE. + + A fresh dict per call, deliberately: a module-level mapping handed to a route is one + `.pop()` away from being a different list for every request after the first. + """ + return {m['topic']: {'key': m['key'], 'label': m['label'], 'subject': m.get('subject')} + for m in REGISTRY + if m.get('topic') and not m.get('archived') and not m.get('group_only')} + + +def subject_of(key): + """The subject a REGISTRY row claims, or None. (`ut_*` databases carry theirs on the stored + definition instead — they are per-tenant data, not compiled rows.)""" + return (BY_KEY.get(key) or {}).get('subject') + + +def subject_conflict(subject, key, claimed=None): + """The OTHER key already presenting `subject`, or None. + + `claimed` is an optional {subject: key} of claims outside the registry — in practice the + caller's own tenant document, whose stored definitions carry a `subject` stamp. It is a + PARAMETER rather than a lookup on purpose: the store is per-tenant, two tenants both holding + an "Odoo customers" is entirely legitimate, and a module-level cache of somebody's tables is + how D-169 happened. This function knows nothing about which tenant it is serving. + + ⭐ THE TEST IS "A DIFFERENT KEY", NOT "IS THIS SUBJECT TAKEN". `_ensure_table_inplace` and + `ut_ensure` re-run on every boot and every 1800 s resync, adopting their table BY KEY — so a + check phrased "is this subject already claimed?" goes red on the second boot against the very + table it created on the first, and the container fails to start. Idempotency is not an + exception to the rule here; it is the common case. + """ + if not subject: + return None # silence is not a claim + for holder in (SUBJECT_CLAIMS, claimed or {}): + other = holder.get(subject) + if other and other != key: + return other + return None diff --git a/platform/core/shared_overlay.py b/platform/core/shared_overlay.py index 3f99d3b6d97cbeac8c610daa309747304aa111cc..7bb2a062a22defdead992b0de67b339858ef6bf4 100644 --- a/platform/core/shared_overlay.py +++ b/platform/core/shared_overlay.py @@ -230,6 +230,44 @@ def put_cells(table_key, pid, values, st=None): return clean +def put_rows(table_key, rows, st=None): + """Write shared cells on MANY rows in ONE store update. Returns `{row_id: {key: value}}`. + + ⭐⭐ THE WHOLE POINT IS THE *ONE*, AND IT IS NOT AN OPTIMISATION. `put_cells` is one row per + store write, so a 1,397-row catalog import is 1,397 download-modify-upload cycles against one + JSON document. This repo has a MEASURED scar for that shape: **18 writes against one document + under the store's coalescing single-flight landed ZERO while answering 200 eighteen times.** + Batching is what makes a bulk import land at all, not what makes it fast. + + ⚠ It deliberately takes the WHOLE SET rather than accepting a stream: a caller that loops over + this function has simply rebuilt `put_cells` with extra steps, and the failure it reintroduces + is silent. If the set does not fit in memory it does not fit in this store either — that is a + signal to change substrate, not to chunk. + + ⛔ NO CAP HERE, ON PURPOSE. The bound belongs at the DOOR, where a caller identity and a + reportable refusal exist (`routes_grid.bulk_cells`). A silent ceiling in a store primitive is + exactly the shape the standing no-cap rule forbids. + """ + clean = {} + for pid, values in dict(rows or {}).items(): + row_id = _pid(pid) + cells_for_row = {str(k): _value(v) for k, v in dict(values or {}).items() if str(k)} + if cells_for_row: + clean.setdefault(row_id, {}).update(cells_for_row) + if not clean: + return {} + + def _patch(data): + for row_id, values in clean.items(): + data['cells'].setdefault(row_id, {}).update(values) + + # `flush='sync'` because a bulk import must be durable when the call returns: the caller is a + # script that will report "1,397 rows written" and exit, and an async flush would make that + # sentence a prediction rather than a fact. + _write(table_key, _patch, st, flush='sync') + return clean + + def clear_row(table_key, pid, st=None): """Forget every shared cell on one row. True when there was something to forget.""" row_id = _pid(pid) diff --git a/platform/core/shares.py b/platform/core/shares.py index 09e691b9543512a9f41880e0e987cf1a4367a748..146302359d42a22e8e701d7cff8d04d94d5d431b 100644 --- a/platform/core/shares.py +++ b/platform/core/shares.py @@ -43,10 +43,32 @@ SHARES_KEY = 'object_shares' #: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken". KINDS = ('view', 'folder', 'database') -#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform" — -#: the module/table wall runs FIRST and this never widens past it. Spelled as a single character -#: so it can never collide with a username (usernames are lower-case and non-empty by -#: `core/users.py`, and are checked against this explicitly below). +#: `*` is "everyone who can already open the surface". It is NOT "every account on the platform". +#: Spelled as a single character so it can never collide with a username (usernames are lower-case +#: and non-empty by `core/users.py`, and are checked against this explicitly below). +#: +#: ⛔⛔ **AND WHAT THAT MEANS DEPENDS ON THE KIND — THE LINE THIS NOTE USED TO CARRY WAS FALSE FOR +#: ONE OF THE THREE** (W33-T30, `waves/wave32/sharing-audit.md` S-8). It read *"the module/table +#: wall runs FIRST and this never widens past it"*, flatly, and the audit's own words for that are +#: *"the third docstring in this audit describing a check that is not on the path"*. Corrected +#: here rather than deleted, because the sentence is TRUE of two kinds and the difference is the +#: whole point: +#: * `kind='view'` / `kind='folder'` on a GOVERNED module (`customer_data`, `product_data`) — +#: the sentence holds. `require_session` plus the topic's own gate run first, and the +#: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is +#: merged, so a grant can only narrow-or-equal what that account could already reach. +#: * `kind='database'` on a `ut_*` table — **THE SENTENCE IS FALSE AND THIS REGISTRY IS THE +#: ONLY WALL.** `routes_admin._PERM_MODULES` is `("customer_data", "product_data")` and +#: `_clean_perms` 400s anything else, so no row filter and no hidden field can even be +#: DECLARED for a user table; `routes_tables.py` makes zero `perm_scope` calls and passes +#: `hidden_keys=frozenset()`. A `database` grant is therefore ALL-OR-NOTHING — every row, +#: every column — and an `*` database grant admits every account in the tenant to all of it. +#: +#: ⚠ THAT IS A REAL CAPABILITY, DELIBERATELY KEPT, not a hole to plug in passing. What was wrong +#: was a docstring promising a second wall that does not exist for this kind; scoping user tables +#: is booked (S-8), not done. `routes_shares.py`'s module docstring says the same thing at the +#: OTHER door — the audit's fix is "say so at both", and one door saying it is how the next reader +#: gets the confident half [[one-question-two-normalizers]]. EVERYONE = '*' ROLES = ('view', 'edit') diff --git a/platform/core/store.py b/platform/core/store.py index 4a5cf6d22a58450480955fbce134fe4c047d36f2..2284c558fc19ccb79aec94a94c012b8b6029871c 100644 --- a/platform/core/store.py +++ b/platform/core/store.py @@ -27,12 +27,35 @@ except Exception: pass from huggingface_hub import HfApi, hf_hub_download -from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError - -REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data') +from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError, + RepositoryNotFoundError) + +import core.data_binding as data_binding + +#: ⛔⛔ D-315 — THE BARE DEFAULT IS GONE. This line used to read +#: `os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')`, i.e. every Space that was never +#: handed the key came up bound to tenant #0's REAL business data — which is how two different +#: builds wrote it on 2026-08-14. The value it resolves to is unchanged (see +#: `data_binding.default_store` for why returning the real id beats `None`); what changed is that +#: WRITING it is now refused unless this deployment is the one that owns it. +REPO = os.environ.get('OS_DATA_REPO') or data_binding.default_store() _FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit _FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget) +#: ⭐⭐ D-305 — HOW MANY TIMES A WRITE MAY REBASE ONTO A MOVED HEAD BEFORE GIVING UP. +#: +#: Bounded rather than unbounded, and the cost is one extra DOWNLOAD per rebase. ⚠ Not an extra +#: commit: a refused write answers 412 and creates nothing, so only the attempt that finally lands +#: spends the 256-commits/hr budget. Four is generous for the measured case +#: (two humans editing one grid) and small enough that a genuinely hot key fails loudly instead of +#: spinning. ⛔ Hitting the bound RAISES — it never falls through to an unconditional upload, which +#: is the pre-fix behaviour wearing a retry loop ([[fallback-that-became-the-rule]]). +_MAX_REBASE = 4 + +#: How many unconfirmed intents one key may hold before the oldest are dropped. See +#: `_journal_append` for why a cap exists at all and what is reported when it bites. +_MAX_JOURNAL = 500 + #: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP. #: #: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a @@ -48,6 +71,51 @@ def _token(): return os.environ.get('HF_TOKEN') or None +class StoreConflict(RuntimeError): + """A write that could not be landed without overwriting somebody else's (D-305). + + Raised only after `_MAX_REBASE` rebases, or when no parent commit can be resolved at all. Both + are "we could not make this safe", and both must reach the caller as a failure: the alternative + — uploading anyway — is exactly the lost update this whole mechanism exists to prevent, and it + would answer 200 while destroying an hour of another person's work. + """ + + +def _is_conflict(exc): + """True for the ONE hub failure that means "the branch moved under you". + + ⛔ NARROW ON PURPOSE, AND THE PROBE IS WHY. Measured against a scratch dataset repo on + 2026-08-18: a stale `parent_commit` returns **HTTP 412** with the server message *"The branch + was updated since you opened this page. Please refresh and try again."*, and the write does NOT + land. A NONSENSE `parent_commit` returns a **404 `RevisionNotFoundError`** — a different fault + that must not be retried, because rebasing on it would loop against a bad sha. A broad + `except HfHubHTTPError` would swallow both, plus every 401/429/500, and turn a network blip + into a silent rebase. + """ + return getattr(getattr(exc, 'response', None), 'status_code', None) == 412 + + +def _sha_from_path(path): + """The commit sha out of a hub cache path, or `None` if it does not look like one. + + ⚠ VALIDATED, NOT ASSUMED. The layout is `…/snapshots/<40-hex>/`, but a caller may have + an unusual `HF_HOME`, symlinks disabled (this repo's own machines warn about exactly that), or + a future hub version. A directory name that is not a 40-character hex sha is reported as "I do + not know", which routes the write through the explicit head lookup — the safe branch. Returning + a bad sha would surface as a 404 `RevisionNotFoundError` on every write instead. + """ + try: + parent = Path(path).parent + if parent.parent.name != 'snapshots': + return None + name = parent.name + if len(name) == 40 and all(c in '0123456789abcdef' for c in name.lower()): + return name + except Exception: # noqa: BLE001 + pass + return None + + class _Projected(dict): """One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped. @@ -111,6 +179,27 @@ def _project(doc, drop): return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()} +def is_projected(value): + """Did this value come back from a PROJECTED read? — the TAKEN-PATH predicate (W33-T01/D-213). + + ⭐⭐ WHY A PREDICATE AND NOT A COUNTER, and it is the whole reason this exists. `all_defs` + FALLS BACK to the whole read on any failure, silently and by design — so a caller that gets the + right answer has learned nothing about which read produced it. A gate asserting only "the + definitions came back" passes identically when the projection worked and when it quietly did + not, which is [[gate-can-report-green-on-nothing]] wearing a green tick. + + ⭐ This is INTRINSIC rather than instrumented: `_project` stamps every top-level value as a + `_Projected`, so the evidence rides the document itself. Nothing has to be wrapped, no counter + can be installed on the wrong object ([[measure-the-real-call]]), and the answer is the same in + production as it is under a gate — a monkey-patched tally is true only while the patch is on. + + ⚠ It answers about ONE VALUE (a table's definition), not about a document: `_project` returns a + plain dict OF `_Projected` values, so `is_projected(doc)` is False and `is_projected(doc[key])` + is True. That is the right grain — the thing a route holds and reads is the definition. + """ + return isinstance(value, _Projected) + + class Store: """One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush. @@ -156,23 +245,51 @@ class Store: self._key_locks = {} self._key_locks_guard = threading.Lock() self._rev_at = {} + # --- ⭐⭐ D-305 — THE TWO FIELDS THAT MAKE A CROSS-PROCESS WRITE SURVIVABLE ----------- + # `_base[name]` the hub commit sha this key's cached document was built ON. Learned for + # free from the download path and from `CommitInfo.oid` after a write, so + # it costs no extra round trip in the common case. + # `_journal[name]` the `fn`s applied to this key since the last CONFIRMED upload — our + # INTENT, not our result. On a conflict the intent is replayed onto the + # hub's newer document instead of the result being uploaded over it. + # + # ⛔ WHY A JOURNAL AND NOT A RETRY. The writers here are DIFFERENT PEOPLE in DIFFERENT + # CONTAINERS, and this instance's cache holds an hour of its own edits over a base that has + # moved. Re-uploading that snapshot — which is what a retry does — reproduces the loss + # exactly. Replaying "add this view" onto the fresher document keeps both writers' work, + # which is the only outcome that answers the defect. + self._base = {} + self._journal = {} def available(self): """True when a token is configured — callers degrade gracefully without one.""" return bool(_token()) def _download(self, name): - """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent. + """`(document, commit_sha)` for {name}.json. `({}, None)` ONLY when it is genuinely absent. Any other failure (network, auth, rate-limit, corrupt JSON) RAISES — a transient read error must never be mistaken for an empty store, or a caller could overwrite real data - with an empty dict (this is what wiped the user registry on restart).""" + with an empty dict (this is what wiped the user registry on restart). + + ⭐⭐ D-305 — THE SHA COMES FROM THE PATH, WHICH COSTS NOTHING. `hf_hub_download` writes into + `…/snapshots//`, so the revision the bytes came from is already in the + return value. Measured 2026-08-18 against a scratch repo: that directory name equals the + `CommitInfo.oid` of the commit that wrote the file, and equals + `get_hf_file_metadata(...).commit_hash` — which would have cost an extra HTTP round trip on + the hottest read in the product. **The parent of our write is therefore known by every + reader, for free.** + + ⚠ A sha we cannot parse comes back as `None`, and `_commit` resolves that with an explicit + head lookup rather than uploading without a precondition. "I do not know my parent" must + take the careful path, never the fast one. + """ try: path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset', token=_token(), force_download=True) with open(path, encoding='utf-8') as f: - return json.load(f) + return json.load(f), _sha_from_path(path) except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError): - return {} + return {}, None except ValueError as e: # Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in # ValueError("Force download failed…") with the 404 as __cause__ — so a brand-new @@ -180,7 +297,7 @@ class Store: # A genuinely-absent repo is the same fact as a genuinely-absent file: {}. cause = getattr(e, '__cause__', None) if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)): - return {} + return {}, None raise def get(self, name, fresh=False): @@ -191,7 +308,7 @@ class Store: if not fresh and name in self._cache: return json.loads(json.dumps(self._cache[name])) # hand back a copy try: - data = self._download(name) + data, sha = self._download(name) except Exception as e: try: # lazy import — core stays layer-clean import harness.telemetry as _tel # noqa: PLC0415 @@ -203,6 +320,10 @@ class Store: return json.loads(json.dumps(self._cache[name])) return {} self._cache[name] = data + # D-305: the cached document and the revision it derives from are set TOGETHER, always. + # A base that can disagree with the cache it describes is worse than no base at all — + # it would make a stale write look like a fresh one and pass the precondition. + self._base[name] = sha return json.loads(json.dumps(data)) def get_projection(self, name, drop=()): @@ -242,7 +363,7 @@ class Store: if name in self._cache: return _project(self._cache[name], drop) try: - data = self._download(name) + data, sha = self._download(name) except Exception as e: try: # lazy import — core stays layer-clean import harness.telemetry as _tel # noqa: PLC0415 @@ -256,6 +377,7 @@ class Store: return {} with self._lock: self._cache[name] = data # ⛔ the WHOLE document, never the projection + self._base[name] = sha # D-305 — set with the cache, never apart from it return _project(data, drop) def _key_lock(self, name): @@ -280,15 +402,25 @@ class Store: So the dirty mark is re-read AFTER the download, under the short lock, and a key that became dirty keeps its cache. """ - data = self._download(name) # network, no lock held + data, sha = self._download(name) # network, no lock held with self._lock: if self._dirty.get(name) and self._owned.get(name) and name in self._cache: - # Our copy is stale by construction: a pending upload is going to overwrite the - # hub anyway, so the fresh read cannot preserve a remote write — it can only - # destroy a local one we have already acknowledged. Same argument, and the same - # three conditions, as `update`'s pending-async branch below. + # ⛔⛔ D-305 REWROTE THIS BRANCH'S JUSTIFICATION, NOT ITS BEHAVIOUR — AND THE OLD + # ONE HAD TO GO, because it asserted the defect. It read: *"a pending upload is + # going to overwrite the hub anyway, so the fresh read cannot preserve a remote + # write."* That sentence IS the lost update: "we are going to overwrite whatever + # the other person did" was stated as a reason to stop worrying about it. + # + # ⭐ It is no longer true. A pending upload now carries a `parent_commit` + # precondition, so it CANNOT overwrite a remote write — it will be refused and + # rebased onto it (`_commit`). The branch survives for its OTHER, still-correct + # reason: this cache holds an async write nobody has flushed yet, and replacing it + # with the hub's copy destroys a row we have already answered 201 for (D-177). + # ⇒ keep the local delta here; preserve the REMOTE work at commit time, which is + # the only place that can see it. return json.loads(json.dumps(self._cache[name])) self._cache[name] = data + self._base[name] = sha return json.loads(json.dumps(data)) def exists(self, name): @@ -345,30 +477,199 @@ class Store: at = self._rev_at.get(n) return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'} - def _upload(self, name, data): - """The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra - API round-trip on every put(), pure latency after the first success.""" - api = HfApi(token=_token()) + def _ensure_repo(self): + """create_repo is confirmed ONCE per instance — it was a full extra API round-trip on + every put(), pure latency after the first success. + + Lifted out of `_upload` (D-305) because `_commit` has to resolve a parent commit BEFORE the + first write, and a repo that does not exist yet has no head to resolve. + """ if not self._repo_ok: - api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True) + HfApi(token=_token()).create_repo(self.repo, repo_type='dataset', private=True, + exist_ok=True) self._repo_ok = True + + def _head_sha(self): + """The repo's current head commit, or `None` if the repo does not exist yet. + + ⚠ ONE EXTRA ROUND TRIP, AND IT IS THE COLD PATH ONLY. `_base[name]` is populated by every + download and by every successful write, so this is reached on a first write into a process + that has never read the key — and on the rebase path, where a round trip is already the + price of correctness. + """ + try: + return HfApi(token=_token()).repo_info(self.repo, repo_type='dataset').sha + except RepositoryNotFoundError: + return None + + def _upload(self, name, data, parent): + """The raw hub write, WITH A PRECONDITION. Returns the new head commit sha. + + ⛔ `parent` IS MANDATORY AND HAS NO DEFAULT, deliberately. A default of `None` would make + "upload unconditionally" the thing you get by forgetting an argument — and unconditional + upload is precisely D-305. Every caller goes through `_commit`, which is the one place + allowed to decide what the parent is. + """ + data_binding.check_write(self.repo, f'write {name!r}') # D-315 backstop, see `update` + self._ensure_repo() buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8')) - api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo, - repo_type='dataset', commit_message=f'update {name}') + info = HfApi(token=_token()).upload_file( + path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo, + repo_type='dataset', + # D-298: under the local opt-in this suffix is what makes a laptop's commit + # distinguishable from the live Space's in the store's git history — the only audit + # trail this product has. + commit_message=f'update {name}' + data_binding.local_commit_suffix(), + parent_commit=parent) + return getattr(info, 'oid', None) + + def _commit(self, name, data, replay=(), max_rebase=None): + """Land `data` for `name` without overwriting anyone — D-305's whole mechanism. + + ⭐⭐ THE SHAPE, and every step of it is measured rather than assumed (scratch-repo probe, + 2026-08-18): + + 1. upload with `parent_commit = the revision our document was built on`; + 2. the hub answers **412** if the branch moved — and the write does NOT land, so there + is nothing to undo; + 3. re-read the head, REPLAY our intent (`replay`, the `fn`s applied since our last + confirmed upload) onto it, and try again against the new parent; + 4. after `_MAX_REBASE` attempts, RAISE. + + ⭐ REPLAYING THE INTENT IS WHAT MAKES THIS DIFFERENT FROM A RETRY. Uploading our snapshot + again would reproduce the loss; re-applying "add this view" to the newer document keeps + both writers. The one place it changes an answer is a `fn` that ALLOCATES from the document + — `table_store._unique_name` picks the first free display name — which on a rebase may pick + a different name than the one already handed to the caller. That is a rename where the + pre-fix behaviour was a DELETION, and the trade is stated rather than hidden. + + ⛔ NO PARENT ⇒ RAISE, NEVER UPLOAD BLIND. If neither the cached base nor `_head_sha()` can + name a revision, the safe answer is "this write cannot be made safe", not "make it the old + way". A fallback that quietly restores the unguarded path is how a fix becomes decorative + ([[fallback-that-became-the-rule]]). + + Returns `(document_that_landed, new_sha)` — the document may differ from the one passed in, + because a rebase rebuilds it. + """ + data_binding.check_write(self.repo, f'write {name!r}') + # ⚠ `_ensure_repo()` IS NOT CALLED HERE, and that is a deliberate reversal. It was, for one + # revision, so that a brand-new repo would have a head to name as a parent — and it broke + # three existing gates, because their fixtures replace `_upload` (which is where + # `create_repo` has always lived) and suddenly found themselves calling the real hub. The + # ordering works without it: a repo that does not exist resolves no head and no `exists`, + # so the first write goes out with `parent=None` and `_upload` creates the repo on its way + # through, exactly as before. + budget = _MAX_REBASE if max_rebase is None else int(max_rebase) + for attempt in range(budget + 1): + with self._lock: + parent = self._base.get(name) + if not parent: + parent = self._head_sha() + if not parent: + # A repo with no commits at all: `_upload` creates it on the way through, so this + # is the genuinely-first write and there is nothing that could be lost. + # ⚠ NO EM DASH IN THIS MESSAGE, OR THE ONE BELOW. Both are raised, and a raised + # store message reaches the client through the 503 handler, which puts them inside + # STANDING RULE 2's scope ("any Python string returned to the client"). `web_prose` + # caught exactly these two. + if self.exists(name): + raise StoreConflict( + f'cannot resolve a parent commit for {self.repo}:{name}, so this write ' + f'cannot be made safe. Refusing to upload without a precondition: an ' + f'unconditional write is what silently erases another session\'s work.') + parent = None + try: + oid = self._upload(name, data, parent) + except HfHubHTTPError as e: + if not _is_conflict(e): + raise # 401/404/429/500 — a different fault, never a rebase + if attempt >= budget: + # ⛔ THE EXHAUSTED CASE RAISES `StoreConflict`, NOT THE RAW 412, and this gate + # caught the first version doing the latter. A caller cannot be asked to know + # that one particular HTTP status out of the hub means "your change was not + # saved and nothing was damaged"; that is a domain fact and it needs a domain + # type. `from e` keeps the hub's own message in the traceback. + raise StoreConflict( + f'{self.repo}:{name} moved under this write {budget + 1} times in a ' + f'row. Refusing to overwrite it. Nothing was saved.') from e + data = self._rebase(name, replay) + continue + with self._lock: + self._base[name] = oid + self._journal[name] = [] + return data, oid + # Unreachable by construction — every iteration returns or raises. Kept so that an edit to + # the loop above cannot make this function fall out and return `None`, which callers would + # unpack into a crash two frames away from the cause. + raise StoreConflict(f'{self.repo}:{name}: the write loop exited without committing.') + + def _journal_append(self, name, fn): + """Record one intent and return the replay list. THE CALLER HOLDS THE CACHE LOCK. + + ⚠ CAPPED, BECAUSE AN UNCONFIRMED WRITE STREAM IS AN UNBOUNDED LIST OF CLOSURES. In normal + operation the cap is unreachable — `_commit` empties this every few seconds — so it only + bites during a sustained hub outage, where the alternative is a process that grows until it + dies. When it bites, the OLDEST intents are dropped and it is reported: a rebase afterwards + replays the most recent `_MAX_JOURNAL` changes rather than all of them. Stated rather than + silently enforced, per the standing rule about limits that cannot be removed. + """ + j = self._journal.setdefault(name, []) + j.append(fn) + if len(j) > _MAX_JOURNAL: + dropped = len(j) - _MAX_JOURNAL + del j[:dropped] + try: # lazy import — core stays layer-clean + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:journal_full:{name}', StoreConflict( + f'{self.repo}:{name} has {_MAX_JOURNAL}+ unconfirmed writes; dropped ' + f'{dropped} oldest replay entr(ies). A rebase will re-apply only the most ' + f'recent {_MAX_JOURNAL}.')) + except Exception: # noqa: BLE001 + pass + return list(j) + + def _rebase(self, name, replay): + """Re-read the hub's newer document and re-apply our intent onto it. Caller holds the key + lock, so nothing else can be mid-write on this key while we do. + + ⚠ THE CACHE IS REPLACED WITH THE REBASED RESULT, and forgetting that line would make the + whole mechanism single-shot: the next write would start from the same stale snapshot and + conflict again, burning a rebase per write forever. It also has a second, welcome effect — + this instance now HOLDS the other container's work, which is the only moment a process + whose cache is never re-read gets to learn that anything changed. + """ + fresh, sha = self._download(name) + for fn in list(replay): + out = fn(fresh) + fresh = out if out is not None else fresh + with self._lock: + self._cache[name] = json.loads(json.dumps(fresh)) + self._base[name] = sha + try: # lazy import — core stays layer-clean + import harness.telemetry as _tel # noqa: PLC0415 + _tel.error(f'store:rebase:{name}', + StoreConflict(f'{self.repo}:{name} moved; replayed {len(replay)} change(s)')) + except Exception: # noqa: BLE001 + pass + return fresh def upload_bytes(self, path_in_repo, data, message=None): """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents). Bytes are never cached in-process: large, cold, fetched on explicit user action.""" if not self.available(): raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') + # D-315: a blob is still this tenant's data. ⚠ No `parent_commit` here and that is correct, + # not an omission: a blob is written at its own path and read whole, so two writers cannot + # silently merge into one damaged document the way `{name}.json` can. The concurrency + # hazard D-305 describes is specific to the shared JSON blob. + data_binding.check_write(self.repo, f'write bytes {path_in_repo!r}') with self._lock: - api = HfApi(token=_token()) - if not self._repo_ok: - api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True) - self._repo_ok = True - api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo, - repo_id=self.repo, repo_type='dataset', - commit_message=message or f'add {path_in_repo}') + self._ensure_repo() + HfApi(token=_token()).upload_file( + path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo, + repo_id=self.repo, repo_type='dataset', + commit_message=(message or f'add {path_in_repo}') + + data_binding.local_commit_suffix()) def download_bytes(self, path_in_repo): """Raw bytes back, or None when absent/unreachable — a missing attachment degrades to @@ -385,13 +686,20 @@ class Store: return None def delete_path(self, path_in_repo): - """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal.""" + """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal. + + ⛔ THE REFUSAL IS RAISED, NOT SWALLOWED INTO THE `return False` BELOW, and the distinction + matters: `False` here means "the delete did not happen", which several callers treat as a + tolerable no-op. "You are not allowed to touch this store" is not a no-op — it is the + thing the operator has to be told. + """ if not self.available(): return False + data_binding.check_write(self.repo, f'delete {path_in_repo!r}') try: - HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo, - repo_type='dataset', - commit_message=f'delete {path_in_repo}') + HfApi(token=_token()).delete_file( + path_in_repo=path_in_repo, repo_id=self.repo, repo_type='dataset', + commit_message=f'delete {path_in_repo}' + data_binding.local_commit_suffix()) return True except (EntryNotFoundError, RepositoryNotFoundError): return True @@ -399,16 +707,44 @@ class Store: return False def put(self, name, data): - """Persist {name}.json (creates the private repo on first write). Updates the cache.""" + """Persist {name}.json (creates the private repo on first write). Updates the cache. + + ⭐ D-305 — `put` NOW TAKES THE KEY LOCK AND GOES THROUGH `_commit`, and both halves are + required. Without the precondition this method is the hole in the whole mechanism: seven + callers outside this file reach it (`users.py`, `modules/ar.py`, `modules/map.py`, + `runtime.py`, `store_backend.py`, `ops/seed_pg_from_hf.py`) and `users.json` is a document + every account shares, so an unguarded whole-file `put` clobbers exactly as `update` used to. + + ⚠ THE CACHE LOCK NO LONGER SPANS THE NETWORK, which is a change to this method's + concurrency and is deliberate. The old body held `self._lock` — the lock every `get()` + takes — across the upload, so one `put` froze every reader for a round trip (D-181's + shape); with up to `_MAX_REBASE` extra round trips now possible that freeze would get + strictly worse. The per-key write lock gives the callers a STRONGER guarantee than they had + (two writers of one key can no longer interleave) while readers stop waiting. Lock order is + key-then-cache, as everywhere else in this class. + + ⚠ REPLAY SEMANTICS FOR A WHOLE-DOCUMENT WRITE ARE LAST-WRITE-WINS, STATED RATHER THAN + IMPLIED. `put` means "the document is this now", so its journal entry replaces rather than + merges. What the precondition buys here is that the replacement is applied to a document we + have actually SEEN — and that any `update` queued behind it replays on top — instead of + being uploaded over an hour of unseen work. + """ if not self.available(): raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') - with self._lock: - self._upload(name, data) - self._cache[name] = json.loads(json.dumps(data)) - self._owned[name] = True - self._last_flush[name] = _time.monotonic() - self._bump(name) # C6 — AFTER the upload: a raise here is not a write - return self._cache[name] + data_binding.check_write(self.repo, f'put {name!r}') + with self._key_lock(name): + def _replace(_current): + return json.loads(json.dumps(data)) + with self._lock: + replay = self._journal_append(name, _replace) + landed, _oid = self._commit(name, data, replay) + with self._lock: + self._cache[name] = json.loads(json.dumps(landed)) + self._owned[name] = True + self._last_flush[name] = _time.monotonic() + self._dirty[name] = False + self._bump(name) # C6 — AFTER the upload: a raise here is not a write + return self._cache[name] def _flush_worker(self, name): """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure — @@ -433,11 +769,17 @@ class Store: this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay OUTSIDE the key lock — a worker that dozed while holding it would block every writer of that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove. + + ⭐⭐ D-305 — THIS WORKER IS ALSO WHERE MOST CROSS-CONTAINER LOSS HAPPENED, because the + table workspace (views, fields, folders, overlays) is written `flush='async'`. It uploads a + SNAPSHOT of a cache that may be minutes old, so it takes the `replay` journal with it: on a + 412 the snapshot is thrown away and the journalled `fn`s are re-applied to the hub's newer + document instead. The journal is cleared inside `_commit`, only on success. """ backoff = 2.0 while True: _time.sleep(_FLUSH_DELAY) - wait, err, snapshot = None, None, None + wait, err, snapshot, replay = None, None, None, () with self._key_lock(name): with self._lock: gap = _time.monotonic() - self._last_flush.get(name, 0.0) @@ -449,10 +791,17 @@ class Store: else: self._dirty[name] = False snapshot = json.loads(json.dumps(self._cache.get(name, {}))) + # Snapshotted WITH the document and under the same lock: a journal that + # could drift from the snapshot it describes would replay the wrong set. + replay = list(self._journal.get(name) or ()) if wait is None: try: - self._upload(name, snapshot) + landed, _oid = self._commit(name, snapshot, replay) with self._lock: + # A rebase rebuilt the document, so the cache must take the result — + # `_rebase` already wrote it, and this keeps the no-conflict path + # identical to it rather than leaving two ways for the cache to be set. + self._cache[name] = json.loads(json.dumps(landed)) self._last_flush[name] = _time.monotonic() backoff = 2.0 except Exception as e: # noqa: BLE001 @@ -507,15 +856,28 @@ class Store: finish, so waiting for its key lock could only turn a partial save into no save at all. Best-effort is the correct contract here, and it is the one case where taking the lock would be worse than skipping it. + + ⭐ D-305 — IT STILL GOES THROUGH `_commit`, so the last write of a dying process cannot be + the one that erases somebody. Exiting is not a licence to overwrite: if the branch has + moved, this rebases like any other write, and if it cannot, it gives up rather than + clobbering. `_commit`'s own `data_binding` check applies too, so a container that was never + allowed to write this store does not get one last unguarded upload on the way out. """ with self._lock: dirty = [k for k, v in self._dirty.items() if v] snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty} + replays = {k: list(self._journal.get(k) or ()) for k in dirty} for k in dirty: self._dirty[k] = False for k, d in snaps.items(): try: - self._upload(k, d) + # ⚠ ONE REBASE, NOT `_MAX_REBASE`, and the docstring above is the argument: this + # runs at interpreter exit with the process going away. The full budget is up to + # four extra downloads and five upload attempts PER DIRTY KEY on a path that may be + # killed at any moment, which turns a best-effort save into a hang. One attempt + # keeps the safety this method exists for (a conflicting write still rebases rather + # than clobbering) at a cost the exit path can actually pay. + self._commit(k, d, replays.get(k, ()), max_rebase=1) except Exception: pass # exiting anyway — nothing left to reschedule @@ -528,9 +890,19 @@ class Store: returns immediately; a coalescing worker commits. The strict read still guards the FIRST async update of a key per process. + ⛔⛔ D-305 — "LAST-WRITE-WINS IS THE CONTRACT" IS NO LONGER TRUE ACROSS PROCESSES, AND THE + SENTENCE BELOW USED TO SAY IT WITHOUT THE QUALIFIER. Two writers in ONE process cannot lose + an update — they share `self._cache`, which is why every gate in this repo passed over the + defect. Two writers in two CONTAINERS could, and did: measured on tenant #0's live store in + commit `dee7481a77`, where one session's write rolled the document back an hour and cost a + real user 3 views, 49 typed cell values and a field id. The commit now carries a + `parent_commit` precondition and rebases (`_commit`), so the contract is: **within a + process, last write wins; across processes, a write that would overwrite unseen work is + refused and re-applied.** A lost update is not the contract at either grain. + ⭐⭐ W31-T13 (D-177(d)) — TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET. `self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key - still cannot interleave (last-write-wins is the contract; a lost update is not). + still cannot interleave. `self._lock` — the one every `get()` takes — is now held only for the microseconds a cache read or write needs, never across a download or an upload. Before this, one sync write froze every reader in the process for two network round trips, on a store whose warm @@ -538,6 +910,13 @@ class Store: ⚠ Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`. """ + # ⛔ D-315 — REFUSED AT THE ACCEPTANCE POINT, NOT AT THE UPLOAD. `_upload` carries the same + # check as a backstop, but refusing only there would let the async branch accept the + # mutation into the cache, answer 200, mark the key dirty, and hand the flush worker a + # write that can never succeed — which retries with backoff forever and reports the refusal + # to nobody. Refuse before anything is accepted, and the caller gets a 503 that tells the + # truth: no change was saved. + data_binding.check_write(self.repo, f'update {name!r}') with self._key_lock(name): return self._update_locked(name, fn, flush) @@ -596,6 +975,15 @@ class Store: result = fn(data) data = result if result is not None else data + # ⭐⭐ D-305 — THE INTENT IS RECORDED, NOT JUST THE RESULT, and this one line is what makes + # a conflict survivable. `data` is our answer over OUR base; `fn` is what we MEANT, and + # only the intent can be re-applied to somebody else's newer document. The journal holds + # every `fn` since the last confirmed upload — which for an async key is a whole flush + # window of edits, not one — and `_commit` clears it only when the hub has accepted them. + # ⚠ Appended AFTER `fn` ran: a `fn` that raised changed nothing and must not be replayed. + with self._lock: + replay = self._journal_append(name, fn) + if flush == 'async': if not self.available(): raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') @@ -606,21 +994,24 @@ class Store: self._schedule_flush(name) return data - # ⛔ THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE. - # `put()` holds `self._lock` across its own `_upload` — which is exactly the freeze this - # ticket removes — but it is PUBLIC with seven callers outside this file (`users.py`, - # `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`, - # `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency - # semantics at once. Duplicating four assignments with this comment beside them is the - # smaller blast radius; if `put` is ever split, this collapses back into a call. + # ⛔ THE SYNC COMMIT. It used to be spelled out here rather than calling `put()`, because + # `put()` held `self._lock` across its own upload — the freeze W31-T13 removed — and is + # PUBLIC with seven callers outside this file whose concurrency semantics changing it would + # alter. D-305 made that duplication untenable: an unguarded whole-file write is the hole + # the precondition exists to close, so BOTH now go through `_commit`, which is the single + # place that resolves a parent, rebases and clears the journal. `put` still keeps its own + # body (its op REPLACES rather than merges), but neither can upload unconditionally. if not self.available(): raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.') - self._upload(name, data) # network, no cache lock held + landed, _oid = self._commit(name, data, replay) # network, no cache lock held with self._lock: - self._cache[name] = json.loads(json.dumps(data)) + # ⚠ `landed`, NOT `data`. A rebase rebuilt the document from the hub's newer copy, so + # caching what we TRIED to write would leave the cache diverged from what is stored — + # and the next write would rebase off that divergence forever. + self._cache[name] = json.loads(json.dumps(landed)) self._owned[name] = True self._last_flush[name] = _time.monotonic() - # ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_upload` raises on failure, + # ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_commit` raises on failure, # and a bump on that path would have `revision()` report a change that never landed. self._bump(name) # The upload above CARRIED whatever was pending, so the key is no longer ahead of @@ -629,9 +1020,9 @@ class Store: # ⚠ AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache # is still the truth and the row must still be committed by the worker). Clearing it # first would turn a transient upload error into the very silent loss the branch above - # exists to stop — which is why it sits here and not before `_upload`. + # exists to stop — which is why it sits here and not before the commit. self._dirty[name] = False - return data + return landed # ------------------------------------------------------------------- instances + module API diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 0b49961db6b89aa6ae624121bad8a4896aff9d2c..d09985b9a785371d40755557102e98c3f4c63e74 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -336,7 +336,7 @@ def limit_report(table_key, st=None): 'cause': f'this database is EDITABLE, so its rows live in the shared `{STORE_KEY}` ' f'document that every request copies; at the widest shipped row (0.318 KB) ' f'{cap:,} rows is 19.1 MB against a 32 MB per-table budget', - 'recommendation': 'connect it to a source instead — connected data is served ' + 'recommendation': 'connect it to a source instead. Connected data is served ' 'read-through from the mirror and is not bounded (R6); for typed data ' 'the increment is D-87, a per-table row key, not a bigger number', } @@ -439,9 +439,15 @@ def records_mutable(table_key, st=None): #: waves because `verify_fields_contract` diffed the client's offer against the ODOO grid's #: acceptance and never against this set. Wave-27 item 13 adds that derived leg, which is what #: found it (measured, not reviewed: `_clean_field({'type':'image'})` returned None). +#: ⭐ WAVE-34 (owner ruling R13) — `ai_enrich` JOINS, and it lands HERE FIRST rather than last. +#: The block above records that `image` reached four of the five surfaces and missed this one, so +#: the client offered a column the server deleted on the next read. `ai_enrich` is added to this +#: set in `W34-T51` (server) while the CLIENT half waits for `W34-T53`, which is the safe order: +#: a kind the server accepts and the client does not offer is invisible, whereas a kind the client +#: offers and the server refuses is a column created, named, configured and gone. UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date', 'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json', - 'link', 'rollup', 'code', 'image', 'formula'} + 'link', 'rollup', 'code', 'image', 'formula', 'ai_enrich'} #: A cell is still a scalar on the wire (`Row` values are strings/numbers) — what this bounds is #: the DOCUMENT inside it. Source data is one already-paid provider response; keep it whole so a #: provider field is never silently discarded merely because the response is rich. 32 MiB matches @@ -902,6 +908,19 @@ def clean_fields(raw): entry = None elif fx is not None: entry['formula'] = fx + # ⭐⭐ WAVE-34 (R13) — the AI ENRICHMENT bag rides THIS door too, and it is the SIXTH time + # this seam has been fixed after the fact (`pinned`, `code`, `link`/`rollup`, `formula`, + # `agg`). It is written in the same change as `_clean_field`'s arm rather than a wave + # later, which is the only difference between this entry and the five above it. + if entry is not None: + ae = (_clean_ai_enrich(f.get('aiEnrich')) + if f.get('aiEnrich') is not None else None) + if (ftype == 'ai_enrich') != (ae is not None): + entry = None + elif ae is not None: + entry['aiEnrich'] = ae + # C3: derived, never typed beside the config. Same call as the other door. + entry['automation'] = _field_agent_binding(entry) if entry is None: seen.discard(key) continue @@ -949,13 +968,26 @@ def all_defs(st=None): fake store installed by a gate, a `_Lent` handle serving a document somebody else read). The fallback is CORRECT-but-slow by construction — it can only ever return MORE than was asked for — which is the one direction a fallback here is allowed to be wrong in. + + ⛔⛔ W33-T01 (D-213) — **THAT PARAGRAPH WAS FALSE ON THIS FUNCTION'S OWN BODY UNTIL 2026-08-14, + AND THE LIE WAS THE DANGEROUS DIRECTION.** Only `AttributeError` fell back; every other + exception returned `{}` — the exact silent-empty answer the paragraph ABOVE it forbids, three + lines earlier, in bold. `/nav`, `/nav/prefs` and `routes_shares` would have rendered ZERO + databases and looked entirely correct doing it ([[empty-answer-vs-unfinished-answer]]). + Nothing had noticed because the projection does not fail on a healthy store — a fallback is + dead code until the day it is the only code running. **Every failure now falls back to the + whole read**, which keeps `all_tables`' own `except: return {}` as the single backstop instead + of two competing ones ([[one-question-two-normalizers]]). This ticket puts the table WALL on + this function, so an empty answer here stops being a blank nav and becomes a 404 on every + database in the tenant. """ try: return dict(_st(st).get_projection(STORE_KEY, drop=DEF_ONLY_DROPS) or {}) - except AttributeError: - return all_tables(st) except Exception: - return {} + # ⚠ `AttributeError` (a store too old to project) is not special-cased any more — it was + # only ever ONE of the ways this can fail, and singling it out is what made the other ways + # silent. `all_tables` is correct for all of them. + return all_tables(st) def get(table_key, st=None): @@ -1651,6 +1683,308 @@ def _clean_rollup(raw): return out +# --------------------------------------------------------------------------------------------- +# THE AI ENRICHMENT COLUMN (wave 34, owner ruling R13) +# --------------------------------------------------------------------------------------------- +# R13: *"a field kind called AI enrichment: a prompt per row that populates text. Detailed +# configuration in the field's own config, including a token-usage limit."* +# +# ⛔ POSTURE: `link`/`rollup`/`formula`'s, NOT `code`'s. The bag DEFINES the column — a prompt is +# the only thing that can ever produce a value here — so the type/bag pairing is enforced BOTH +# WAYS and either half alone refuses the field. An `ai_enrich` column with no prompt is a column +# that renders, sorts, filters and can never hold anything, which is the silent kind of broken +# every other bag on this door is paired to prevent. +# +# ⚠ AND THE VALUE IS STORED, NOT COMPUTED. `formula` is client-recomputed on every paint and +# `rollup` is read-through; neither persists. An enrichment that vanished on reload would not be +# the feature (PRD assumption A6), so an `ai_enrich` cell is an ordinary stored string, a human +# may type over it, and it is deliberately NOT in `is_computed_cell`. + +#: The bag's own key set, read by BOTH the cleaner and the reporter below so they cannot disagree +#: about what "unknown" means. ⛔ ONE constant, because T51's contract is that an unknown key is +#: DROPPED **and NAMED** — two hand-maintained lists would eventually drop a key the report did +#: not mention, which reads to a user as the config silently not saving. +AI_ENRICH_KEYS = ('prompt', 'model', 'maxTokens', 'trigger', 'overwrite') +#: When the column runs. `manual` = only when a person asks; `on_change` = when a referenced cell +#: moves; `schedule` = on a cadence. The community ask R13 cites ("enrichment on a schedule") is +#: the third of these, so it is in the vocabulary from the start rather than added later. +AI_ENRICH_TRIGGER_MODES = ('manual', 'on_change', 'schedule') +#: What an AUTOMATIC run may replace. ⛔ A HUMAN-EDITED CELL IS NEVER OVERWRITTEN AND THAT IS NOT +#: IN THIS VOCABULARY — it is a law (`ai_enrich_may_write`), not a policy, so no configuration can +#: turn it off. These three decide only what happens to cells the AGENT itself wrote. +AI_ENRICH_OVERWRITE = ('blank', 'stale', 'always') +#: Bounded because the ceiling is the feature (R13). The floor is one useful sentence; the roof is +#: what one cell can cost before the run has to say so out loud. +AI_ENRICH_MIN_TOKENS, AI_ENRICH_MAX_TOKENS = 16, 4000 +AI_ENRICH_DEFAULT_TOKENS = 300 +#: A prompt is a template, not an essay. Long enough for real instructions plus a few `{field}` +#: tokens, short enough that the per-cell input cost is predictable. +AI_ENRICH_PROMPT_MAX = 2000 +#: `cron` is stored as an opaque bounded string. ⚠ It is NOT parsed here on purpose: this module +#: owns the field contract, and the scheduler that reads it owns the cadence vocabulary. Parsing +#: it in two places is how one door starts refusing a cadence the other accepts. +AI_ENRICH_CRON_MAX = 120 + + +def _clean_ai_enrich(raw): + """The `ai_enrich` config bag → the stored shape, or None if the column cannot run. + + ⚠ Every key is normalised to a canonical form, so a definition saved and re-read is + byte-identical (`_clean_ai_enrich(_clean_ai_enrich(x)) == _clean_ai_enrich(x)`). That + idempotence is what makes the round-trip clause testable rather than a claim. + """ + if not isinstance(raw, dict): + return None + prompt = str(raw.get('prompt') or '').strip()[:AI_ENRICH_PROMPT_MAX] + if not prompt: + return None # the prompt IS the column; see the posture note above + out = {'prompt': prompt} + # The model is OPTIONAL and an unknown name is DROPPED rather than refused: it selects a + # PROVIDER for a column that already knows what to ask, so a stale name should fall back to + # the ladder's own order, never destroy the column. (`code`'s language argument, same shape.) + model = str(raw.get('model') or '').strip().lower()[:60] + if model: + out['model'] = model + tokens = raw.get('maxTokens') + # `bool` is an `int` subclass — `True` would otherwise validate as a 1-token ceiling. + if isinstance(tokens, bool) or not isinstance(tokens, (int, float)): + tokens = AI_ENRICH_DEFAULT_TOKENS + out['maxTokens'] = max(AI_ENRICH_MIN_TOKENS, min(int(tokens), AI_ENRICH_MAX_TOKENS)) + trig = raw.get('trigger') if isinstance(raw.get('trigger'), dict) else {} + mode = str(trig.get('mode') or '').strip().lower() + trigger = {'mode': mode if mode in AI_ENRICH_TRIGGER_MODES else 'manual'} + # ⛔ `cron` is kept ONLY on a scheduled column. A cadence stored under `manual` is a setting + # the user can see and nothing will ever read — the shape D-246 wears on the Odoo sync, where + # a `manual` preset does nothing because no branch reads it. + cron = str(trig.get('cron') or '').strip()[:AI_ENRICH_CRON_MAX] + if trigger['mode'] == 'schedule' and cron: + trigger['cron'] = cron + out['trigger'] = trigger + overwrite = str(raw.get('overwrite') or '').strip().lower() + out['overwrite'] = overwrite if overwrite in AI_ENRICH_OVERWRITE else 'blank' + return out + + +def ai_enrich_dropped_keys(raw): + """The config keys `_clean_ai_enrich` will NOT keep, sorted — the NAMING half of T51's + "an unknown config key is DROPPED and NAMED, never silently eaten". + + ⛔ IT EXISTS BECAUSE EVERY OTHER BAG CLEANER IN THIS FILE HAS NO ERROR CHANNEL. `_clean_link`, + `_clean_rollup`, `_clean_profile` and `_clean_format` all return `dict | None` and drop + unknown keys in silence — `verify_fields_contract` even asserts that they do. Copying one of + them as a template gets the DROP right and the NAMING wrong by construction, so the report is + a separate function reading the SAME `AI_ENRICH_KEYS` constant rather than a second list. + + ⚠ It reports UNKNOWN keys only. A known key whose VALUE was normalised (an out-of-range + ceiling clamped, an unknown model dropped) is not a surprise worth a sentence — the stored + definition comes straight back to the editor, so the user sees the accepted value itself. + """ + if not isinstance(raw, dict): + return [] + return sorted(str(k) for k in raw if k not in AI_ENRICH_KEYS) + + +def ai_enrich_fields(defn): + """Every `ai_enrich` column in this table definition, in declaration order. + + One reader for "which columns are field agents", so the Agents module (contract C3), the + runner and the scheduler cannot disagree about the set ([[one-evaluator-per-question]]). + """ + return [f for f in ((defn or {}).get('fields') or []) + if isinstance(f, dict) and f.get('type') == 'ai_enrich' + and isinstance(f.get('aiEnrich'), dict)] + + +def _field_agent_binding(field): + """Contract C3's `field.automation` bag, DERIVED from the column rather than typed beside it. + + ⛔⛔ THIS IS DERIVED ON PURPOSE AND IT IS THE ONE DESIGN CALL IN T51 WORTH ARGUING WITH. + C3 as drafted has F *store* `{kind, table, field, trigger}` — but `field` is already + `field['key']` and `trigger` is already `aiEnrich['trigger']`, so two of its four keys would + be a second copy of a fact this same dict already carries. Two copies of one fact is + [[one-question-two-normalizers]] with a guaranteed drift date: the first PATCH that changes + the trigger through the enrichment editor and not through the automation editor. + Computing it inside the SINGLE validator both write doors run means they cannot diverge. + + ⚠ `table` IS ABSENT, and that is the deviation E must know about: this function is handed one + field dict and no table context, while `_clean_field(f) == f` is asserted by two gates over + engine-seeded field lists — stamping the table key from a caller that happens to know it would + make a stored field unequal to its own validator's output. E iterates tables to build the + synthetic rows, so E already holds the key that would go here. + """ + bag = (field or {}).get('aiEnrich') + if not isinstance(bag, dict): + return None + return {'kind': 'field_agent', 'field': str((field or {}).get('key') or ''), + 'trigger': dict(bag.get('trigger') or {'mode': 'manual'})} + + +# --------------------------------------------------------------------------------------------- +# PER-CELL PROVENANCE (wave 34, W34-T51 — "stamp it FROM DAY ONE") +# --------------------------------------------------------------------------------------------- +# ⛔⛔ IT CANNOT LIVE IN THE ROW, AND THAT IS MEASURED RATHER THAN ASSUMED. `add_row` builds its +# row as `{k: str(v) for k, v in values.items() if k in valid}` where `valid` is the declared +# field keys — a row dict is field-keys-only by construction, so a reserved `__ai__` key inside +# one would be stripped on the way in. Widening that filter to admit a private key would weaken +# the wall that stops a client inventing columns. So provenance is a SIBLING STRATUM of `rows`, +# exactly as `fields` and `rows` are siblings today. +# +# ⛔ AND `stale` IS DERIVED, NEVER STORED. A stored `stale` flag needs a sweep to stay true, and a +# sweep nobody runs is [[flag-shipped-without-its-writer]]. What IS stored is the INPUT HASH the +# value was produced from; `ai_enrich_is_stale` compares it to today's inputs, so staleness is a +# question asked at read time and can never be out of date. + +#: The stratum's key inside a table document, beside `fields` and `rows`. +AI_ENRICH_MARK_KEY = 'aiEnrich' +#: What a mark can say. ⚠ `stale` is absent ON PURPOSE (see above) and so is `generating`: an +#: in-flight run is a fact about a REQUEST, not about stored data, and persisting it would leave +#: a cell stuck on "generating" forever the first time a process dies mid-run. +AI_ENRICH_STATES = ('agent', 'human', 'error') +#: A bound on the stratum, in the shape `row_limit` uses: this document is copied whole on every +#: read, so a mark set that can outgrow the rows it describes is a store problem wearing a +#: feature's name. One mark per (row, column) is the natural size; the cap is the guard against a +#: column deleted without its marks. +AI_ENRICH_MAX_MARKS = MAX_ROWS +# +# ⚠ THERE IS DELIBERATELY NO `drop_marks(column)`, AND THE ABSENCE IS THE DECISION. `delete_field` +# leaves a deleted column's CELLS in the rows on purpose — an accidental delete stays recoverable, +# and re-adding the column under the same key brings the values back. Marks must follow the same +# rule or a restored column's agent-written values would come back looking human-authored, which +# is the one distinction this whole stratum exists to make. A table that is deleted takes its +# document (and therefore its marks) with it, so neither lifecycle leaks. + + +def ai_enrich_input_hash(prompt, row, refs): + """The fingerprint a stored value was produced FROM: the prompt plus the referenced cells. + + ⚠ `refs` is passed in rather than re-parsed here, so the token vocabulary is owned by one + module (`api/ai_enrich.py::prompt_refs`) instead of two. A hash built from a second parse + would disagree with the runner's the first time the token syntax gains a form. + """ + payload = [str(prompt or '')] + for key in sorted(str(r) for r in (refs or ())): + payload.append(f'{key}={(row or {}).get(key, "")}') + return hashlib.sha256('\x1f'.join(payload).encode('utf-8')).hexdigest()[:32] + + +def ai_enrich_marks(table_key, col_id, st=None): + """`{row_id: mark}` for ONE column. Absent = a cell nobody has written yet.""" + stratum = (get(table_key, st) or {}).get(AI_ENRICH_MARK_KEY) or {} + got = stratum.get(str(col_id)) + return dict(got) if isinstance(got, dict) else {} + + +def ai_enrich_is_stale(mark, fresh_hash): + """Was this value produced from inputs that have since changed? + + A human edit is never stale — a person's own words do not go out of date because a + neighbouring cell moved. Only an agent-written value can be. + """ + if not isinstance(mark, dict) or mark.get('state') != 'agent': + return False + return bool(fresh_hash) and str(mark.get('hash') or '') != str(fresh_hash) + + +def ai_enrich_human_authored(mark, has_value=False): + """Did somebody other than the agent put this value here? + + ⛔ TWO CASES, AND THE SECOND IS THE ONE THAT GETS MISSED. An explicit `human` mark is obvious. + The other is a cell that holds a value with NO mark at all: a value typed, pasted or imported + before the column became an enrichment column. The agent did not write it, so the agent does + not own it. Without this arm, `overwrite: 'always'` would eat exactly the pre-existing work + R8's *"we are helping the team put their work into the system"* is about. + + ⚠ ONE PREDICATE, because `ai_enrich_may_write` and `api/ai_enrich.py::cell_state` both answer + "is this the human's?" and a cell the UI labels human-written while the runner overwrites it + is the worst of both ([[one-question-two-normalizers]]). + """ + if isinstance(mark, dict) and mark.get('state') == 'human': + return True + return bool(has_value) and not (isinstance(mark, dict) and mark.get('state') == 'agent') + + +def ai_enrich_may_write(mark, policy, fresh_hash, has_value=False): + """May an AUTOMATIC run write this cell? The law, in one place. + + ⛔ THE FIRST CLAUSE IS NOT CONFIGURABLE. A cell a person typed into is never overwritten by an + agent, whatever `overwrite` says — that is why `human` is not one of the `AI_ENRICH_OVERWRITE` + values, and why `always` means "always refresh what the AGENT owns" rather than "overwrite + everything". A policy that could turn it off would make the column unsafe to hand to a team. + ⚠ A MANUAL, per-cell run is a different act with a different door; this predicate governs the + automatic paths (`on_change`, `schedule`, and a bulk "all rows" sweep). + """ + if ai_enrich_human_authored(mark, has_value): + return False + if not has_value: + return True # a blank cell is fillable under every policy + if policy == 'always': + return True + if policy == 'stale': + return ai_enrich_is_stale(mark, fresh_hash) + return False # 'blank': an agent never replaces a value it wrote + + +def stamp_ai_enrich(table_key, col_id, marks, st=None): + """Write MANY marks for one column in ONE store update. Returns the number stored. + + ⛔ BULK BY CONSTRUCTION, and the signature is what enforces it. Every write here is a + read-modify-write of the whole tenant document, so a per-row stamp inside a run loop is the + same shape as the 2,000-full-document-copies problem `add_rows` exists to avoid. There is + deliberately no `stamp_one`. + """ + col_id = str(col_id) + clean = {} + for row_id, mark in (marks or {}).items(): + if not isinstance(mark, dict): + continue + state = str(mark.get('state') or '') + if state not in AI_ENRICH_STATES: + continue + entry = {'state': state, 'at': _dt.datetime.now(_dt.timezone.utc) + .replace(microsecond=0).isoformat()} + for key, cap in (('hash', 64), ('model', 60), ('error', 300)): + got = str(mark.get(key) or '').strip()[:cap] + if got: + entry[key] = got + tokens = mark.get('tokens') + if isinstance(tokens, int) and not isinstance(tokens, bool) and tokens >= 0: + entry['tokens'] = tokens + clean[str(row_id)] = entry + if not clean: + return 0 + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + col = t.setdefault(AI_ENRICH_MARK_KEY, {}).setdefault(col_id, {}) + col.update(clean) + # The cap is enforced by DROPPING THE OLDEST, never by refusing the write: a run that + # produced values and could not record where they came from would leave the cells + # looking human-authored, which is the one state this stratum exists to distinguish. + if len(col) > AI_ENRICH_MAX_MARKS: + for rid in sorted(col, key=lambda r: str(col[r].get('at') or ''))[ + :len(col) - AI_ENRICH_MAX_MARKS]: + col.pop(rid, None) + return cur + + _st(st).update(STORE_KEY, _set, flush='async') + return len(clean) + + +def note_human_edit(table_key, values, row_id, st=None): + """A person typed into these cells: mark any `ai_enrich` column among them `human`. + + ⛔ THE STAMP HAS TO HAPPEN AT THE WRITE DOOR, not at read time, because "did a human write + this" is not recoverable from the value afterwards. Called by `routes_tables.patch_row`; + a no-op (and cheap) for the overwhelming case where the edited columns are ordinary. + """ + defn = get(table_key, st) or {} + touched = {f['key'] for f in ai_enrich_fields(defn)} & {str(k) for k in (values or {})} + for col_id in touched: + stamp_ai_enrich(table_key, col_id, {str(row_id): {'state': 'human'}}, st=st) + return sorted(touched) + + def _clean_field(raw, previous=None): """One field dict → the stored shape, or None. The single validator for create AND patch, so a column cannot be typed one way on the way in and another on the way back.""" @@ -1886,6 +2220,28 @@ def _clean_field(raw, previous=None): out['formula'] = fx elif ftype == 'formula': return None + # ⭐⭐ WAVE-34 (owner ruling R13) — THE AI ENRICHMENT BAG. Posture and inheritance copied from + # `formula` two lines up, deliberately and for the same reason: the bag defines the column, so + # the pairing is enforced both ways, and an inherited bag is only inherited while the type + # still wants it (retyping an enrichment column to text must not resurrect its prompt through + # `prev` and then refuse the whole write, which is the bug the `prev_type == ftype` guard on + # `link`/`rollup`/`formula` exists to record). + enrich_raw = (raw.get('aiEnrich') if 'aiEnrich' in raw + else (prev.get('aiEnrich') if prev_type == ftype else None)) + if enrich_raw is not None: + ae = _clean_ai_enrich(enrich_raw) + if ae is None or ftype != 'ai_enrich': + return None + out['aiEnrich'] = ae + elif ftype == 'ai_enrich': + return None + # C3 (wave 34): the Agents module reads field agents off this bag. DERIVED here, inside the + # one validator both write doors run, so the binding can never drift from the config it + # describes — see `_field_agent_binding` for why it is computed rather than stored beside it. + # ⛔ It is written AFTER the generic `automation` passthrough above, so a caller cannot hand us + # a `field_agent` bag that disagrees with the column. Every OTHER automation bag is untouched. + if ftype == 'ai_enrich': + out['automation'] = _field_agent_binding(out) return out @@ -2081,6 +2437,16 @@ def flow_bound(bag, st=None): lives in `aios_grid._clean_automation` — this guards the WRITE doors only.)""" if not isinstance(bag, dict): return True # no bag, no law — an ordinary column + # ⭐⭐ WAVE-34 (R13 / contract C3) — A FIELD AGENT IS BOUND BY ITS OWN COLUMN, NOT BY A FLOW, + # and without this arm the whole feature is unsavable. `_clean_field` stamps + # `{kind:'field_agent', ...}` on every `ai_enrich` column; `add_field` then asks this + # predicate, which read `flowId` and only `flowId` — so every AI enrichment column would have + # been refused at the create door by the guard for a different feature that happens to share + # the `automation` key. ⚠ The binding is REAL, not waived: the bag names the column it belongs + # to, and `_clean_field` derives that name from the field itself, so it cannot name a column + # that does not exist the way a stale `flowId` can name a deleted flow. + if bag.get('kind') == 'field_agent': + return bool(str(bag.get('field') or '').strip()) flow = str(bag.get('flowId') or '').strip() if not flow: return False @@ -2386,6 +2752,41 @@ def patch_cells(table_key, row_id, values, st=None): return True +def patch_many_cells(table_key, updates, st=None): + """Write cells across MANY rows in ONE store update. Returns the number of rows touched. + + ⛔ WHY THIS EXISTS RATHER THAN A LOOP OVER `patch_cells`, which is the same argument + `add_rows` makes one screen up: every write here is a read-modify-write of the WHOLE tenant + document, so filling a 1,000-row enrichment column row by row is 1,000 full-document copies + under one lock on the single uvicorn process this product runs. One `update()` writes them + all. + + ⚠ `updates` is `{row_id: {field_key: value}}`, and a row id that does not exist is SKIPPED + rather than created: this is a cell writer, not an insert door, and inventing a row here would + let a stale plan resurrect a record somebody deleted mid-run. + """ + if not is_user_table(table_key, st): + return 0 + have = set((get(table_key, st) or {}).get('rows') or {}) + clean = {str(r): {str(k): str(v) for k, v in (cells or {}).items()} + for r, cells in (updates or {}).items() + if str(r) in have and isinstance(cells, dict) and cells} + if not clean: + return 0 + + def _set(cur): + t = cur.get(str(table_key)) + if t is not None: + rows = t.setdefault('rows', {}) + for rid, cells in clean.items(): + if rid in rows: + rows[rid].update(cells) + return cur + + _st(st).update(STORE_KEY, _set, flush='sync') + return len(clean) + + def patch_link_cell(table_key, row_id, fkey, value, st=None): """Persist one user-picked Link cell in the shared row and return its canonical id string. @@ -2647,6 +3048,28 @@ def lend(st=None, **docs): return _Lent(_st(st), {k: v for k, v in docs.items() if v is not None}) +def lend_defs(st=None): + """`lend`, but the `user_tables` document it serves is the PROJECTION — definitions, no rows. + + ⭐⭐ W33-T01 (D-213). `nav_entries` has done exactly this inline since W32-T02 + (`all_defs` then `lend(st, user_tables=tables)`); this is that pair named, so the TABLE WALL + can take it too. `routes_tables._defn_or_refuse` is the wall a database SWITCH goes through and + it read the whole 28.6 MB document to answer two questions — "does this key exist" and + "may this session open it" — neither of which has ever looked at a row. + + ⛔ **HAND THIS TO A READ, NEVER TO A WRITE.** Every value it serves is a `_Projected`, so a + caller that reaches `defn['rows']` gets a `KeyError` naming the projection — loud, which is the + point — and a read-modify-write handed this document would upload a tenant with its rows + removed. `store.update`/`_read_strict` are unaffected (they never project), and the six write + walls in `routes_tables` deliberately do not opt in. C5's second clause is the same rule from + the other end: a projected snapshot is never handed to a post-write read-back. + + ⚠ It is not free — it is ~0.1% of the bytes, not zero. A caller that will need rows ANYWAY + should take the whole read once rather than this plus that. + """ + return lend(st, **{STORE_KEY: all_defs(st)}) + + def nav_entries(viewer=None, is_admin=False, st=None): """Registry-SHAPED dicts so the nav can render user tables beside real modules with no special-casing: the same keys the flat nav reads (`key`, `label`, `source`). diff --git a/platform/core/users.py b/platform/core/users.py index a29435e3b3b9fe9b2c047f9f8d7bcda08d7375a5..0d37dfb63aea46b27d87a65720d0ce05c30f5463 100644 --- a/platform/core/users.py +++ b/platform/core/users.py @@ -445,21 +445,20 @@ def assignable_people(tenant=None): Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a - cross-tenant name leak. None = unscoped (the Streamlit host, tenant #0's process). + cross-tenant name leak. + + ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and + return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was + written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at + EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform + roster one forgotten kwarg away, with no error to notice + ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing). + ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass + `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate + changes — and a future caller that forgets gets an empty picker somebody notices instead of a + leak nobody does. """ - try: - reg = registry() or {} - except Exception: - return [] - want = str(tenant or '').strip().lower() - out = [] - for username, u in reg.items(): - if not isinstance(u, dict) or u.get('active') is False: - continue - if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: - continue - out.append(str(u.get('name') or username)) - return sorted(set(out)) + return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)}) def set_avatar(username, data_url): @@ -486,20 +485,41 @@ def avatar_map(tenant=None): name, so the display name is the only join a renderer has. Two active accounts sharing a display name share one option; the first WITH a photo wins the key rather than a coin flip deciding whether the option has a face. `tenant` scopes it exactly as - `assignable_people(tenant)` does, and for the same leak.""" - try: - reg = registry() or {} - except Exception: - return {} - want = str(tenant or '').strip().lower() + `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed + blank, which both take from the ONE resolver below rather than each writing `if want and …`. + """ out = {} - for username, u in sorted(reg.items()): - if not isinstance(u, dict) or u.get('active') is False: - continue - if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want: - continue + for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]): av = u.get('avatar') nm = str(u.get('name') or username) if av and nm not in out: out[nm] = str(av) return out + + +def _tenant_accounts(tenant): + """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED. + + ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice, + identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a + blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of + them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of + the same predicate and is lane C's to fold in. + + ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a + pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public` + normalises it the same way. What changed is the WANT side — a blank want is now a refusal + rather than a wildcard. + """ + want = str(tenant or '').strip().lower() + if not want: + # Not an error and not everything: nobody. See `assignable_people`'s note — the one + # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6. + return [] + try: + reg = registry() or {} + except Exception: + return [] + return [(n, u) for n, u in reg.items() + if isinstance(u, dict) and u.get('active') is not False + and str(u.get('tenant') or 'royal-imports').strip().lower() == want] diff --git a/platform/harness/datastore.py b/platform/harness/datastore.py index c4be70047eeaea72f2055498b3a8159364893ce1..2892c73329ad239d426a7a702f749413ad6a9fe9 100644 --- a/platform/harness/datastore.py +++ b/platform/harness/datastore.py @@ -188,14 +188,35 @@ ENTITIES = { # backfill below has walked every partner (a partner's `write_date` did not move because # WE changed the schema). Every projection degrades a missing/NULL column to blank rather # than raising, so declaring early is safe — it is just not yet true. + # ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS + # ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors' + # etc."* He was right, and the census + # (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how + # much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76 + # on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here + # was measured non-empty on the live population first — none is added on the strength of + # "Odoo probably has that", which is how a schema grows columns nobody can fill. + # ⚠ THE MIRROR IS THE CEILING, NOT THE GRID. Every reader in `odoo_relational.py` reads + # THIS file, so a column added to a `*_fields()` contract with no entry here renders as a + # blank column forever — which is D-165's shape exactly, and the reason the widening starts + # on this line rather than on the grid. "fields": ["name", "team_id", "city", "state_id", "country_id", "active", "agent", "salesman_as_agent", "customer_rank", - "street", "street2", "zip", "write_date"], + "street", "street2", "zip", + "email", "phone", "mobile", "website", "vat", "ref", + "write_date"], # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next # sync_all() instead of needing a reseed. + # ⛔ D-165 IS ABOUT THIS LIST, AND THE CENSUS SETTLED WHICH HALF IS WRONG: `street`, + # `street2` and `zip` are `store=True` on `res.partner` and POPULATED in live Odoo — they + # are declared here and still NULL in the mirror purely because a backfill pass has not + # walked every partner (changing OUR schema does not move a partner's `write_date`, so an + # incremental sync never revisits them). Declaring is not filling; the fix is a run. "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28 "customer_rank", # added 2026-08-11 (W29-T54) - "street", "street2", "zip"], # added 2026-08-12 (W30-T33) + "street", "street2", "zip", # added 2026-08-12 (W30-T33) + "email", "phone", "mobile", # added 2026-08-14 (W33-T48) + "website", "vat", "ref"], # m2m fields land as _id (the FIRST linked id — the Customers-module convention: # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + as a # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be diff --git a/platform/harness/tools.py b/platform/harness/tools.py index d046a442ba05436fe62116a22beae87809c2a74b..94e74480c49c1545e8c352b0eed06908e890e569 100644 --- a/platform/harness/tools.py +++ b/platform/harness/tools.py @@ -65,13 +65,22 @@ def _err(msg): # ------------------------------------------------------------------ schema tools def list_topics(): - """The 'what data exists' tool.""" + """The 'what data exists' tool. + + W33-T49 adds `grid` — the store key of the database this topic describes, when it describes + one. It is what lets the agent connect "the thing the person is looking at" to "the topic I + can query", which is the whole point of the entity topics: before this, the semantic layer + could see `sale_order_line` and `account_move` and had never heard of `ut_odoo_customers`. + """ out = [] for k, t in SEM.topics().items(): - out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"), - "grain": t.get("grain"), - "dims": list((t.get("store") or {}).get("dims") or {}), - "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]}) + row = {"topic": k, "label": t.get("label"), "entity": t.get("entity"), + "grain": t.get("grain"), + "dims": list((t.get("store") or {}).get("dims") or {}), + "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]} + if t.get("grid"): + row["grid"] = t["grid"] + out.append(row) return out @@ -83,10 +92,30 @@ def describe_topic(topic): mets = {k: {"label": m.get("label"), "description": m.get("description"), "format": m.get("format"), "ai_context": m.get("ai_context")} for k, m in SEM.metrics().items() if m["topic"] == topic} - return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"), - "grain": t.get("grain"), "ai_context": t.get("ai_context"), - "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()}, - "metrics": mets} + out = {"topic": topic, "label": t.get("label"), "scope": t.get("scope"), + "grain": t.get("grain"), "ai_context": t.get("ai_context"), + "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()}, + "metrics": mets} + # ⭐⭐ W33-T49 — THE FIELD SEMANTICS, which this tool did not carry AT ALL. + # + # ⛔ THE GAP WAS NOT THE TOPIC FILES, IT WAS THIS RETURN. Before this, `describe_topic` + # answered with scope, grain, dims, metrics and `ai_context` — and **no vocabulary of + # COLUMNS**. So an agent told to describe a database could name what it MEASURES and never + # what it HOLDS: the owner's item 13 asks for "ALL populated fields … including its semantic + # layer", and adding eight topic files without this would have left `describe_topic` returning + # the same field-free shape it always did, one screen away from the fields. + # + # ⚠ `kind` is the part worth reading, and it is the difference between a column an agent can + # FILTER and one it cannot: `data` is stored on the row; `rollup` is computed from another + # topic (`from_topic` names it, and that is where a windowed or reconciled number must come + # from); `link` points at another database. An agent that treats a rollup as a stored column + # will ask this topic for a number this topic does not hold. + if t.get("fields"): + out["fields"] = [dict(f) for f in t["fields"]] + for k in ("grid", "subject", "entity"): + if t.get(k): + out[k] = t[k] + return out # ------------------------------------------------------------------ query tools diff --git a/platform/model/topics/odoo_accounts.yml b/platform/model/topics/odoo_accounts.yml new file mode 100644 index 0000000000000000000000000000000000000000..6a1bb6f3190085d603f45df74bce9d2cb8b8ef71 --- /dev/null +++ b/platform/model/topics/odoo_accounts.yml @@ -0,0 +1,60 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_accounts +label: "Odoo GL accounts" +entity: account.account +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: ut_odoo_accounts +subject: "odoo:account.account" +grain: "one row per GL account in the chart of accounts" +scope: + population: "every account.account record (192 measured)" +store: + table: account_account + alias: a + # NO date_col — a registry is not a dated event stream. Stated rather than + # omitted, so its absence reads as a fact and not as an unfinished file. + dims: + account_type: {label: "Account type"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: account_code + label: "Code" + type: text + kind: data + - key: account_name + label: "Account" + type: text + kind: data + - key: odoo_id + label: "Odoo ID" + type: int + kind: data + means: "The `account.account` id. Also this row's id." + - key: account_type + label: "Type" + type: select + kind: data + - key: is_expense + label: "Expense account" + type: checkbox + kind: data + means: "Ticked for the expense family - the same predicate the semantic layer's gl_lines topic uses, so this column and that topic cannot disagree." + - key: refreshed + label: "Refreshed" + type: date + kind: data + means: "When this row was last reconciled against Odoo." + +ai_context: > + The chart of accounts — the registry gl_lines posts against. + Use it to resolve an account code to a name or a type. + ⚠ `is_expense` is a DERIVED boolean standing in for Odoo's five-value `internal_group` (asset/liability/equity/income/expense), which the mirror does not yet carry. diff --git a/platform/model/topics/odoo_agents.yml b/platform/model/topics/odoo_agents.yml new file mode 100644 index 0000000000000000000000000000000000000000..03de0e544ecff625e5741014ff14381b18fae212 --- /dev/null +++ b/platform/model/topics/odoo_agents.yml @@ -0,0 +1,59 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_agents +label: "Odoo agents" +entity: res.partner +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: ut_odoo_agents +subject: "odoo:res.partner.agent" +grain: "one row per sales agent" +scope: + population: "a UNION of two disagreeing sources — partners carrying commission lines, and partners flagged res_partner.agent = TRUE" + why_union: "measured 2026-08-09: 16 carry commission lines, 17 carry the flag, and the union is 19 — either source alone silently drops real agents" +store: + table: res_partner + alias: p + # NO date_col — a registry is not a dated event stream. Stated rather than + # omitted, so its absence reads as a fact and not as an unfinished file. + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: agent + label: "Agent" + type: text + kind: data + - key: odoo_id + label: "Odoo ID" + type: int + kind: data + means: "The `res.partner` id. Also this row's id." + - key: agent_id + label: "Odoo agent id" + type: int + kind: data + - key: flagged + label: "Flagged in Odoo" + type: checkbox + kind: data + means: "Ticked = `res.partner.agent` is set. Unticked agents were found by their commission lines instead - both are real, which is why this table is the union of the two." + - key: commissioned + label: "Has commission lines" + type: checkbox + kind: data + - key: refreshed + label: "Refreshed" + type: date + kind: data + means: "When this row was last reconciled against Odoo." + +ai_context: > + The sales-agent registry. + `flagged` and `commissioned` are the two SOURCES, kept as separate columns rather than merged, because they disagree and the disagreement is information. + For an agent's BOOK use sales_lines/commission_lines with the agent dim. diff --git a/platform/model/topics/odoo_bills.yml b/platform/model/topics/odoo_bills.yml new file mode 100644 index 0000000000000000000000000000000000000000..2db0aacf017c98721287aee9c95fd980fa962f8f --- /dev/null +++ b/platform/model/topics/odoo_bills.yml @@ -0,0 +1,86 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_bills +label: "Odoo vendor bills" +entity: account.move +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: ut_odoo_bills +subject: "odoo:account.move.vendor" +grain: "one row per posted VENDOR bill or vendor credit note (document grain)" +scope: + population: "move_type in (in_invoice, in_refund) AND state = posted" + signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure" +store: + table: account_move + alias: m + date_col: "m.invoice_date" + dims: + vendor: {label: "Vendor"} + payment_state: {label: "Payment state"} + move_type: {label: "Document type"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: bill_no + label: "Bill" + type: text + kind: data + - key: odoo_id + label: "Odoo ID" + type: int + kind: data + means: "The `account.move` id. Also this row's id." + - key: vendor + label: "Vendor" + type: text + kind: data + - key: vendor_id + label: "Odoo vendor id" + type: int + kind: data + - key: invoice_date + label: "Bill date" + type: date + kind: data + - key: due_date + label: "Due date" + type: date + kind: data + - key: amount_untaxed + label: "Billed $" + type: currency + kind: data + - key: residual + label: "Outstanding $" + type: currency + kind: data + - key: payment_state + label: "Payment state" + type: select + kind: data + - key: move_type + label: "Document" + type: select + kind: data + - key: vendor_link + label: "Vendor record" + type: link + kind: link + to_grid: ut_odoo_vendors + - key: refreshed + label: "Refreshed" + type: date + kind: data + means: "When this row was last reconciled against Odoo." + +ai_context: > + Vendor bills at DOCUMENT grain — the payables side of account_move. + Use it for AP questions. + ⚠ The census found `ref` (the VENDOR's own invoice number), `journal_id` and `currency_id` populated in Odoo and absent from the mirror, so they cannot be answered from here yet. diff --git a/platform/model/topics/odoo_customers.yml b/platform/model/topics/odoo_customers.yml new file mode 100644 index 0000000000000000000000000000000000000000..9fb4872efe738205e584309c22cee6ba4ff6e628 --- /dev/null +++ b/platform/model/topics/odoo_customers.yml @@ -0,0 +1,213 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_customers +label: "Odoo customers" +entity: res.partner +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: customer_data +subject: "odoo:res.partner" +grain: "one row per customer in tenant #0's scoped book — a REGISTRY, not a dated event stream. Identity is the Odoo `res.partner` id (the row's `pid` IS that id)." +scope: + population: "the customer book `modules/customer_data.pool()` builds; the grid's own contract states it as one row per customer who ordered in the last 24 months" + identity: "the Odoo res.partner id — exposed as the `partner_id` column (W33-T43) and used as the row pid, so there is ONE id per row and no second one" + merged: "W33-T44 retired `ut_odoo_customers`, which presented this same subject. This topic describes the SURVIVING database, `customer_data`, which keeps its store bucket so no saved view, grant, cohort or formula moved" + no_date: "a registry has no date dimension; ask sales_lines or customer_invoices for anything time-windowed about a customer" +store: + table: res_partner + alias: p + # NO date_col — a registry is not a dated event stream. Stated rather than + # omitted, so its absence reads as a fact and not as an unfinished file. + dims: + agent: {label: "Sales agent"} + state: {label: "State"} + country: {label: "Country"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: customer + label: "Customer" + type: text + kind: data + means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months." + - key: partner_id + label: "Odoo ID" + type: int + kind: data + means: "The Odoo res.partner id — this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact." + - key: odoo_status + label: "Odoo record" + type: status + kind: data + means: "Whether this customer still exists in Odoo. Archived means deleted there." + - key: agent + label: "Agent" + type: text + kind: data + means: "The sales agent who owns this account." + - key: dba + label: "DBA" + type: select + kind: data + means: "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA." + - key: salesperson + label: "Salesperson" + type: text + kind: data + means: "Who keyed in most of this customer's orders — not the Agent, who owns the account." + - key: street + label: "Street" + type: text + kind: data + means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." + - key: street2 + label: "Street 2" + type: text + kind: data + means: "Second address line (suite, unit, floor) on the customer's Odoo address." + - key: city + label: "City" + type: text + kind: data + means: "City on the customer's Odoo address." + - key: state + label: "State" + type: text + kind: data + means: "State or province on the customer's Odoo address." + - key: country + label: "Country" + type: text + kind: data + means: "Country on the customer's Odoo address." + - key: zip + label: "ZIP" + type: text + kind: data + means: "Postal code on the customer's Odoo address." + - key: customer_since + label: "Customer since" + type: date + kind: data + means: "When this customer was first set up in Odoo." + - key: tags + label: "Tags" + type: text + kind: data + means: "Odoo labels on this customer, comma-separated." + - key: pricelist + label: "Price list" + type: text + kind: data + means: "The price list this customer buys on." + - key: payment_terms + label: "Payment terms" + type: text + kind: data + means: "Payment terms on this customer's account — Net 30, for example." + - key: last_order + label: "Last order" + type: date + kind: data + means: "Date of the most recent confirmed order." + - key: overdue_days + label: "Overdue days" + type: int + kind: data + means: "How many days late this customer is running against their own usual ordering rhythm." + - key: est_missed + label: "Est. missed $" + type: currency + kind: data + means: "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed." + - key: ar_open + label: "AR current $" + type: currency + kind: data + means: "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)." + - key: ar_overdue + label: "AR overdue $" + type: currency + kind: data + means: "Invoiced money past due — same basis as the Collections page." + - key: ar_outstanding + label: "AR outstanding $" + type: currency + kind: data + means: "Total invoiced money owed right now: AR current $ plus AR overdue $." + - key: ar_exposure + label: "Credit exposure $" + type: currency + kind: data + means: "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced." + - key: ar_aged_1_30 + label: "1-30 days $" + type: currency + kind: data + means: "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $." + - key: ar_aged_31_60 + label: "31-60 days $" + type: currency + kind: data + means: "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $." + - key: ar_aged_61_90 + label: "61-90 days $" + type: currency + kind: data + means: "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $." + - key: ar_aged_90_plus + label: "90+ days $" + type: currency + kind: data + means: "Overdue by more than 90 days. The four aging buckets sum to AR overdue $." + - key: days_to_pay + label: "Days to pay" + type: int + kind: data + means: "Average days to pay an invoice in full. Blank means no fully paid invoice yet." + - key: top_category + label: "Top category" + type: text + kind: data + means: "The category this customer spent the most on in the last 12 months." + - key: top_category_pct + label: "Top category %" + type: pct + kind: data + means: "Share of last-12-months spend that went to the top category." + - key: sku_count + label: "SKUs bought" + type: int + kind: data + means: "Distinct products bought in the last 12 months." + - key: top_sku + label: "Top SKU" + type: text + kind: data + means: "The product this customer spent the most on in the last 12 months." + - key: days_since + label: "Days since order" + type: int + kind: data + means: "Days since the last confirmed order." + - key: typical_gap_days + label: "Typical gap days" + type: int + kind: data + means: "Days this customer usually goes between orders, from their own history." + - key: notes + label: "Notes" + type: text + kind: data + means: "Your notes on this customer. Saved in this app only, visible only to you." + +ai_context: > + The customer REGISTRY as a person sees it in the app — the database labelled 'Odoo customers'. + Use it to answer 'who is this customer', 'which customers exist', 'which agent owns them', 'where are they', and for the AR columns it carries (ar_open, ar_overdue, days_to_pay), which are reconciled by modules/ar.py. + ⛔ For anything WINDOWED or at line grain go to sales_lines / customer_invoices / receivables — those topics own the time dimension and this one has none. diff --git a/platform/model/topics/odoo_invoices.yml b/platform/model/topics/odoo_invoices.yml new file mode 100644 index 0000000000000000000000000000000000000000..e9571fac1dd4de200b1fccfba7c35261dfe160e5 --- /dev/null +++ b/platform/model/topics/odoo_invoices.yml @@ -0,0 +1,98 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_invoices +label: "Odoo invoices" +entity: account.move +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: ut_odoo_invoices +subject: "odoo:account.move.customer" +grain: "one row per posted CUSTOMER invoice or credit note (document grain)" +scope: + population: "move_type in (out_invoice, out_refund) AND state = posted" + signs: "amount_residual_signed is POSITIVE for an invoice and NEGATIVE for a credit note, so summing it nets the credit notes correctly" +store: + table: account_move + alias: m + date_col: "m.invoice_date" + dims: + customer: {label: "Customer"} + payment_state: {label: "Payment state"} + move_type: {label: "Document type"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: invoice_no + label: "Invoice" + type: text + kind: data + - key: odoo_id + label: "Odoo ID" + type: int + kind: data + means: "The `account.move` id. Also this row's id." + - key: customer + label: "Customer" + type: text + kind: data + - key: partner_id + label: "Odoo partner id" + type: int + kind: data + - key: invoice_date + label: "Invoice date" + type: date + kind: data + - key: due_date + label: "Due date" + type: date + kind: data + - key: residual + label: "Outstanding $" + type: currency + kind: data + means: "Odoo's signed residual. Exactly 0 on every settled document, which is why AR rollups need no filter." + - key: amount_untaxed + label: "Invoiced $" + type: currency + kind: data + - key: payment_state + label: "Payment state" + type: select + kind: data + - key: move_type + label: "Document" + type: select + kind: data + - key: wholesale_scope + label: "In wholesale scope" + type: checkbox + kind: data + means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page." + - key: origin_order + label: "Source order" + type: text + kind: data + means: "Odoo's `invoice_origin` - usually the order name, sometimes blank." + - key: order_link + label: "Order record" + type: link + kind: link + to_grid: ut_odoo_orders + - key: refreshed + label: "Refreshed" + type: date + kind: data + means: "When this row was last reconciled against Odoo." + +ai_context: > + Customer invoices at DOCUMENT grain — one row per invoice, not per line. + Use it for AR questions: what is open, what is overdue, what was invoiced. + For anything at product or line grain use invoice_lines; for the reconciled receivables view use receivables. + `origin_order` is the single stored key that joins an invoice back to the order it was raised from. diff --git a/platform/model/topics/odoo_orders.yml b/platform/model/topics/odoo_orders.yml new file mode 100644 index 0000000000000000000000000000000000000000..926f78c7d39eb15fcebf304e965c755e36053348 --- /dev/null +++ b/platform/model/topics/odoo_orders.yml @@ -0,0 +1,87 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_orders +label: "Odoo orders" +entity: sale.order +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: ut_odoo_orders +subject: "odoo:sale.order" +grain: "one row per confirmed sales order (document grain)" +scope: + population: "state in (sale, done) — quotations and cancellations excluded" + vs_sales_orders: "⚠ DISTINCT from the `sales_orders` topic: that one is the wholesale-scoped analytical view (teams 5/6, house accounts excluded); THIS one is the registry a person opens as a grid, and carries the row set the app shows" +store: + table: sale_order + alias: o + date_col: "o.date_order" + dims: + customer: {label: "Customer"} + team: {label: "Business unit"} + state: {label: "State"} + invoice_status: {label: "Invoice status"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: order_no + label: "Order" + type: text + kind: data + - key: odoo_id + label: "Odoo ID" + type: int + kind: data + means: "The `sale.order` id. Also this row's id." + - key: customer + label: "Customer" + type: text + kind: data + - key: partner_id + label: "Odoo partner id" + type: int + kind: data + - key: order_date + label: "Order date" + type: date + kind: data + - key: amount_untaxed + label: "Order $" + type: currency + kind: data + - key: team + label: "Business unit" + type: select + kind: data + - key: state + label: "State" + type: select + kind: data + - key: invoice_status + label: "Invoice status" + type: select + kind: data + - key: wholesale_scope + label: "In wholesale scope" + type: checkbox + kind: data + means: "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale metric in this product excludes. The row is kept so no Odoo id is missing; filter on this column to reconcile against the AR page." + - key: invoices + label: "Invoices" + type: link + kind: link + to_grid: ut_odoo_invoices + - key: refreshed + label: "Refreshed" + type: date + kind: data + means: "When this row was last reconciled against Odoo." + +ai_context: > + Sales orders at DOCUMENT grain, as the app's grid shows them. + ⛔ If you are asked for revenue, prefer `sales_lines` (line grain, wholesale scope) or `sales_orders` — this topic exists to describe the ORDER RECORD a person can open, and its scope is the grid's, not the analytical model's. diff --git a/platform/model/topics/odoo_products.yml b/platform/model/topics/odoo_products.yml new file mode 100644 index 0000000000000000000000000000000000000000..fb83723704c208872fe2bcf922d6213e38c968f1 --- /dev/null +++ b/platform/model/topics/odoo_products.yml @@ -0,0 +1,147 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_products +label: "Odoo products" +entity: product.product +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: product_data +subject: "odoo:product.product" +grain: "one row per SKU in the active catalogue — a CATALOGUE, not a dated event stream" +scope: + population: "every ACTIVE product, sold or not (measured 5,862). Deliberately not the sales-window universe: that lands at 2,717 and hides ~2,550 SKUs that have never sold in wholesale scope" + identity: "the SKU code (`default_code`). ⚠ 33 active products carry NO code and are keyed `pid:` instead, so every active product has exactly one row and none is dropped" + merged: "W33-T44 retired `ut_odoo_products`, which presented this same subject on Odoo's product_id. This topic describes the SURVIVING database, `product_data`" + no_date: "a catalogue has no date dimension; ask sales_lines for movement" +store: + table: product_product + alias: pp + # NO date_col — a registry is not a dated event stream. Stated rather than + # omitted, so its absence reads as a fact and not as an unfinished file. + dims: + category: {label: "Category"} + supplier: {label: "Supplier"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: code + label: "SKU" + type: text + kind: data + means: "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer." + - key: product + label: "Product" + type: text + kind: data + means: "Product name as it appears in Odoo." + - key: category + label: "Category" + type: select + kind: data + means: "Product category; '(uncategorized)' when Odoo carries none." + - key: supplier + label: "Supplier" + type: text + kind: data + means: "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet." + - key: origin_country + label: "Country" + type: text + kind: data + means: "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet." + - key: lead_days + label: "Lead time (days)" + type: int + kind: data + means: "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone." + - key: first_cost + label: "First cost" + type: currency + kind: data + means: "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone." + - key: price_fisch + label: "Fisch price" + type: currency + kind: data + means: "Fisch pricelist price for this SKU. Blank when that list prices it nowhere." + - key: price_royal_1 + label: "Royal 1 price" + type: currency + kind: data + means: "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere." + - key: price_royal_2 + label: "Royal 2 price" + type: currency + kind: data + means: "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere." + - key: rev_ytd + label: "Revenue YTD" + type: currency + kind: data + means: "Year-to-date revenue for this SKU, BU-scoped when the caller is." + - key: rev_ly + label: "Revenue LY" + type: currency + kind: data + means: "Same period last year — seasonal wholesale compares like for like." + - key: yoy_pct + label: "YoY %" + type: pct + kind: data + means: "Year-over-year change; null when last year was zero (a ratio to zero is not a number)." + - key: qty_ytd + label: "Units YTD" + type: int + kind: data + means: "Units sold year to date." + - key: orders_ytd + label: "Orders YTD" + type: int + kind: data + means: "Distinct orders containing this SKU, year to date." + - key: on_hand + label: "On hand" + type: int + kind: data + means: "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide." + - key: unit_cost + label: "Unit cost" + type: currency + kind: data + means: "Inventory unit cost. Consolidated; absent for a BU-scoped caller." + - key: inv_value + label: "Stock value" + type: currency + kind: data + means: "On-hand value at cost. Consolidated; absent for a BU-scoped caller." + - key: qty_ltm + label: "Units LTM" + type: int + kind: data + means: "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller." + - key: dos + label: "Days of supply" + type: int + kind: data + means: "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller." + - key: cover_gap_d + label: "Cover gap (days)" + type: int + kind: data + means: "Days of supply minus lead time. Negative means it runs out before a reorder lands." + - key: stock_bucket + label: "Stock status" + type: select + kind: data + means: "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller." + +ai_context: > + The product CATALOGUE — the database labelled 'Odoo products'. + Use it for 'what do we sell', 'what does it cost' (first_cost/unit_cost), 'who supplies it', 'what is on hand' and the stock-coverage columns. + ⛔ Revenue and unit columns here are WINDOWED SNAPSHOTS (rev_ytd, qty_ltm); for any other window, or for line grain, go to sales_lines. diff --git a/platform/model/topics/odoo_vendors.yml b/platform/model/topics/odoo_vendors.yml new file mode 100644 index 0000000000000000000000000000000000000000..ca0067ecfb02fd1ee817ca461aded684b3a62e8d --- /dev/null +++ b/platform/model/topics/odoo_vendors.yml @@ -0,0 +1,103 @@ +# ⭐⭐ W33-T49 (owner item 13) — an ENTITY topic: the semantic layer can finally see +# the databases a person actually opens, not only the line and document grains. +# +# ⛔ THE `fields:` BLOCK IS GENERATED FROM THE GRID'S OWN FIELD CONTRACT +# and is held to it by `verify_query.py::section_entity_topics`. Do not hand-edit a +# field row: add the column to the grid contract and re-emit, or the agent is being +# trained on a schema the product does not have. +key: odoo_vendors +label: "Odoo vendors" +entity: res.partner +# The database this topic DESCRIBES — the same store key the nav opens (W33-T46). +grid: ut_odoo_vendors +subject: "odoo:res.partner.vendor" +grain: "one row per partner we have posted a vendor bill to" +scope: + population: "DERIVED FROM THE BILLS — a partner with no posted vendor bill has no payable history to show, so the bill is the population" + asymmetry: "deliberately unlike odoo_customers, which is NOT derived from its documents; there is no second document universe for vendors" + overlap: "measured: only 9 of the vendors also appear in the customer population, so a vendor is not a customer row" +store: + table: res_partner + alias: p + # NO date_col — a registry is not a dated event stream. Stated rather than + # omitted, so its absence reads as a fact and not as an unfinished file. + dims: + country: {label: "Country"} + +# key / label / type / kind, derived from the grid contract. `kind` says where the +# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed +# from another topic; a `link` points at another database. +fields: + - key: vendor + label: "Vendor" + type: text + kind: data + - key: odoo_id + label: "Odoo ID" + type: int + kind: data + means: "The `res.partner` id. Also this row's id." + - key: vendor_id + label: "Odoo vendor id" + type: int + kind: data + - key: country + label: "Country" + type: text + kind: data + - key: email + label: "Email" + type: text + kind: data + - key: phone + label: "Phone" + type: text + kind: data + - key: mobile + label: "Mobile" + type: text + kind: data + - key: vat + label: "Tax ID" + type: text + kind: data + means: "Odoo `vat` — the vendor's tax/VAT registration number." + - key: vendor_ref + label: "Vendor reference" + type: text + kind: data + means: "Odoo `res.partner.ref` — our internal reference for this vendor." + - key: website + label: "Website" + type: url + kind: data + - key: street + label: "Street" + type: text + kind: data + - key: street2 + label: "Street 2" + type: text + kind: data + - key: city + label: "City" + type: text + kind: data + - key: zip + label: "ZIP" + type: text + kind: data + - key: bills + label: "Bills" + type: link + kind: link + to_grid: ut_odoo_bills + - key: refreshed + label: "Refreshed" + type: date + kind: data + means: "When this row was last reconciled against Odoo." + +ai_context: > + The vendor/supplier registry, with contact and tax identity (W33-T48 widened it from 5 columns to 15 after a census found 76 populated fields on the underlying partner) + ⚠ The contact columns are SPARSE by nature — measured 65/393 with an email, 13/393 with a tax id — so 'blank' means Odoo has no value, never that the sync failed. diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index eaf5ebac17decb7fc7029da5fe05c0cd5658c0f2..92902252b3f1a59ad973bdd3ffe987387422f0bf 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -410,6 +410,13 @@ def pool(team_id=None, t=None): # SKU that is not re-SKUed. `directory()` derives the category identically. "product": meta.get("product") or code, "category": meta.get("category") or "(uncategorized)", + # ⭐⭐ W33-T43 (R2 / amendment A2) — ODOO'S OWN PRODUCT ID, beside the hashed `pid`. + # R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this + # is that column. ⛔ NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here, + # unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover + # it and it has to be carried from `products.catalogue()`. + # ⚠ `None`, never 0, when the catalogue somehow has no id — 0 is a real Odoo id. + "product_id": meta.get("id"), # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev` # keeps its measured numbers including a real 0.0 (it sold last year, not this one). "rev_ytd": r.get("rev_ytd", 0.0) if r else None, diff --git a/platform/modules/products.py b/platform/modules/products.py index f5c1ea9cfe3c52359185878c84b6d368bbc9eaac..3c0f73444439149ed26d21fe6a36c8e1aef31731 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -167,7 +167,7 @@ def directory(t=None, team_id=None): def catalogue(): - """`{sku_code: {'product', 'category'}}` — EVERY ACTIVE product, sold or not. + """`{sku_code: {'product', 'category', 'id'}}` — EVERY ACTIVE product, sold or not. ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU @@ -207,8 +207,17 @@ def catalogue(): code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}") # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero # active products share a code, so this branch is a guard, not a merge policy. + # ⭐⭐ W33-T43 (R2 / amendment A2) — `id` RIDES THE ROW. Odoo's `product.product` id is + # already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is + # the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the + # legacy key and keeps every data column, so retiring the twin without this would lose a + # real identifier rather than a navigational link. + # ⚠ A product's grid pid is `crc32(default_code)`, NOT the Odoo id — unlike a CUSTOMER row, + # whose pid IS the partner id. That asymmetry is exactly why this cannot be derived + # downstream in `aios_grid.py` and has to be carried from the source read. out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code, - 'category': code_cat.get(code, '(uncategorized)')}) + 'category': code_cat.get(code, '(uncategorized)'), + 'id': p['id']}) return out diff --git a/web/public/brand/SOURCES.md b/web/public/brand/SOURCES.md new file mode 100644 index 0000000000000000000000000000000000000000..1b99c7befe8aa464eb42569555a375ab4132c591 --- /dev/null +++ b/web/public/brand/SOURCES.md @@ -0,0 +1,83 @@ +# Connector brand marks — where every file came from + +**Wave 33 · owner item 6 · ruling R4 · ticket `W33-T60`. Fetched 2026-08-14.** + +> Owner, verbatim: *"Do not generate your own logo for Connectors, they look really uncanny +> /browse them the PNG, cut appropriately for our needs, verify visually"*. + +⛔ **EVERY FILE IN THIS DIRECTORY IS THE VENDOR'S OWN ARTWORK, FETCHED FROM A HOST THE VENDOR +CONTROLS.** Nothing here is drawn, traced, redrawn or "inspired by". That is the whole point of the +ruling: the marks that shipped before this wave were hand-authored *simplifications* — honest about +being simplifications in their own header comment, and the owner still read them as uncanny. **A +hand-drawn substitute filed here as official artwork would be the violation**, so a brand we cannot +obtain officially keeps its hand-drawn mark and is listed under MISSES below rather than faked. + +⚠ **`webhooks` and `scraper` are deliberately NOT here.** They are house glyphs in `currentColor` +(`connectors/brandMarks.tsx`) because they are *ours* — there is no vendor and no official artwork to +fetch. Adding a picture for either would be inventing a brand, which is the thing this file exists to +prevent. + +## The cut (identical for all nine, so the marks sit on one grid) + +`_cut_brand.py`, beside these files (the `_` prefix is what keeps it out of the deploy — `deploy_web. +py::_web_source_files` skips any path PART starting with one, so the recipe lives with the artwork +without riding into the image). Four steps: load RGBA → *(lockups only)* keep the leftmost ink block → +trim to the alpha bounding box → fit the longest edge to **116 px** and centre it on a transparent +**128×128** canvas. So every mark has the same optical size and the same 6 px breathing room, which +is what R4's *"trimmed to a consistent square"* means in practice. **The originals are never +modified** — the transform reads the vendor's file and writes a new one. + +⚠ **Two marks carry their OWN ground and that is the vendor's design, not our padding**: TikTok's app +icon is a black rounded square and Stripe's press icon is a solid `#635BFF` square. They are left +exactly as the vendor draws them — squaring them onto a transparent canvas would be redrawing the +mark, and both already read correctly on a light *and* a dark chip, which is the property the +hand-drawn set was built for. + +## The nine + +| key | file | vendor's brand/press page | the exact asset fetched | what was cut | +|---|---|---|---|---| +| `instagram` | `instagram.png` | https://about.meta.com/brand/resources/instagram/instagram-brand/ | `IG_brand_asset_pack_2023.zip` → `01 Static Glyph/01 Gradient Glyph/Instagram_Glyph_Gradient.png` (5000×5000) | trim + fit only | +| `tiktok` | `tiktok.png` | https://www.tiktok.com/about/brand | `https://www.tiktok.com/apple-touch-icon.png` (400×400) | trim + fit only | +| `odoo` | `odoo.png` | https://www.odoo.com/page/brand-assets | `https://odoocdn.com/openerp_website/static/src/img/assets/png/odoo_logo.png` (621×196) | **the leading `o`**, isolated by Odoo's own brand purple `#714B67` — see the note below | +| `google` | `google.png` | https://about.google/brand-resource-center/ | `https://www.google.com/images/branding/googleg/1x/googleg_standard_color_128dp.png` (128×128) | trim + fit only | +| `gmail` | `gmail.png` | https://workspace.google.com/ (gstatic product branding) | `https://www.gstatic.com/images/branding/product/2x/gmail_2020q4_48dp.png` (96×96) | trim + fit only | +| `slack` | `slack.png` | https://slack.com/media-kit | `https://a.slack-edge.com/80588/marketing/img/icons/icon_slack_hash_colored.png` (400×400) | trim + fit only | +| `meta` | `meta.png` | https://about.meta.com/brand/resources/meta/company-brand/ | `Meta_Company-Lockup.zip` → `1 Positive Primary/RGB/Meta_lockup_positive primary_RGB.png` (29011×12501) | **first-block crop**: the pack ships the *lockup* (glyph + wordmark); the glyph is the leftmost ink block, 6272 px wide | +| `stripe` | `stripe.png` | https://stripe.com/newsroom/brand-assets | `https://images.ctfassets.net/fzn2n1nzq965/HTTOloNPhisV9P4hlMPNA/…/Stripe_icon_-_square.svg` (400×400, rasterised at 1024) | trim + fit only | +| `shopify` | `shopify.png` | https://www.shopify.com/brand-assets | `shopify-shopping-bag.zip` → `02 - Glyph/png/shopify_glyph.png` (600×683) | trim + fit only | + +## ⚠ The one judgement call, stated rather than buried — `odoo` + +**Odoo publishes no square glyph.** Its brand-assets page offers the wordmark (`odoo_logo.png`, its +inverted twin, and an SVG) and ~200 *app* icons — and the app icons are not the company mark: the one +served by `odoo.com/web/image/website/1/logo` is a grey camera placeholder and +`icons/base/…/icon.png` is a teal cube, both of which were fetched, looked at, and rejected here. A +621×196 wordmark fitted into a 128 px square renders as an illegible smear at the 15 px the chips +actually use. + +So the cut is **the leading `o`**, isolated from the official PNG by selecting the pixels in Odoo's +own brand purple `#714B67` — which is exactly the part of that wordmark Odoo draws in the brand colour +while the trailing `doo` is grey `#8f8f8f` (visible in the official SVG as two separate `` +elements). It is a CUT of the vendor's artwork, at the vendor's colour, which is the owner's +instruction; it is not a redraw. **If a square Odoo glyph is ever published, this is the one row in +the table to revisit.** + +## MISSES — none + +All nine marks in `BRAND_OF_CONNECTOR` that name a real company were obtained. Recorded explicitly +because "no misses" has to be a stated finding rather than an absence a reader has to infer — the +standing rule (wave-30 R6) is that a limit which cannot be removed is *reported* with its cause and a +recommended fix, and its second sentence is the half that gets dropped. + +## How to add one + +1. Find the vendor's own brand/press page and take the **glyph** (not the lockup) at the largest size + offered — SVG if there is one, PNG otherwise. +2. Drop it beside `_cut_brand.py`, add a `JOBS` row (second element `True` if it is a lockup), + run it, and **look at the contact sheet on both grounds** before committing. +3. Add the row to the table above with the page URL, the asset URL, and what was cut. +4. Add the key to `BRAND_OF_CONNECTOR` in `connectors/brandMarks.tsx`. + +⛔ Steps 3 and 4 are not optional and `aios-web/api/verify_meta.py` checks both: a file with no row +here has no provenance, and a file with no map entry is served by a door nothing asks. diff --git a/web/public/brand/gmail.png b/web/public/brand/gmail.png new file mode 100644 index 0000000000000000000000000000000000000000..5cce2656c1d9e0746a6f00e96d781368d9b24b58 Binary files /dev/null and b/web/public/brand/gmail.png differ diff --git a/web/public/brand/google.png b/web/public/brand/google.png new file mode 100644 index 0000000000000000000000000000000000000000..0ce59803b5242a4b3fe5f888fec980b91a7205cb Binary files /dev/null and b/web/public/brand/google.png differ diff --git a/web/public/brand/instagram.png b/web/public/brand/instagram.png new file mode 100644 index 0000000000000000000000000000000000000000..7b7b8851ebf529ab418affb179d5af61af3c8725 Binary files /dev/null and b/web/public/brand/instagram.png differ diff --git a/web/public/brand/meta.png b/web/public/brand/meta.png new file mode 100644 index 0000000000000000000000000000000000000000..3c7ce66a8cd8ee4a38c11ec4c23bd7ac055d92bb Binary files /dev/null and b/web/public/brand/meta.png differ diff --git a/web/public/brand/odoo.png b/web/public/brand/odoo.png new file mode 100644 index 0000000000000000000000000000000000000000..97a9500232d403dc9bce81396431376fc5867938 Binary files /dev/null and b/web/public/brand/odoo.png differ diff --git a/web/public/brand/shopify.png b/web/public/brand/shopify.png new file mode 100644 index 0000000000000000000000000000000000000000..2fca99069d85ef6026aa9fe3d1a7b715714948c8 Binary files /dev/null and b/web/public/brand/shopify.png differ diff --git a/web/public/brand/slack.png b/web/public/brand/slack.png new file mode 100644 index 0000000000000000000000000000000000000000..32e84c0eeaa2c75c90161f456a03c7d86ef92b79 Binary files /dev/null and b/web/public/brand/slack.png differ diff --git a/web/public/brand/stripe.png b/web/public/brand/stripe.png new file mode 100644 index 0000000000000000000000000000000000000000..a4711636729d4a7b4bebfa0a57915fff468049ff Binary files /dev/null and b/web/public/brand/stripe.png differ diff --git a/web/public/brand/tiktok.png b/web/public/brand/tiktok.png new file mode 100644 index 0000000000000000000000000000000000000000..59bde802b20635acdd9483df477f09aa7f5118f3 Binary files /dev/null and b/web/public/brand/tiktok.png differ diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index 22262f3b280cd1d73fe8393d0fbce2d1be869b22..3d70964f7782c4c55380f593138cbd7aae8de7a8 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -9,12 +9,25 @@ "default": true, "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months." }, + { + "key": "partner_id", + "label": "Odoo ID", + "type": "int", + "source": "odoo", + "derived": true, + "default": false, + "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source." + }, { "key": "odoo_status", "label": "Odoo record", "type": "status", "source": "odoo", "default": false, + "options": [ + "Active", + "Archived" + ], "description": "Whether this customer still exists in Odoo. Archived means deleted there." }, { diff --git a/web/src/account/FeedbackPage.tsx b/web/src/account/FeedbackPage.tsx new file mode 100644 index 0000000000000000000000000000000000000000..c301215e32d99d072e96cd1ca0e43cd4d8cd6f06 --- /dev/null +++ b/web/src/account/FeedbackPage.tsx @@ -0,0 +1,246 @@ +// --------------------------------------------------------------------------- +// account/FeedbackPage.tsx — WAVE 35 · W35-T18 (owner item 13, ruling R8, C6). +// +// One composer, one category, one send. R8 puts every submission on the +// platform-wide operator plane: no tenant sees another's, and the submitting +// tenant cannot edit or delete what it sent — so there is deliberately no "my +// feedback" list here, and the 201 IS the receipt. +// +// ⛔ THE SHAPE IS THE ASSISTANT'S CENTRED COLUMN, READ AND NOT IMPORTED. +// `assistant/assistant.css` belongs to another session this wave, so the anatomy +// is borrowed (a measure-bounded column; a rounded field with the control row +// inside it; a round send button that goes dark when there is something to send) +// while every rule here is `.acct-*` in this module's own stylesheet. Same system +// to the eye, no shared file to collide over. +// +// ⛔ FOUR STATES, AND THE FIRST TWO ARE THE ONES THAT GET SKIPPED. The category +// list is the SERVER'S (B-1/E-9) — there is no local copy to fall back to — so +// while it is in flight there is no form, and if it FAILS there is a sentence and +// a retry, never an empty dropdown and never an invented list. A page that offers +// a category the door refuses is worse than a page that says it cannot load. +// +// ⚠ THE SURFACE IS SPLIT FROM THE READ, and the reason is evidence rather than +// taste: `renderToStaticMarkup` never runs an effect, so a shot of the fetching +// component can only photograph its own spinner. With the whole state as ONE +// prop, every state this page has — loading, failed, ready, sending, sent, +// refused — can be rendered and looked at without a server, which is the only +// evidence a lane with no server can produce. +// --------------------------------------------------------------------------- + +import { useCallback, useEffect, useRef, useState } from "react"; + +import { loadFeedbackForm, sendFeedback } from "./accountApi"; +import type { FeedbackForm } from "./accountApi"; +import "./account.css"; + +/** The send arrow, in the same stroke vocabulary as the rest of the app's small glyphs. */ +function SendIcon() { + return ( + + ); +} + +export type FeedbackLoad = + | { phase: "loading" } + | { phase: "ready"; form: FeedbackForm } + | { phase: "failed"; message: string }; + +/** Everything this page holds, in one value, so the surface below is a function of it. */ +export interface FeedbackState { + load: FeedbackLoad; + category: string; + text: string; + sending: boolean; + /** The last send SUCCEEDED. Cleared the moment the person types again. */ + sent: boolean; + /** The last send was REFUSED, and this is the server's own sentence. */ + error: string; +} + +export const EMPTY_FEEDBACK: FeedbackState = { + load: { phase: "loading" }, + category: "", + text: "", + sending: false, + sent: false, + error: "", +}; + +export function FeedbackSurface({ + state, + onText, + onCategory, + onSend, + onRetry, +}: { + state: FeedbackState; + onText: (text: string) => void; + onCategory: (key: string) => void; + onSend: () => void; + onRetry: () => void; +}) { + const { load, category, text, sending, sent, error } = state; + const form = load.phase === "ready" ? load.form : null; + const tooLong = !!form && form.maxChars > 0 && text.length > form.maxChars; + const canSend = !!form && !!category && text.trim() !== "" && !tooLong && !sending; + + return ( +
+

Feedback

+
+ {load.phase === "loading" ? ( +
+ +
+ ) : load.phase === "failed" ? ( + // ⛔ NOT AN EMPTY FORM. The categories are the server's and there is no local copy, so + // a failed read means there is nothing to offer; drawing the composer anyway would + // invite somebody to write a paragraph into a control that cannot send it. +
+

{load.message}

+ +
+ ) : ( + <> + {sent ? ( + // The confirmation. It sits ABOVE the composer rather than replacing it, because + // the next thought is often the same person's, and a page that has to be navigated + // back to is a page that collects less of what R8 exists to collect. +

+ Thanks. That went to the Loopable team. +

+ ) : null} + {error ? ( +

+ {error} +

+ ) : null} + +
+